notes-structured-text-json-messages 0.1.1

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/.rspec

@@ -0,0 +1 @@
+--color

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2011 Trampoline Systems Ltd
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,29 @@
+= notes-structured-text-json-messages
+
+A command-line tool for transforming Lotus Notes Structured Text exports into JSON message structures for SONAR import
+
+ notes_structured_text_json_messages <output_dir> <input_file> [<input_file>*]
+
+= Dependencies
+ * actionmailer ~> 2.3.11
+ * RubyGems
+
+= Install
+
+ gem install notes-structured-text-json-messages
+
+== Contributing to notes-structured-text-json-messages
+ 
+* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
+* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it
+* Fork the project
+* Start a feature/bugfix branch
+* Commit and push until you are happy with your contribution
+* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
+* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
+
+== Copyright
+
+Copyright (c) 2011 Trampoline Systems Ltd. See LICENSE.txt for
+further details.
+

data/Rakefile

@@ -0,0 +1,47 @@
+require 'rubygems'
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "notes-structured-text-json-messages"
+  gem.homepage = "http://github.com/trampoline/notes-structured-text-json-messages"
+  gem.license = "MIT"
+  gem.summary = %Q{produces json message descriptions from lotus notes structured text exports}
+  gem.description = %Q{parses lotus notes structured text exports, producing a json message file for each each message}
+  gem.email = "craig@trampolinesystems.com"
+  gem.authors = ["craig mcmillan"]
+  # Include your dependencies below. Runtime dependencies are required when using your gem,
+  # and development dependencies are only needed for development (ie running rake tasks, tests, etc)
+  #  gem.add_runtime_dependency 'jabber4r', '> 0.1'
+  #  gem.add_development_dependency 'rspec', '> 1.2.3'
+  gem.add_runtime_dependency "actionmailer", "~> 2.3.11"
+  gem.add_development_dependency "rspec", "~> 1.3.0"
+  gem.add_development_dependency "jeweler", "~> 1.5.2"
+  gem.add_development_dependency "rcov", ">= 0"
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'spec/rake/spectask'
+Spec::Rake::SpecTask.new(:spec) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.spec_files = FileList['spec/**/*_spec.rb']
+end
+
+Spec::Rake::SpecTask.new(:rcov) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+task :default => :spec
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "notes-structured-text-json-messages #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.1.1

data/bin/notes_structured_text_json_messages

@@ -0,0 +1,41 @@
+#!/usr/bin/env ruby
+$: << File.expand_path("../../lib", __FILE__)
+require 'rubygems'
+require 'action_mailer'
+require 'notes_structured_text_json_messages'
+require 'optparse'
+require 'logger'
+
+options={:verbose=>true}
+
+USAGE = "Usage: #{File.basename(__FILE__)} <output_dir> <input_files> [<input_file>]* "
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: #{File.basename(__FILE__)} <output_dir> <input_files> [<input_file>]* "
+
+  opts.on("-v", "--[no-]verbose", "Run verbosely (default: true)") do |v|
+    options[:verbose] = v
+  end
+end.parse!
+
+NotesStructuredTextJsonMessages.logger = Logger.new($stderr)
+if options[:verbose]
+  NotesStructuredTextJsonMessages.logger.level=Logger::INFO
+else
+  NotesStructuredTextJsonMessages.logger.level=Logger::WARN
+end
+  
+begin
+  raise "insufficient arguments" if ARGV.length<2
+
+  output_dir = ARGV[0]
+  input_files = ARGV[1..-1]
+
+  NotesStructuredTextJsonMessages.json_messages(output_dir, input_files, options)
+  NotesStructuredTextJsonMessages.log{|logger| logger.info("complete")}
+rescue Exception=>e
+  NotesStructuredTextJsonMessages.log{|logger| logger.error(e)}
+  NotesStructuredTextJsonMessages.log{|logger| logger.info(USAGE)}
+  exit(1)
+end
+exit(0)

data/lib/notes_structured_text_json_messages.rb

@@ -0,0 +1,228 @@
+require 'md5'
+require 'tmail'
+
+module NotesStructuredTextJsonMessages
+  class << self
+    attr_accessor :logger
+    attr_accessor :stats
+  end
+
+  module_function
+
+  def log
+    yield logger if logger
+  end
+
+  def reset_stats
+    self.stats={}
+  end
+
+  def increment_stats(key)
+    self.stats[key] = (self.stats[key]||0) + 1
+  end
+  
+  def json_messages(output_dir, input_files, options={})
+    reset_stats
+    [*input_files].each do |input_file|
+      File.open(input_file, "r") do |input|
+        json_messages_from_stream(output_dir, input, options)
+      end
+    end
+    stats.each do |k,v|
+      log{|logger| logger.info("#{k}: #{v}")}
+    end
+  end
+
+  def json_messages_from_stream(output_dir, input, options={})
+    block = nil
+    process_block(output_dir, block, options) while block=read_block(input)
+  end
+
+  def read_block(input)
+    return nil if input.eof?
+    block = []
+    begin
+      l = input.readline.chomp
+      block << l if l.length>0
+    end while !input.eof? && l != ""
+    block
+  end
+
+  def is_message_block?(block)
+    !!header_value(block, MESSAGE_ID)
+  end
+
+  def is_distinguished_name?(addr)
+    !!(addr =~ /CN=/)
+  end
+
+  def header_value(block, header)
+    patt = /^#{Regexp.quote(header)}: /i
+    h = block.find{|l| l =~ patt}
+    h.gsub(patt, '').strip if h
+  end
+
+  def header_values(block, header, split_on=",")
+    h = header_value(block, header)
+    if h
+      if split_on.is_a?(Symbol)
+        self.send(split_on, h)
+      else
+        h.split(split_on)
+      end.map(&:strip)
+    end
+  end
+
+  def split_rfc822_addresses(header)
+    addresses = []
+    quoted_pair = false
+    quoted_string = false
+    buf = ""
+    header.each_char do |c|
+      if quoted_pair
+        buf << c
+        quoted_pair = false
+      elsif quoted_string && c=='\\'
+        buf << c
+        quoted_pair = true
+      elsif !quoted_string && c==','
+        addresses << buf
+        buf = ""
+      elsif !quoted_string && c=='"'
+        buf << c
+        quoted_string = true
+      elsif quoted_string && c=='"'
+        buf << c
+        quoted_string = false
+      else
+        buf << c
+      end
+    end
+    addresses << buf if buf.length>0
+    addresses
+  end
+  
+  def strip_angles(value)
+    value.gsub(/<([^>]*)>/, '\1')
+  end
+
+  def process_block(output_dir, block, options={})
+    if is_message_block?(block)
+      json_message = extract_json_message(block, options)
+      output_json_message(output_dir, json_message)
+      increment_stats(:message)
+    else
+      increment_stats(:non_message)
+    end
+  rescue Exception=>e
+    increment_stats(:failed_message)
+    log do |logger| 
+      logger.error(e)
+      logger.error(block.join("\n"))
+    end
+  end
+
+  MESSAGE_ID = "$MessageID"
+  POSTED_DATE = "PostedDate"
+  IN_REPLY_TO = "in_reply_to"
+  REFERENCES = "references"
+  FROM = "From"
+  TO = "SendTo"
+  CC = "CopyTo"
+  BCC = "BlindCopyTo"
+  INET_FROM = "InetFrom"
+  INET_TO = "InetSendTo"
+  INET_CC = "InetCopyTo"
+  INET_BCC = "InetBlindCopyTo"
+  
+  def process_address(addr)
+    if is_distinguished_name?(addr)
+      name = addr[/CN=([^\/]*)/, 1]
+      { :name=>name,
+        :notes_dn=>addr}
+    else
+      ta = TMail::Address.parse(addr)
+      if ta.is_a?(TMail::Address)
+        { :name=>ta.name,
+          :email_address=>ta.address.downcase}
+      else
+        log{|logger| logger.warn("addr does not parse to a TMail::Address: #{addr}")}
+      end
+    end
+  end
+
+  def process_address_pair(inet_addr, notes_addr)
+    if inet_addr == "."
+      process_address(notes_addr)
+    else
+      process_address(inet_addr)
+    end
+  end
+
+  def process_addresses(block, inet_field, notes_field)
+    inet_h = header_values(block, inet_field, :split_rfc822_addresses)
+    notes_h = header_values(block, notes_field, :split_rfc822_addresses)
+
+    if inet_h && notes_h
+      if inet_h.length == notes_h.length
+        inet_h.zip(notes_h).map do |inet_addr, notes_addr|
+          process_address_pair(inet_addr, notes_addr)
+        end
+      else
+        raise "#{inet_field}: does not match #{notes_field}:"
+      end
+    elsif inet_h
+      inet_h.map{|addr| process_address(addr)}
+    elsif notes_h
+      notes_h.map{|addr| process_address(addr)}
+    else
+      nil
+    end
+  end
+
+  NOTES_US_DATE_FORMAT = "%m/%d/%Y %I:%M:%S %p"
+
+  def parse_date(date, options={})
+    DateTime.strptime(date, NOTES_US_DATE_FORMAT)
+  end
+
+  def extract_json_message(block, options={})
+    message_id_h = header_value(block, MESSAGE_ID) 
+    raise "no #{MESSAGE_ID}" if !message_id_h
+    message_id = strip_angles(message_id_h)
+
+    posted_date_h = header_value(block, POSTED_DATE)
+    raise "no #{POSTED_DATE}" if !posted_date_h
+    posted_date = parse_date(posted_date_h, options)
+
+    in_reply_to_h = header_value(block, IN_REPLY_TO)
+    in_reply_to = strip_angles(in_reply_to_h) if in_reply_to_h
+
+    references_h = header_values(block, REFERENCES, " ")
+    references = references_h.map{|r| strip_angles(r)} if references_h
+
+    froms = process_addresses(block, INET_FROM, FROM)
+    raise "no From:, or more than one From:" if !froms || froms.size>1
+    from = froms[0]
+    to = process_addresses(block, INET_TO, TO)
+    cc = process_addresses(block, INET_CC, CC)
+    bcc = process_addresses(block, INET_BCC, BCC)
+
+    raise "no recipients" if (to||[]).size + (cc||[]).size + (bcc||[]).size == 0
+    
+    { :message_type=>"email",
+      :message_id=>message_id,
+      :sent_at=>posted_date,
+      :in_reply_to=>in_reply_to,
+      :references=>references,
+      :from=>from,
+      :to=>to,
+      :cc=>cc,
+      :bcc=>bcc}
+  end
+
+  def output_json_message(output_dir, json_message)
+    fname = File.join(output_dir, MD5.hexdigest(json_message[:message_id]))
+    File.open(fname, "w"){|out| out << json_message.to_json}
+  end
+end

data/spec/notes_structured_text_json_messages_spec.rb

@@ -0,0 +1,507 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe NotesStructuredTextJsonMessages do
+  describe "json_messages" do
+    it "should open each file and call json_messages_from_stream with it" do
+      output_dir = Object.new
+      input_files = [Object.new, Object.new]
+      options = Object.new
+
+      input0 = Object.new
+      input1 = Object.new
+      mock(File).open(input_files[0], "r"){|f,opts,block| block.call(input0)}
+      mock(File).open(input_files[1], "r"){|f,opts,block| block.call(input1)}
+      
+      mock(NotesStructuredTextJsonMessages).json_messages_from_stream(output_dir, input0, options)
+      mock(NotesStructuredTextJsonMessages).json_messages_from_stream(output_dir, input1, options)
+
+      NotesStructuredTextJsonMessages.json_messages(output_dir, input_files, options)
+    end
+  end
+
+  describe "json_messages_from_stream" do
+    it "should call process_block for each block retrieved from the stream" do
+      output_dir = Object.new
+      input = Object.new
+      blocks = [nil, ["foo", "bar"], ["baz", "boo"]]
+
+      mock(NotesStructuredTextJsonMessages).read_block(input).times(3){blocks.pop}
+
+      mock(NotesStructuredTextJsonMessages).process_block(output_dir, ["baz", "boo"], anything)
+      mock(NotesStructuredTextJsonMessages).process_block(output_dir, ["foo", "bar"], anything)
+
+      NotesStructuredTextJsonMessages.json_messages_from_stream(output_dir, input)
+    end
+  end
+
+  describe "readblock" do
+    it "should return lines read from a stream until the first empty line" do
+      input = <<-EOF
+foo
+bar
+
+baz
+boo
+EOF
+      io = StringIO.new(input)
+      NotesStructuredTextJsonMessages.read_block(io).should == ["foo", "bar"]
+      NotesStructuredTextJsonMessages.read_block(io).should == ["baz", "boo"]
+    end
+
+    it "should return nil if the input stream is at EOF" do
+      io = StringIO.new("foo\nbar")
+      NotesStructuredTextJsonMessages.read_block(io).should == ["foo", "bar"]
+      NotesStructuredTextJsonMessages.read_block(io).should == nil
+    end
+  end
+
+  describe "is_distinguished_name?" do
+    it "should return true if the address contains a CN=... string" do
+      NotesStructuredTextJsonMessages.is_distinguished_name?("CN=foo bar/OU=before/O=after").should == true
+    end
+
+    it "should return false if there is no CN=... " do
+      NotesStructuredTextJsonMessages.is_distinguished_name?("foo@bar.com").should == false
+    end
+  end
+
+  describe "is_message_block?" do
+    it "should return true if the block contains a line which start with '$MessageID: ' " do
+      NotesStructuredTextJsonMessages.is_message_block?( ["foo", "$MessageID: bar", "baz"] ).should == true
+    end
+
+    it "should return false if there are no lines starting with '$MessageID: ' in the block" do
+      NotesStructuredTextJsonMessages.is_message_block?( ["foo", "bar", "baz"] ).should == false
+    end
+
+    it "should not be case-sensitive" do
+      NotesStructuredTextJsonMessages.is_message_block?( ["foo", "$MESSAGEID: bar", "baz"] ).should == true
+      NotesStructuredTextJsonMessages.is_message_block?( ["foo", "$messageID: bar", "baz"] ).should == true
+      NotesStructuredTextJsonMessages.is_message_block?( ["foo", "$messageid: bar", "baz"] ).should == true
+    end
+  end
+
+  describe "header_value" do
+    it "should extract the first occurence of a header from the block" do
+      NotesStructuredTextJsonMessages.header_value(["Foo: foo", "Bar: bar", "Baz: baz"], "Foo").should == "foo"
+    end
+
+    it "should strip whitespace" do
+      NotesStructuredTextJsonMessages.header_value(["Foo:   foo   ", "Bar: bar", "Baz: baz"], "Foo").should == "foo"
+    end
+    it "should return nil if there are no occurences of the heaer in the block" do
+      NotesStructuredTextJsonMessages.header_value(["Boo: foo", "Bar: bar", "Baz: baz"], "Foo").should == nil
+    end
+
+    it "should not be case-sensitive" do
+      NotesStructuredTextJsonMessages.header_value(["fOO: foo", "Bar: bar", "Baz: baz"], "Foo").should == "foo"
+    end
+  end
+
+  describe "header_values" do
+    it "should extract an array of values for the header if present" do
+      NotesStructuredTextJsonMessages.header_values(["Foo: foo", "Bar: bar", "Baz: baz"], "Foo").should == ["foo"]
+      NotesStructuredTextJsonMessages.header_values(["Foo: a,b,c", "Bar: bar", "Baz: baz"], "Foo").should == ["a", "b", "c"]
+    end
+
+    it "should return nil if there are no occurences of the header" do
+      NotesStructuredTextJsonMessages.header_values(["Boo: a,b,c", "Bar: bar", "Baz: baz"], "Foo").should == nil
+    end
+
+    it "should strip whitespace" do
+      NotesStructuredTextJsonMessages.header_values(["Foo:   a ,\tb  ,  c  ", "Bar: bar", "Baz: baz"], "Foo").should == ["a", "b", "c"]
+    end
+
+    it "should strip on a given character" do
+      NotesStructuredTextJsonMessages.header_values(["Foo: a b c", "Bar: c d e", "Baz: f g h"], "Bar", " ").should == ["c", "d", "e"]
+    end
+
+    it "should strip with a given method" do
+      NotesStructuredTextJsonMessages.header_values(['SendTo: "mcfoo, foo" <foo.mcfoo@foo.com>,"bar \\"barry\\" mcbar" <bar.mcbar@bar.com>'],
+                                                    "SendTo",
+                                                    :split_rfc822_addresses).should ==
+        ['"mcfoo, foo" <foo.mcfoo@foo.com>',
+         '"bar \\"barry\\" mcbar" <bar.mcbar@bar.com>']
+    end
+  end
+
+  describe "split_rfc822_addresses" do
+    it "should do nothing to a single address" do
+      NotesStructuredTextJsonMessages.split_rfc822_addresses('"foo mcfoo" <foo.mcfoo@foo.com>').should ==
+        ['"foo mcfoo" <foo.mcfoo@foo.com>']
+    end
+    
+    it "should split a simple case" do
+      NotesStructuredTextJsonMessages.split_rfc822_addresses('"foo mcfoo" <foo.mcfoo@foo.com>,"bar mcbar" <bar.mcbar@bar.com>').should ==
+        ['"foo mcfoo" <foo.mcfoo@foo.com>','"bar mcbar" <bar.mcbar@bar.com>']
+    end
+    
+    it "should split if there is a comma inside a quoted string" do
+      NotesStructuredTextJsonMessages.split_rfc822_addresses('"mcfoo, foo" <foo.mcfoo@foo.com>,"bar mcbar" <bar.mcbar@bar.com>').should ==
+        ['"mcfoo, foo" <foo.mcfoo@foo.com>','"bar mcbar" <bar.mcbar@bar.com>']
+    end
+
+    it "should split if there is a quoted double-qoute within a string" do
+      NotesStructuredTextJsonMessages.split_rfc822_addresses('"foo \\"foo foo\\" mcfoo" <foo.mcfoo@foo.com>,"bar mcbar" <bar.mcbar@bar.com>').should ==
+        ['"foo \\"foo foo\\" mcfoo" <foo.mcfoo@foo.com>','"bar mcbar" <bar.mcbar@bar.com>']
+    end
+  end
+
+  describe "strip_angles" do
+    it "should remove angle-brackets if present" do
+      NotesStructuredTextJsonMessages.strip_angles("<foo>").should == "foo"
+    end
+
+    it "should do nothing if no angle brackets present" do
+      NotesStructuredTextJsonMessages.strip_angles("foo").should == "foo"
+    end
+
+    it "should do nothing if a single angle bracket present" do
+      NotesStructuredTextJsonMessages.strip_angles("<foo").should == "<foo"
+      NotesStructuredTextJsonMessages.strip_angles("foo>").should == "foo>"
+    end
+  end
+
+  describe "process_block" do
+    it "should output_json_message if is_message_block?" do
+      output_dir = Object.new
+      block = Object.new
+      json_message = Object.new
+
+      stub(NotesStructuredTextJsonMessages).is_message_block?(block){true}
+      mock(NotesStructuredTextJsonMessages).extract_json_message(block, anything){json_message}
+      mock(NotesStructuredTextJsonMessages).output_json_message(output_dir, json_message)
+
+      NotesStructuredTextJsonMessages.process_block(output_dir, block)
+    end
+
+    it "should ignore if !is_message_block?" do
+      output_dir = Object.new
+      block = Object.new
+
+      stub(NotesStructuredTextJsonMessages).is_message_block?(block){false}
+      dont_allow(NotesStructuredTextJsonMessages).extract_json_message.with_any_args
+      dont_allow(NotesStructuredTextJsonMessages).output_json_message.with_any_args
+
+      NotesStructuredTextJsonMessages.process_block(output_dir, block)
+    end
+
+    it "should catch and log exceptions during processing" do
+      output_dir = Object.new
+      block = ["foo", "bar"]
+      logger = Object.new
+
+      stub(NotesStructuredTextJsonMessages).logger{logger}
+      stub(NotesStructuredTextJsonMessages).is_message_block?(block){true}
+      stub(NotesStructuredTextJsonMessages).extract_json_message(block, anything){raise "boo"}
+
+      mock(logger).error(anything) do |err| 
+        err.is_a?(Exception).should == true
+        err.message.should =~ /boo/
+      end
+      mock(logger).error(block.join("\n"))
+
+      lambda {
+        NotesStructuredTextJsonMessages.process_block(output_dir, block)
+      }.should_not raise_error
+    end
+  end
+
+  describe "process_address" do
+    it "should produce a notes_dn hash if is_distinguished_name?" do
+      NotesStructuredTextJsonMessages.process_address("CN=foo bar/OU=here/O=there").should == 
+        {:name=>"foo bar", :notes_dn=>"CN=foo bar/OU=here/O=there"}
+    end
+
+    it "should be case-preserving for distinguished names" do
+      NotesStructuredTextJsonMessages.process_address("CN=Foo Bar/OU=Here/O=There").should == 
+        {:name=>"Foo Bar", :notes_dn=>"CN=Foo Bar/OU=Here/O=There"}
+    end
+
+    it "should parse with TMail::Address if !is_distinguished_name?" do
+      NotesStructuredTextJsonMessages.process_address('"foo bar" <foo@bar.com>').should ==
+        {:name=>"foo bar", :email_address=>"foo@bar.com"}
+    end
+
+    it "should downcase internet email addresses" do
+      NotesStructuredTextJsonMessages.process_address('"Foo Bar" <Foo@Bar.com>').should ==
+        {:name=>"Foo Bar", :email_address=>"foo@bar.com"}
+    end
+
+    it "should log a warning if an internet email address does not parse to a TMail::Address" do
+      logger = Object.new
+      stub(NotesStructuredTextJsonMessages).logger{logger}
+      
+      mock(logger).warn(/does not parse .* TMail::Address/)
+
+      NotesStructuredTextJsonMessages.process_address('Undisclosed recipients:;').should == nil
+    end
+  end
+
+  describe "process_address_pair" do
+    it "should process_address the notes_address if the inet_address is '.'" do
+      NotesStructuredTextJsonMessages.process_address_pair(".", "CN=foo bar/OU=here/O=there").should == 
+        {:name=>"foo bar", :notes_dn=>"CN=foo bar/OU=here/O=there"}
+    end
+
+    it "should process_address the inet_addr if the inet_address is not '.'" do
+      NotesStructuredTextJsonMessages.process_address_pair('"foo bar" <foo@bar.com>', "CN=foo bar/OU=here/O=there").should ==     
+        {:name=>"foo bar", :email_address=>"foo@bar.com"}
+    end
+  end
+
+  describe "process_addresses" do
+    it "should raise an exception if the notes and inet headers do not match" do
+      inet_field = Object.new
+      notes_field = Object.new
+      block = Object.new
+
+      stub(NotesStructuredTextJsonMessages).header_values(block, inet_field, :split_rfc822_addresses){["foo", "bar"]}
+      stub(NotesStructuredTextJsonMessages).header_values(block, notes_field, :split_rfc822_addresses){["foo"]}
+      
+      lambda {
+        NotesStructuredTextJsonMessages.process_addresses(block, inet_field, notes_field)
+      }.should raise_error(/does not match/)
+    end
+
+    it "should call process_address_pair if notes and inet headers are both present" do
+      inet_field = Object.new
+      notes_field = Object.new
+      block = Object.new
+
+      stub(NotesStructuredTextJsonMessages).header_values(block, inet_field, :split_rfc822_addresses){["foo.mcfoo@foo.com", "bar.mcbar@bar.com"]}
+      stub(NotesStructuredTextJsonMessages).header_values(block, notes_field, :split_rfc822_addresses){["CN=foo mcfoo/OU=main/O=foo", "CN=bar mcbar/OU=main/O=bar"]}
+
+      mock(NotesStructuredTextJsonMessages).process_address_pair("foo.mcfoo@foo.com", "CN=foo mcfoo/OU=main/O=foo")
+      mock(NotesStructuredTextJsonMessages).process_address_pair("bar.mcbar@bar.com", "CN=bar mcbar/OU=main/O=bar")
+
+      NotesStructuredTextJsonMessages.process_addresses(block, inet_field, notes_field)
+    end
+
+    it "should call process_address if an inet header is present" do
+      inet_field = Object.new
+      notes_field = Object.new
+      block = Object.new
+
+      stub(NotesStructuredTextJsonMessages).header_values(block, inet_field, :split_rfc822_addresses){["foo.mcfoo@foo.com", "bar.mcbar@bar.com"]}
+      stub(NotesStructuredTextJsonMessages).header_values(block, notes_field, :split_rfc822_addresses){nil}
+
+      mock(NotesStructuredTextJsonMessages).process_address("foo.mcfoo@foo.com")
+      mock(NotesStructuredTextJsonMessages).process_address("bar.mcbar@bar.com")
+
+      NotesStructuredTextJsonMessages.process_addresses(block, inet_field, notes_field)
+    end
+
+    it "should call process_address if a notes header is present" do
+      inet_field = Object.new
+      notes_field = Object.new
+      block = Object.new
+
+      stub(NotesStructuredTextJsonMessages).header_values(block, inet_field, :split_rfc822_addresses){nil}
+      stub(NotesStructuredTextJsonMessages).header_values(block, notes_field, :split_rfc822_addresses){["CN=foo mcfoo/OU=main/O=foo", "CN=bar mcbar/OU=main/O=bar"]}
+
+      mock(NotesStructuredTextJsonMessages).process_address("CN=foo mcfoo/OU=main/O=foo")
+      mock(NotesStructuredTextJsonMessages).process_address("CN=bar mcbar/OU=main/O=bar")
+
+      NotesStructuredTextJsonMessages.process_addresses(block, inet_field, notes_field)
+    end
+    
+  end
+
+  describe "output_json_message" do
+    it "should write the json encoded message to a file named by the MD5 of the message_id" do
+      output_dir = "/a/b/c/d"
+      json_message = Object.new
+      json_struct = {:message_id=>"foo123@foo.com"}
+
+      stub(json_message).[](:message_id){"foo123@foo.com"}
+      stub(json_message).to_json{json_struct.to_json}
+
+      output_stream = Object.new
+      mock(output_stream).<<(json_struct.to_json){output_stream}
+
+      mock(File).open("/a/b/c/d/#{MD5.hexdigest("foo123@foo.com")}", "w") do |fn,m,block|
+        block.call(output_stream)
+      end
+
+      NotesStructuredTextJsonMessages.output_json_message(output_dir, json_message)
+    end
+  end
+
+  describe "parse_date" do
+    it "should parse a US morning date correctly" do
+      dt = NotesStructuredTextJsonMessages.parse_date("01/25/2011 05:21:37 AM")
+      dt.is_a?(DateTime).should == true
+      dt.mday.should == 25
+      dt.mon.should == 01
+      dt.year.should == 2011
+      dt.hour.should == 5
+      dt.min.should == 21
+      dt.sec.should == 37
+      dt.zone.should == "+00:00"
+    end
+
+    it "should parse a US evening date correctly" do
+      dt = NotesStructuredTextJsonMessages.parse_date("12/01/2011 05:21:37 PM")
+      dt.is_a?(DateTime).should == true
+      dt.mday.should == 1
+      dt.mon.should == 12
+      dt.year.should == 2011
+      dt.hour.should == 17
+      dt.min.should == 21
+      dt.sec.should == 37
+      dt.zone.should == "+00:00"
+    end
+  end
+
+  describe "extract_json_message" do
+    def notes_message(options={})
+      h = { 
+        "$MessageID" => "<foo123@foo.com>",
+        "PostedDate" => "02/25/2011 08:06:10 PM",
+        "In_Reply_To" => "<bar456@bar.com>",
+        "References" => "<bar456@bar.com> <ear789@ear.com>",
+        "From" => "CN=foo mcfoo/OU=fooclub/O=foo",
+        "SendTo" => "CN=bar mcbar/OU=barclub/O=bar,CN=baz mcbaz/OU=bazclub/O=baz",
+        "CopyTo" => "CN=dar mcdar/OU=darclub/O=dar,CN=ear mcear/OU=earclub/O=ear",
+        "BlindCopyTo" => "CN=far mcfar/OU=farclub/O=far,CN=gar mcgar/OU=garclub/O=gar"}.merge(options)
+      h.map{|k,v| "#{k}:  #{v}" if v}
+    end
+
+    it "should raise an exception if there is no message-id" do
+      block = notes_message("$MessageID"=>nil)
+      lambda {
+        NotesStructuredTextJsonMessages.extract_json_message(block)
+      }.should raise_error(/no \$MessageID/)
+    end
+
+    it "should raise an exception if there is no From: or InetFrom:" do
+      block = notes_message("From"=>nil)
+     
+      lambda {
+        NotesStructuredTextJsonMessages.extract_json_message(block)
+      }.should raise_error(/no From/)
+    end
+
+    it "should raise an exception if there are no recipients" do
+      block = notes_message("SendTo"=>nil, "CopyTo"=>nil, "BlindCopyTo"=>nil)
+      
+      lambda {
+        NotesStructuredTextJsonMessages.extract_json_message(block)
+      }.should raise_error(/no recipients/)
+    end
+
+    it "should raise an exception if there is no PostedDate" do
+      block = notes_message("PostedDate"=>nil)
+     
+      lambda {
+        NotesStructuredTextJsonMessages.extract_json_message(block)
+      }.should raise_error(/no PostedDate/)
+    end
+
+    it "should remove angle-brackets from message_id, in_reply_to and references" do
+      block = notes_message
+      j = NotesStructuredTextJsonMessages.extract_json_message(block)
+      j[:message_id].should == "foo123@foo.com"
+    end
+
+    it "should parse a US date correctly" do
+      block = notes_message
+      j = NotesStructuredTextJsonMessages.extract_json_message(block)
+      d = j[:sent_at]
+      d.is_a?(DateTime).should == true
+      d.mday.should == 25
+      d.month.should == 2
+      d.year.should == 2011
+      d.hour.should == 20
+      d.min.should == 6
+      d.sec.should == 10
+    end
+
+    it "should parse the From / InetFrom fields" do
+      block = notes_message
+      j = NotesStructuredTextJsonMessages.extract_json_message(block)
+      j[:from].should == {:notes_dn=>"CN=foo mcfoo/OU=fooclub/O=foo", :name=>"foo mcfoo"}
+
+      block = notes_message("From"=>'"foo mcfoo" <foo.mcfoo@foo.com>')
+      j = NotesStructuredTextJsonMessages.extract_json_message(block)
+      j[:from].should == {:email_address=>"foo.mcfoo@foo.com", :name=>"foo mcfoo"}
+      
+      block = notes_message("InetFrom"=>'"foo mcfoo" <foo.mcfoo@foo.com>', "From"=>"CN=foo mcfoo/OU=fooclub/O=foo")
+      j = NotesStructuredTextJsonMessages.extract_json_message(block)
+      j[:from].should == {:email_address=>"foo.mcfoo@foo.com", :name=>"foo mcfoo"}
+
+      block = notes_message("InetFrom"=>'.', "From"=>'"foo mcfoo" <foo.mcfoo@foo.com>')
+      j = NotesStructuredTextJsonMessages.extract_json_message(block)
+      j[:from].should == {:email_address=>"foo.mcfoo@foo.com", :name=>"foo mcfoo"}
+    end
+
+    it "should parse the SendTo / InetSendTo fields" do
+      block = notes_message
+      j = NotesStructuredTextJsonMessages.extract_json_message(block)
+      j[:to].should == [{:notes_dn=>"CN=bar mcbar/OU=barclub/O=bar", :name=>"bar mcbar"},
+                        {:notes_dn=>"CN=baz mcbaz/OU=bazclub/O=baz", :name=>"baz mcbaz"}]
+
+      block = notes_message("SendTo"=>'"bar mcbar" <bar.mcbar@bar.com>,"baz mcbaz" <baz.mcbaz@baz.com>')
+      j = NotesStructuredTextJsonMessages.extract_json_message(block)
+      j[:to].should == [{:email_address=>"bar.mcbar@bar.com", :name=>"bar mcbar"},
+                        {:email_address=>"baz.mcbaz@baz.com", :name=>"baz mcbaz"}]
+
+      block = notes_message("InetSendTo"=>'"bar mcbar" <bar.mcbar@bar.com>,"baz mcbaz" <baz.mcbaz@baz.com>')
+      j = NotesStructuredTextJsonMessages.extract_json_message(block)
+      j[:to].should == [{:email_address=>"bar.mcbar@bar.com", :name=>"bar mcbar"},
+                        {:email_address=>"baz.mcbaz@baz.com", :name=>"baz mcbaz"}]
+
+      block = notes_message("InetSendTo"=>'.,"baz mcbaz" <baz.mcbaz@baz.com>', "SendTo"=>'"bar mcbar" <bar.mcbar@bar.com>,"CN=baz mcbaz/OU=bazclub/O=baz"')
+      j = NotesStructuredTextJsonMessages.extract_json_message(block)
+      j[:to].should == [{:email_address=>"bar.mcbar@bar.com", :name=>"bar mcbar"},
+                        {:email_address=>"baz.mcbaz@baz.com", :name=>"baz mcbaz"}]
+    end
+
+    it "should parse the CopyTo / InetCopyTo fields" do
+      block = notes_message
+      j = NotesStructuredTextJsonMessages.extract_json_message(block)
+      j[:cc].should == [{:notes_dn=>"CN=dar mcdar/OU=darclub/O=dar", :name=>"dar mcdar"},
+                        {:notes_dn=>"CN=ear mcear/OU=earclub/O=ear", :name=>"ear mcear"}]
+
+      block = notes_message("CopyTo" => '"dar mcdar" <dar.mcdar@dar.com>,"ear mcear" <ear.mcear@ear.com>')
+      j = NotesStructuredTextJsonMessages.extract_json_message(block)
+      j[:cc].should == [{:email_address=>"dar.mcdar@dar.com", :name=>"dar mcdar"},
+                        {:email_address=>"ear.mcear@ear.com", :name=>"ear mcear"}]
+
+      block = notes_message("InetCopyTo" => '"dar mcdar" <dar.mcdar@dar.com>,"ear mcear" <ear.mcear@ear.com>')
+      j = NotesStructuredTextJsonMessages.extract_json_message(block)
+      j[:cc].should == [{:email_address=>"dar.mcdar@dar.com", :name=>"dar mcdar"},
+                        {:email_address=>"ear.mcear@ear.com", :name=>"ear mcear"}]
+
+      block = notes_message("InetCopyTo" => '.,"ear mcear" <ear.mcear@ear.com>',
+                            "CopyTo" => '"dar mcdar" <dar.mcdar@dar.com>,CN=ear mcear/OU=earclub/O=ear')
+      j = NotesStructuredTextJsonMessages.extract_json_message(block)
+      j[:cc].should == [{:email_address=>"dar.mcdar@dar.com", :name=>"dar mcdar"},
+                        {:email_address=>"ear.mcear@ear.com", :name=>"ear mcear"}]
+    end
+
+    it "should parse the BlindCopyTo / InetBlindCopyTo fields" do
+      block = notes_message
+      j = NotesStructuredTextJsonMessages.extract_json_message(block)
+      j[:bcc].should == [{:notes_dn=>"CN=far mcfar/OU=farclub/O=far", :name=>"far mcfar"},
+                         {:notes_dn=>"CN=gar mcgar/OU=garclub/O=gar", :name=>"gar mcgar"}]
+
+      block = notes_message("BlindCopyTo" => '"far mcfar" <far.mcfar@far.com>,"gar mcgar" <gar.mcgar@gar.com>')
+      j = NotesStructuredTextJsonMessages.extract_json_message(block)
+      j[:bcc].should == [{:email_address=>"far.mcfar@far.com", :name=>"far mcfar"},
+                         {:email_address=>"gar.mcgar@gar.com", :name=>"gar mcgar"}]
+
+      block = notes_message("InetBlindCopyTo" => '"far mcfar" <far.mcfar@far.com>,"gar mcgar" <gar.mcgar@gar.com>')
+      j = NotesStructuredTextJsonMessages.extract_json_message(block)
+      j[:bcc].should == [{:email_address=>"far.mcfar@far.com", :name=>"far mcfar"},
+                         {:email_address=>"gar.mcgar@gar.com", :name=>"gar mcgar"}]
+
+      block = notes_message("InetBlindCopyTo" => '.,"gar mcgar" <gar.mcgar@gar.com>',
+                            "BlindCopyTo" => '"far mcfar" <far.mcfar@far.com>,CN=gar mcgar/OU=garclub/O=gar')
+      j = NotesStructuredTextJsonMessages.extract_json_message(block)
+      j[:bcc].should == [{:email_address=>"far.mcfar@far.com", :name=>"far mcfar"},
+                         {:email_address=>"gar.mcgar@gar.com", :name=>"gar mcgar"}]
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,12 @@
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'rubygems'
+require 'spec'
+require 'spec/autorun'
+require 'rr'
+require 'action_mailer'
+require 'notes_structured_text_json_messages'
+
+Spec::Runner.configure do |config|
+  config.mock_with RR::Adapters::Rspec
+end

metadata

@@ -0,0 +1,139 @@
+--- !ruby/object:Gem::Specification 
+name: notes-structured-text-json-messages
+version: !ruby/object:Gem::Version 
+  hash: 25
+  prerelease: 
+  segments: 
+  - 0
+  - 1
+  - 1
+  version: 0.1.1
+platform: ruby
+authors: 
+- craig mcmillan
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-04-26 00:00:00 +01:00
+default_executable: notes_structured_text_json_messages
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: actionmailer
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 21
+        segments: 
+        - 2
+        - 3
+        - 11
+        version: 2.3.11
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 27
+        segments: 
+        - 1
+        - 3
+        - 0
+        version: 1.3.0
+  type: :development
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: jeweler
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 7
+        segments: 
+        - 1
+        - 5
+        - 2
+        version: 1.5.2
+  type: :development
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: rcov
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id004
+description: parses lotus notes structured text exports, producing a json message file for each each message
+email: craig@trampolinesystems.com
+executables: 
+- notes_structured_text_json_messages
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE.txt
+- README.rdoc
+files: 
+- .document
+- .rspec
+- LICENSE.txt
+- README.rdoc
+- Rakefile
+- VERSION
+- bin/notes_structured_text_json_messages
+- lib/notes_structured_text_json_messages.rb
+- spec/notes_structured_text_json_messages_spec.rb
+- spec/spec_helper.rb
+has_rdoc: true
+homepage: http://github.com/trampoline/notes-structured-text-json-messages
+licenses: 
+- MIT
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.6.2
+signing_key: 
+specification_version: 3
+summary: produces json message descriptions from lotus notes structured text exports
+test_files: 
+- spec/notes_structured_text_json_messages_spec.rb
+- spec/spec_helper.rb