sonar_ews_pull_connector 0.2.0

data/.document

@@ -0,0 +1,5 @@
+README.rdoc
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Trampoline Systems Ltd
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,17 @@
+= sonar-ews-pull-connector
+
+Description goes here.
+
+== Note on Patches/Pull Requests
+ 
+* Fork the project.
+* Make your feature addition or bug fix.
+* Add tests for it. This is important so I don't break it in a
+  future version unintentionally.
+* Commit, do not mess with rakefile, version, or history.
+  (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
+* Send me a pull request. Bonus points for topic branches.
+
+== Copyright
+
+Copyright (c) 2011 Trampoline Systems Ltd. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,52 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "sonar_ews_pull_connector"
+    gem.summary = %Q{Exchange Web Services connector for Exchange 2007/2010}
+    gem.description = %Q{A sonar-connector for extracting emails from Exchange 2007/2010 through Exchange Web Services}
+    gem.email = "craig@trampolinesystems.com"
+    gem.homepage = "http://github.com/trampoline/sonar-ews-pull-connector"
+    gem.authors = ["mccraigmccraig"]
+    gem.add_dependency "sonar_connector", ">= 0.7.2"
+    gem.add_dependency "savon", ">= 0.8.6"
+    gem.add_dependency "ntlm-http", ">= 0.1.1"
+    gem.add_dependency "httpclient", ">= 2.1.6.1.1"
+    gem.add_dependency "fetch_in", ">= 0.2.0"
+    gem.add_dependency "rfc822_util", ">= 0.1.1"
+    gem.add_development_dependency "rspec", ">= 1.2.9"
+    gem.add_development_dependency "rr", ">= 0.10.5"
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+end
+
+require 'spec/rake/spectask'
+Spec::Rake::SpecTask.new(:spec) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.spec_files = FileList['spec/**/*_spec.rb']
+end
+
+Spec::Rake::SpecTask.new(:rcov) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+task :spec => :check_dependencies
+
+task :default => :spec
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "sonar-ews-pull-connector #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.2.0

data/lib/ews_pull_connector/ews_pull_connector.rb

@@ -0,0 +1,208 @@
+require 'date'
+require 'json'
+require 'sonar_connector'
+require 'rfc822_util'
+require 'base64'
+require 'md5'
+require 'rews'
+
+module Sonar
+  module Connector
+    class EwsPullConnector < Sonar::Connector::Base
+
+      MIN_BATCH_SIZE = 2
+      DEFAULT_BATCH_SIZE = 100
+
+      attr_accessor :url
+      attr_accessor :auth
+      attr_accessor :user
+      attr_accessor :password
+      attr_accessor :distinguished_folders
+      attr_accessor :batch_size
+      attr_accessor :delete
+      attr_accessor :is_journal
+
+      def parse(settings)
+        ["name", "repeat_delay", "url", "auth", "user", "password", "distinguished_folders", "batch_size"].each do |param|
+          raise Sonar::Connector::InvalidConfig.new("#{self.class}: param '#{param}' is blank") if settings[param].blank?
+        end
+        
+        @url = settings["url"]
+        @auth = settings["auth"]
+        @user = settings["user"]
+        @password = settings["password"]
+        @mailbox_email = settings["mailbox_email"]
+        @distinguished_folders = settings["distinguished_folders"]
+        @batch_size = [settings["batch_size"] || DEFAULT_BATCH_SIZE, MIN_BATCH_SIZE].max
+        @delete = !!settings["delete"]
+        @is_journal = !!settings["is_journal"]
+      end
+      
+      def inspect
+        "#<#{self.class} @url=#{url}, @auth=#{auth}, @user=#{user}, @password=#{password}, @distinguished_folders=#{distinguished_folders}, @batch_size=#{batch_size}, @delete=#{delete}, @is_journal=#{is_journal}>"
+      end
+
+      def distinguished_folder_ids
+        return @distinguished_folder_ids if @distinguished_folder_ids
+        client ||= Rews::Client.new(url, auth, user, password)
+
+        @distinguished_folder_ids = @distinguished_folders.inject([]) do |ids, (name, mailbox_email)|
+          ids << client.distinguished_folder_id(name, mailbox_email)
+        end
+      end
+
+      # find message ids from a folder
+      def find(folder_id, offset)
+        find_opts = {
+          :sort_order=>[["item:DateTimeReceived", "Ascending"]],
+          :indexed_page_item_view=>{
+            :max_entries_returned=>batch_size, 
+            :offset=>offset},
+          :item_shape=>{
+            :base_shape=>:IdOnly}}
+        
+        restriction = [:==, "item:ItemClass", "IPM.Note"]
+        if state[folder_id.key]
+          restriction = [:and,
+                         restriction,
+                         [:>= , "item:DateTimeReceived", state[folder_id.key]]]
+        end
+        find_opts[:restriction] =  restriction
+        
+        folder_id.find_item(find_opts)
+      end
+
+      def get(folder_id, msg_ids)
+        get_opts = {
+          :item_shape=>{
+            :base_shape=>:IdOnly, 
+            :additional_properties=>[[:field_uri, "item:ItemClass"],
+                                     [:field_uri, "item:DateTimeSent"],
+                                     [:field_uri, "item:DateTimeReceived"],
+                                     [:field_uri, "item:InReplyTo"],
+                                     [:field_uri, "message:InternetMessageId"],
+                                     [:field_uri, "message:References"],
+                                     [:field_uri, "message:From"],
+                                     [:field_uri, "message:Sender"],
+                                     [:field_uri, "message:ToRecipients"],
+                                     [:field_uri, "message:CcRecipients"],
+                                     [:field_uri, "message:BccRecipients"]]}}
+
+        # we have to retrieve the journal message content and unwrap the 
+        # original message if this
+        # is an exchange journal mailbox
+        if is_journal
+          get_opts[:item_shape][:additional_properties] << [:field_uri, "item:MimeContent"]
+        end
+        
+        folder_id.get_item(msg_ids, get_opts)
+      end
+
+      def action
+        distinguished_folder_ids.each do |fid|
+          log.info "processing: #{fid.inspect}"
+
+          offset = 0
+
+          begin
+            msg_ids = find(fid, offset)
+
+            if msg_ids && msg_ids.length>0
+              msgs = get(fid, msg_ids)
+
+              # if there is no state, then state is set to the first message timestamp
+              state[fid.key] ||= msgs.first[:date_time_received].to_s if msgs.first[:date_time_received]
+
+              save_messages(msgs)
+
+              if msgs.last[:date_time_received] != state[fid.key]
+                finished=true
+                state[fid.key] = msgs.last[:date_time_received].to_s
+              end
+              
+              delete_messages(fid, msgs) if delete
+
+              offset += msg_ids.length
+            end
+          end while msg_ids.length>0 && !finished
+
+          save_state
+
+          log.info "finished processing: #{fid.inspect}"
+        end
+        log.info "finished action"
+      end
+
+      def save_messages(messages)
+        messages.each do |msg|
+          h = if is_journal
+                extract_journalled_message(msg)
+              else
+                message_to_hash(msg)
+              end
+          
+          if !h
+            log.warn("no data extracted from message. could be a decoding eror")
+            return
+          end
+
+          h[:type] = "email"
+          h[:connector] = name
+          h[:source] = url
+          h[:source_id]=msg[:item_id][:id]
+          h[:received_at] = msg[:date_time_received]
+
+
+          fname = MD5.hexdigest(msg[:item_id][:id])
+          filestore.write(:complete, "#{fname}.json", h.to_json)
+        end
+      end
+
+      def mailbox_to_hash(mailbox)
+        [:name, :email_address].inject({}) do |h, k|
+          h[k] = mailbox[k]
+          h
+        end
+      end
+
+      def mailbox_recipients_to_hashes(recipients)
+        mailboxes = recipients[:mailbox] if recipients
+        mailboxes = [mailboxes] if !mailboxes.is_a?(Array)
+        mailboxes.compact.map{|a| mailbox_to_hash(a)}
+      end
+
+      def message_to_hash(msg)
+        message_id = Rfc822Util.strip_header(msg[:internet_message_id]) if msg[:internet_message_id]
+        in_reply_to = Rfc822Util.strip_headers(msg[:in_reply_to]).first if msg[:in_reply_to]
+        references = Rfc822Util.strip_headers(msg[:references]) if msg[:references]
+        
+          json_hash = {
+            :message_id=>message_id,
+            :sent_at=>msg[:date_time_sent].to_s,
+            :in_reply_to=>in_reply_to,
+            :references=>references,
+            :from=>mailbox_recipients_to_hashes(msg[:from]).first,
+            :sender=>mailbox_recipients_to_hashes(msg[:sender]).first,
+            :to=>mailbox_recipients_to_hashes(msg[:to_recipients]),
+            :cc=>mailbox_recipients_to_hashes(msg[:cc_recipients]),
+            :bcc=>mailbox_recipients_to_hashes(msg[:bcc_recipients])
+          }
+          
+      end
+
+      def extract_journalled_message(message)
+        mime_msg = Base64::decode64(message[:mime_content])
+        journal_msg = Rfc822Util.extract_journalled_mail(mime_msg)
+        Rfc822Util.mail_to_hash(journal_msg)
+      rescue Exception=>e
+        log.warn("problem extracting journalled message from wrapper message")
+        log.warn(e)
+      end
+
+      def delete_messages(folder_id, messages)
+        log.info "deleting #{messages.length} messages from #{folder_id.inspect}"
+        folder_id.delete_item(messages, :delete_type=>:HardDelete)
+      end
+    end
+  end
+end

data/lib/sonar_ews_pull_connector.rb

@@ -0,0 +1 @@
+require File.expand_path("../ews_pull_connector/ews_pull_connector", __FILE__)

data/spec/ews_pull_connector/ews_pull_connector_spec.rb

@@ -0,0 +1,490 @@
+require File.expand_path("../../spec_helper", __FILE__)
+
+module Sonar
+  module Connector
+    describe EwsPullConnector do
+
+      before do
+        setup_valid_config_file
+        @base_config = Sonar::Connector::Config.load(valid_config_filename)
+      end
+
+      def one_folder_config
+        {
+          'name'=>'foobarcom-exchange',
+          'repeat_delay'=>60,
+          'url'=>"https://foo.com/EWS/Exchange.asmx",
+          'auth'=>'ntlm',
+          'user'=>"foo",
+          'password'=>"foopass",
+          'distinguished_folders'=>[["inbox", "foo@foo.com"]],
+          'batch_size'=>100,
+          'delete'=>true
+        }
+      end
+
+      def two_folder_config
+        {
+          'name'=>'foobarcom-exchange',
+          'repeat_delay'=>60,
+          'url'=>"https://foo.com/EWS/Exchange.asmx",
+          'auth'=>'ntlm',
+          'user'=>"foo",
+          'password'=>"foopass",
+          'distinguished_folders'=>[["inbox", "foo@foo.com"], "inbox"],
+          'batch_size'=>100,
+          'delete'=>true
+        } 
+      end
+
+      it "should parse config" do
+        Sonar::Connector::EwsPullConnector.new(two_folder_config, @base_config)
+      end
+      
+      describe "distinguished_folder_ids" do
+        it "should create Rews::Clients for each configured distinguished folder" do
+          c=Sonar::Connector::EwsPullConnector.new(two_folder_config, @base_config)
+          fids = c.distinguished_folder_ids
+          fids.size.should == 2
+
+          fids[0].client.should be(fids[1].client)
+          client = fids[0].client
+          client.endpoint.should == "https://foo.com/EWS/Exchange.asmx"
+          client.auth_type.should == "ntlm"
+          client.user.should == 'foo'
+          client.password.should == 'foopass'
+          
+          fid0 = fids[0]
+          fid0.id.should == 'inbox'
+          fid0.mailbox_email.should == 'foo@foo.com'
+          fid0.key.should == ['distinguished_folder', 'inbox', 'foo@foo.com']
+
+          fid1 = fids[1]
+          fid1.id.should == 'inbox'
+          fid1.mailbox_email.should == nil 
+          fid1.key.should == ['distinguished_folder', 'inbox']
+        end
+
+        it "should cache the Rews::Clients" do
+          c=Sonar::Connector::EwsPullConnector.new(two_folder_config, @base_config)
+          fids = c.distinguished_folder_ids
+          fid0 = fids[0]
+          fid1 = fids[1]
+
+          fids = c.distinguished_folder_ids
+          fid0.should be(fids[0])
+          fid1.should be(fids[1])
+        end
+      end
+
+      describe "find" do
+        it "should include batch_size and offset but not item:DateTimeReceived restriction if there is no folder state" do
+          c=Sonar::Connector::EwsPullConnector.new(one_folder_config, @base_config)
+          state={}
+          stub(c.state){state}
+          stub(c).batch_size{17}
+
+          folder_id = Object.new
+          folder_key = Object.new
+          stub(folder_id).key{folder_key}
+
+          mock(folder_id).find_item.with_any_args do |find_opts|
+            find_opts.should == {
+              :sort_order=>[["item:DateTimeReceived", "Ascending"]],
+              :indexed_page_item_view=>{
+                :max_entries_returned=>17, 
+                :offset=>123},
+              :item_shape=>{
+                :base_shape=>:IdOnly},
+              :restriction=>[:==, "item:ItemClass", "IPM.Note"]}
+          end
+
+          c.find(folder_id, 123)
+        end
+
+        it "should include a item:DateTimeReceived restriction if there is folder state" do
+          c=Sonar::Connector::EwsPullConnector.new(one_folder_config, @base_config)
+          stub(c).batch_size{17}
+
+          folder_id = Object.new
+          folder_key = Object.new
+          stub(folder_id).key{folder_key}
+
+          state_time = DateTime.now.to_s
+          stub(c).state.stub!.[](folder_key){state_time}
+
+          mock(folder_id).find_item.with_any_args do |find_opts|
+            find_opts.should == {
+              :sort_order=>[["item:DateTimeReceived", "Ascending"]],
+              :indexed_page_item_view=>{
+                :max_entries_returned=>17, 
+                :offset=>123},
+              :item_shape=>{
+                :base_shape=>:IdOnly},
+              :restriction=>[:and,
+                             [:==, "item:ItemClass", "IPM.Note"],
+                             [:>=, "item:DateTimeReceived", state_time]]}
+          end
+
+          c.find(folder_id, 123)
+        end
+      end
+
+      describe "get" do
+        it "should not fetch message content if !is_journal" do
+          c=Sonar::Connector::EwsPullConnector.new(one_folder_config, @base_config)
+
+          folder_id = Object.new
+          msg_ids = Object.new
+
+          mock(folder_id).get_item.with_any_args do |mids, get_opts|
+            mids.should be(msg_ids)
+            get_opts.should == {
+              :item_shape=>{
+                :base_shape=>:IdOnly, 
+                :additional_properties=>[[:field_uri, "item:ItemClass"],
+                                         [:field_uri, "item:DateTimeSent"],
+                                         [:field_uri, "item:DateTimeReceived"],
+                                         [:field_uri, "item:InReplyTo"],
+                                         [:field_uri, "message:InternetMessageId"],
+                                         [:field_uri, "message:References"],
+                                         [:field_uri, "message:From"],
+                                         [:field_uri, "message:Sender"],
+                                         [:field_uri, "message:ToRecipients"],
+                                         [:field_uri, "message:CcRecipients"],
+                                         [:field_uri, "message:BccRecipients"]]}}
+          end
+
+          c.get(folder_id, msg_ids)
+        end
+
+        it "should fetch message content if is_journal" do
+          c=Sonar::Connector::EwsPullConnector.new(one_folder_config, @base_config)
+          stub(c).is_journal{true}
+
+          folder_id = Object.new
+          msg_ids = Object.new
+
+          mock(folder_id).get_item.with_any_args do |mids, get_opts|
+            mids.should be(msg_ids)
+            get_opts.should == {
+              :item_shape=>{
+                :base_shape=>:IdOnly, 
+                :additional_properties=>[[:field_uri, "item:ItemClass"],
+                                         [:field_uri, "item:DateTimeSent"],
+                                         [:field_uri, "item:DateTimeReceived"],
+                                         [:field_uri, "item:InReplyTo"],
+                                         [:field_uri, "message:InternetMessageId"],
+                                         [:field_uri, "message:References"],
+                                         [:field_uri, "message:From"],
+                                         [:field_uri, "message:Sender"],
+                                         [:field_uri, "message:ToRecipients"],
+                                         [:field_uri, "message:CcRecipients"],
+                                         [:field_uri, "message:BccRecipients"],
+                                         [:field_uri, "item:MimeContent"]]}}
+          end
+
+          c.get(folder_id, msg_ids)
+        end
+      end
+
+      describe "action" do
+        it "should make a Rews find_item request, save, update state, delete" do
+          c=Sonar::Connector::EwsPullConnector.new(two_folder_config, @base_config)
+          state = {}
+          stub(c).state{state}
+          
+          c.distinguished_folder_ids.each do |fid|
+            msg_ids = Object.new
+            stub(msg_ids).length{1}
+
+            msgs = Object.new
+            stub(msgs).first.stub!.[](:date_time_received){ DateTime.now-1 }
+            stub(msgs).last.stub!.[](:date_time_received){ DateTime.now }
+
+            mock(fid).find_item(anything){msg_ids}
+            mock(fid).get_item(msg_ids, anything){msgs}
+            
+            mock(c).save_messages(msgs)
+            mock(c).delete_messages(fid, msgs)
+          end
+          c.action
+        end
+
+        it "should have a single item:ItemClass Restriction clause if fstate is nil" do
+          c=Sonar::Connector::EwsPullConnector.new(one_folder_config, @base_config)
+          fid = c.distinguished_folder_ids.first
+
+          msg_ids = Object.new
+          stub(msg_ids).length{1}
+          mock(fid).find_item(anything) do |query_opts|
+            r = query_opts[:restriction]
+            r.should == [:==, "item:ItemClass", "IPM.Note"] 
+            msg_ids
+          end
+
+          msgs=Object.new
+          stub(msgs).first.stub!.[](:date_time_received){ DateTime.now-1 }
+          stub(msgs).last.stub!.[](:date_time_received){DateTime.now}
+
+          mock(fid).get_item(msg_ids, anything){msgs}
+          mock(c).save_messages(msgs)
+          mock(c).delete_messages(fid, msgs)
+
+          c.action
+        end
+        
+        it "should have a second item:DateTimeReceived Restriction clause if fstate is non-nil" do
+          c=Sonar::Connector::EwsPullConnector.new(one_folder_config, @base_config)
+
+          fid = c.distinguished_folder_ids.first
+
+          state_time = DateTime.now - 1
+          state = {fid.key => state_time.to_s}
+          stub(c).state{state}
+
+          msg_ids = Object.new
+          stub(msg_ids).length{1}
+          mock(fid).find_item(anything) do |query_opts|
+            r = query_opts[:restriction]
+            r[0].should == :and
+              r[1].should == [:==, "item:ItemClass", "IPM.Note"] 
+            r[2].should == [:>=, "item:DateTimeReceived", state_time.to_s]
+            msg_ids
+          end
+
+          msgs = Object.new
+          stub(msgs).first.stub!.[](:date_time_received){state_time}
+          stub(msgs).last.stub!.[](:date_time_received){DateTime.now}
+
+          mock(fid).get_item(msg_ids, anything){msgs}
+          mock(c).save_messages(msgs)
+          mock(c).delete_messages(fid, msgs)
+
+          c.action
+        end
+
+        it "should cycle through messages with identical item:DateTimeReceived, so state is always updated even if 'delete' option is false" do
+          c=Sonar::Connector::EwsPullConnector.new(one_folder_config, @base_config)
+          
+          fid = c.distinguished_folder_ids.first
+
+          state_time = (DateTime.now - 2).to_s
+          state = {fid.key => state_time}
+          stub(c).state{state}
+
+          msg_ids = Object.new
+          stub(msg_ids).length{10}
+          msgs = Object.new
+          stub(msgs).first.stub!.[](:date_time_received){state_time}
+          stub(msgs).last.stub!.[](:date_time_received){state_time}
+
+          more_msg_ids = Object.new
+          stub(more_msg_ids).length{1}
+          more_msgs = Object.new
+          stub(more_msgs).first.stub!.[](:date_time_received){state_time}
+          stub(more_msgs).last.stub!.[](:date_time_received){DateTime.now}
+
+          mock(fid).find_item.with_any_args.twice do |opts| 
+            if opts[:indexed_page_item_view][:offset]==0
+              msg_ids
+            elsif opts[:indexed_page_item_view][:offset]==10
+              more_msg_ids
+            else
+              raise "oops"
+            end
+          end
+
+          mock(fid).get_item(msg_ids, anything){msgs}
+          mock(c).save_messages(msgs)
+          mock(c).delete_messages(fid, msgs)
+
+          mock(fid).get_item(more_msg_ids, anything){more_msgs}
+          mock(c).save_messages(more_msgs)
+          mock(c).delete_messages(fid, more_msgs)
+
+          c.action
+        end
+
+        it "should terminate the fetch loop if no messages are returned" do
+          c=Sonar::Connector::EwsPullConnector.new(one_folder_config, @base_config)
+          
+          fid = c.distinguished_folder_ids.first
+
+          state_time = (DateTime.now - 1).to_s
+          state = {fid.key => state_time}
+          stub(c).state{state}
+
+          msg_ids = Object.new
+          stub(msg_ids).length{0}
+          msgs = Object.new
+
+          mock(fid).find_item.with_any_args do |opts| 
+            opts[:indexed_page_item_view][:offset].should == 0
+            msg_ids
+          end
+
+          c.action
+        end
+
+
+      end
+
+      describe "mailbox_to_hash" do
+        it "should keep only :name and :email_address keys of a Rews address hash" do
+          c=Sonar::Connector::EwsPullConnector.new(one_folder_config, @base_config)
+          mb = {:name=>"foo bar", :email_address=>"foo@bar.com", :blah=>"blah"}
+          c.mailbox_to_hash(mb).should == {:name=>"foo bar", :email_address=>"foo@bar.com"}
+        end
+      end
+
+      describe "mailbox_recipients_to_hashes" do
+        it "should convert a nil recipient to an empty list" do
+          c=Sonar::Connector::EwsPullConnector.new(one_folder_config, @base_config)
+          mbr = {:mailbox=>nil}
+          c.mailbox_recipients_to_hashes(mbr).should == []
+        end
+
+        it "should convert a single recipient to a list of one hash" do
+          c=Sonar::Connector::EwsPullConnector.new(one_folder_config, @base_config)
+          mbr = {:mailbox=>{:name=>"foo bar", :email_address=>"foo@bar.com", :blah=>"blah"}}
+          c.mailbox_recipients_to_hashes(mbr).should == [{:name=>"foo bar", :email_address=>"foo@bar.com"}]
+        end
+
+        it "should convert multiple recipients to a list of hashes" do
+          c=Sonar::Connector::EwsPullConnector.new(one_folder_config, @base_config)
+          mbr = {:mailbox=>[{:name=>"foo bar", :email_address=>"foo@bar.com", :blah=>"blah"},
+                            {:name=>"baz mcbaz", :email_address=>"baz.mcbaz@baz.com"}]}
+          c.mailbox_recipients_to_hashes(mbr).should == [{:name=>"foo bar", :email_address=>"foo@bar.com"},
+                                                         {:name=>"baz mcbaz", :email_address=>"baz.mcbaz@baz.com"}]
+        end
+      end
+
+      describe "message_to_hash" do
+        it "should convert a message Rews::Item::Item to a hash" do
+          c=Sonar::Connector::EwsPullConnector.new(one_folder_config, @base_config)
+          sent_at = DateTime.now-1
+          m = Rews::Item::Item.new(c, 
+                                   :message, 
+                                   :item_id=>{:id=>"abc", :change_key=>"def"},
+                                   :internet_message_id=>"<abc123>",
+                                   :date_time_sent=>sent_at,
+                                   :date_time_received=>DateTime.now,
+                                   :in_reply_to=>"<foo>",
+                                   :references=>["<foo>", "<bar>"],
+                                   :from=>{:mailbox=>{:name=>"foo mcfoo", :email_address=>"foo.mcfoo@foo.com"}},
+                                   :sender=>{:mailbox=>{:name=>"mrs mcmrs", :email_address=>"mrs.mcmrs@foo.com"}},
+                                   :to_recipients=>{:mailbox=>[{:name=>"bar mcbar", :email_address=>"bar.mcbar@bar.com"}, {:name=>"baz mcbaz", :email_address=>"baz.mcbaz@baz.com"}]},
+                                   :cc_recipients=>{:mailbox=>{:name=>"woo wuwoo", :email_address=>"woo.wuwoo@woo.com"}},
+                                   :bcc_recipients=>{:mailbox=>{:name=>"fee mcfee", :email_address=>"fee.mcfee@fee.com"}})
+          h=c.message_to_hash(m)
+          h.should == {
+            :message_id=>"abc123",
+            :sent_at=>sent_at.to_s,
+            :in_reply_to=>"foo",
+            :references=>["foo", "bar"],
+            :from=>{:name=>"foo mcfoo", :email_address=>"foo.mcfoo@foo.com"},
+            :sender=>{:name=>"mrs mcmrs", :email_address=>"mrs.mcmrs@foo.com"},
+            :to=>[{:name=>"bar mcbar", :email_address=>"bar.mcbar@bar.com"},
+                             {:name=>"baz mcbaz", :email_address=>"baz.mcbaz@baz.com"}],
+            :cc=>[{:name=>"woo wuwoo", :email_address=>"woo.wuwoo@woo.com"}],
+            :bcc=>[{:name=>"fee mcfee", :email_address=>"fee.mcfee@fee.com"}]
+          }
+        end
+      end
+
+      describe "save_messages" do
+        def check_saved_msg(c, msg, json_msg)
+            h = JSON.parse(json_msg)
+            h["type"].should == "email"
+            h["connector"].should == c.name
+            h["source"].should == c.url
+            h["source_id"].should == msg[:item_id][:id]
+          
+        end
+
+        it "should save a file for each message result" do
+          c=Sonar::Connector::EwsPullConnector.new(one_folder_config, @base_config)
+
+          
+          msg1 = {
+            :item_id=>{:id=>"abc", :change_key=>"def"},
+            "item:DateTimeSent"=>DateTime.now-1,
+            "item:DateTimeReceived"=>DateTime.now,
+            "item:InReplyTo"=>"foo",
+            "message:InternetMessageId"=>"foobar",
+            "message:References"=>"barbar",
+            "messsage:From"=>"foo@bar.com",
+            "message:Sender"=>"foo@bar.com",
+            "message:ToRecipients"=>"baz@bar.com",
+            "message:CcRecipients"=>"abc@def.com",
+            "message:BccRecipients"=>"boss@foo.com"
+          }
+          msg2 = {
+            :item_id=>{:id=>"ghi", :change_key=>"jkl"}}
+
+          msgs = [msg1, msg2]
+
+          mock(c.filestore).write(:complete, "#{MD5.hexdigest('abc')}.json", anything) do |*args|
+            check_saved_msg(c, msg1, args.last)
+          end
+          mock(c.filestore).write(:complete, "#{MD5.hexdigest('ghi')}.json", anything) do |*args|
+            check_saved_msg(c, msg2, args.last)
+          end
+
+          c.save_messages(msgs)
+        end
+
+        it "should log and continue if the extracted message is nil" do
+          c=Sonar::Connector::EwsPullConnector.new(one_folder_config, @base_config)
+          stub(c).is_journal{true}
+
+          msg = Object.new
+          msgs=[msg]
+          stub(c).extract_journalled_message(msg){nil}
+          
+          stub(c.log).warn(/no data extracted/)
+          dont_allow(c.filestore).write
+
+          c.save_messages(msgs)
+        end
+      end
+
+      describe "extract_journalled_message" do
+        it "should catch any exception extracting the message, log a warning and return nil" do
+          c=Sonar::Connector::EwsPullConnector.new(one_folder_config, @base_config)
+
+          msg = Object.new
+          mime_content = Object.new
+          stub(msg).[](:mime_content){mime_content}
+          mime_msg = Object.new
+          stub(Base64).decode64(mime_content){mime_msg}
+          e = begin ; raise "bang" ; rescue Exception=>e ; e ; end
+          stub(Rfc822Util).extract_journalled_mail(mime_msg){raise e}
+
+          stub(c.log).warn(/problem/)
+          stub(c.log).warn(e)
+          
+          lambda{
+            c.extract_journalled_message(msg)
+          }.should_not raise_error
+        end
+      end
+
+      describe "delete_messages" do
+        it "should HardDelete all messages from a result" do
+          c=Sonar::Connector::EwsPullConnector.new(one_folder_config, @base_config)
+          fid = c.distinguished_folder_ids.first
+
+          msgs = Object.new
+          mock(msgs).length{1}
+
+          mock(fid).delete_item(msgs, :delete_type=>:HardDelete)
+
+          c.delete_messages(fid, msgs)
+        end
+      end
+      
+    end
+  end
+end

data/spec/spec.opts

@@ -0,0 +1 @@
+--color

data/spec/spec_helper.rb

@@ -0,0 +1,12 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+require 'rubygems'
+require 'spec'
+require 'spec/autorun'
+require 'rr'
+require 'ews_pull_connector/ews_pull_connector'
+require 'sonar_connector/rspec/spec_helper'
+
+Spec::Runner.configure do |config|
+  config.mock_with RR::Adapters::Rspec
+end

metadata

@@ -0,0 +1,207 @@
+--- !ruby/object:Gem::Specification 
+name: sonar_ews_pull_connector
+version: !ruby/object:Gem::Version 
+  hash: 23
+  prerelease: 
+  segments: 
+  - 0
+  - 2
+  - 0
+  version: 0.2.0
+platform: ruby
+authors: 
+- mccraigmccraig
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-03-25 00:00:00 +00:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: sonar_connector
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 7
+        segments: 
+        - 0
+        - 7
+        - 2
+        version: 0.7.2
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: savon
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 51
+        segments: 
+        - 0
+        - 8
+        - 6
+        version: 0.8.6
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: ntlm-http
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 25
+        segments: 
+        - 0
+        - 1
+        - 1
+        version: 0.1.1
+  type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: httpclient
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 217
+        segments: 
+        - 2
+        - 1
+        - 6
+        - 1
+        - 1
+        version: 2.1.6.1.1
+  type: :runtime
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: fetch_in
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 23
+        segments: 
+        - 0
+        - 2
+        - 0
+        version: 0.2.0
+  type: :runtime
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  name: rfc822_util
+  prerelease: false
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 25
+        segments: 
+        - 0
+        - 1
+        - 1
+        version: 0.1.1
+  type: :runtime
+  version_requirements: *id006
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id007 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 13
+        segments: 
+        - 1
+        - 2
+        - 9
+        version: 1.2.9
+  type: :development
+  version_requirements: *id007
+- !ruby/object:Gem::Dependency 
+  name: rr
+  prerelease: false
+  requirement: &id008 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 61
+        segments: 
+        - 0
+        - 10
+        - 5
+        version: 0.10.5
+  type: :development
+  version_requirements: *id008
+description: A sonar-connector for extracting emails from Exchange 2007/2010 through Exchange Web Services
+email: craig@trampolinesystems.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.rdoc
+files: 
+- .document
+- LICENSE
+- README.rdoc
+- Rakefile
+- VERSION
+- lib/ews_pull_connector/ews_pull_connector.rb
+- lib/sonar_ews_pull_connector.rb
+- spec/ews_pull_connector/ews_pull_connector_spec.rb
+- spec/spec.opts
+- spec/spec_helper.rb
+has_rdoc: true
+homepage: http://github.com/trampoline/sonar-ews-pull-connector
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.6.2
+signing_key: 
+specification_version: 3
+summary: Exchange Web Services connector for Exchange 2007/2010
+test_files: 
+- spec/ews_pull_connector/ews_pull_connector_spec.rb
+- spec/spec_helper.rb