mms2r 3.7.1 → 3.8.0

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    mms2r (3.7.1)
+    mms2r (3.8.0)
       exifr (>= 1.0.3)
       json (>= 1.6.0)
       mail (>= 2.4.0)
@@ -19,11 +19,11 @@ GEM
       mime-types (~> 1.16)
       treetop (~> 1.4.8)
     metaclass (0.0.1)
-    mime-types (1.18)
+    mime-types (1.19)
     mocha (0.11.4)
       metaclass (~> 0.0.1)
     multi_json (1.3.5)
-    nokogiri (1.5.2)
+    nokogiri (1.5.5)
     polyglot (0.3.3)
     rake (0.9.2.2)
     rdoc (3.12)

data/History.txt

@@ -1,3 +1,8 @@
+### 3.8.0 / 2012-07-04 (Dr. Donald Gorfield – Comedy specialist)
+
+* 1 major enhancement
+  * Handle MMS from Sprint that have their media attached rather than CDN'd
+
 ### 3.7.1 / 2012-06-04 (Abrigail Remeltindtdrinc - The Record Cleaner)
 
 * 2 minor enhancements

data/README.rdoc

@@ -9,7 +9,7 @@ https://github.com/monde/mms2r
 https://rubygems.org/gems/mms2r
 http://peepcode.com/products/mms2r-pdf
 
-MMS2R is a library that decodes the parts of an MMS message to disk while
+MMS2R is a library that decodes the parts of a MMS message to disk while
 stripping out advertising injected by the mobile carriers.  MMS messages are
 multipart email and the carriers often inject branding into these messages.  Use
 MMS2R if you want to get at the real user generated content from a MMS without

data/conf/pm.sprint.com.yml

@@ -4,6 +4,8 @@ ignore:
   - !ruby/regexp /We're sorry, this page is not available. We apologize for the inconvenience./mi
   text/plain:
   - !ruby/regexp /You have new [Picture|Video] Mail!\s+Click Go\/View to see now./mi
+  - !ruby/regexp /You have received a Picture Mail from /mi
+  - !ruby/regexp /Click Go\/View to see/mi
 transform:
   text/plain:
   - - !ruby/regexp /^You have new Picture Mail!$/i

data/lib/mms2r.rb

@@ -42,14 +42,67 @@ module MMS2R
     USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.120 Safari/535.2"
   end
 
+  ##
   # Simple convenience function to make it a one-liner:
-  # MMS2R.parse raw_mail or MMS2R.parse File.load(raw_mail)
+  # MMS2R.parse raw_mail or
+  # MMS2R.parse File.load(file)
+  # MMS2R.parse File.load(path_to_file)
   # Combined w/ the method_missing delegation, this should behave as an enhanced Mail object, more or less.
-  def self.parse raw_mail, options = {}
-    mail = Mail.new raw_mail
+
+  def self.parse thing, options = {}
+    mail = case
+           when File.exist?(thing); Mail.new(open(thing).read)
+           when thing.respond_to?(:read); Mail.new(thing.read)
+           else
+             Mail.new(thing)
+           end
+
     MMS2R::Media.new(mail, options)
   end
 
+  ##
+  # Compare original MMS2R results with original mail values and other metrics.
+  #
+  # Takes a file path, mms2r object, mail object, or mail text blob.
+
+  def self.debug(thing, options = {})
+    mms = case thing
+          when MMS2R::Media; thing
+          when Mail::Message; MMS2R::Media.new(thing, options)
+          else
+            self.parse(thing, options)
+          end
+
+    <<OUT
+#{'-' * 80}
+
+  original mail
+    #{'from:'.ljust(15)} #{mms.mail.from}
+    #{'to:'.ljust(15)} #{mms.mail.to}
+    #{'subject:'.ljust(15)} #{mms.mail.subject}
+
+  mms2r
+    #{'from:'.ljust(15)} #{mms.from}
+    #{'to:'.ljust(15)} #{mms.to}
+    #{'subject:'.ljust(15)} #{mms.subject}
+    #{'number:'.ljust(15)} #{mms.number}
+
+    default media
+      #{mms.default_media.inspect}
+
+    default text
+      #{mms.default_text.inspect}
+      #{mms.default_text.read if mms.default_text}
+
+    all plain text
+      #{(mms.media['text/plain']||[]).each {|t| open(t).read}.join("\n\n")}
+
+    media hash
+      #{mms.media.inspect}
+
+OUT
+  end
+
 end
 
 %W{ yaml mail fileutils pathname tmpdir yaml digest/sha1 iconv exifr }.each do |g|

data/lib/mms2r/media.rb

@@ -343,33 +343,16 @@ module MMS2R
     # note: purge must be explicitly called to remove the media files
     #       mms2r extracts from an mms message.
 
-    def process() # :yields: media_type, file
+    def process # :yields: media_type, file
       unless @was_processed
         log("#{self.class} processing", :info)
 
-        parts = mail.multipart? ? mail.parts : [mail]
-
-        # Double check for multipart/related, if it exists replace it with its
-        # children parts.  Do this twice as multipart/alternative can have
-        # children and we want to fold everything down
-        for i in 1..2
-          flat = []
-          parts.each do |p|
-            if p.multipart?
-              p.parts.each {|mp| flat << mp }
-            else
-              flat << p
-            end
-          end
-          parts = flat.dup
-        end
-
-        # get to work
-        parts.each do |p|
-          t = p.part_type?
-          unless ignore_media?(t,p)
-            t,f = process_media(p)
-            add_file(t,f) unless t.nil? || f.nil?
+        parts = self.folded_parts(mail)
+        parts.each do |part|
+          if part.part_type? == 'text/html'
+            process_html_part(part)
+          else
+            process_part(part)
           end
         end
 
@@ -436,6 +419,25 @@ module MMS2R
       return type, file
     end
 
+    ##
+    # Helper to decide if a part should be kept or ignored
+
+    def process_part(part)
+      return if ignore_media?(part.part_type?, part)
+
+      type, file = process_media(part)
+      add_file(type, file) unless type.nil? || file.nil?
+    end
+
+    ##
+    # Helper to decide if a html part should be kept or ignored.
+    # We are defining it here primarily for the benefit so that Sprint
+    # can override a special case for processing.
+
+    def process_html_part(part)
+      process_part(part)
+    end
+
     ##
     # Helper for process_media template method to transform text.
     # See the transform section in the discussion of the built-in
@@ -490,7 +492,7 @@ module MMS2R
     # Purges the unique MMS2R::Media.media_dir directory created
     # for this producer and all of the media that it contains.
 
-    def purge()
+    def purge
       log("#{self.class} purging #{@media_dir} and all its contents", :info)
       FileUtils.rm_rf(@media_dir)
     end
@@ -507,7 +509,7 @@ module MMS2R
     # Helper to temp_file to create a unique temporary directory that is a
     # child of tmp_dir  This version is based on the message_id of the mail.
 
-    def msg_tmp_dir()
+    def msg_tmp_dir
       @dir_count += 1
       dir = File.expand_path(File.join(@media_dir, "#{@dir_count}"))
       FileUtils.mkdir_p(dir)
@@ -762,7 +764,7 @@ module MMS2R
       self.class.initialize_config(config)
     end
 
-    private
+    protected
 
     ##
     # accessor for the config
@@ -780,6 +782,19 @@ module MMS2R
       ent.nil? ? nil : ent.first
     end
 
+    ##
+    # Helper to fold all the parts of multipart mail down into a flat array.
+    # multipart/related and multipart/alternative parts can have child parts.
+    def folded_parts(parts)
+      return folded_parts([parts]) unless parts.respond_to?(:each)
+
+      result = [] # NOTE could use #tap but want 1.8.7 compat
+      parts.each do |part|
+        result << (part.multipart? ? folded_parts(part.parts) : part)
+      end
+      result.flatten
+    end
+
     ##
     # used by #default_media and #text to return the biggest attachment type
     # listed in the types array

data/lib/mms2r/media/sprint.rb

@@ -28,53 +28,25 @@ module MMS2R
 
     module Sprint
 
-      ##
-      # Override process() because Sprint doesn't attach media (images, video,
-      # etc.) to its MMS.  Media such as images and videos are hosted on a
-      # Sprint content server.   MMS2R::Media::Sprint has to pick apart an
-      # HTML attachment to find the URL to the media on Sprint's content
-      # server and download each piece of content.  Any text message part of
-      # the MMS if it exists is embedded in the html.
-
-      def process
-        unless @was_processed
-          log("#{self.class} processing", :info)
-          #sprint MMS are multipart
-          parts = @mail.parts
-
-          #find the payload html
-          doc = nil
-          parts.each do |p|
-            next unless p.part_type? == 'text/html'
-            d = Nokogiri(p.body.decoded)
-            title = d.at('title').inner_html
-            if title =~ /You have new Picture Mail!/
-              doc = d
-              @is_video = (p.body.decoded =~ /type="VIDEO">/m ? true : false)
-            end
-          end
-          return if doc.nil? # it was a dud
-          @is_video ||= false
-
-          # break it down
-          sprint_phone_number(doc)
-          sprint_process_text(doc)
-          sprint_process_media(doc)
-
-          @was_processed = true
-        end
-
-        # when process acts upon a block
-        if block_given?
-          media.each do |k, v|
-            yield(k, v)
-          end
-        end
+      protected
 
+      ##
+      # Helper to process old style media on the Sprint CDN which didn't attach
+      # media (images, video, etc.) to its MMS.  Media such as images and
+      # videos are hosted on a Sprint content server.  MMS2R::Media::Sprint has
+      # to pick apart an HTML attachment to find the URL to the media on
+      # Sprint's content server and download each piece of content.  Any text
+      # message part of the MMS if it exists is embedded in the html.
+
+      def process_html_part(part)
+        doc = Nokogiri(part.body.decoded)
+
+        is_video = (part.body.decoded =~ /type="VIDEO">/m ? true : false)
+        sprint_process_media(doc, is_video)
+        sprint_process_text(doc)
+        sprint_phone_number(doc)
       end
 
-      private
-
       ##
       # Digs out where Sprint hides the phone number
 
@@ -82,7 +54,8 @@ module MMS2R
         c = doc.search("/html/head/comment()").last
         t = c.content.gsub(/\s+/m," ").strip
         #@number returned in parent's #number
-        @number = / name="MDN">(\d+)</.match(t)[1]
+        matched = / name="MDN">(\d+)</.match(t)
+        @number = matched[1] if matched
       end
 
       ##
@@ -157,7 +130,7 @@ module MMS2R
       ##
       # Fetch all the media that is referred to in the doc
 
-      def sprint_process_media(doc)
+      def sprint_process_media(doc, is_video=false)
         srcs = Array.new
         # collect all the images in the document, even though
         # they are <img> tag some might actually refer to video.
@@ -183,14 +156,14 @@ module MMS2R
           begin
 
             uri = URI.parse(CGI.unescapeHTML(src))
-            unless @is_video
+            unless is_video
               query={}
               uri.query.split('&').each{|a| p=a.split('='); query[p[0]] = p[1]}
               query.delete_if{|k, v| k == 'limitsize' || k == 'squareoutput' }
               uri.query = query.map{|k,v| "#{k}=#{v}"}.join("&")
             end
             # sprint is a ghetto, they expect to see &amp; for video request
-            uri.query = uri.query.gsub(/&/, "&amp;") if @is_video
+            uri.query = uri.query.gsub(/&/, "&amp;") if is_video
 
             connection = Net::HTTP.new(uri.host, uri.port)
             #connection.set_debug_output $stdout

data/lib/mms2r/version.rb

@@ -6,11 +6,11 @@ module MMS2R
     end
 
     def self.minor
-      7
+      8
     end
 
     def self.patch
-      1
+      0
     end
 
     def self.pre

data/mms2r.gemspec

@@ -21,7 +21,7 @@ Gem::Specification.new do |gem|
   gem.email       = ["mikemondragon@gmail.com"]
   gem.homepage    = "https://github.com/monde/mms2r"
   gem.summary     = "Extract user media from MMS (and not carrier cruft)"
-  gem.description = "MMS2R is a library that decodes the parts of an MMS message to disk while stripping out advertising injected by the mobile carriers."
+  gem.description = "MMS2R is a library that decodes the parts of a MMS message to disk while stripping out advertising injected by the mobile carriers."
   gem.rubyforge_project = "mms2r"
   gem.rubygems_version = ">= 1.3.6"
   gem.files         = `git ls-files`.split("\n")

data/test/test_mms2r_media.rb

@@ -311,18 +311,10 @@ class TestMms2rMedia < Test::Unit::TestCase
 
   def test_attachment_should_return_duck_typed_file
     mms = MMS2R::Media.new stub_mail
-    temp_big = temp_text_file("hello world")
-    size = File.size(temp_text_file("hello world"))
-    temp_small = temp_text_file("hello")
-    mms.stubs(:media).returns({'text/plain' => [temp_small, temp_big]})
     duck_file = mms.send(:attachment, ['text'])
-    assert_not_nil duck_file
-    assert_equal true, File::exist?(duck_file)
-    assert_equal true, File::exist?(temp_big)
-    assert_equal temp_big, duck_file.local_path
-    assert_equal File.basename(temp_big), duck_file.original_filename
-    assert_equal size, duck_file.size
+    assert_equal 1, duck_file.size
     assert_equal 'text/plain', duck_file.content_type
+    assert_equal "a", open(mms.media['text/plain'].first).read
   end
 
   def test_empty_body
@@ -632,86 +624,24 @@ class TestMms2rMedia < Test::Unit::TestCase
     mms.purge
   end
 
-  def test_process_with_multipart_alternative_parts
-    mail = stub_mail
-
-    plain = stub('plain', :filename => 'message.txt', :content_type => 'text/plain', :part_type? => 'text/plain', :body => Mail::Body.new('a'), :main_type => 'text')
-    plain.stubs(:multipart?).at_least_once.returns(false)
-
-    html = stub('html', :filename => 'message.html', :content_type => 'text/html', :part_type? => 'text/html', :body => Mail::Body.new('a'), :main_type => 'text')
-    html.stubs(:multipart?).at_least_once.returns(false)
-
-    multi = stub('multi', :content_type => 'multipart/alternative', :part_type? => 'multipart/alternative', :parts => [plain, html])
-    multi.stubs(:multipart?).at_least_once.returns(true)
-
-    mail.stubs(:multipart?).at_least_once.returns(true)
-    mail.stubs(:parts).at_least_once.returns([multi])
-
-    # the multipart/alternative should get flattend to text and html
-    mms = MMS2R::Media.new(mail)
-    assert_equal 2, mms.media.size
-    assert_equal 2, mms.media.size
-    assert_not_nil mms.media['text/plain']
-    assert_not_nil mms.media['text/html']
-    assert_equal 1, mms.media['text/plain'].size
-    assert_equal 1, mms.media['text/html'].size
-    assert_equal 'message.txt', File.basename(mms.media['text/plain'].first)
-    assert_equal 'message.html', File.basename(mms.media['text/html'].first)
-    assert_equal true, File.exist?(mms.media['text/plain'].first)
-    assert_equal true, File.exist?(mms.media['text/html'].first)
-    assert_equal 1, File.size(mms.media['text/plain'].first)
-    assert_equal 1, File.size(mms.media['text/html'].first)
-    mms.purge
+  def test_folding_with_multipart_alternative_parts
+    mail = mail('helio-message-01.mail')
+    mms = MMS2R::Media.new(Mail.new)
+    assert_equal 5, mms.send(:folded_parts, mail.parts).size
   end
 
   def test_process_when_media_is_ignored
-    mail = stub_mail
-    plain = stub('plain', :filename => 'message.txt', :content_type => 'text/plain', :part_type? => 'text/plain', :body => Mail::Body.new(''), :main_type => 'text')
-    plain.stubs(:multipart?).at_least_once.returns(false)
-
-    html = stub('html', :filename => 'message.html', :content_type => 'text/html', :part_type? => 'text/html', :body => Mail::Body.new(''), :main_type => 'text')
-    html.stubs(:multipart?).at_least_once.returns(false)
-
-
-    multi = stub('multi', :content_type => 'multipart/alternative', :part_type? => 'multipart/alternative', :parts => [plain, html])
-    multi.stubs(:multipart?).at_least_once.returns(true)
-
-    mail.stubs(:multipart?).at_least_once.returns(true)
-    mail.stubs(:parts).at_least_once.returns([multi])
-
-    mms = MMS2R::Media.new(mail, :process => :lazy)
-    mms.stubs(:config).returns({'ignore' => {'text/plain' => ['message.txt'],
-                                             'text/html' => ['message.html']}})
-    assert_nothing_raised { mms.process }
-    # the multipart/alternative should get flattend to text and html and then
-    # what's flattened is ignored
-    assert_equal 0, mms.media.size
-    mms.purge
+    # TODO - I'd like to get away from mocks and test on real data, and
+    # this is covered repeatedly for various samples from the carrier
   end
 
   def test_process_when_yielding_to_a_block
-    mail = stub_mail
-
-    plain = stub('plain', :filename => 'message.txt', :content_type => 'text/plain', :part_type? => 'text/plain', :body => Mail::Body.new('a'), :main_type => 'text')
-    plain.stubs(:multipart?).at_least_once.returns(false)
-
-    html = stub('html', :filename => 'message.html', :content_type => 'text/html', :part_type? => 'text/html', :body => Mail::Body.new('b'), :main_type => 'text')
-    html.stubs(:multipart?).at_least_once.returns(false)
-
-    multi = stub('multi', :content_type => 'multipart/alternative', :part_type? => 'multipart/alternative', :parts => [plain, html])
-    multi.stubs(:multipart?).at_least_once.returns(true)
-
-    mail.stubs(:multipart?).at_least_once.returns(true)
-    mail.stubs(:parts).at_least_once.returns([multi])
-
-    # the multipart/alternative should get flattend to text and html
+    mail = mail('att-image-01.mail')
     mms = MMS2R::Media.new(mail)
-    assert_equal 2, mms.media.size
     mms.process do |type, files|
       assert_equal 1, files.size
-      assert_equal true, type == 'text/plain' || type == 'text/html'
-      assert_equal true, File.basename(files.first) == 'message.txt' ||
-                         File.basename(files.first) == 'message.html'
+      assert_equal true, type == 'image/jpeg'
+      assert_equal true, File.basename(files.first) == 'Photo_12.jpg'
       assert_equal true, File::exist?(files.first)
     end
     mms.purge

data/test/test_mms_myhelio_com.rb

@@ -21,6 +21,7 @@ class TestMmsMyhelioCom < Test::Unit::TestCase
     assert_equal "mms.myhelio.com", mms.carrier
     assert_equal 1, mms.media.size
     assert_equal 1, mms.media['text/plain'].size
+    assert_equal "Test message", open(mms.media['text/plain'].first).read
     mms.purge
   end
 

data/test/test_pm_sprint_com.rb

@@ -259,7 +259,6 @@ class TestPmSprintCom < Test::Unit::TestCase
 
     assert_equal '5135455555', mms.number
     assert_equal "pm.sprint.com", mms.carrier
-
     assert_equal 0, mms.media.size
 
     mms.purge

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mms2r
 version: !ruby/object:Gem::Version
-  version: 3.7.1
+  version: 3.8.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-06-05 00:00:00.000000000 Z
+date: 2012-07-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -155,7 +155,7 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-description: MMS2R is a library that decodes the parts of an MMS message to disk while
+description: MMS2R is a library that decodes the parts of a MMS message to disk while
   stripping out advertising injected by the mobile carriers.
 email:
 - mikemondragon@gmail.com
@@ -374,7 +374,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -3061222747631140380
+      hash: 2515700881121403429
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -383,7 +383,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -3061222747631140380
+      hash: 2515700881121403429
 requirements: []
 rubyforge_project: mms2r
 rubygems_version: 1.8.24