mailparser 0.4.22a

data/lib/mailparser/conv_charset.rb

@@ -0,0 +1,27 @@
+# Copyright (C) 2007-2010 TOMITA Masahiro
+# mailto:tommy@tmtm.org
+
+require "iconv"
+require "nkf"
+
+class MailParser::ConvCharset
+  CHARSET = {
+    "sjis"      => "cp932",
+    "x-sjis"    => "cp932",
+    "shift_jis" => "cp932",
+    "shift-jis" => "cp932",
+  }
+  def self.conv_charset(from, to, str)
+    from = CHARSET[from.downcase] || from.downcase
+    to = CHARSET[to.downcase] || to.downcase
+    s = str
+    if from == "iso-2022-jp" then
+      s = NKF.nkf("-m0Jxs", str)
+      from = "cp932"
+    end
+    if to == "iso-2022-jp" then
+      return NKF.nkf("-m0Sxj", Iconv.iconv("cp932", from, s)[0])
+    end
+    return Iconv.iconv(to, from, s)[0]
+  end
+end

data/lib/mailparser/error.rb

@@ -0,0 +1,7 @@
+# Copyright (C) 2006-2010 TOMITA Masahiro
+# mailto:tommy@tmtm.org
+
+module MailParser
+  class ParseError < StandardError
+  end
+end

data/lib/mailparser/loose.rb

@@ -0,0 +1,292 @@
+# -*- coding: utf-8 -*-
+# Copyright (C) 2007-2010 TOMITA Masahiro
+# mailto:tommy@tmtm.org
+
+require "time"
+require "strscan"
+require "mailparser/rfc2822"
+require "mailparser/rfc2045"
+
+module MailParser
+  module Loose
+    HEADER_PARSER = {
+      "date"                      => :parse_date,
+      "from"                      => :parse_mailbox_list,
+      "sender"                    => :parse_mailbox,
+      "reply-to"                  => :parse_mailbox_list,
+      "to"                        => :parse_mailbox_list,
+      "cc"                        => :parse_mailbox_list,
+      "bcc"                       => :parse_mailbox_list,
+      "message-id"                => :parse_msg_id,
+      "in-reply-to"               => :parse_msg_id_list,
+      "references"                => :parse_msg_id_list,
+      "keywords"                  => :parse_phrase_list,
+      "resent-date"               => :parse_date,
+      "resent-from"               => :parse_mailbox_list,
+      "resent-sender"             => :parse_mailbox,
+      "resent-to"                 => :parse_mailbox_list,
+      "resent-cc"                 => :parse_mailbox_list,
+      "resent-bcc"                => :parse_mailbox_list,
+      "resent-message-id"         => :parse_msg_id,
+      "return-path"               => :parse_return_path,
+      "received"                  => :parse_received,
+      "content-type"              => :parse_content_type,
+      "content-transfer-encoding" => :parse_content_transfer_encoding,
+      "content-id"                => :parse_msg_id,
+      "mime-version"              => :parse_mime_version,
+      "content-disposition"       => :parse_content_disposition,
+    }
+
+    module_function
+    # ヘッダをパースした結果のオブジェクトを返す
+    # hname:: ヘッダ名(String)
+    # hbody:: ヘッダ本文(String)
+    # opt:: オプション(Hash)
+    def parse(hname, hbody, opt={})
+      if HEADER_PARSER.key? hname then
+        return method(HEADER_PARSER[hname]).call(hbody, opt)
+      else
+        r = hbody.gsub(/\s+/, " ")
+        if opt[:decode_mime_header] then
+          return RFC2047.decode(r, opt)
+        else
+          return r
+        end
+      end
+    end
+
+    # Date ヘッダをパースして、RFC2822::DateTime を返す
+    def parse_date(str, opt={})
+      begin
+        t = Time.rfc2822(str) rescue Time.parse(str)
+      rescue
+        t = Time.now
+      end
+      return RFC2822::DateTime.new(t.year, t.month, t.day, t.hour, t.min, t.sec, t.zone)
+    end
+
+    # From,To,Cc 等のヘッダをパースして RFC2822::Mailbox の配列を返す
+    def parse_mailbox_list(str, opt={})
+      mailbox_list(str, opt)
+    end
+
+    # Sender,Resent-Sender ヘッダをパースして RFC2822::Mailbox を返す
+    def parse_mailbox(str, opt={})
+      mailbox_list(str, opt)[0]
+    end
+
+    # Message-Id,Resent-Message-Id ヘッダをパースして RFC2822::MsgId を返す
+    def parse_msg_id(str, opt={})
+      msg_id_list(str)[0]
+    end
+
+    # In-Reply-To,References 等のヘッダを RFC2822::MsgIdList を返す
+    def parse_msg_id_list(str, opt={})
+      msg_id_list(str)
+    end
+
+    # Keywords ヘッダをパースして文字列の配列を返す
+    def parse_phrase_list(str, opt={})
+      s = split_by(Tokenizer.token(str), ",")
+      s.map!{|i| i.join(" ")}
+      if opt[:decode_mime_header] then
+        s.map!{|i| RFC2047.decode(i, opt)}
+      end
+      s
+    end
+
+    # Return-Path ヘッダをパースして RFC2822:ReturnPath を返す
+    def parse_return_path(str, opt={})
+      mailbox_list(str, opt)[0]
+    end
+
+    # Received ヘッダをパースして RFC2822::Received を返す
+    def parse_received(str, opt={})
+      a = split_by(Tokenizer.token_received(str), ";")
+      date = a.length > 1 ? parse_date(a.last.join(" ")) : RFC2822::DateTime.now
+      name_val = {}
+      i = 0
+      v = ""
+      unless a.empty?
+        while i < a[0].length do
+          if a[0][i] =~ /\A[a-z0-9]+\z/ino then
+            v = a[0][i+1]
+            name_val[a[0][i].downcase] = v
+            i += 1
+          else
+            v << a[0][i]
+          end
+          i += 1
+        end
+      end
+      RFC2822::Received.new(name_val, date)
+    end
+
+    # Content-Type ヘッダをパースして RFC2045::ContentType を返す
+    def parse_content_type(str, opt={})
+      token = split_by(Tokenizer.token(str), ";")
+      type, subtype = token.shift.to_s.split("/", 2)
+      params = {}
+      token.map do |param|
+        pn, pv = param.to_s.split(/=/, 2)
+        params[pn.to_s] = pv.to_s.gsub(/\A"|"\z/,"")
+      end
+      type = "text" if type.nil? or type.empty?
+      if subtype.nil? or subtype.empty?
+        subtype = type == "text" ? "plain" : ""
+      end
+      RFC2045::ContentType.new(type, subtype, params)
+    end
+
+    # Content-Transfer-Encoding ヘッダをパースして RFC2045::ContentTransferEncoding を返す
+    def parse_content_transfer_encoding(str, opt={})
+      RFC2045::ContentTransferEncoding.new(Tokenizer.token(str).first.to_s)
+    end
+
+    # Mime-Version ヘッダをパースして文字列を返す
+    def parse_mime_version(str, opt={})
+      Tokenizer.token(str).join
+    end
+
+    # Content-Disposition ヘッダをパースして RFC2183::ContentDisposition を返す
+    def parse_content_disposition(str, opt={})
+      token = split_by(Tokenizer.token(str), ";")
+      type = token.shift.to_s
+      params = {}
+      token.map do |param|
+        pn, pv = param.to_s.split(/=/, 2)
+        params[pn.to_s] = pv.to_s.gsub(/\A"|"\z/,"")
+      end
+      RFC2183::ContentDisposition.new(type, params)
+    end
+
+    # array を delim で分割した配列(要素は配列)を返す
+    def split_by(array, delim)
+      ret = []
+      a = []
+      array.each do |i|
+        if i == delim then
+          ret << a
+          a = []
+        else
+          a << i
+        end
+      end
+      ret << a unless a.empty?
+      return ret
+    end
+
+    # Mailbox のリストを返す
+    def mailbox_list(str, opt)
+      ret = []
+      split_by(Tokenizer.token(str), ",").each do |m|
+        if a1 = m.index("<") and a2 = m.rindex(">") and a2 > a1 then
+          display_name = m[0..a1-1].join(" ")
+          if opt[:decode_mime_header] then
+            display_name = RFC2047.decode(display_name, opt)
+          end
+          mailaddr = m[a1+1..a2-1].to_s
+          local_part, domain = mailaddr.split(/@/, 2)
+          ret << RFC2822::Mailbox.new(RFC2822::AddrSpec.new(local_part, domain), display_name)
+        else
+          local_part, domain = m.to_s.split(/@/, 2)
+          ret << RFC2822::Mailbox.new(RFC2822::AddrSpec.new(local_part, domain))
+        end
+      end
+      return ret
+    end
+
+    # MsgId のリストを返す
+    def msg_id_list(str)
+      ret = []
+      flag = false
+      msgid = nil
+      Tokenizer.token(str).each do |m|
+        case m
+        when "<"
+          unless flag
+            flag = true
+            msgid = ""
+          end
+        when ">"
+          if flag
+            flag = false
+            ret << RFC2822::MsgId.new(msgid)
+          end
+        else
+          msgid << m if flag
+        end
+      end
+      if ret.empty?
+        ret = str.split.map{|s| RFC2822::MsgId.new(s)}
+      end
+      return ret
+    end
+
+    class Tokenizer < RFC2822::Scanner
+      def initialize(str)
+        @comments = []
+        @ss = StringScanner.new(str)
+      end
+
+      # トークンに分割(コメント部は削除)
+      def token()
+        token = []
+        while @ss.rest? do
+          if s = @ss.scan(/\s+/nmo) then
+            # ignore
+          elsif s = @ss.scan(/\(/nmo) then
+            begin
+              pos = @ss.pos
+              cfws(@ss)
+            rescue ParseError
+              @ss.pos = pos
+              token << s
+            end
+          elsif s = @ss.scan(/\"(\s*(\\[#{TEXT_RE}]|[#{QTEXT_RE}]))*\s*\"/nmo) ||
+              @ss.scan(/\[(\s*(\\[#{TEXT_RE}]|[#{DTEXT_RE}]))*\s*\]/nmo) ||
+              @ss.scan(/[#{ATEXT_RE}]+/no)
+            token << s
+          else
+            token << @ss.scan(/./no)
+          end
+        end
+        return token
+      end
+
+      # Received 用に分割
+      def token_received()
+        ret = []
+        while @ss.rest? do
+          if s = @ss.scan(/[\s]+/nmo) then
+            # ignore blank
+          elsif s = @ss.scan(/\(/nmo) then
+            begin
+              pos = @ss.pos
+              cfws(@ss)
+            rescue ParseError
+              @ss.pos = pos
+              ret.last << s unless ret.empty?
+            end
+          elsif s = @ss.scan(/\"([\s]*(\\[#{TEXT_RE}]|[#{QTEXT_RE}]))*[\s]*\"/nmo)
+            ret << s
+          elsif s = @ss.scan(/;/)
+            ret << s
+          else
+            ret << @ss.scan(/[^\s\(\;]+/nmo)
+          end
+        end
+        return ret
+      end
+
+      def self.token(str)
+        Tokenizer.new(str).token
+      end
+
+      def self.token_received(str)
+        Tokenizer.new(str).token_received
+      end
+    end
+
+  end
+end

data/lib/mailparser/obsolete.rb

@@ -0,0 +1,403 @@
+# Copyright (C) 2003-2010 TOMITA Masahiro
+# mailto:tommy@tmtm.org
+
+require "nkf"
+require "date"
+
+module MailParser
+
+  @@output_charset = "euc-jp"
+  @@text_body_only = false
+  @@extract_message_type = true
+
+  ConvertMethods = {
+    "JE" => :jistoeuc,
+    "SE" => :sjistoeuc,
+    "UE" => :utf8toeuc,
+    "EU" => :euctoutf8,
+    "SU" => :sjistoutf8,
+    "JU" => :jistoutf8,
+  }
+
+  Charsets = {
+    "iso-2022-jp" => "J",
+    "euc-jp"      => "E",
+    "shift_jis"   => "S",
+    "sjis"        => "S",
+    "x-sjis"      => "S",
+    "utf-8"       => "U",
+    "us-ascii"    => "N",
+  }
+
+  module_function
+
+  def euctoutf8(s)
+    NKF.nkf("-m0Ewx", s)
+  end
+
+  def sjistoutf8(s)
+    NKF.nkf("-m0Swx", s)
+  end
+
+  def jistoutf8(s)
+    NKF.nkf("-m0Jwx", s)
+  end
+
+  def sjistoeuc(s)
+    NKF.nkf("-m0Sex", s)
+  end
+
+  def jistoeuc(s)
+    NKF.nkf("-m0Jex", s)
+  end
+
+  def utf8toeuc(s)
+    NKF.nkf("-m0Wex", s)
+  end
+
+  def output_charset=(c)
+    @@output_charset = c
+  end
+
+  def text_body_only=(f)
+    @@text_body_only = f
+  end
+
+  def extract_message_type=(f)
+    @@extract_message_type = f
+  end
+
+  def b64_hdecode(str)
+    str.unpack("m")[0]
+  end
+
+  def b64_decode(str)
+    str.unpack("m")[0]
+  end
+
+  def qp_hdecode(str)
+    str.gsub("_", " ").gsub(/=([0-9A-F][0-9A-F])/no) do $1.hex.chr end
+  end
+
+  def qp_decode(str)
+    str.gsub(/[ \t]+$/no, "").gsub(/=\r?\n/no, "").
+      gsub(/=([0-9A-F][0-9A-F])/no) do $1.hex.chr end
+  end
+
+  def mdecode_token(s)
+    if s !~ /\A=\?([a-z0-9_-]+)\?(Q|B)\?([^?]+)\?=\Z/nio then
+      s
+    else
+      charset, encoding, text = $1, $2, $3
+      fc = MailParser::Charsets[charset.downcase]
+      if fc == nil then return s end
+      if encoding.downcase == 'q' then
+        s2 = qp_hdecode(text)
+      else
+        s2 = b64_hdecode(text)
+      end
+      tc = @@output_charset && MailParser::Charsets[@@output_charset.downcase]
+      if fc == "N" or tc.nil? or fc == tc then return s2 end
+      MailParser.send(MailParser::ConvertMethods[fc+tc], s2)
+    end
+  end
+
+  def mime_header_decode(str)
+    return str.gsub(/\s+/no, " ").gsub(/\?=\s+=\?/no, "?==?").gsub(/=\?[a-z0-9_-]+\?(Q|B)\?[^?]+\?=/nio){mdecode_token $&}
+  end
+
+  def trunc_comment(v)
+    ret = ""
+    after = v
+    while not after.empty? and after =~ /^(\\.|\"(\\.|[^\\\"])*\"|[^\\\(])*/no do
+      ret << $&
+      after = $'
+      if after =~ /^\(/no then
+        a = trunc_comment_sub(after[1..-1])
+        if a == nil then
+          return ret+after
+        end
+        after = a
+      end
+      if after == "\\" then
+        break
+      end
+    end
+    ret+after
+  end
+
+  def trunc_comment_sub(orig)
+    after = orig
+    loop do
+      if after =~ /^(\\.|[^\\\(\)])*/no then
+        after = $'
+      end
+      if after =~ /^\)/no then
+        return after[1..-1]
+      end
+      if after =~ /^\(/no then
+        after = trunc_comment_sub(after[1..-1])
+        if after == nil then
+          return nil
+        end
+        next
+      end
+      return nil
+    end
+  end
+
+  def split_address(v)
+    a = []
+    r = ""
+    while not v.empty? do
+      if v =~ /^(\s+|[0-9A-Za-z\!\#\$\%\&\'\*\+\-\/\=\?\^\_\`\{\|\}\~]+|\"(\\.|[^\\\"])*\")/ then
+        r << $&
+        v = $'
+      elsif v[0] == ?, then
+        a << r.strip
+        r = ""
+        v.slice!(0,1)
+      else
+        r << v.slice!(0,1)
+      end
+    end
+    a << r.strip
+    return a
+  end
+
+  def get_mail_address(v)
+    v = trunc_comment(v)
+    a = split_address(v)
+    return a.map{|i| i.strip =~ /<([^<>]*)>$/ ? $1 : i.strip}
+  end
+
+  def get_date(s)
+    if s =~ /^[A-Z][A-Z][A-Z]\s*,\s*/i then
+      s = $'
+    end
+    d = ::DateTime._strptime(s, "%d %b %Y %X")
+    return unless d
+    Time.mktime(d[:year], d[:mon], d[:mday], d[:hour], d[:min], d[:sec]) rescue nil
+  end
+
+  def parse_content_type(str)
+    hash = {}
+    hash[:parameter] = {}
+    if str.strip =~ /^([a-z0-9_-]+)(?:\/([a-z0-9_-]+))?\s*/nio then
+      hash[:type] = $1.downcase
+      hash[:subtype] = $2.downcase if $2
+      params = $'	#'
+      pending = {}
+      while true do
+        if params =~ /\A\s*;\s*([a-z0-9_-]+)(?:\*(\d+))?\s*=\s*(?:\"((?:\\\"|[^\"])*)\"|([^\s\(\)\<\>\@\,\;\:\\\"\/\[\]\?\=]*))\s*/nio then
+          pn, ord, pv = $1, $2, $3||$4
+          params = $'
+          if ord then
+            pending[pn] = [] unless pending.key? pn
+            pending[pn] << [ord.to_i, pv]
+          else
+            hash[:parameter][pn.downcase] = pv
+          end
+        elsif params =~ /\A\s*;\s*([a-z0-9_-]+)\*\s*=\s*([a-z0-9_-]+)?\'(?:[a-z0-9_-]+)?\'(?:\"((?:\\\"|[^\"])*)\"|([^\s\(\)\<\>\@\,\;\:\\\"\/\[\]\?\=]*))\s*/nio then
+          pn, charset, pv = $1, $2, $3||$4
+          params = $'
+          pending[pn] = [[0, pv, charset, true]]
+        elsif params =~ /\A\s*;\s*([a-z0-9_-]+)\*0\*\s*=\s*([a-z0-9_-]+)?\'(?:[a-z0-9_-]+)?\'(?:\"((?:\\\"|[^\"])*)\"|([^\s\(\)\<\>\@\,\;\:\\\"\/\[\]\?\=]*))\s*/nio then
+          pn, charset, pv = $1, $2, $3||$4
+          params = $'
+          pending[pn] = [[0, pv, charset, true]]
+        elsif params =~ /\A\s*;\s*([a-z0-9_-]+)\*(\d+)\*\s*=\s*(?:\"((?:\\\"|[^\"])*)\"|([^\s\(\)\<\>\@\,\;\:\\\"\/\[\]\?\=]*))\s*/nio then
+          pn, ord, pv = $1, $2, $3||$4
+          params = $'
+          pending[pn] = [] unless pending.key? pn
+          pending[pn] << [ord.to_i, pv, nil, true]
+        else
+          break
+        end
+      end
+      pending.each do |pn, pv|
+        pv = pv.sort{|a,b| a[0]<=>b[0]}
+        charset = pv[0][2]
+        v = pv.map{|a|a[3] ? a[1].gsub(/%([0-9A-F][0-9A-F])/nio){$1.hex.chr} : a[1]}.join
+        fc = MailParser::Charsets[charset.downcase] if charset
+        tc = @@output_charset && MailParser::Charsets[@@output_charset.downcase]
+        if fc and fc != "N" and fc != tc then
+          v = MailParser.send(MailParser::ConvertMethods[fc+tc], v)
+        end
+        hash[:parameter][pn.downcase] = v
+      end
+    end
+    return hash
+  end
+
+  def parse_content_disposition(str)
+    return parse_content_type(str)
+  end
+
+  def parse_message(msg)
+    class << msg
+      def _each_with_multiple_delimiter(delim=[])
+        @found_boundary = false
+        loop do
+          @l = gets
+          if @l == nil then
+            return
+          end
+          ll = @l.chomp
+          if delim.include? ll then
+            @found_boundary = true
+            return
+          end
+          yield @l
+        end
+      end
+      def last_line()
+        @l && @l.chomp
+      end
+      attr_reader :found_boundary
+    end
+
+    m = parse_message2(msg)
+    class << m
+      def to_s()
+        return <<EOS
+From: #{self[:from].join(",")}
+To: #{self[:to].join(",")}
+Subject:#{self[:subject]}
+Date: #{self[:date]}
+
+#{self[:body]}
+
+#{if self[:parts] then self[:parts].map{|p| "[#{p[:type]}/#{p[:subtype]}]<#{p[:filename]}>"}.join("\n") end}
+EOS
+      end
+    end
+    return m
+  end
+
+  def parse_message2(msg, boundary=[])
+    ret = parse_header(msg, boundary)
+    return ret if msg.found_boundary
+
+    if ret[:type] == "message" and @@extract_message_type then
+      m = parse_message2(msg, boundary)
+      ret[:message] = m
+    elsif ret[:multipart] and ret[:boundary] then
+      parts = []
+      b = ret[:boundary]
+      bd = boundary + ["--"+b+"--", "--"+b]
+      msg._each_with_multiple_delimiter(bd) do end	# skip preamble
+      while msg.last_line == bd[-1] do
+        m = parse_message2(msg, bd)
+        parts << m
+      end
+      if msg.last_line == bd[-2] then
+        msg._each_with_multiple_delimiter(boundary) do end
+      end
+      ret[:parts] = parts
+    else
+      if not @@text_body_only or ret[:type] == "text" or ret[:type].nil? then
+        body = ""
+        msg._each_with_multiple_delimiter(boundary) do |l|
+          body << l 
+        end
+        ret[:body] = decode_body(body, ret[:encoding], ret[:charset])
+      else
+        msg._each_with_multiple_delimiter(boundary) do end
+      end
+    end
+    return ret
+  end
+
+  def parse_header(msg, boundary=[])
+    ret = {}
+    raw = ""
+    header = []
+    msg._each_with_multiple_delimiter(boundary) do |l|
+      l.chomp!
+      break if l.empty?
+      raw << l+"\n"
+      if l =~ /^\s/no and not header.empty? then
+        header[-1] << l
+      elsif not l.include? ":"
+        next			# skip garbage
+      else
+        header << l
+      end
+    end
+
+    from = []
+    to = []
+    cc = []
+    date = nil
+    subject = ""
+    encoding = ct = charset = multipart = body = filename = bd = nil
+    h = {}
+
+    header.each do |str|
+      hn, hb = str.split(/:\s*/no, 2)
+      hn.downcase!
+      h[hn] = [] unless h.key? hn
+      h[hn] << mime_header_decode(hb)
+      case hn.downcase
+      when "from"
+        from.concat get_mail_address(hb)
+      when "to"
+        to.concat get_mail_address(hb)
+      when "cc"
+        cc.concat get_mail_address(hb)
+      when "date"
+        date = get_date(hb)
+      when "subject"
+        subject.concat hb
+      when "content-type"
+        ct = parse_content_type(hb)
+        if ct[:type] == "text" then
+          charset = ct[:parameter]["charset"]
+        elsif ct[:type] == "multipart" then
+          multipart = true
+          bd = ct[:parameter]["boundary"]
+        end
+        filename = mime_header_decode(ct[:parameter]["name"]) if ct[:parameter]["name"]
+      when "content-disposition"
+        cd = parse_content_disposition(hb)
+        filename = mime_header_decode(cd[:parameter]["filename"]) if cd[:parameter]["filename"]
+      when "content-transfer-encoding"
+        encoding = hb.strip.downcase
+      end
+    end
+
+    ret[:from] = from
+    ret[:to] = to
+    ret[:cc] = cc
+    ret[:date] = date
+    ret[:subject] = mime_header_decode subject
+    if ct then
+      ret[:type] = ct[:type].downcase if ct[:type]
+      ret[:subtype] = ct[:subtype].downcase if ct[:subtype]
+      ret[:charset] = charset.downcase if charset
+    end
+    ret[:encoding] = encoding if encoding
+    ret[:multipart] = multipart
+    ret[:boundary] = bd
+    ret[:filename] = filename if filename
+    ret[:header] = h
+    ret[:rawheader] = raw
+    return ret
+  end
+
+  def decode_body(body, encoding, charset)
+    case encoding
+    when "base64"
+      body = b64_decode body
+    when "quoted-printable"
+      body = qp_decode body
+    end
+    if charset == nil then return body end
+    fc = MailParser::Charsets[charset.downcase]
+    if fc == nil then return body end
+    tc = @@output_charset && MailParser::Charsets[@@output_charset.downcase]
+    if fc == "N" or tc.nil? or fc == tc then return body end
+    MailParser.send(MailParser::ConvertMethods[fc+tc], body)
+  end
+end