virginity 0.3.31

data/lib/virginity/encoding_decoding.rb

@@ -0,0 +1,177 @@
+module Virginity
+
+  class Error < StandardError; end
+  class InvalidEncoding < Error; end
+
+  module EncodingDecoding
+    extend Encodings
+
+    # VALUE-CHAR   = WSP / VCHAR / NON-ASCII
+    WSP = [0x20, 0x09] # WSP = SP / HTAB
+    VCHAR = 0x21..0x7E # VCHAR =  %x21-7E ; visible (printing) characters
+    NONASCII = 0x80..0xFF # NON-ASCII = %x80-FF
+
+    CR_AND_LF = /\r\n/
+    CR = "\r"
+    LF = "\n"
+    def self.decode_quoted_printable(text)
+      text.gsub(CR_AND_LF, LF).gsub(/\=([0-9a-fA-F])?\n\s+([0-9a-fA-F])/, "=\\1\\2").unpack('M*').first
+    end
+
+    QP_ALSO_ENCODE = "\x0A\x20"
+    def self.encode_quoted_printable(text, options = {})
+      options[:also_encode] ||= QP_ALSO_ENCODE
+      # special_chars = /[\t ](?:[\v\t ]|$)|[=\x00-\x08\x0B-\x1F\x7F-\xFF#{options[:also_encode]}]/
+      special_chars = /[=\x00-\x08\x0B-\x1F\x7F-\xFF#{options[:also_encode]}]/n
+      encoded = to_binary(text).gsub(special_chars) do |char|
+        char[0 ... -1] + "=%02X" % char[-1].ord
+      end
+      fold_quoted_printable(encoded, options[:width] || 76, options[:initial_position])
+    end
+
+    QPCHAR = /[^=]|=[\dABCDEF]{2}/
+    QPFOLD = "=\r\n" # only vCard 2.1 uses encode_quoted_printable so we always use windows line endings
+    def self.fold_quoted_printable(qp_text, width = 76, initial_position = 0)
+      return qp_text unless width > 5
+      pos = initial_position.to_i
+      scanner = StringScanner.new(qp_text)
+      folded = ""
+      while !scanner.eos?
+        char = scanner.scan(QPCHAR)
+        charsize = char.size
+        if pos + charsize > width - 3
+          folded << QPFOLD
+          pos = 0
+        end
+        folded << char
+        pos += charsize
+      end
+      folded
+    end
+
+
+    def self.normalize_newlines!(text)
+      text.gsub!(/\r?\n|\r/, "\n")
+    end
+
+    # "text": The "text" value type should be used to identify values that
+    # contain human-readable text. The character set and language in which
+    # the text is represented is controlled by the charset content-header
+    # and the language type parameter and content-header.
+    #
+    # A formatted text line break in a text value type MUST be represented
+    # as the character sequence backslash (ASCII decimal 92) followed by a
+    # Latin small letter n (ASCII decimal 110) or a Latin capital letter N
+    # (ASCII decimal 78), that is "\n" or "\N".
+    #
+    # TODO options for saving to ascii (convert to quoted printable) or storing plain utf-8
+    ENCODED_LF = "\\n"
+    CRLF = CR + LF
+    BACKSLASH = "\\"
+    COMMA = ","
+    SEMICOLON = ";"
+    STUFF_TO_ENCODE = /[\n\\\,\;]/
+    STUFF_NOT_TO_ENCODE = %r{[^\n\\\,\;]*}
+    def self.encode_text(text)
+      raise "#{text.inspect} must be a String" unless text.is_a? String
+      normalize_newlines!(text)
+      encoded = ""
+      s = StringScanner.new(text)
+      while !s.eos?
+        encoded << s.scan(STUFF_NOT_TO_ENCODE)
+        # 5.8.4 Backslashes, newlines, and commas must be encoded.
+        case x = s.scan(STUFF_TO_ENCODE)
+        when LF
+          encoded << ENCODED_LF
+        when BACKSLASH, COMMA, SEMICOLON
+          # RFC2426 tells us to encode ":" too, which is needed for structured text fields
+          encoded << BACKSLASH << x
+        end
+      end
+      encoded
+    end
+
+    def self.decode_text(text)
+      text.gsub(/\\(.)/) { $1.casecmp('n') == 0 ? LF : $1 }
+    end
+
+    def self.encode_text_list(list, separator = COMMA)
+      list.map { |value| encode_text(value) }.join(separator)
+    end
+
+#     # TODO: port to C someday
+      # This is the old simple implementation that is easy to port to another language
+      # this can be a lot faster if we don't create a new string object for each char
+#     def self.decode_text_list(text_list, separator = COMMA)
+#       strings = []
+#       state = :normal # there are two states: :normal and :escaped
+#       string = ""
+#       text_list.each_char do |char|
+#         if state == :escaped
+#           string << (%w(n N).include?(char) ? LF : char)
+#           state = :normal
+#         else
+#           case char
+#           when BACKSLASH
+#             state = :escaped
+#           when separator
+#             strings << string
+#             string = ""
+#           else
+#             string << char
+#           end
+#         end
+#       end
+#       strings << string
+#       strings
+#     end
+
+
+    NON_ESCAPE_OR_SEPARATOR_REGEXP = {}
+    def self.non_escape_or_separator_regexp(separator)
+      NON_ESCAPE_OR_SEPARATOR_REGEXP[separator] ||= %r{[^\\#{separator}\\\\]*}
+    end
+
+    def self.decode_text_list(text_list, separator = COMMA)
+      not_special = non_escape_or_separator_regexp(separator)
+      list = []
+      text = ""
+      s = StringScanner.new(text_list)
+      while !s.eos?
+        text << s.scan(not_special)
+        break if s.eos?
+        case s.getch
+        when BACKSLASH
+          char = s.getch
+          # what do I do when char is nil? ignore the backslash too? I don't know...
+          raise InvalidEncoding, "text list \"#{text_list}\" ends after escape char" if char.nil?
+          text << (char.casecmp('n') == 0 ? LF : char)
+        when separator
+          list << text
+          text = ""
+        else
+          raise InvalidEncoding, "read #{s.matched.inspect} at #{s.pos} in #{s.string.inspect} (#{s.string.size}) using #{not_special.inspect}"
+        end
+      end
+      list << text
+      list
+    end
+
+    # Compound type values are delimited by a field delimiter, specified by the SEMI-COLON character (ASCII decimal 59). A SEMI-COLON in a component of a compound property value MUST be escaped with a BACKSLASH character (ASCII decimal 92).
+    #
+    # Lists of values are delimited by a list delimiter, specified by the COMMA character (ASCII decimal 44). A COMMA character in a value MUST be escaped with a BACKSLASH character (ASCII decimal 92).
+    #
+    # This profile supports the type grouping mechanism defined in [MIME-DIR]. Grouping of related types is a useful technique to communicate common semantics concerning the properties of a vCard.
+    def self.decode_structured_text(value, size, separator = SEMICOLON)
+      list = decode_text_list(value, separator)
+      list << "" while list.size < size
+      list.pop while list.size > size
+      list
+    end
+
+    def self.encode_structured_text(list, separator = SEMICOLON)
+      encode_text_list(list, separator)
+    end
+
+  end
+end

data/lib/virginity/encodings.rb

@@ -0,0 +1,36 @@
+module Virginity
+  module Encodings
+
+    def binary?(s)
+      s.encoding == Encoding::BINARY
+    end
+
+    def to_binary(s)
+      s.dup.force_encoding(Encoding::BINARY)
+    end
+
+    def to_ascii(s)
+      s.dup.force_encoding(Encoding::ASCII)
+    end
+
+    def to_default(s)
+      s.encode
+    end
+
+    def to_default!(s)
+      s.encode!
+    end
+
+    def verify_utf8ness(string)
+      if string.encoding == Encoding::UTF_8 || string.encoding == Encoding::US_ASCII
+        unless string.valid_encoding?
+          # puts "*"*100, "incorrectly encoded String", string, "*"*100
+          raise InvalidEncoding, "incorrectly encoded String"
+        end
+      else
+        raise InvalidEncoding, "expected UTF-8 or ASCII"
+      end
+    end
+
+  end
+end

data/lib/virginity/fixes.rb

@@ -0,0 +1,230 @@
+module Virginity
+  module Fixes
+    def unfold_faulty_qp_lines(faulty_lines)
+      lines = faulty_lines.dup
+      loop do # unfold line that do not begin with " " but are encoded as QP
+        changed = false
+        lines.each_with_index do |line, i|
+          # if line is QP, ends with equals and is not the last line
+          if line =~ /ENCODING=QUOTED-PRINTABLE/i and line =~ /=$/ and lines.length > i+1
+            lines[i] = lines[i].chomp('=') # remove the soft-line break, the last character, (=)...
+            lines[i] += lines.delete_at(i+1) # ...and add the next line to the failing line.
+            changed = true
+          end
+        end
+        break unless changed
+      end
+      lines
+    end
+
+    def non_empty_lines(faulty_lines) # (needed for simon3.vcf)
+      faulty_lines.reject {|line| line.empty? }
+    end
+
+    # for osx
+    # more info on this Base64-line: http://www.imc.org/imc-vcard/mail-archive/msg00555.html
+    def remove_spaces_from_base64_lines(faulty_lines)
+      faulty_lines.map do |line|
+        if line =~ /BASE64/ # FIXME, this will break if the value contains this string. --> so let's do this in the field-class?
+          line.gsub(/\s/, "")
+        else
+          line
+        end
+      end
+    end
+
+    def self.line_parts(line)
+      ContentLine::line_parts(line)
+    rescue InvalidEncoding
+      # so, it's invalid 3.0 encoded, could it be a 2.1-encoding?
+      DirectoryInformation::line21_parts(line)
+    end
+
+    # FIXME --> width can't be greater than width in the normal folding method... but it is now :-(
+    def self.photo_folding_like_apple(value, options = {})
+      width = options[:width] || 78
+      line_ending = (options[:windows_line_endings] ? "\r\n" : "\n")
+      s = line_ending + "  " + value.gsub(/.{#{width-2},#{width-2}}/) {|x| x + line_ending + "  "}
+      s.sub(/#{line_ending}  $/,"") # remove the last line ending if if it so happens to be that the last line is 'empty'
+    end
+
+    def self.sane_line_endings(s)
+      s.gsub(LineFolding::LINE_ENDING, "\n")
+    end
+
+    def self.should_be_folded?(line)
+      return false if line =~ /\A $/ # TODO: clarify this, it was line.first, which takes the first string.
+      if line.include?(":") # probably a field
+        unless line[0] == ":" or line.split(":").first.match(/[\(\)]/)
+          # it is a name then, so it's a new field
+          return false
+        end
+      end
+      true
+    end
+
+    def self.unfold_wrongly_folded_lines(s)
+      x = ""
+      sane_line_endings(s).split("\n").each do |line|
+        if should_be_folded?(line.dup)
+          x << " " + line
+        else
+          x << line
+        end
+        x << "\n"
+      end
+      x
+    end
+
+    def self.remove_ascii_ctl_chars(s)
+      ctl = (0..31).to_a + [127]
+      v = ""
+      s.each_byte do |c|
+        v << c unless ctl.include? c
+      end
+      v
+    end
+
+    def self.reencode_qp(qp, type=:value)
+      decoded = EncodingDecoding::decode_quoted_printable(qp)
+      case type
+      when nil
+        @value = EncodingDecoding::encode_text decoded
+      when :separated
+        @value = EncodingDecoding::encode_text_list [decoded]
+      when :structured
+        @value = EncodingDecoding::encode_structured_text [decoded]
+      else
+        raise TypeError, type
+      end
+    end
+
+    EQUALS_SIGN_WITHOUT_HEX = /=(([^0-9^A-F])|(.[^0-9^A-F]))/
+
+    def self.fix_faulty_qp_chars(s)
+      x = Virginity::LineFolding::unfold_and_split(s)
+      y = x.map do |l|
+        if l =~ /QUOTED\-PRINTABLE/
+          l =~ /(.*):(.*)/
+          prevalue, value = $1, $2
+          value = value.gsub(EQUALS_SIGN_WITHOUT_HEX) { |s| " #{$1}" }
+          "#{prevalue}:#{value}"
+        else
+          l
+        end
+      end
+      y.join("\n")
+    end
+
+    def self.guess_latin1(s)
+      x = Vcard.new(s).super_clean!
+      x.fields.each do |f|
+        begin
+          f.to_s.encode('UTF-8//TRANSLIT', Encoding::UTF_8)
+          f.to_s.force_encoding(LATIN1).encode
+        rescue EncodingError
+          print "\tguessing Latin1 for #{f.to_s.inspect}"
+          f.params << Param.new("CHARSET", "Latin1")
+          f.clean_charsets!
+          raise "GAAAAAAAAH!" unless f.to_s.is_utf8?
+          puts "\t-->\t" + f.to_s.inspect
+        end
+      end
+      x.super_clean!.to_s
+    end
+
+#     def faulty?(lns=lines)
+#       lns.any? { |line| not line =~ %r{#{Bnf::LINE}}i }
+#     end
+
+#     def fix!(lines)
+#       ls = lines.dup
+#       ls = unfold_faulty_qp_lines(ls)
+#       ls = non_empty_lines(ls)
+#       raise InvalidEncoding.new("I do not know how to repair this vcard:\n" + inspect) if faulty?(ls)
+#       ls
+#     end
+
+#     def fixed!
+#       original = lines.join("\n")
+#       ls = lines.dup
+#       before = original.dup
+#       after = ""
+#       while before != after and faulty_lines?(lines)
+#         before = ls.join("\n")
+#         ls = unfold_faulty_qp_lines(ls)
+#         ls = non_empty_lines(ls)
+#         after = ls.join("\n")
+#       end
+#       raise InvalidEncoding.new("I do not know how to repair this vcard:\n" + original) if faulty_lines?(lines)
+#       lines
+#     end
+
+  end
+end
+
+
+class Virginity::Vcard < Virginity::DirectoryInformation
+
+  def self.fields_from_broken_vcard(vcard_as_string)
+    lines = Virginity::LineFolding::unfold_and_split(vcard_as_string.lstrip).map do |line|
+      begin
+        # if it is not valid 3.0, could it be a 2.1-line?
+        Virginity::Field.parse(line)
+      rescue
+        group, name, params, value = DirectoryInformation::line21_parts(line)
+        Virginity::Field[name].new(name, value, params, group, :no_deep_copy => true)
+      end
+    end
+  end
+
+  def self.valid_utf8?(v)
+    if v.to_s.dup.force_encoding(Encoding::UTF_8).valid_encoding?
+      true
+    else
+      false
+    end
+  rescue EncodingError
+    return false
+  end
+
+  # NB. this only works for vCard 3.0, not for 2.1!
+  def self.fix_and_clean(vcard_as_string)
+    fixes ||= []
+    # puts vcard_as_string
+    # puts "fixes => #{fixes.inspect}"
+    lines = fields_from_broken_vcard(vcard_as_string)
+    v = Virginity::Vcard.new(lines)
+    v.super_clean!
+    valid_utf8?(v)
+    v
+  rescue EncodingError
+    #puts e.class, e
+    if !fixes.include?(:faulty_qp)
+      newcard = Fixes::fix_faulty_qp_chars(vcard_as_string)
+      fixes << :faulty_qp
+    elsif !fixes.include?(:guess_latin1)
+      newcard = Fixes::guess_latin1(lines)
+      fixes << :guess_latin1
+    else
+      File.open("illegal_sequence_#{vcard_as_string.hash}.vcf", "wb") do |f|
+        f.puts Virginity::Vcard.new(vcard_as_string).super_clean!
+      end
+      raise
+    end
+    vcard_as_string = newcard
+    retry
+  rescue Virginity::InvalidEncoding, Virginity::Vcard21::ParseError => e
+    if !fixes.include?(:fix_folding)
+      vcard_as_string = Fixes::unfold_wrongly_folded_lines(vcard_as_string)
+      fixes << :fix_folding
+      retry
+    elsif !fixes.include?(:weird_mac_16bit_encoding_that_is_not_utf16) && vcard_as_string.include?("\000")
+      vcard_as_string.gsub!("\000", "")
+      fixes << :weird_mac_16bit_encoding_that_is_not_utf16
+      retry
+    else
+      raise
+    end
+  end
+end

data/lib/virginity/vcard.rb

@@ -0,0 +1,244 @@
+require 'digest'
+require 'virginity/encodings'
+require 'virginity/dir_info'
+require 'virginity/vcard/cleaning'
+require 'virginity/vcard/categories'
+require 'virginity/vcard/name_handler'
+require 'virginity/vcard/fields'
+require 'virginity/vcard/fields_osx'
+require 'virginity/vcard/fields_soocial'
+require 'virginity/vcard/patching'
+
+module Virginity
+
+  class InvalidVcard < Error
+    attr_reader :original
+
+    def initialize(msg, original = $!)
+      super(msg)
+      @original = original
+    end
+  end
+
+  # rfc 2426, vCard MIME Directory Profile
+  class Vcard < DirectoryInformation
+    include VcardCleaning
+    include VcardCategories
+    include Patching
+    include Encodings
+
+    def self.diff(old, new)
+      Patching::diff(old, new)
+    end
+
+    EMPTY = "BEGIN:VCARD\nN:;;;;\nVERSION:3.0\nEND:VCARD\n"
+
+    VCARD_REGEX = /^[\ \t]*VCARD[\ \t]*$/i
+    def initialize(lines = EMPTY, options = {})
+      if lines.is_a? Array
+        @lines = lines.map {|line| line.to_field }
+      else # it's expected to be a String
+        verify_utf8ness lines
+        @lines = LineFolding::unfold_and_split(lines.lstrip).map { |line| Field.parse(line) }
+      end
+      raise InvalidVcard, "missing BEGIN:VCARD" unless @lines.first.name =~ Field::BEGIN_REGEX
+      raise InvalidVcard, "missing END:VCARD" unless @lines.last.name =~ Field::END_REGEX
+      raise InvalidVcard, "missing BEGIN:VCARD" unless @lines.first.raw_value =~ VCARD_REGEX
+      raise InvalidVcard, "missing END:VCARD" unless @lines.last.raw_value =~ VCARD_REGEX
+    rescue => error
+      raise InvalidVcard, error.message
+    end
+
+    # the empty vcard
+    def self.empty
+      new(LineFolding::unfold_and_split(EMPTY.lstrip).map { |line| Field.parse(line) })
+    end
+
+    # import vcard21 and convert it to 3.0
+    def self.from_vcard21(vcf, options = {})
+      verify_utf8ness vcf
+      vcard = new(lines_from_vcard21(vcf, options), options)
+      vcard.clean_version!
+    rescue => error
+      raise InvalidVcard, error.message
+    end
+
+    VERSION21 = /^VERSION\:2\.1(\s*)$/i
+    def self.parse(vcf, options = {})
+      vcf = vcf.to_s
+      verify_utf8ness vcf
+      if vcf =~ VERSION21
+        from_vcard21(vcf, options)
+      else
+        new(vcf, options)
+      end
+    end
+
+    def self.from_vcard(vcf, options = {})
+      parse(vcf, options)
+    end
+
+    def self.load_all_from(filename, options = {})
+      list(File.read(filename), options)
+    end
+
+    END_VCARD = /^END\:VCARD\s*$/i
+    # split a given string of concatenated vcards to an array containing one vcard per element
+    def self.vcards_in_list(vcf)
+      s = StringScanner.new(vcf)
+      array = []
+      while !s.eos?
+        if v = s.scan_until(END_VCARD)
+          v.lstrip!
+          array << v
+        else
+          puts s.peek(100)
+          break
+        end
+      end
+      array
+    end
+
+    # returns an array of Vcards for a string of concatenated vcards
+    def self.list(vcf, options = {})
+      vcards_in_list(vcf).map do |v|
+        from_vcard(v, options)
+      end
+    end
+
+    def inspect
+      super.chomp(">") + " name=" + name.to_s.inspect + ">"
+    end
+
+    def dir_info
+      DirectoryInformation.new(to_s)
+    end
+
+    def deep_copy
+      Marshal::load(Marshal::dump(self))
+    end
+
+    alias_method :fields, :lines
+    alias_method :delete_field, :delete_content_line
+
+    # add a field and return it
+    # if a block is given, then the new field is yielded so it can be changed
+    def add_field(line)
+      end_vcard = @lines.pop
+      raise InvalidVcard, "there is no last line? ('END:VCARD')" if end_vcard.nil?
+      @lines << (field = Field.parse(line))
+      @lines << end_vcard
+      yield field if block_given?
+      field
+    end
+
+    # add a field, returns the vcard
+    def <<(line)
+      add_field(line)
+      self
+    end
+    alias_method :push, :<<
+
+    # a vCard 2.1 string representation of this vCard
+    # if :windows_line_endings => true then lines end with \r\n otherwise with \n
+    CRLF = "\r\n"
+    def to_vcard21(options = {})
+      line_ending = options[:windows_line_endings] ? CRLF : LF
+      fields.map { |field| field.encode21(options) }.join(line_ending)
+    end
+
+    # are all fields also present or supersets in other?
+    IGNORE_IN_SUBSET_COMPARISON = %w(BEGIN END FN)
+    def subset_of?(other)
+      fields.all? do |field|
+        if IGNORE_IN_SUBSET_COMPARISON.include?(field.name)
+          true
+        else
+          # puts "-----\n"
+          # puts "#{field} in #{other}?\n"
+          other.lines_with_name(field.name).any? do |f|
+            begin
+              # puts "considering #{f} ==> #{f.raw_value == field.raw_value or field.subset_of?(f)}"
+              f.raw_value == field.raw_value or field.subset_of?(f)
+            rescue NoMethodError
+              false
+            end
+          end
+        end
+      end
+    end
+
+    SINGLETON_FIELDS = %w(N FN BEGIN END VERSION)
+    # import all fields except N, FN, and VERSION from other (another Vcard)
+    # duplicate fields are deduped
+    def assimilate_fields_from!(other)
+      other.fields.each do |field|
+        next if SINGLETON_FIELDS.include? field.name.upcase
+        push(field)
+      end
+      clean_same_value_fields!
+      self
+    end
+
+    def add(name)
+      add_field(name + ":") do |field|
+        yield field if block_given?
+      end
+    end
+
+    def add_email(address = nil)
+      add(EMAIL) do |email|
+        email.address = address.to_s
+        yield email if block_given?
+      end
+    end
+
+    def add_telephone(number = nil)
+      add(TEL) do |tel|
+        tel.number = number.to_s
+        yield tel if block_given?
+      end
+    end
+
+    def name
+      @name ||= NameHandler.new(self)
+    end
+
+    ADR = "ADR"
+    BDAY = "BDAY"
+    CATEGORIES = "CATEGORIES"
+    EMAIL = "EMAIL"
+    IMPP = "IMPP"
+    LOGO = "LOGO"
+    NICKNAME = "NICKNAME"
+    TEL = "TEL"
+    NOTE = "NOTE"
+    ORG = "ORG"
+    PHOTO = "PHOTO"
+    TITLE = "TITLE"
+    URL = "URL"
+    def addresses; lines_with_name(ADR); end
+    def birthdays; lines_with_name(BDAY); end
+    def categories; lines_with_name(CATEGORIES); end
+    def emails; lines_with_name(EMAIL); end
+    def impps; lines_with_name(IMPP); end
+    def logos; lines_with_name(LOGO); end
+    def nicknames; lines_with_name(NICKNAME); end
+    def telephones; lines_with_name(TEL); end
+    def notes; lines_with_name(NOTE); end
+    def organizations; lines_with_name(ORG); end
+    alias_method :organisations, :organizations # Britania rules the waves!
+    def photos; lines_with_name(PHOTO); end
+    def titles; lines_with_name(TITLE); end
+    def urls; lines_with_name(URL); end
+    def custom_im_fields; @lines.select{|line| line.is_a? Virginity::Vcard::CustomImField}; end
+
+    XABRELATEDNAMES = "X-ABRELATEDNAMES"
+    def related_names; lines_with_name(XABRELATEDNAMES); end
+    XABDATE = "X-ABDATE"
+    def dates; lines_with_name(XABDATE); end
+    XANNIVERSARY = "X-ANNIVERSARY"
+    def anniversaries; lines_with_name(XANNIVERSARY); end
+  end
+
+end