phonelib 0.3.6 → 0.4.0

data/data/libphonenumber/tools/java/java-build/test/com/google/i18n/phonenumbers/buildtools/GenerateTimeZonesMapDataTest.java

@@ -0,0 +1,136 @@
+/*
+ * Copyright (C) 2012 The Libphonenumber Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.i18n.phonenumbers.buildtools;
+
+import com.google.i18n.phonenumbers.prefixmapper.PrefixTimeZonesMap;
+
+import junit.framework.TestCase;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.util.Map;
+import java.util.SortedMap;
+
+/**
+ * Unittests for GenerateTimeZonesMapData.java
+ *
+ * @author Walter Erquinigo
+ */
+public class GenerateTimeZonesMapDataTest extends TestCase {
+  private static final String BRUSSELS_TZ = "Europe/Brussels";
+  private static final String PARIS_TZ = "Europe/Paris";
+  private static final String PARIS_BRUSSELS_LINES =
+        "322|" + BRUSSELS_TZ + "\n331|" + PARIS_TZ + "\n";
+
+  private static SortedMap<Integer, String> parseTextFileHelper(String input) throws IOException {
+    return GenerateTimeZonesMapData.parseTextFile(new ByteArrayInputStream(input.getBytes()));
+  }
+
+  public void testParseTextFile() throws IOException {
+    Map<Integer, String> result = parseTextFileHelper(PARIS_BRUSSELS_LINES);
+    assertEquals(2, result.size());
+    assertEquals(PARIS_TZ, result.get(331));
+    assertEquals(BRUSSELS_TZ, result.get(322));
+  }
+
+  public void testParseTextFileIgnoresComments() throws IOException {
+    Map<Integer, String> result = parseTextFileHelper("# Hello\n" + PARIS_BRUSSELS_LINES);
+    assertEquals(2, result.size());
+    assertEquals(PARIS_TZ, result.get(331));
+    assertEquals(BRUSSELS_TZ, result.get(322));
+  }
+
+  public void testParseTextFileIgnoresBlankLines() throws IOException {
+    Map<Integer, String> result = parseTextFileHelper("\n" + PARIS_BRUSSELS_LINES);
+    assertEquals(2, result.size());
+    assertEquals(PARIS_TZ, result.get(331));
+    assertEquals(BRUSSELS_TZ, result.get(322));
+  }
+
+  public void testParseTextFileIgnoresTrailingWhitespaces() throws IOException {
+    Map<Integer, String> result = parseTextFileHelper(
+        "331|" + PARIS_TZ + "\n322|" + BRUSSELS_TZ + "  \n");
+    assertEquals(2, result.size());
+    assertEquals(PARIS_TZ, result.get(331));
+    assertEquals(BRUSSELS_TZ, result.get(322));
+  }
+
+  public void testParseTextFileThrowsExceptionWithMalformattedData() throws IOException {
+    try {
+      parseTextFileHelper("331");
+      fail();
+    } catch (RuntimeException e) {
+      // Expected.
+    }
+  }
+
+  public void testParseTextFileThrowsExceptionWithMissingTimeZone() throws IOException {
+    try {
+      parseTextFileHelper("331|");
+      fail();
+    } catch (RuntimeException e) {
+      // Expected.
+    }
+  }
+
+  // Returns a String representing the input after serialization and deserialization by
+  // PrefixTimeZonesMap.
+  private static String convertDataHelper(String input) throws IOException {
+    ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(input.getBytes());
+    ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+
+    SortedMap<Integer, String> prefixTimeZonesMapping = parseTextFileHelper(input);
+    GenerateTimeZonesMapData.writeToBinaryFile(prefixTimeZonesMapping, byteArrayOutputStream);
+    // The byte array output stream now contains the corresponding serialized prefix to time zones
+    // map. Try to deserialize it and compare it with the initial input.
+    PrefixTimeZonesMap prefixTimeZonesMap = new PrefixTimeZonesMap();
+    prefixTimeZonesMap.readExternal(
+        new ObjectInputStream(new ByteArrayInputStream(byteArrayOutputStream.toByteArray())));
+
+    return prefixTimeZonesMap.toString();
+  }
+
+  public void testConvertData() throws IOException {
+    String input = PARIS_BRUSSELS_LINES;
+
+    String dataAfterDeserialization = convertDataHelper(input);
+    assertEquals(input, dataAfterDeserialization);
+  }
+
+  public void testConvertThrowsExceptionWithMissingTimeZone() throws IOException {
+    String input = PARIS_BRUSSELS_LINES + "3341|\n";
+
+    try {
+      String dataAfterDeserialization = convertDataHelper(input);
+    } catch (RuntimeException e) {
+      // Expected.
+    }
+  }
+
+  public void testConvertDataThrowsExceptionWithDuplicatedPrefixes() throws IOException {
+    String input = "331|" + PARIS_TZ + "\n331|" + BRUSSELS_TZ + "\n";
+
+    try {
+      convertDataHelper(input);
+      fail();
+    } catch (RuntimeException e) {
+      // Expected.
+    }
+  }
+}

data/data/libphonenumber/tools/java/pom.xml

@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>com.google.i18n.phonenumbers</groupId>
+  <artifactId>tools</artifactId>
+  <packaging>pom</packaging>
+  <version>1.0-SNAPSHOT</version>
+  <name>Libphonenumber build tools</name>
+
+  <properties>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+  </properties>
+
+  <licenses>
+    <license>
+      <name>Apache 2</name>
+      <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+      <comments>Copyright (C) 2011 The Libphonenumber Authors</comments>
+    </license>
+  </licenses>
+
+  <modules>
+    <module>common</module>
+    <module>cpp-build</module>
+    <module>data</module>
+    <module>java-build</module>
+  </modules>
+
+</project>

data/data/libphonenumber/tools/script/continuous-integration.sh

@@ -0,0 +1,96 @@
+#! /bin/sh
+
+# Copyright (C) 2011 The Libphonenumber Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Author: Philippe Liard
+
+# Countinuous integration script that tests the different versions and
+# configurations of libphonenumber.
+
+# Check geocoding resource files encoding.
+(find resources/geocoding -type f | xargs file | egrep -v 'UTF-8|ASCII') && exit 1
+
+# Test the C++ version with the provided CMake parameter.
+CXX=g++
+INSTALL_PREFIX=/tmp/libphonenumber
+
+test_cpp_version() {
+  CC_TEST_FILE=`mktemp`.cc
+  CC_TEST_BINARY=`mktemp`
+  CMAKE_FLAGS="$1"
+  LD_FLAGS="-L${INSTALL_PREFIX}/lib -lphonenumber -lboost_thread $2"
+
+  # Write the program that tests the installation of the library to a temporary
+  # source file.
+  > $CC_TEST_FILE echo '
+    #include <cassert>
+
+    #include <base/memory/scoped_ptr.h>
+
+    // Include all the public headers.
+    #include <phonenumbers/asyoutypeformatter.h>
+    #include <phonenumbers/phonenumber.pb.h>
+    #include <phonenumbers/phonenumbermatch.h>
+    #include <phonenumbers/phonenumbermatcher.h>
+    #include <phonenumbers/phonenumberutil.h>
+
+    using i18n::phonenumbers::AsYouTypeFormatter;
+    using i18n::phonenumbers::PhoneNumberUtil;
+
+    int main() {
+      PhoneNumberUtil* const phone_util = PhoneNumberUtil::GetInstance();
+      const scoped_ptr<AsYouTypeFormatter> asytf(
+          phone_util->GetAsYouTypeFormatter("US"));
+
+      assert(phone_util != NULL);
+      assert(asytf != NULL);
+    }'
+
+  # Run the build and tests.
+  (
+    set +e
+    rm -rf cpp/build /tmp/libphonenumber
+    mkdir cpp/build /tmp/libphonenumber
+    cd cpp/build
+    cmake "${CMAKE_FLAGS}" -DCMAKE_INSTALL_PREFIX=$INSTALL_PREFIX ..
+    make test
+    make install
+    $CXX -o $CC_TEST_BINARY $CC_TEST_FILE -I${INSTALL_PREFIX}/include $LD_FLAGS
+    LD_LIBRARY_PATH="${INSTALL_PREFIX}/lib" $CC_TEST_BINARY
+  )
+  STATUS=$?
+  # Remove the temporary files.
+  rm -f $CC_TEST_FILE
+  rm -f $CC_TEST_BINARY
+
+  [ $STATUS -ne 0 ] && exit $STATUS
+}
+
+BASE_LIBS='-licuuc -lprotobuf'
+
+test_cpp_version '' "$BASE_LIBS"
+test_cpp_version '-DUSE_ICU_REGEXP=ON' "$BASE_LIBS"
+test_cpp_version '-DUSE_LITE_METADATA=ON' '-licuuc -lprotobuf-lite'
+test_cpp_version '-DUSE_RE2=ON' "$BASE_LIBS -lre2"
+test_cpp_version '-DUSE_STD_MAP=ON' "$BASE_LIBS"
+
+# Test Java version using Ant.
+(cd java && ant clean jar && ant junit) || exit $?
+
+# Test Java version using Maven.
+(cd java && mvn clean package) || exit $?
+
+# Test build tools.
+(cd tools/java && mvn clean package) || exit $?

data/lib/phonelib/core.rb

@@ -9,6 +9,14 @@ module Phonelib
       @@phone_data ||= load_data
     end
 
+    # variable for storing geo/carrier/timezone data
+    @@phone_ext_data = nil
+
+    # getter for extended phone data
+    def phone_ext_data
+      @@phone_ext_data ||= load_ext_data
+    end
+
     # default country for parsing variable setting
     @@default_country = nil
 
@@ -23,6 +31,12 @@ module Phonelib
     end
 
     # gem constants definition
+
+    # Main data file
+    FILE_MAIN_DATA = 'data/phone_data.dat'
+    # Extended data file
+    FILE_EXT_DATA = 'data/extended_data.dat'
+
     # constants for phone types
 
     # Validation patterns keys constants
@@ -101,6 +115,21 @@ module Phonelib
       fixed_or_mobile: 'Fixed Line or Mobile'
     }
 
+    # Extended data prefixes hash key
+    EXT_PREFIXES = :prefixes
+    # Extended data geo names array key
+    EXT_GEO_NAMES = :geo_names
+    # Extended data timezones array key
+    EXT_TIMEZONES = :timezones
+    # Extended data carriers array key
+    EXT_CARRIERS = :carriers
+    # Extended data key for geoname in prefixes hash
+    EXT_GEO_NAME_KEY = :g
+    # Extended data key for timezone in prefixes hash
+    EXT_TIMEZONE_KEY = :t
+    # Extended data key for carrier in prefixes hash
+    EXT_CARRIER_KEY = :c
+
     # method for parsing phone number.
     # On first run fills @@phone_data with data present in yaml file
     def parse(phone, passed_country = nil)
@@ -141,7 +170,13 @@ module Phonelib
 
     # Load data file into memory
     def load_data
-      data_file = File.dirname(__FILE__) + '/../../data/phone_data.dat'
+      data_file = "#{File.dirname(__FILE__)}/../../#{FILE_MAIN_DATA}"
+      Marshal.load(File.binread(data_file))
+    end
+
+    # Load extended data file into memory
+    def load_ext_data
+      data_file = "#{File.dirname(__FILE__)}/../../#{FILE_EXT_DATA}"
       Marshal.load(File.binread(data_file))
     end
   end

data/lib/phonelib/data_importer.rb

@@ -0,0 +1,268 @@
+module Phonelib
+  # module processes creation of data files needed for this gem
+  module DataImporter
+    require 'nokogiri'
+
+    # official libphonenumber repo for cloning
+    REPO = 'https://github.com/googlei18n/libphonenumber.git'
+
+    # importing function
+    def self.import
+      Importer.new
+    end
+
+    # class with functionality for importing data
+    class Importer
+      # main data file in repo
+      MAIN_FILE = 'resources/PhoneNumberMetadata.xml'
+      # alternate formats data file in repo
+      FORMATS_FILE = 'resources/PhoneNumberAlternateFormats.xml'
+      # geocoding data dir in repo
+      GEOCODING_DIR = 'resources/geocoding/en/'
+      # carrier data dir in repo
+      CARRIER_DIR = 'resources/carrier/en/'
+      # timezones data dir in repo
+      TIMEZONES_DIR= 'resources/timezones/'
+
+      # xml comments attributes names that should not be parsed
+      XML_COMMENT_ATTRIBUTES = %w(text comment)
+      # xml format attributes names
+      XML_FORMAT_NAMES = %w(intlFormat format)
+
+      # class initialization method
+      def initialize
+        @destination = File.path(
+            "#{File.dirname(__FILE__)}/../../data/libphonenumber/")
+        @data = {}
+        @prefixes = {}
+        @geo_names = []
+        @timezones = []
+        @carriers = []
+
+        clone_repo
+        import_main_data
+        import_alternate_formats
+        import_geocoding_data
+        import_timezone_data
+        import_carrier_data
+        save_data_file
+      end
+
+      private
+
+      # method saves parsed data to data files
+      def save_data_file
+        data_file =
+            "#{File.dirname(__FILE__)}/../../#{Phonelib::Core::FILE_MAIN_DATA}"
+
+        File.open(data_file, 'wb+') do |f|
+          Marshal.dump(@data, f)
+        end
+
+        ext_file =
+            "#{File.dirname(__FILE__)}/../../#{Phonelib::Core::FILE_EXT_DATA}"
+        extended = {
+          Phonelib::Core::EXT_PREFIXES => @prefixes,
+          Phonelib::Core::EXT_GEO_NAMES => @geo_names,
+          Phonelib::Core::EXT_TIMEZONES => @timezones,
+          Phonelib::Core::EXT_CARRIERS => @carriers
+        }
+        File.open(ext_file, 'wb+') do |f|
+          Marshal.dump(extended, f)
+        end
+        puts 'DATA SAVED'
+      end
+
+      # method clones libphonenumber repo to local dir
+      def clone_repo
+        repo = Phonelib::DataImporter::REPO
+
+        system("rm -rf #{@destination}")
+        cloned = system("git clone #{repo} #{@destination} --depth 1 -b master")
+        raise 'Could not clone repo' unless cloned
+      end
+
+      # method parses main data file
+      def import_main_data
+        puts 'IMPORTING MAIN DATA'
+        main_from_xml("#{@destination}#{MAIN_FILE}").elements.each do |el|
+          # each country
+          country = get_hash_from_xml(el, :attributes)
+          country[:types] = {}
+
+          without_comments(el.children).each do | phone_type |
+            if phone_type.name == 'availableFormats'
+              country[:formats] = parse_formats(phone_type.children)
+            else
+              country[:types][name2sym(phone_type.name)] =
+                  get_hash_from_xml(phone_type, :element)
+            end
+          end
+
+          @data[country[:id]] = country
+        end
+      end
+
+      # method parses alternate formats file
+      def import_alternate_formats
+        puts 'IMPORTING ALTERNATE FORMATS'
+
+
+        main_from_xml("#{@destination}#{FORMATS_FILE}").elements.each do |el|
+          el.children.each do | phone_type |
+            if phone_type.name == 'availableFormats'
+              formats = parse_formats(phone_type.children)
+
+              country_code = el.attribute('countryCode').value
+              @data[get_country_by_code(country_code)][:formats] += formats
+            end
+          end
+        end
+      end
+
+      # method parses geocoding data dir
+      def import_geocoding_data
+        puts 'IMPORTING GEOCODING DATA'
+        import_raw_files_data("#{@destination}#{GEOCODING_DIR}*",
+                              @geo_names,
+                              :g)
+      end
+
+      # method parses timezones data dir
+      def import_timezone_data
+        puts 'IMPORTING TIMEZONES DATA'
+        import_raw_files_data("#{@destination}#{TIMEZONES_DIR}*",
+                              @timezones,
+                              :t)
+      end
+
+      # method parses carriers data dir
+      def import_carrier_data
+        puts 'IMPORTING CARRIER DATA'
+        import_raw_files_data("#{@destination}#{CARRIER_DIR}*",
+                              @carriers,
+                              :c)
+      end
+
+      # method filters xml elements excluding comments elements
+      def without_comments(data)
+        data.select do |el|
+          !XML_COMMENT_ATTRIBUTES.include? el.name
+        end
+      end
+
+      # method creates hash from xml elements/element attributes
+      def get_hash_from_xml(data, type)
+        hash = {}
+        case type
+          when :attributes
+            data.attributes.each do |k, v|
+              hash[name2sym(k)] = str_clean(v)
+            end
+          when :children
+            data.each do |f|
+              hash[name2sym(f[0])] = f[1]
+            end
+          when :element
+            data.elements.each do |child|
+              hash[name2sym(child.name)] = str_clean(child.children.first)
+            end
+        end
+        hash
+      end
+
+      # method parses xml for formats data
+      def parse_formats(formats_children)
+        without_comments(formats_children).map do |format|
+          current_format = get_hash_from_xml(format, :children)
+
+          without_comments(format.children).each do |f|
+            current_format[name2sym(f.name)] =
+                str_clean(f.children.first, is_not_format(f.name))
+          end
+
+          current_format
+        end.compact
+      end
+
+      # method updates data from raw files
+      def import_raw_files_data(dir, var, key)
+        name2index = {}
+        Dir["#{dir}"].each do |file|
+          parse_raw_file(file).each do |prefix, name|
+            unless name2index[name]
+              var.push name
+              name2index[name] = var.size - 1
+            end
+
+            @prefixes = fill_prefixes(key, name2index[name], prefix, @prefixes)
+          end
+        end
+      end
+
+      # method updates prefixes hash recursively
+      def fill_prefixes(key, value, prefix, prefixes)
+        prefixes = {} if prefixes.nil?
+        if prefix.size == 1
+          prefixes[prefix.to_i] = {} unless prefixes[prefix.to_i]
+          prefixes[prefix.to_i][key] = value
+        else
+          pr = prefix[0].to_i
+          prefixes[pr] = fill_prefixes(key, value, prefix[1..-1], prefixes[pr])
+        end
+        prefixes
+      end
+
+      # method parses raw data file
+      def parse_raw_file(file)
+        data = {}
+        File.readlines(file).each do |line|
+          line = str_clean line
+          next if line.empty? || line[0] == '#'
+          prefix, line_data = line.split('|')
+          data[prefix] = line_data && line_data.split('&')
+        end
+        data
+      end
+
+      # method for checking if element name is not a format element
+      def is_not_format(name)
+        !XML_FORMAT_NAMES.include? name
+      end
+
+      # get main body from parsed xml document
+      def main_from_xml(file)
+        xml_data = File.read(file)
+        xml_data.force_encoding("utf-8")
+
+        doc = Nokogiri::XML(xml_data)
+        doc.elements.first.elements.first
+      end
+
+      # method finds country by country prefix
+      def get_country_by_code(country_code)
+        match = @data.select { |k, v| v[:country_code] == country_code }
+        if match.size > 1
+          match = match.select { |k, v| v[:main_country_for_code] == 'true' }
+        end
+
+        match.keys.first
+      end
+
+      # helper that cleans string
+      def str_clean(s, white_space = true)
+        s.to_s.tr(white_space ? " \n" : "\n", '')
+      end
+
+      # helper that converts xml element name to symbol
+      def name2sym(name)
+        camel2snake(name).to_sym
+      end
+
+      # method that converts camel case to snake case
+      def camel2snake(s)
+        s.gsub(/[A-Z]+/) { |m| "_#{m.downcase}" }
+      end
+    end
+  end
+end