swissmatch-location 0.1.2.201409 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4b41908ad58903a56c562d0ad66b1f0f0518c4f4
-  data.tar.gz: 2273c6732c40767514b1ca058bad38706292967f
+  metadata.gz: 561e487d2ba8a0c726877e72e89c26bef87931a1
+  data.tar.gz: 22332920078c7e433fd68f34c7716eea73e7f9fe
 SHA512:
-  metadata.gz: 0dc83aefbf0466108498dfba097b120fcc6794cfff5cb13f8df1ff5c08078b765ac7933986e5326463f31864b3545b8186c1d26e87dd63890255c947a48e3780
-  data.tar.gz: acd33e0b2bfa223c271370a899726bc01f86929227b48e65d92b44346e4798500223f62685906d35610f43af469c1e04444b554ef29731bb6028036e1cbf440b
+  metadata.gz: b2a72500cce5454dcb9731c68647af586f0e44def5b8fece74db640be777825fa72682782f57e6a0873416d12bf35af6b4e03283805d3b1d49f5a36bd685348d
+  data.tar.gz: fe23fbc5b3d87ba1160931d6cee52abd7cf9479c7e4f64701df8dce15e2511eca9c1ac7655c6068415bc7e059dc8f375e100f51b43a9cdace7df5e19054bcc8d

data/README.markdown

@@ -4,21 +4,50 @@ README
 
 Summary
 -------
+
 Deal with swiss zip codes, cantons and communities, using the official swiss post mat[ch]
 database.
 
 
 Installation
 ------------
+
 Install the gem: `gem install swissmatch-location`  
 Depending on how you installed rubygems, you have to use `sudo`:
 `sudo gem install swissmatch-location`  
 In Ruby: `require 'swissmatch/location'`  
 To automatically load the datafiles: `require 'swissmatch/location/autoload'`
 
+**IMPORTANT!**
+
+Due to a change in the license agreement of the swiss post, I'm no longer
+allowed to ship the data together with the gem. Here's a guide on how to
+install and update your swissmatch data:
+
+1. Go to https://www.post.ch/de/pages/downloadcenter-match
+2. **In the pop-up menu top-left** select "Register"
+3. Once you're registered (you'll get a snail-mail letter from the post to sign),
+   you visit the same page again and this time you choose "Login"
+   **from the pop-up menu top-left**, the login button top right **does not work
+   for this!** (the former logs you into the downloadcenter, the latter into
+   the customer center).
+3. After login, you choose the download page for "Address master data"
+   (de: "Adressstammdaten", fr: "Base de données d'adresses de référence", it:
+   "Banca dati indirizzi di riferimento")
+4. Download "Existing data" (de: "Bestand", fr: "Etat", it: "Versione completa")
+5. Unzip the file
+6. Open a shell and cd into the directory with the unzipped master data
+7. Run `swissmatch-location install-data PATH_TO_MASTER_DATA_FILE`
+
+You can test your installation by running `swissmatch-location stats`. It should
+tell you the age of the data and a number >0 of zip codes.  
+A negative age is possible since the swiss post provides files which start to be
+valid in the future.
+
 
 Usage
 -----
+
     require 'swissmatch/location/autoload' # use this to automatically load the data
 
     # Get all zip codes for a given code, the example returns the official name of the first
@@ -42,19 +71,40 @@ Usage
     SwissMatch.canton("Zurigo").name                        # => "Zürich"
 
     # SwissMatch also provides data over swiss communities (Gemeinden)
-    SwissMatch.communities("Zürich").first.community_number # => 261
-    SwissMatch.community(261).name                          # => "Zürich"
+    SwissMatch.community("Zürich").community_number # => 261
+    SwissMatch.community(261).name                  # => "Zürich"
 
 
 SwissMatch and Rails/Databases
 ------------------------------
+
 If you want to load the data into your database, or use it in a rails project,
 then you should look at swissmatch-rails. It provides a couple of models and
 a data loading script.
 
 
+Notable Recent Changes
+----------------------
+
+### 0.1.2.x -> 1.0.0
+
+* Zip code master data is no longer bundled with the gem. Check the installation
+  guide for how to obtain, install and update the data.
+* swissmatch-location executable added to support the installation and updating
+  of the master data
+* SwissMatch.communities and SwissMatch::Location.communities no longer return
+  communities by name. This has moved to the .community methods since community
+  names are unique now.
+* SwissMatch::Location::Converter has been added to convert the new data format
+  into a compact data file (~140MB down to ~400KB)
+* SwissMatch::Location::DataFiles rewritten to read the new compact master data
+  file.
+* Dropped rubyzip dependency.
+
+
 Relevant Classes and Modules
 ----------------------------
+
 * __{SwissMatch}__
   Convenience methods to access cantons, communities and zip codes
 * __{SwissMatch::Cantons}__
@@ -68,7 +118,8 @@ Relevant Classes and Modules
 * __{SwissMatch::ZipCodes}__
   Swiss zip code collection
 * __{SwissMatch::ZipCode}__
-  A swiss zip code (a zip code can be described and uniquely identified by either code and city, code and add-on or the swiss posts ONRP)
+  A swiss zip code (a zip code can be described and uniquely identified by
+  either code and city, code and add-on or the swiss posts ONRP)
 
 
 Links

data/bin/swissmatch-location

@@ -0,0 +1,51 @@
+#!/usr/bin/env ruby
+
+lib_dir = File.expand_path("#{__dir__}/../lib")
+$LOAD_PATH << lib_dir if File.directory?(lib_dir) && !$LOAD_PATH.include?(lib_dir)
+
+require "fileutils"
+require "swissmatch/location"
+require "swissmatch/location/converter"
+
+begin
+  is_empty = false
+  SwissMatch::Location.load
+rescue SwissMatch::LoadError, ArgumentError
+  is_empty = true
+  SwissMatch::Location.load(SwissMatch::Location::DataFiles.empty)
+end
+
+case ARGV[0]
+  when "stats"
+    puts "SwissMatch::Location Statistics"
+    puts "Master Data from #{SwissMatch::Location.data.date} (age #{(Date.today - SwissMatch::Location.data.date).floor} days), random code #{SwissMatch::Location.data.random_code}"
+    puts "Zip Codes:   #{SwissMatch.zip_codes.size}"
+    puts "Cantons:     #{SwissMatch.cantons.size}"
+    puts "Communities: #{SwissMatch.communities.size}"
+    puts "Districts:   #{SwissMatch.districts.size}"
+
+  when "install-data"
+    master_file = ARGV[1]
+    install_dir = ARGV[2]
+
+    if !master_file
+      abort("Please supply a master file (`swissmatch-location install-data MASTER_FILE [INSTALL_DIRECTORY]`)")
+    elsif !File.exist?(master_file)
+      abort("Could not find #{master_file.inspect}")
+    elsif !File.readable?(master_file)
+      abort("Could not read #{master_file.inspect}")
+    end
+    unless install_dir
+      install_dir = File.expand_path('~/.swissmatch')
+      FileUtils.mkdir_p(install_dir)
+    end
+
+    puts "Installing data from #{master_file} in #{install_dir}"
+    binary_file = "#{install_dir}/locations_#{Time.now.strftime('%F')}.binary"
+    SwissMatch::Location::Converter.new(master_file).convert.write(binary_file)
+    puts "Done"
+
+  else
+    puts "Please supply either `stats` or `install-data MASTER_FILE [INSTALL_DIRECTORY]` as arguments"
+    exit(1)
+end

data/lib/swissmatch/location.rb

@@ -72,22 +72,26 @@ module SwissMatch
       @data.districts
     end
 
-    # @param [Integer] key
-    #   The community number of the community
+    # @param [Integer, String] key
+    #   The name or community number of the community
     #
     # @return [SwissMatch::Community]
-    #   The community with the community number
+    #   The community with the given name or community number
     def self.community(key)
-      @data.communities.by_community_number(key)
+      case key
+        when Integer
+          @data.communities.by_community_number(key)
+        when String
+          @data.communities.by_name(key)
+        else
+          raise TypeError, "Expected Integer or String, but got #{key.inspect}:#{key.class}"
+      end
     end
 
-    # @param [String] name
-    #   The name of the communities
-    #
     # @return [SwissMatch::Communities]
-    #   All communities, or those matching the given name
-    def self.communities(name=nil)
-      name ? @data.communities.by_name(name) : @data.communities
+    #   All communities
+    def self.communities
+      @data.communities
     end
 
     # @param [String, Integer] code_or_name

data/lib/swissmatch/location/converter.rb

@@ -0,0 +1,155 @@
+module SwissMatch
+  module Location
+
+    # SwissMatch::Location::Converter
+    #
+    # Converts the files supplied by post.ch and bfs.admin.ch into a single
+    # binary file which is faster to load
+    #
+    # Format:
+    #   Byte 0...4: PostMatch master file date, in Date.jd format
+    #   Byte 4...8: PostMach master file random code
+    #   Byte 8...18: zip1_count, zip2_count, community1_count, community2_count, district_count; packed with N*
+    #   Byte 18...34: bytesizes of int1_columns, int2_columns, int4_columns and text_columns
+    #   Byte 34...-1: int1_columns + int2_columns + int4_columns + text_columns
+    #   
+    #   int1_columns: packed with C* the columns
+    #   * zip1_type
+    #   * zip1_addon
+    #   * zip1_language
+    #   * zip1_language_alternative
+    #   * zip2_region
+    #   * zip2_type
+    #   * zip2_lang
+    #   * com2_PLZZ
+    #
+    #   int2_columns: packed with n* the columns
+    #   * zip1_onrp
+    #   * zip1_code
+    #   * zip1_delivery_by
+    #   * zip1_largest_community_number
+    #   * zip2_onrp
+    #   * com1_bfsnr
+    #   * com1_agglomeration
+    #   * com2_GDENR
+    #   * com2_PLZ4
+    #   * district_GDEBZNR
+    #
+    #   int4_columns: packed with N* the columns
+    #   * zip1_valid_from
+    #
+    #   text_columns: joined with \x1f
+    #   * zip1_name_short
+    #   * zip1_name
+    #   * zip1_canton
+    #   * zip2_short
+    #   * zip2_name
+    #   * com1_name
+    #   * com1_canton
+    #   * district_GDEKT
+    #   * district_GDEBZNA
+    #
+    class Converter
+      def initialize(match_path, districts_path=nil, communities_path=nil)
+        @match_path       = match_path
+        @districts_path   = districts_path || gem_districts_path
+        @communities_path = communities_path || gem_communities_path
+        @data             = nil
+      end
+
+      def gem_data_path
+        data_directory = File.expand_path('../../../../data/swissmatch-location', __FILE__)
+        data_directory = Gem.datadir 'swissmatch-location' if defined?(Gem) && !File.directory?(data_directory)
+
+        data_directory
+      end
+
+      def gem_districts_path
+        Dir.enum_for(:glob, "#{gem_data_path}/districts_*.csv").sort.last
+      end
+
+      def gem_communities_path
+        Dir.enum_for(:glob, "#{gem_data_path}/communities_*.csv").sort.last
+      end
+
+      def generate_expression(size, separator, terminator)
+        /^#{Array.new(size) { "([^#{separator}]*)" }.join(eval("'#{separator}'"))}#{terminator}/
+      end
+
+      def convert
+        match_data       = File.read(@match_path, encoding: Encoding::Windows_1252).encode(Encoding::UTF_8)
+        districts_data   = File.read(@districts_path, encoding: Encoding::Windows_1252).encode(Encoding::UTF_8)
+        communities_data = File.read(@communities_path, encoding: Encoding::Windows_1252).encode(Encoding::UTF_8)
+
+        r_base        = generate_expression(3, ';', '\r\n')
+        r_zip_1       = generate_expression(16, ';', '\r\n')
+        r_zip_2       = generate_expression(7, ';', '\r\n')
+        r_community1  = generate_expression(5, ';', '\r\n')
+        r_community2  = generate_expression(10, ',', '(?:\n|\z)')
+        r_district    = generate_expression(3, ',', '\n')
+
+        start_zip1 = match_data.index(/^01/)
+        start_zip2 = match_data.index(/^02/, start_zip1)
+        start_com  = match_data.index(/^03/, start_zip2)
+        end_com    = match_data.index(/^04/, start_com)
+
+        base      = match_data[0...start_zip1].scan(r_base).first
+        zip1      = match_data[start_zip1...start_zip2].scan(r_zip_1); zip1.size
+        zip2      = match_data[start_zip2...start_com].scan(r_zip_2); zip2.size
+        com1      = match_data[start_com...end_com].scan(r_community1); com1.size
+        com2      = communities_data.scan(r_community2); com2.size
+        districts = districts_data.scan(r_district); districts.size
+
+        zip1_columns = zip1.transpose; 0
+        zip2_columns = zip2.transpose; 0
+        com1_columns  = com1.transpose; 0
+        com2_columns  = com2.transpose; 0
+        dist_columns  = districts.transpose; 0
+
+        int1_columns = (
+          zip1_columns.values_at(3,5,10,11).flatten+
+          zip2_columns.values_at(2,3,4).flatten+
+          com2_columns[8]
+        ).map(&:to_i).pack("C*")
+
+        int2_columns = (
+          zip1_columns.values_at(1,4,12,2).flatten+
+          zip2_columns[1]+
+          com1_columns.values_at(1,4).flatten+
+          com2_columns[4]+
+          com2_columns[7]+
+          dist_columns[1]
+        ).map(&:to_i).pack("n*")
+
+        int4_columns = (
+          zip1_columns[13].map { |date| Date.civil(*date.match(/^(\d{4})(\d\d)(\d\d)$/).captures.map(&:to_i)).jd }
+        ).pack("N*")
+
+        text_columns = (
+          zip1_columns.values_at(7,8,9).flatten+
+          zip2_columns[5]+
+          zip2_columns[6]+
+          com1_columns[2]+
+          com1_columns[3]+
+          dist_columns[0]+
+          dist_columns[2]
+        ).join("\x1f").force_encoding(Encoding::BINARY)
+
+        @data =
+          [Date.civil(*base[1].match(/^(\d{4})(\d\d)(\d\d)$/).captures.map(&:to_i)).jd, base[2].to_i].pack("NN")+
+          [zip1.size, zip2.size, com1.size, com2.size, districts.size].pack("n*")+
+          [int1_columns.bytesize, int2_columns.bytesize, int4_columns.bytesize, text_columns.bytesize].pack("N*")+
+          int1_columns+
+          int2_columns+
+          int4_columns+
+          text_columns
+
+        self
+      end
+
+      def write(path)
+        File.write(path, @data, encoding: Encoding::BINARY)
+      end
+    end
+  end
+end

data/lib/swissmatch/location/datafiles.rb

@@ -12,12 +12,15 @@ require 'swissmatch/community'
 require 'swissmatch/communities'
 require 'swissmatch/zipcode'
 require 'swissmatch/zipcodes'
+require 'swissmatch/location/ruby'
 
 
 
 module SwissMatch
   module Location
 
+    # SwissMatch::Location::DataFiles
+    #
     # Deals with retrieving and updating the files provided by the swiss postal service,
     # and loading the data from them.
     #
@@ -26,73 +29,58 @@ module SwissMatch
     #   change over iterations.
     class DataFiles
 
-      # Used to generate the regular expressions used to parse the data files.
-      # Generates a regular expression, that matches +size+ tab separated fields,
-      # delimited by \r\n.
-      # @private
-      def self.generate_expression(size, separator, terminator)
-        /^#{Array.new(size) { "([^#{separator}]*)" }.join(eval("'#{separator}'"))}#{terminator}/
-      end
-
-      # Regular expressions used to parse the different files.
-      # @private
-      Expressions = {
-        :community    => generate_expression(4, '\t', '\r\n'),
-        :zip_2        => generate_expression(6, '\t', '\r\n'),
-        :zip_1        => generate_expression(13, '\t', '\r\n'),
-        :districts    => generate_expression(3, ',', '\n'),
-        :communities  => generate_expression(10, ',', '\n'),
-      }
-
-      # @private
-      # The URL of the plz_p1 file
-      URLZip1         = "https://match.post.ch/download?file=10001&tid=11&rol=0"
+      # Used to convert numerical language codes to symbols
+      LanguageCodes = [nil, :de, :fr, :it, :rt]
 
+      # The data of all cantons
       # @private
-      # The URL of the plz_p2 file
-      URLZip2         = "https://match.post.ch/download?file=10002&tid=14&rol=0"
+      AllCantons = Cantons.new([
+        Canton.new("AG", "Aargau",                    "Aargau",                   "Argovie",                      "Argovia",                  "Argovia"),
+        Canton.new("AI", "Appenzell Innerrhoden",     "Appenzell Innerrhoden",    "Appenzell Rhodes-Intérieures", "Appenzello Interno",       "Appenzell Dadens"),
+        Canton.new("AR", "Appenzell Ausserrhoden",    "Appenzell Ausserrhoden",   "Appenzell Rhodes-Extérieures", "Appenzello Esterno",       "Appenzell Dadora"),
+        Canton.new("BE", "Bern",                      "Bern",                     "Berne",                        "Berna",                    "Berna"),
+        Canton.new("BL", "Basel-Landschaft",          "Basel-Landschaft",         "Bâle-Campagne",                "Basilea Campagna",         "Basilea-Champagna"),
+        Canton.new("BS", "Basel-Stadt",               "Basel-Stadt",              "Bâle-Ville",                   "Basilea Città",            "Basilea-Citad"),
+        Canton.new("FR", "Freiburg",                  "Fribourg",                 "Fribourg",                     "Friburgo",                 "Friburg"),
+        Canton.new("GE", "Genève",                    "Genf",                     "Genève",                       "Ginevra",                  "Genevra"),
+        Canton.new("GL", "Glarus",                    "Glarus",                   "Glaris",                       "Glarona",                  "Glaruna"),
+        Canton.new("GR", "Graubünden",                "Graubünden",               "Grisons",                      "Grigioni",                 "Grischun"),
+        Canton.new("JU", "Jura",                      "Jura",                     "Jura",                         "Giura",                    "Giura"),
+        Canton.new("LU", "Luzern",                    "Luzern",                   "Lucerne",                      "Lucerna",                  "Lucerna"),
+        Canton.new("NE", "Neuchâtel",                 "Neuenburg",                "Neuchâtel",                    "Neuchâtel",                "Neuchâtel"),
+        Canton.new("NW", "Nidwalden",                 "Nidwalden",                "Nidwald",                      "Nidvaldo",                 "Sutsilvania"),
+        Canton.new("OW", "Obwalden",                  "Obwalden",                 "Obwald",                       "Obvaldo",                  "Sursilvania"),
+        Canton.new("SG", "St. Gallen",                "St. Gallen",               "Saint-Gall",                   "San Gallo",                "Son Gagl"),
+        Canton.new("SH", "Schaffhausen",              "Schaffhausen",             "Schaffhouse",                  "Sciaffusa",                "Schaffusa"),
+        Canton.new("SO", "Solothurn",                 "Solothurn",                "Soleure",                      "Soletta",                  "Soloturn"),
+        Canton.new("SZ", "Schwyz",                    "Schwyz",                   "Schwytz",                      "Svitto",                   "Sviz"),
+        Canton.new("TG", "Thurgau",                   "Thurgau",                  "Thurgovie",                    "Turgovia",                 "Turgovia"),
+        Canton.new("TI", "Ticino",                    "Tessin",                   "Tessin",                       "Ticino",                   "Tessin"),
+        Canton.new("UR", "Uri",                       "Uri",                      "Uri",                          "Uri",                      "Uri"),
+        Canton.new("VD", "Vaud",                      "Waadt",                    "Vaud",                         "Vaud",                     "Vad"),
+        Canton.new("VS", "Valais",                    "Wallis",                   "Valais",                       "Vallese",                  "Vallais"),
+        Canton.new("ZG", "Zug",                       "Zug",                      "Zoug",                         "Zugo",                     "Zug"),
+        Canton.new("ZH", "Zürich",                    "Zürich",                   "Zurich",                       "Zurigo",                   "Turitg"),
+        Canton.new("FL", "Fürstentum Liechtenstein",  "Fürstentum Liechtenstein", "Liechtenstein",                "Liechtenstein",            "Liechtenstein"),
+        Canton.new("DE", "Deutschland",               "Deutschland",              "Allemagne",                    "Germania",                 "Germania"),
+        Canton.new("IT", "Italien",                   "Italien",                  "Italie",                       "Italia",                   "Italia"),
+      ])
+
+      def self.empty
+        data = new
+        data.load_empty!
+
+        data
+      end
 
-      # @private
-      # The URL of the plz_c file
-      URLCommunity    = "https://match.post.ch/download?file=10003&tid=13&rol=0"
+      # @return [Date]
+      #   The date from when the data from the swiss post master data file
+      #   starts to be valid
+      attr_reader :date
 
-      # @private
-      # An array of all urls
-      URLAll          = [URLZip1, URLZip2, URLCommunity]
-
-      # The data of all cantons
-      # @private
-      CantonData = [
-        ["AG", "Aargau",                    "Aargau",                   "Argovie",                      "Argovia",                  "Argovia"],
-        ["AI", "Appenzell Innerrhoden",     "Appenzell Innerrhoden",    "Appenzell Rhodes-Intérieures", "Appenzello Interno",       "Appenzell Dadens"],
-        ["AR", "Appenzell Ausserrhoden",    "Appenzell Ausserrhoden",   "Appenzell Rhodes-Extérieures", "Appenzello Esterno",       "Appenzell Dadora"],
-        ["BE", "Bern",                      "Bern",                     "Berne",                        "Berna",                    "Berna"],
-        ["BL", "Basel-Landschaft",          "Basel-Landschaft",         "Bâle-Campagne",                "Basilea Campagna",         "Basilea-Champagna"],
-        ["BS", "Basel-Stadt",               "Basel-Stadt",              "Bâle-Ville",                   "Basilea Città",            "Basilea-Citad"],
-        ["FR", "Freiburg",                  "Fribourg",                 "Fribourg",                     "Friburgo",                 "Friburg"],
-        ["GE", "Genève",                    "Genf",                     "Genève",                       "Ginevra",                  "Genevra"],
-        ["GL", "Glarus",                    "Glarus",                   "Glaris",                       "Glarona",                  "Glaruna"],
-        ["GR", "Graubünden",                "Graubünden",               "Grisons",                      "Grigioni",                 "Grischun"],
-        ["JU", "Jura",                      "Jura",                     "Jura",                         "Giura",                    "Giura"],
-        ["LU", "Luzern",                    "Luzern",                   "Lucerne",                      "Lucerna",                  "Lucerna"],
-        ["NE", "Neuchâtel",                 "Neuenburg",                "Neuchâtel",                    "Neuchâtel",                "Neuchâtel"],
-        ["NW", "Nidwalden",                 "Nidwalden",                "Nidwald",                      "Nidvaldo",                 "Sutsilvania"],
-        ["OW", "Obwalden",                  "Obwalden",                 "Obwald",                       "Obvaldo",                  "Sursilvania"],
-        ["SG", "St. Gallen",                "St. Gallen",               "Saint-Gall",                   "San Gallo",                "Son Gagl"],
-        ["SH", "Schaffhausen",              "Schaffhausen",             "Schaffhouse",                  "Sciaffusa",                "Schaffusa"],
-        ["SO", "Solothurn",                 "Solothurn",                "Soleure",                      "Soletta",                  "Soloturn"],
-        ["SZ", "Schwyz",                    "Schwyz",                   "Schwytz",                      "Svitto",                   "Sviz"],
-        ["TG", "Thurgau",                   "Thurgau",                  "Thurgovie",                    "Turgovia",                 "Turgovia"],
-        ["TI", "Ticino",                    "Tessin",                   "Tessin",                       "Ticino",                   "Tessin"],
-        ["UR", "Uri",                       "Uri",                      "Uri",                          "Uri",                      "Uri"],
-        ["VD", "Vaud",                      "Waadt",                    "Vaud",                         "Vaud",                     "Vad"],
-        ["VS", "Valais",                    "Wallis",                   "Valais",                       "Vallese",                  "Vallais"],
-        ["ZG", "Zug",                       "Zug",                      "Zoug",                         "Zugo",                     "Zug"],
-        ["ZH", "Zürich",                    "Zürich",                   "Zurich",                       "Zurigo",                   "Turitg"],
-        ["FL", "Fürstentum Liechtenstein",  "Fürstentum Liechtenstein", "Liechtenstein",                "Liechtenstein",            "Liechtenstein"],
-        ["DE", "Deutschland",               "Deutschland",              "Allemagne",                    "Germania",                 "Germania"],
-        ["IT", "Italien",                   "Italien",                  "Italie",                       "Italia",                   "Italia"],
-      ]
+      # @return [Integer]
+      #   The random code from the swiss post master data file
+      attr_reader :random_code
 
       # The directory in which the post mat[ch] files reside
       attr_accessor :data_directory
@@ -116,14 +104,13 @@ module SwissMatch
       #   The directory in which the post mat[ch] files reside
       def initialize(data_directory=nil)
         reset_errors!
+        @loaded = false
         if data_directory then
           @data_directory = data_directory
         elsif ENV['SWISSMATCH_DATA'] then
           @data_directory = ENV['SWISSMATCH_DATA']
         else
-          data_directory  = File.expand_path('../../../../data/swissmatch-location', __FILE__)
-          data_directory  = Gem.datadir 'swissmatch-location' if defined?(Gem) && !File.directory?(data_directory)
-          @data_directory = data_directory
+          @data_directory  = File.expand_path('~/.swissmatch')
         end
       end
 
@@ -134,120 +121,116 @@ module SwissMatch
         self
       end
 
-      # Load new files
-      #
-      # @return [Array<String>]
-      #   An array with the absolute file paths of the extracted files.
-      def load_updates
-        URLAll.flat_map { |url|
-          http_get_zip_file(url, @data_directory)
-        }
+      def latest_binary_file
+        Dir.enum_for(:glob, "#{@data_directory}/locations_*.binary").last
       end
 
-      # Performs an HTTP-GET for the given url, extracts it as a zipped file into the
-      # destination directory.
-      #
-      # @return [Array<String>]
-      #   An array with the absolute file paths of the extracted files.
-      def http_get_zip_file(url, destination)
-        require 'open-uri'
-        require 'swissmatch/zip' # patched zip/zip
-        require 'fileutils'
-
-        files = []
-
-        open(url) do |zip_buffer|
-          Zip::ZipFile.open(zip_buffer) do |zip_file|
-            zip_file.each do |f|
-              target_path = File.join(destination, f.name)
-              FileUtils.mkdir_p(File.dirname(target_path))
-              zip_file.extract(f, target_path) unless File.exist?(target_path)
-              files << target_path
-            end
-          end
-        end
-
-        files
-      end
+      def load_empty!
+        return if @loaded
 
-      # Unzips it as a zipped file into the destination directory.
-      def unzip_file(file, destination)
-        require 'swissmatch/zip'
-        Zip::ZipFile.open(file) do |zip_file|
-          zip_file.each do |f|
-            target_path = File.join(destination, f.name)
-            FileUtils.mkdir_p(File.dirname(target_path))
-            zip_file.extract(f, target_path) unless File.exist?(target_path)
-          end
-        end
+        @loaded      = true
+        @date        = Date.new(0)
+        @random_code = 0
+        @cantons     = AllCantons
+        @districts   = Districts.new([])
+        @communities = Communities.new([])
+        @zip_codes   = ZipCodes.new([])
       end
 
-      # Used to convert numerical language codes to symbols
-      LanguageCodes = [nil, :de, :fr, :it, :rt]
-
       # Loads the data into this DataFiles instance
       #
       # @return [self]
       #   Returns self.
-      def load!
-        @cantons, @districts, @communities, @zip_codes = *load
-        self
-      end
+      def load!(file=nil)
+        return if @loaded
+
+        file ||= latest_binary_file
+
+        raise LoadError.new("File #{file.inspect} not found or not readable", nil) unless file && File.readable?(file)
+
+        data = File.read(file, encoding: Encoding::BINARY)
+        date, random_code, zip1_count, zip2_count, com1_count, com2_count, district_count = *data[0,18].unpack("NNn*")
+        int1_size, int2_size, int4_size, text_size = *data[18,16].unpack("N*")
+
+        offset    = 34
+        int1_cols = data[offset, int1_size].unpack("C*")
+        int2_cols = data[offset+=int1_size, int2_size].unpack("n*")
+        int4_cols = data[offset+=int2_size, int4_size].unpack("N*")
+        text_cols = data[offset+=int4_size, text_size].force_encoding(Encoding::UTF_8).split("\x1f")
+
+        offset                    = 0
+        zip1_type                 = int1_cols[offset, zip1_count]
+        zip1_addon                = int1_cols[offset += zip1_count, zip1_count]
+        zip1_language             = int1_cols[offset += zip1_count, zip1_count]
+        zip1_language_alternative = int1_cols[offset += zip1_count, zip1_count]
+        zip2_region               = int1_cols[offset += zip1_count, zip2_count]
+        zip2_type                 = int1_cols[offset += zip2_count, zip2_count]
+        zip2_lang                 = int1_cols[offset += zip2_count, zip2_count]
+        com2_PLZZ                 = int1_cols[offset += zip2_count, com2_count]
+
+        offset                        = 0
+        zip1_onrp                     = int2_cols[offset, zip1_count]
+        zip1_code                     = int2_cols[offset += zip1_count, zip1_count]
+        zip1_delivery_by              = int2_cols[offset += zip1_count, zip1_count]
+        zip1_largest_community_number = int2_cols[offset += zip1_count, zip1_count]
+        zip2_onrp                     = int2_cols[offset += zip1_count, zip2_count]
+        com1_bfsnr                    = int2_cols[offset += zip2_count, com1_count]
+        com1_agglomeration            = int2_cols[offset += com1_count, com1_count]
+        com2_GDENR                    = int2_cols[offset += com1_count, com2_count]
+        com2_PLZ4                     = int2_cols[offset += com2_count, com2_count]
+        district_GDEBZNR              = int2_cols[offset += com2_count, district_count]
+
+        zip1_valid_from = int4_cols
+
+        offset           = 0
+        zip1_name_short  = text_cols[offset, zip1_count]
+        zip1_name        = text_cols[offset += zip1_count, zip1_count]
+        zip1_canton      = text_cols[offset += zip1_count, zip1_count]
+        zip2_short       = text_cols[offset += zip1_count, zip2_count]
+        zip2_name        = text_cols[offset += zip2_count, zip2_count]
+        com1_name        = text_cols[offset += zip2_count, com1_count]
+        com1_canton      = text_cols[offset += com1_count, com1_count]
+        district_GDEKT   = text_cols[offset += com1_count, district_count]
+        district_GDEBZNA = text_cols[offset += district_count, district_count]
+
+        zip1     = [
+          zip1_onrp, zip1_type, zip1_canton, zip1_code, zip1_addon,
+          zip1_delivery_by, zip1_language, zip1_language_alternative,
+          zip1_name_short, zip1_name, zip1_largest_community_number,
+          zip1_valid_from
+        ].transpose
+        zip2     = [zip2_onrp, zip2_region, zip2_type, zip2_lang, zip2_short, zip2_name].transpose
+        com1     = [com1_bfsnr, com1_name, com1_canton, com1_agglomeration].transpose
+        com2     = [com2_PLZ4, com2_PLZZ, com2_GDENR].transpose
+        district = [district_GDEKT, district_GDEBZNR, district_GDEBZNA].transpose
+
+        @date        = Date.jd(date)
+        @random_code = random_code
+        @cantons     = AllCantons
+        @districts   = load_districts(district)
+        @communities = load_communities(com1)
+        @zip_codes   = load_zipcodes(zip1, zip2, com2)
 
-      # @return [Array]
-      #   Returns an array of the form [SwissMatch::Cantons, SwissMatch::Communities,
-      #   SwissMatch::ZipCodes].
-      def load
-        reset_errors!
-
-        cantons     = load_cantons
-        districts   = load_districts(cantons)
-        communities = load_communities(cantons)
-        zip_codes   = load_zipcodes(cantons, communities)
-
-        [cantons, districts, communities, zip_codes]
+        self
       end
 
-      # @return [SwissMatch::Cantons]
-      #   A SwissMatch::Cantons containing all cantons used by the swiss postal service.
-      def load_cantons
-        Cantons.new(
-          CantonData.map { |tag, name, name_de, name_fr, name_it, name_rt|
-            Canton.new(tag, name, name_de, name_fr, name_it, name_rt)
-          }
-        )
-      end
-
-      def load_districts(cantons)
-        # File format: GDEKT,GDEBZNR,GDEBZNA
-        path      = Dir.enum_for(:glob, "#{@data_directory}/districts_*.csv").last
-        data      = File.read(path, encoding: Encoding::UTF_8.to_s).scan(Expressions[:districts])
-        districts = data[1..-1].map { |canton_tag, district_number, district_name|
-          district_number = Integer(district_number, 10)
-          canton          = cantons.by_license_tag(canton_tag)
-
-          District.new(district_number, district_name, canton, SwissMatch::Communities.new([]))
-        }
-
-        Districts.new(districts)
+      def load_districts(data)
+        Districts.new(data.map { |data|
+          District.new(*data, SwissMatch::Communities.new([]))
+        })
       end
 
       # @return [SwissMatch::Communities]
       #   An instance of SwissMatch::Communities containing all communities defined by the
       #   files known to this DataFiles instance.
-      def load_communities(cantons)
-        raise "Must load cantons first" unless cantons
-
-        file      = Dir.enum_for(:glob, "#{@data_directory}/plz_c_*.txt").last
+      def load_communities(data)
         temporary = []
         complete  = {}
-        load_table(file, :community).each do |bfsnr, name, canton, agglomeration|
-          bfsnr         = bfsnr.to_i
-          agglomeration = agglomeration.to_i
-          canton        = cantons.by_license_tag(canton)
+        data.each do |bfsnr, name, canton, agglomeration|
+          canton = @cantons.by_license_tag(canton)
           if agglomeration == bfsnr then
             complete[bfsnr] = Community.new(bfsnr, name, canton, :self)
-          elsif agglomeration.nil? then
+          elsif agglomeration.zero? then
             complete[bfsnr] = Community.new(bfsnr, name, canton, nil)
           else
             temporary << [bfsnr, name, canton, agglomeration]
@@ -268,48 +251,31 @@ module SwissMatch
       # @return [SwissMatch::ZipCodes]
       #   An instance of SwissMatch::ZipCodes containing all zip codes defined by the
       #   files known to this DataFiles instance.
-      def load_zipcodes(cantons, communities)
-        raise "Must load cantons first" unless cantons
-        raise "Must load communities first" unless communities
-
+      def load_zipcodes(zip1_data, zip2_data, com2_data)
         community_mapping = Hash.new { |h,k| h[k] = [] }
         self_delivered    = []
         others            = []
-        zip1_file         = Dir.enum_for(:glob, "#{@data_directory}/plz_p1_*.txt").last
-        zip2_file         = Dir.enum_for(:glob, "#{@data_directory}/plz_p2_*.txt").last
-        communities_file  = Dir.enum_for(:glob, "#{@data_directory}/communities_*.csv").last
-
-        # KTKZ,OHW,ORTNAME,GHW,GDENR,GDENAMK,PHW,PLZ4,PLZZ,PLZNAMK
-        communities_data  = File.read(
-          communities_file,
-          encoding: Encoding::UTF_8.to_s
-        ).scan(Expressions[:communities])[1..-1].transpose.values_at(4,7,8)
-        communities_data[0].map!(&:to_i)
-        communities_data[1].map!(&:to_i)
-        communities_data[2].map!(&:to_i)
-        communities_data.transpose.each do |data|
-          community_mapping[data.last(2)] << data.at(0)
+        temporary         = {}
+
+        com2_data.each do |*key, value|
+          community_mapping[key] << value
         end
 
-        temporary         = {}
-        load_table(zip1_file, :zip_1).each do |row|
-          onrp                      = row.at(0).to_i
-          code                      = row.at(2).to_i
-          addon                     = row.at(3).to_i
-          delivery_by               = row.at(10).to_i
+        zip1_data.each do |onrp, type, canton, code, addon, delivery_by, lang, lang_alt, name_short, name, largest_community_number, valid_from|
           delivery_by               = case delivery_by when 0 then nil; when onrp then :self; else delivery_by; end
-          language                  = LanguageCodes[row.at(7).to_i]
-          language_alternative      = LanguageCodes[row.at(8).to_i]
-          name_short                = Name.new(row.at(4), language)
-          name                      = Name.new(row.at(5), language)
-          largest_community_number  = row.at(11).to_i
+          language                  = LanguageCodes[lang]
+          language_alternative      = LanguageCodes[lang_alt]
+          name_short                = Name.new(name_short, language)
+          name                      = Name.new(name, language)
+
           # compact, because some communities already no longer exist, so by_community_numbers can
           # contain nils which must be removed
-          community_numbers         = (community_mapping[[code, addon]]|[largest_community_number]).sort
-          communities               = Communities.new(communities.by_community_numbers(*community_numbers).compact)
+          community_numbers         = (community_mapping[[code, addon]] | [largest_community_number]).sort
+          communities               = Communities.new(@communities.by_community_numbers(*community_numbers).compact)
+
           data                      = [
             onrp,                              # ordering_number
-            row.at(1).to_i,                    # type
+            type,                              # type
             code,
             addon,
             name,                              # name (official)
@@ -318,14 +284,14 @@ module SwissMatch
             [name_short],                      # names_short (official + alternative)
             [],                                # PLZ2 type 3 short names (additional region names)
             [],                                # PLZ2 type 3 names (additional region names)
-            cantons.by_license_tag(row.at(6)), # canton
+            cantons.by_license_tag(canton),    # canton
             language,
             language_alternative,
-            row.at(9) == "1",                  # sortfile_member
+            false,                             # sortfile_member TODO: remove
             delivery_by,                       # delivery_by
             communities.by_community_number(largest_community_number),  # community_number
             communities,
-            Date.civil(*row.at(12).match(/^(\d{4})(\d\d)(\d\d)$/).captures.map(&:to_i)) # valid_from
+            Date.jd(valid_from) # valid_from
           ]
           temporary[onrp] = data
           if :self == delivery_by then
@@ -335,26 +301,26 @@ module SwissMatch
           end
         end
 
-        load_table(zip2_file, :zip_2).each do |onrp, rn, type, lang, short, name|
-          onrp      = onrp.to_i
-          lang_code = lang.to_i
+        zip2_data.each do |onrp, rn, type, lang, short, name|
+          onrp      = onrp
+          lang_code = lang
           language  = LanguageCodes[lang_code]
           entry     = temporary[onrp]
-          if type == "2"
-            entry[5] << Name.new(name, language, rn.to_i)
-            entry[7] << Name.new(short, language, rn.to_i)
-          elsif type == "3"
-            entry[8] << Name.new(name, language, rn.to_i)
-            entry[9] << Name.new(short, language, rn.to_i)
+          if type == 2
+            entry[5] << Name.new(name, language, rn)
+            entry[7] << Name.new(short, language, rn)
+          elsif type == 3
+            entry[8] << Name.new(name, language, rn)
+            entry[9] << Name.new(short, language, rn)
           end
         end
 
         self_delivered.each do |row|
-          temporary[row.at(0)] = ZipCode.new(*row)
+          temporary[row[0]] = ZipCode.new(*row)
         end
         others.each do |row|
-          if row.at(14) then
-            raise "Delivery not found:\n#{row.inspect}" unless tmp = temporary[row.at(14)]
+          if row[14] then
+            raise "Delivery not found:\n#{row.inspect}" unless tmp = temporary[row[14]]
             if tmp.kind_of?(Array) then
               @errors << LoadError.new("Invalid reference: onrp #{row.at(0)} delivery by #{row.at(14)}", row)
               row[14] = nil
@@ -362,26 +328,11 @@ module SwissMatch
               row[14] = tmp
             end
           end
-          temporary[row.at(0)] = ZipCode.new(*row)
+          temporary[row[0]] = ZipCode.new(*row)
         end
 
         ZipCodes.new(temporary.values)
       end
-
-      # Reads a file and parses using the pattern of the given name.
-      #
-      # @param [String] path
-      #   The path of the file to parse
-      # @param [Symbol] pattern
-      #   The pattern-name used to parse the file (see Expressions)
-      #
-      # @return [Array<Array<String>>]
-      #   A 2 dimensional array representing the tabular data contained in the given file.
-      def load_table(path, pattern)
-        File.read(path, :encoding => Encoding::Windows_1252.to_s). # to_s because sadly, ruby 1.9.2 can't handle an Encoding instance as argument
-          encode(Encoding::UTF_8).
-          scan(Expressions[pattern])
-      end
     end
   end
 end