zip-codes-pl 0.1.1

data/lib/zip-codes-pl.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require_relative "zip_codes/pl"

data/lib/zip_codes/pl/builder.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "time"
+
+module ZipCodes
+  module PL
+    # Downloads the source and writes the dataset into the configured directory.
+    class Builder
+      Result = Data.define(:status, :data_path, :manifest) do
+        def built? = status == :built
+        def up_to_date? = status == :up_to_date
+      end
+
+      # `administrative_names_source: false` builds with the raw upstream labels.
+      # The two sources default independently: tying the names to whether a custom
+      # row source was injected silently dropped the enrichment, and the manifest
+      # attribution with it.
+      def initialize(config: ZipCodes::PL.config, source: nil, administrative_names_source: nil)
+        @config = config
+        @source = source || Sources::Geonames.new(config: config)
+        @administrative_names_source =
+          case administrative_names_source
+          when nil then Sources::PocztaPolska.new(config: config)
+          when false then nil
+          else administrative_names_source
+          end
+      end
+
+      def call
+        previous = Manifest.read(config.manifest_path)
+        download = source.download(etag: reusable_etag(previous))
+        return Result.new(status: :up_to_date, data_path: config.data_path, manifest: previous) if download.nil?
+
+        names = administrative_names_source&.fetch
+        write(download, names)
+      end
+
+      private
+
+      attr_reader :administrative_names_source, :config, :source
+
+      # Only claim a cached copy when the data file it describes is still there;
+      # a 304 with no file on disk would leave the caller with nothing.
+      def reusable_etag(previous)
+        return nil if previous.nil? || !File.exist?(config.data_path)
+
+        previous.etag
+      end
+
+      def write(download, names)
+        FileUtils.mkdir_p(config.output_dir)
+        row_count = write_data(download, names)
+        manifest = build_manifest(download, row_count, names).write(config.manifest_path)
+
+        Result.new(status: :built, data_path: config.data_path, manifest: manifest)
+      end
+
+      # Written to a temporary file and renamed, so an interrupted run never
+      # leaves a half-written dataset where a complete one used to be.
+      def write_data(download, names)
+        temporary_path = "#{config.data_path}.tmp"
+        row_count = 0
+
+        File.open(temporary_path, "w") do |file|
+          file.puts(Record::COLUMNS.join("\t"))
+          source.each_record(download.body) do |record|
+            record = apply_administrative_names(record, names) if names
+            file.puts(record.to_row.join("\t"))
+            row_count += 1
+          end
+        end
+
+        File.rename(temporary_path, config.data_path)
+        row_count
+      ensure
+        FileUtils.rm_f(temporary_path) if temporary_path && File.exist?(temporary_path)
+      end
+
+      def apply_administrative_names(record, names)
+        county = name_for!(names.counties, record.county_teryt, "county")
+        commune = name_for!(names.communes, record.commune_teryt, "commune")
+        county = "powiat #{county}" if county && record.county_teryt[2, 2].to_i < 60
+
+        Record.new(**record.to_h, county: county, commune: commune)
+      end
+
+      def name_for!(names, code, level)
+        return nil if code.nil?
+
+        names.fetch(code) do
+          raise DownloadError, "no #{level} name for TERYT code #{code} in the Poczta Polska response"
+        end
+      end
+
+      def build_manifest(download, row_count, names)
+        attribution = Sources::Geonames::ATTRIBUTION
+        attribution = "#{attribution}; #{Sources::PocztaPolska::ATTRIBUTION}" if names
+
+        Manifest.new(
+          source_url: config.source_url,
+          attribution: attribution,
+          etag: download.etag,
+          last_modified: download.last_modified,
+          row_count: row_count,
+          built_at: Time.now.utc.iso8601,
+          gem_version: VERSION,
+          format_version: Manifest::FORMAT_VERSION
+        )
+      end
+    end
+  end
+end

data/lib/zip_codes/pl/city.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module ZipCodes
+  module PL
+    # A place rather than a postal code: what a "cities" table wants.
+    class City < Data.define(
+      :name, :voivodeship, :voivodeship_teryt, :commune, :commune_teryt,
+      :latitude, :longitude, :postal_codes
+    )
+      class << self
+        # Keyed by commune as well as name, because a voivodeship can hold dozens
+        # of distinct villages sharing one - mazowieckie has 28 called "Nowa Wieś".
+        # Collapsing them by name alone puts the averaged point in a field, up to
+        # 158 km from the farthest place it claims to be.
+        def group_key(record)
+          [record.city, record.voivodeship_teryt, record.commune_teryt]
+        end
+
+        def sort_key(city)
+          [Normalize.key(city.name), city.voivodeship_teryt, city.commune_teryt.to_s]
+        end
+      end
+
+      # Holds running sums rather than the rows behind them, so building every
+      # place in the country does not mean holding every postal code first. The
+      # coordinate ends up the mean of the group's rows, because the source gives
+      # one point per postal code and Bydgoszcz has 679 of them.
+      class Accumulator
+        def initialize(record)
+          # Copies the identity fields instead of holding the row, so the rest of
+          # each Record can be collected while the country is being aggregated.
+          @name = record.city
+          @voivodeship = record.voivodeship
+          @voivodeship_teryt = record.voivodeship_teryt
+          @commune = record.commune
+          @commune_teryt = record.commune_teryt
+          @latitude = 0.0
+          @longitude = 0.0
+          @count = 0
+          @postal_codes = {}
+        end
+
+        def add(record)
+          @latitude += record.latitude
+          @longitude += record.longitude
+          @count += 1
+          @postal_codes[record.postal_code] = true
+          self
+        end
+
+        def to_city
+          City.new(
+            name: @name,
+            voivodeship: @voivodeship,
+            voivodeship_teryt: @voivodeship_teryt,
+            commune: @commune,
+            commune_teryt: @commune_teryt,
+            latitude: (@latitude / @count).round(6),
+            longitude: (@longitude / @count).round(6),
+            postal_codes: @postal_codes.keys.sort.freeze
+          )
+        end
+      end
+    end
+  end
+end

data/lib/zip_codes/pl/configuration.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module ZipCodes
+  module PL
+    class Configuration
+      DEFAULT_OUTPUT_DIR = "data"
+      DATA_FILENAME = "zip-codes-pl.tsv"
+      MANIFEST_FILENAME = "zip-codes-pl.manifest.json"
+      BUNDLED_DIR = File.expand_path("../../../data", __dir__)
+
+      attr_accessor :output_dir, :source_url, :user_agent, :open_timeout, :read_timeout, :poczta_request_interval
+
+      def initialize
+        @output_dir = ENV.fetch("ZIP_CODES_PL_DIR", DEFAULT_OUTPUT_DIR)
+        @source_url = Sources::Geonames::URL
+        @user_agent = "zip-codes-pl/#{VERSION} (+https://github.com/carlos-1410/zip-codes-pl)"
+        @open_timeout = 10
+        @read_timeout = 60
+        @poczta_request_interval = Sources::PocztaPolska::DEFAULT_REQUEST_INTERVAL
+      end
+
+      # Where a refresh writes.
+      def data_path
+        File.join(output_dir, DATA_FILENAME)
+      end
+
+      def manifest_path
+        File.join(output_dir, MANIFEST_FILENAME)
+      end
+
+      # Where a read looks. Your own refreshed copy wins; without one the dataset
+      # shipped inside the gem answers, so nothing has to be built before first use.
+      def readable_data_path
+        File.exist?(data_path) ? data_path : bundled_data_path
+      end
+
+      def readable_manifest_path
+        File.exist?(data_path) ? manifest_path : File.join(BUNDLED_DIR, MANIFEST_FILENAME)
+      end
+
+      def bundled_data_path
+        File.join(BUNDLED_DIR, DATA_FILENAME)
+      end
+    end
+  end
+end

data/lib/zip_codes/pl/errors.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module ZipCodes
+  module PL
+    class Error < StandardError; end
+
+    class DownloadError < Error; end
+
+    class DatasetError < Error; end
+  end
+end

data/lib/zip_codes/pl/manifest.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require "json"
+require "time"
+
+module ZipCodes
+  module PL
+    # Sits next to the dataset and records where it came from. The etag is what
+    # lets a later run ask the server "changed?" instead of downloading again.
+    class Manifest < Data.define(
+      :source_url, :attribution, :etag, :last_modified,
+      :row_count, :built_at, :gem_version, :format_version
+    )
+      FORMAT_VERSION = 1
+
+      def self.read(path)
+        return nil unless File.exist?(path)
+
+        payload = JSON.parse(File.read(path), symbolize_names: true)
+        return nil unless payload[:format_version] == FORMAT_VERSION
+
+        new(**payload.slice(*members))
+      rescue JSON::ParserError, ArgumentError
+        nil
+      end
+
+      def write(path)
+        File.write(path, "#{JSON.pretty_generate(to_h)}\n")
+        self
+      end
+
+      def built_at_time
+        Time.iso8601(built_at)
+      rescue ArgumentError, TypeError
+        nil
+      end
+    end
+  end
+end

data/lib/zip_codes/pl/normalize.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module ZipCodes
+  module PL
+    # Folding used for lookups only. Stored names keep their diacritics; this is
+    # what makes "zlotow", "Złotów" and "ZŁOTÓW" find the same rows.
+    module Normalize
+      DIACRITICS = {
+        "ą" => "a", "ć" => "c", "ę" => "e", "ł" => "l", "ń" => "n",
+        "ó" => "o", "ś" => "s", "ź" => "z", "ż" => "z"
+      }.freeze
+
+      POSTAL_CODE_PATTERN = /\A(\d{2})-?(\d{3})\z/
+
+      module_function
+
+      def key(value)
+        value.to_s.strip.downcase.gsub(/[ąćęłńóśźż]/, DIACRITICS)
+      end
+
+      # Accepts "86-010" and "86010"; returns nil for anything that is not a PNA.
+      def postal_code(value)
+        match = POSTAL_CODE_PATTERN.match(value.to_s.strip)
+        return nil if match.nil?
+
+        "#{match[1]}-#{match[2]}"
+      end
+    end
+  end
+end

data/lib/zip_codes/pl/railtie.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module ZipCodes
+  module PL
+    class Railtie < Rails::Railtie
+      rake_tasks do
+        load File.expand_path("tasks/zip_codes_pl.rake", __dir__)
+      end
+    end
+  end
+end

data/lib/zip_codes/pl/reader.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+module ZipCodes
+  module PL
+    # Reads the dataset by streaming the file, holding nothing between calls.
+    #
+    # One scan of 72 899 rows costs about 15 ms and 1.6 MB, against 222 ms and
+    # 99 MB to hold the same rows in indexed memory. The break-even is around
+    # fifteen lookups and the ordinary use of this gem is three, so there is no
+    # in-memory mode: past that point the rows belong in the caller's own database,
+    # and `cities` is the shape to import.
+    class Reader
+      HEADER = Record::COLUMNS.map(&:to_s).freeze
+      MIN_CITY_FRAGMENT_LENGTH = 3
+
+      attr_reader :path
+
+      def initialize(path)
+        @path = path
+      end
+
+      def each(&block)
+        return enum_for(:each) unless block
+
+        open_data do |file|
+          file.each_line { |line| block.call(Record.from_row(line.chomp.split("\t", -1))) }
+        end
+      end
+
+      include Enumerable
+
+      def find_by_postal_code(code)
+        normalized = Normalize.postal_code(code)
+        return [] if normalized.nil?
+
+        prefix = "#{normalized}\t"
+        scan { |line| line.start_with?(prefix) }
+      end
+
+      def find_by_city(name, voivodeship: nil)
+        wanted_name = Normalize.key(name)
+        wanted_teryt = resolve_voivodeship(voivodeship)
+        # Asked to narrow to a voivodeship that does not exist: nothing matches.
+        # Treating it as "no filter" would answer with every place of that name.
+        return [] if wanted_teryt == :unknown
+
+        scan do |line|
+          fields = line.split("\t", 5)
+          # Folding every name costs more than reading the file, and the folding
+          # used here is length preserving (ł->l, ó->o, NFD plus mark stripping),
+          # so a length mismatch rules a row out before any of that work.
+          fields[1].length == wanted_name.length &&
+            Normalize.key(fields[1]) == wanted_name &&
+            (wanted_teryt.nil? || fields[3] == wanted_teryt)
+        end
+      end
+
+      def search_by_city(fragment, voivodeship: nil)
+        wanted_fragment = Normalize.key(fragment)
+        return [] if wanted_fragment.length < 2
+
+        wanted_teryt = resolve_voivodeship(voivodeship)
+        return [] if wanted_teryt == :unknown
+
+        scan do |line|
+          fields = line.split("\t", 5)
+          name = Normalize.key(fields[1])
+          matches = if wanted_fragment.length < MIN_CITY_FRAGMENT_LENGTH
+                      name == wanted_fragment
+                    else
+                      name.include?(wanted_fragment)
+                    end
+
+          matches && (wanted_teryt.nil? || fields[3] == wanted_teryt)
+        end
+      end
+
+      # Accumulates as it streams, so the 72 899 source rows are never all resident
+      # at once - only the 52 325 places they collapse into.
+      def cities
+        accumulators = {}
+        each { |record| (accumulators[City.group_key(record)] ||= City::Accumulator.new(record)).add(record) }
+        accumulators.each_value.map(&:to_city).sort_by { |city| City.sort_key(city) }
+      end
+
+      private
+
+      def resolve_voivodeship(name)
+        return nil if name.nil?
+
+        Voivodeship.find_by_name(name)&.teryt_code || :unknown
+      end
+
+      def scan
+        found = []
+        open_data do |file|
+          file.each_line do |line|
+            found << Record.from_row(line.chomp.split("\t", -1)) if yield(line)
+          end
+        end
+        found
+      end
+
+      def open_data
+        raise DatasetError, "dataset missing: #{path} (run `rake zip_codes:pl:update`)" unless File.exist?(path)
+
+        File.open(path) do |file|
+          header = file.gets&.chomp&.split("\t")
+          raise DatasetError, "unexpected header in #{path}" unless header == HEADER
+
+          yield file
+        end
+      end
+    end
+  end
+end

data/lib/zip_codes/pl/record.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module ZipCodes
+  module PL
+    # One postal code in one place. A place with several codes has several
+    # records, and so does a code shared by several places.
+    #
+    # county and commune carry the source's own labels, which are uneven for
+    # Poland ("Powiat bydgoski", "Leszno County"). The *_teryt codes are the
+    # dependable identifiers and the join key for official names.
+    class Record < Data.define(
+      :postal_code, :city,
+      :voivodeship, :voivodeship_teryt,
+      :county, :county_teryt,
+      :commune, :commune_teryt,
+      :latitude, :longitude, :accuracy
+    )
+      COLUMNS = members.freeze
+
+      def self.from_row(row)
+        new(
+          postal_code: row[0],
+          city: row[1],
+          voivodeship: row[2],
+          voivodeship_teryt: row[3],
+          county: presence(row[4]),
+          county_teryt: presence(row[5]),
+          commune: presence(row[6]),
+          commune_teryt: presence(row[7]),
+          latitude: row[8].to_f,
+          longitude: row[9].to_f,
+          accuracy: presence(row[10])&.to_i
+        )
+      end
+
+      def self.presence(value)
+        value.nil? || value.empty? ? nil : value
+      end
+      private_class_method :presence
+
+      def to_row
+        COLUMNS.map { |column| public_send(column) }
+      end
+
+      def coordinates
+        [latitude, longitude]
+      end
+    end
+  end
+end

data/lib/zip_codes/pl/sources/geonames.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "stringio"
+require "uri"
+require "zip"
+
+module ZipCodes
+  module PL
+    module Sources
+      # The GeoNames postal-code export for Poland: one tab separated row per
+      # (postal code, place), carrying WGS84 coordinates and the TERYT county and
+      # commune codes.
+      class Geonames
+        URL = "https://download.geonames.org/export/zip/PL.zip"
+        ENTRY = "PL.txt"
+        ATTRIBUTION = "GeoNames (https://www.geonames.org), CC BY 4.0"
+        MAX_REDIRECTS = 3
+        EXPECTED_FIELDS = 12
+
+        Download = Data.define(:body, :etag, :last_modified)
+
+        def initialize(config: ZipCodes::PL.config)
+          @config = config
+        end
+
+        # Returns nil when the server reports the file is unchanged, which is what
+        # makes re-running the rake task cheap.
+        def download(etag: nil)
+          response = get(URI.parse(config.source_url), etag: etag)
+          return nil if response.is_a?(Net::HTTPNotModified)
+
+          raise DownloadError, "#{config.source_url} answered #{response.code}" unless response.is_a?(Net::HTTPSuccess)
+
+          Download.new(
+            body: response.body,
+            etag: response["etag"],
+            last_modified: response["last-modified"]
+          )
+        end
+
+        def each_record(archive, &block)
+          return enum_for(:each_record, archive) unless block
+
+          Zip::File.open_buffer(StringIO.new(archive)) do |zip|
+            entry = zip.find_entry(ENTRY)
+            raise DownloadError, "the archive has no #{ENTRY}" if entry.nil?
+
+            entry.get_input_stream do |stream|
+              # The entry stream yields ASCII-8BIT, and Polish names have to survive it.
+              stream.each_line { |line| parse_line(line.force_encoding(Encoding::UTF_8), &block) }
+            end
+          end
+        end
+
+        private
+
+        attr_reader :config
+
+        def get(uri, etag:, redirects: 0)
+          response = perform(uri, build_request(uri, etag))
+
+          # 304 is a subclass of Net::HTTPRedirection, and it carries no Location.
+          # Following it as a redirect is how the second run used to die.
+          return response if response.is_a?(Net::HTTPNotModified)
+          return response unless response.is_a?(Net::HTTPRedirection)
+          raise DownloadError, "too many redirects from #{config.source_url}" if redirects >= MAX_REDIRECTS
+
+          get(URI.parse(response["location"]), etag: etag, redirects: redirects + 1)
+        end
+
+        def build_request(uri, etag)
+          request = Net::HTTP::Get.new(uri)
+          request["user-agent"] = config.user_agent
+          request["if-none-match"] = etag if etag
+          request
+        end
+
+        def perform(uri, request)
+          Net::HTTP.start(
+            uri.host, uri.port,
+            use_ssl: uri.scheme == "https",
+            open_timeout: config.open_timeout,
+            read_timeout: config.read_timeout
+          ) { |http| http.request(request) }
+        end
+
+        def parse_line(line)
+          return if line.strip.empty?
+
+          fields = line.chomp.split("\t", -1)
+          unless fields.length == EXPECTED_FIELDS
+            raise DownloadError,
+                  "malformed GeoNames row: expected #{EXPECTED_FIELDS} fields, got #{fields.length}"
+          end
+
+          voivodeship = Voivodeship.find_by_geonames_code(fields[4])
+          # An unknown admin1 code means the canonical table above is stale, and a
+          # silently dropped row would quietly corrupt the output.
+          raise DownloadError, "unknown GeoNames voivodeship code: #{fields[4].inspect}" if voivodeship.nil?
+
+          yield build_record(fields, voivodeship)
+        end
+
+        def build_record(fields, voivodeship)
+          Record.new(
+            postal_code: fields[1],
+            city: fields[2],
+            voivodeship: voivodeship.name,
+            voivodeship_teryt: voivodeship.teryt_code,
+            county: blank_to_nil(fields[5]&.tr("_", " ")),
+            county_teryt: blank_to_nil(fields[6]),
+            commune: blank_to_nil(fields[7]&.tr("_", " ")),
+            commune_teryt: blank_to_nil(fields[8]),
+            latitude: fields[9].to_f,
+            longitude: fields[10].to_f,
+            accuracy: blank_to_nil(fields[11])&.to_i
+          )
+        end
+
+        def blank_to_nil(value)
+          value.nil? || value.strip.empty? ? nil : value.strip
+        end
+      end
+    end
+  end
+end