zip-codes-pl 0.1.1

data/lib/zip_codes/pl/sources/poczta_polska/client.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "time"
+
+module ZipCodes
+  module PL
+    module Sources
+      class PocztaPolska
+        class Client
+          MAX_ATTEMPTS = 5
+          MAX_RETRY_DELAY = 30.0
+
+          def initialize(
+            config:,
+            sleeper: ->(seconds) { sleep(seconds) },
+            clock: -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) },
+            transport: nil
+          )
+            @request_interval = config.poczta_request_interval
+            @sleeper = sleeper
+            @clock = clock
+            @transport = transport || ->(uri, params) { post(uri, params, config) }
+            @last_request_at = nil
+          end
+
+          # Retries rate limiting a bounded number of times. An unbounded loop here
+          # would turn sustained throttling into a refresh that never finishes and
+          # never says why.
+          def post_form(uri, params)
+            MAX_ATTEMPTS.times do
+              throttle
+              response = transport.call(uri, params)
+              @last_request_at = clock.call
+
+              return response if response.is_a?(Net::HTTPSuccess)
+
+              raise DownloadError, failure_message(response, uri) unless response.is_a?(Net::HTTPTooManyRequests)
+
+              sleeper.call(retry_after(response))
+            end
+
+            raise DownloadError, "Poczta Polska is rate limiting - gave up after #{MAX_ATTEMPTS} attempts for #{uri}"
+          end
+
+          private
+
+          attr_reader :clock, :request_interval, :sleeper, :transport
+
+          def throttle
+            return if @last_request_at.nil?
+
+            remaining = request_interval - (clock.call - @last_request_at)
+            sleeper.call(remaining) if remaining.positive?
+          end
+
+          # Clamped at both ends: a hostile or broken header must not make the
+          # build sleep backwards, which raises, nor park it for an hour.
+          def retry_after(response)
+            value = response["retry-after"]
+            seconds = Float(value, exception: false) || (Time.httpdate(value.to_s) - Time.now)
+            seconds.clamp(0.0, MAX_RETRY_DELAY)
+          rescue ArgumentError
+            [request_interval, 1.0].max
+          end
+
+          # A moved endpoint is reported rather than followed: this is a POST to a
+          # form, and replaying it against an unknown location is not a safe guess.
+          def failure_message(response, uri)
+            message = "Poczta Polska answered #{response.code} for #{uri}"
+            location = response["location"] if response.is_a?(Net::HTTPRedirection)
+            location ? "#{message} (redirected to #{location})" : message
+          end
+
+          def post(uri, params, config)
+            request = Net::HTTP::Post.new(uri)
+            request["user-agent"] = config.user_agent
+            request.set_form_data(params)
+
+            Net::HTTP.start(
+              uri.host, uri.port,
+              use_ssl: uri.scheme == "https",
+              open_timeout: config.open_timeout,
+              read_timeout: config.read_timeout
+            ) { |http| http.request(request) }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/zip_codes/pl/sources/poczta_polska.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+require "json"
+require "uri"
+require_relative "poczta_polska/client"
+
+module ZipCodes
+  module PL
+    module Sources
+      class PocztaPolska
+        DISTRICTS_URL = "https://www.poczta-polska.pl/wp-content/themes/pp/inc/pna-form/pna-search-district.php"
+        COMMUNES_URL = "https://www.poczta-polska.pl/wp-content/themes/pp/inc/pna-form/pna-search-commune.php"
+        ATTRIBUTION = "county and commune names: Poczta Polska (https://www.poczta-polska.pl)"
+        DEFAULT_REQUEST_INTERVAL = 0.25
+        LEGACY_COMMUNES = { "320304" => "Ostrowice" }.freeze
+        Names = Data.define(:counties, :communes)
+
+        def initialize(config: ZipCodes::PL.config, client: nil, voivodeships: Voivodeship.all)
+          @client = client || Client.new(config: config)
+          @voivodeships = voivodeships
+        end
+
+        def fetch
+          counties, districts = fetch_counties
+          communes = fetch_communes(districts)
+
+          Names.new(counties: counties.freeze, communes: communes.freeze)
+        end
+
+        private
+
+        attr_reader :client, :voivodeships
+
+        def fetch_counties
+          counties = {}
+          districts = []
+
+          voivodeships.each do |voivodeship|
+            entries(DISTRICTS_URL, province: voivodeship.teryt_code).each do |entry|
+              code, name = entry.values_at("value", "name")
+              validate_code!(code, 4, voivodeship.teryt_code)
+              add!(counties, code, name)
+              districts << [voivodeship.teryt_code, code]
+            end
+          end
+
+          [counties, districts]
+        end
+
+        def fetch_communes(districts)
+          communes = LEGACY_COMMUNES.dup
+          districts.each do |province, district|
+            entries(COMMUNES_URL, province: province, district: district).each do |entry|
+              terc, name = entry.values_at("value", "name")
+              validate_commune_code!(terc, district)
+              add!(communes, terc[0, 6], normalize_commune_name(name))
+            end
+          end
+
+          communes
+        end
+
+        def entries(url, params)
+          response = client.post_form(URI(url), params)
+          # A success with no body is still a success to Net::HTTP, and JSON.parse
+          # raises TypeError rather than ParserError on nil.
+          payload = JSON.parse(response.body.to_s)
+          unless payload.is_a?(Array) && payload.all? { |entry| valid_entry?(entry) }
+            raise DownloadError, "malformed Poczta Polska response from #{url}"
+          end
+
+          payload
+        rescue JSON::ParserError
+          raise DownloadError, "malformed Poczta Polska JSON from #{url}"
+        end
+
+        # Strings are demanded rather than coerced, so a number where a code should
+        # be is refused here instead of reaching String methods further down and
+        # surfacing as NoMethodError instead of a controlled failure.
+        def valid_entry?(entry)
+          entry.is_a?(Hash) &&
+            entry["name"].is_a?(String) && !entry["name"].empty? &&
+            entry["value"].is_a?(String) && !entry["value"].empty?
+        end
+
+        def validate_code!(code, length, prefix)
+          return if code.match?(/\A\d{#{length}}\z/) && code.start_with?(prefix)
+
+          raise DownloadError, "malformed Poczta Polska TERYT code: #{code.inspect}"
+        end
+
+        def validate_commune_code!(code, district)
+          return if code.match?(/\A\d{6,7}\z/) && code.start_with?(district)
+
+          raise DownloadError, "malformed Poczta Polska TERYT code: #{code.inspect}"
+        end
+
+        def add!(names, code, name)
+          normalized = name.strip
+          previous = names[code]
+          if previous && previous != normalized
+            raise DownloadError,
+                  "conflicting Poczta Polska names for TERYT #{code}: #{previous.inspect} and #{normalized.inspect}"
+          end
+
+          names[code] = normalized
+        end
+
+        def normalize_commune_name(name)
+          name.sub(/ \((?:miejska|wiejska|miejsko-wiejska)\)\z/, "")
+        end
+      end
+    end
+  end
+end

data/lib/zip_codes/pl/tasks/zip_codes_pl.rake

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+namespace :zip_codes do
+  namespace :pl do
+    desc "Download and refresh the postal code dataset (rake zip_codes:pl:update[directory])"
+    task :update, [:output_dir] do |_task, args|
+      require "zip_codes/pl"
+
+      result = ZipCodes::PL.update(output_dir: args[:output_dir])
+      manifest = result.manifest
+
+      if result.up_to_date?
+        puts "Source unchanged - the dataset is already current: #{result.data_path}"
+      else
+        puts "Wrote #{manifest.row_count} rows to #{result.data_path}"
+      end
+      puts "Source: #{manifest.attribution}" if manifest
+    end
+
+    desc "Check the bundled dataset against its manifest and the record columns"
+    task :verify, [:output_dir] do |_task, args|
+      require "zip_codes/pl"
+
+      ZipCodes::PL.config.output_dir = args[:output_dir] if args[:output_dir]
+      path = ZipCodes::PL.config.readable_data_path
+      manifest = ZipCodes::PL.manifest
+      abort("No dataset at #{path}") unless File.exist?(path)
+      abort("No manifest next to #{path}") if manifest.nil?
+
+      header = File.open(path, &:gets).to_s.chomp.split("\t")
+      expected = ZipCodes::PL::Record::COLUMNS.map(&:to_s)
+      abort("Header is #{header.inspect}, expected #{expected.inspect}") unless header == expected
+
+      rows = ZipCodes::PL.reader.count
+      abort("Manifest claims #{manifest.row_count} rows, the file holds #{rows}") unless rows == manifest.row_count
+
+      known = ZipCodes::PL::Voivodeship.all.to_h { |voivodeship| [voivodeship.teryt_code, true] }
+      unknown = ZipCodes::PL.reader.reject { |record| known.key?(record.voivodeship_teryt) }
+      abort("Unknown voivodeship codes: #{unknown.first(5).map(&:voivodeship_teryt).uniq.inspect}") if unknown.any?
+
+      puts "Dataset consistent: #{rows} rows, #{known.size} voivodeships, built #{manifest.built_at}"
+    end
+
+    desc "Show the state of the built dataset"
+    task :info, [:output_dir] do |_task, args|
+      require "zip_codes/pl"
+
+      ZipCodes::PL.config.output_dir = args[:output_dir] if args[:output_dir]
+      manifest = ZipCodes::PL.manifest
+
+      if manifest.nil?
+        puts "No dataset in #{ZipCodes::PL.config.output_dir} - run rake zip_codes:pl:update"
+        next
+      end
+
+      puts "File:       #{ZipCodes::PL.config.data_path}"
+      puts "Rows:       #{manifest.row_count}"
+      puts "Built:      #{manifest.built_at}"
+      puts "Source:     #{manifest.source_url}"
+      puts "Modified:   #{manifest.last_modified}"
+      puts "Attribution: #{manifest.attribution}"
+    end
+  end
+end

data/lib/zip_codes/pl/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module ZipCodes
+  module PL
+    VERSION = "0.1.1"
+  end
+end

data/lib/zip_codes/pl/voivodeship.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module ZipCodes
+  module PL
+    # The sixteen voivodeships, pairing the GeoNames admin1 code that appears in
+    # the source export with the official TERYT code and the Polish name.
+    #
+    # GeoNames ships English and inconsistent labels for every one of them
+    # ("Lower Silesia", "Warmia-Masuria", "Łódź Voivodeship"), so the name used by
+    # this gem is defined here rather than read from the download. The
+    # geonames_code => teryt_code pairing is verified against the source data by
+    # the test suite, because it is the join key everything else hangs off.
+    class Voivodeship < Data.define(:teryt_code, :geonames_code, :name, :slug)
+      ALL = [
+        new(teryt_code: "02", geonames_code: "72", name: "dolnośląskie", slug: "dolnoslaskie"),
+        new(teryt_code: "04", geonames_code: "73", name: "kujawsko-pomorskie", slug: "kujawsko-pomorskie"),
+        new(teryt_code: "06", geonames_code: "75", name: "lubelskie", slug: "lubelskie"),
+        new(teryt_code: "08", geonames_code: "76", name: "lubuskie", slug: "lubuskie"),
+        new(teryt_code: "10", geonames_code: "74", name: "łódzkie", slug: "lodzkie"),
+        new(teryt_code: "12", geonames_code: "77", name: "małopolskie", slug: "malopolskie"),
+        new(teryt_code: "14", geonames_code: "78", name: "mazowieckie", slug: "mazowieckie"),
+        new(teryt_code: "16", geonames_code: "79", name: "opolskie", slug: "opolskie"),
+        new(teryt_code: "18", geonames_code: "80", name: "podkarpackie", slug: "podkarpackie"),
+        new(teryt_code: "20", geonames_code: "81", name: "podlaskie", slug: "podlaskie"),
+        new(teryt_code: "22", geonames_code: "82", name: "pomorskie", slug: "pomorskie"),
+        new(teryt_code: "24", geonames_code: "83", name: "śląskie", slug: "slaskie"),
+        new(teryt_code: "26", geonames_code: "84", name: "świętokrzyskie", slug: "swietokrzyskie"),
+        new(teryt_code: "28", geonames_code: "85", name: "warmińsko-mazurskie", slug: "warminsko-mazurskie"),
+        new(teryt_code: "30", geonames_code: "86", name: "wielkopolskie", slug: "wielkopolskie"),
+        new(teryt_code: "32", geonames_code: "87", name: "zachodniopomorskie", slug: "zachodniopomorskie")
+      ].freeze
+
+      BY_GEONAMES_CODE = ALL.to_h { |voivodeship| [voivodeship.geonames_code, voivodeship] }.freeze
+      BY_TERYT_CODE = ALL.to_h { |voivodeship| [voivodeship.teryt_code, voivodeship] }.freeze
+      BY_LOOKUP_NAME = ALL.flat_map { |v| [[v.name, v], [v.slug, v]] }.to_h.freeze
+
+      class << self
+        def all = ALL
+
+        def find_by_geonames_code(code) = BY_GEONAMES_CODE[code.to_s]
+
+        def find_by_teryt_code(code) = BY_TERYT_CODE[code.to_s]
+
+        def find_by_name(name) = BY_LOOKUP_NAME[name.to_s.strip.downcase]
+      end
+    end
+  end
+end

data/lib/zip_codes/pl.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require_relative "pl/errors"
+require_relative "pl/version"
+require_relative "pl/normalize"
+require_relative "pl/voivodeship"
+require_relative "pl/record"
+require_relative "pl/city"
+require_relative "pl/sources/geonames"
+require_relative "pl/sources/poczta_polska"
+require_relative "pl/configuration"
+require_relative "pl/manifest"
+require_relative "pl/builder"
+require_relative "pl/reader"
+require_relative "pl/railtie" if defined?(Rails::Railtie)
+
+module ZipCodes
+  module PL
+    class << self
+      def config
+        @config ||= Configuration.new
+      end
+
+      def configure
+        yield config
+        reset!
+        config
+      end
+
+      # Drops the loaded dataset so the next read picks up a freshly built file.
+      def reset!
+        @reader = nil
+        self
+      end
+
+      # Streams the file. Holds nothing, costs about 15 ms per call.
+      def reader
+        @reader ||= Reader.new(config.readable_data_path)
+      end
+
+      def update(output_dir: nil)
+        config.output_dir = output_dir if output_dir
+        Builder.new(config: config).call.tap { reset! }
+      end
+
+      def manifest = Manifest.read(config.readable_manifest_path)
+
+      def find_by_postal_code(code) = reader.find_by_postal_code(code)
+
+      def find_by_city(name, voivodeship: nil) = reader.find_by_city(name, voivodeship: voivodeship)
+
+      def search_by_city(fragment, voivodeship: nil) = reader.search_by_city(fragment, voivodeship: voivodeship)
+
+      def cities = reader.cities
+
+      def each_record(&) = reader.each(&)
+
+      def voivodeships = Voivodeship.all
+    end
+  end
+end

metadata

@@ -0,0 +1,86 @@
+--- !ruby/object:Gem::Specification
+name: zip-codes-pl
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Łukasz Kerl
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rubyzip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.3'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.3'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4'
+description: Builds and refreshes a local dataset of Polish postal codes (PNA), including
+  locality, voivodeship, county and commune TERYT codes, and coordinates. PNA and
+  coordinates come from GeoNames (CC BY 4.0), while administrative names come from
+  Poczta Polska.
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE.txt
+- README.md
+- README.pl.md
+- data/zip-codes-pl.manifest.json
+- data/zip-codes-pl.tsv
+- lib/zip-codes-pl.rb
+- lib/zip_codes/pl.rb
+- lib/zip_codes/pl/builder.rb
+- lib/zip_codes/pl/city.rb
+- lib/zip_codes/pl/configuration.rb
+- lib/zip_codes/pl/errors.rb
+- lib/zip_codes/pl/manifest.rb
+- lib/zip_codes/pl/normalize.rb
+- lib/zip_codes/pl/railtie.rb
+- lib/zip_codes/pl/reader.rb
+- lib/zip_codes/pl/record.rb
+- lib/zip_codes/pl/sources/geonames.rb
+- lib/zip_codes/pl/sources/poczta_polska.rb
+- lib/zip_codes/pl/sources/poczta_polska/client.rb
+- lib/zip_codes/pl/tasks/zip_codes_pl.rake
+- lib/zip_codes/pl/version.rb
+- lib/zip_codes/pl/voivodeship.rb
+homepage: https://github.com/carlos-1410/zip-codes-pl
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/carlos-1410/zip-codes-pl
+  source_code_uri: https://github.com/carlos-1410/zip-codes-pl
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.2.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.9
+specification_version: 4
+summary: Polish postal codes, localities, voivodeships, and coordinates
+test_files: []