atlas_engine 0.1.0

data/app/jobs/atlas_engine/address_importer/open_address/geo_json_import_job.rb

@@ -0,0 +1,212 @@
+# typed: true
+# frozen_string_literal: true
+
+# The GeoJsonImportJob loads a GeoJSON file into the PostAddress table.
+# It is designed to be resumable, according to JobIteration best practices.
+# The input file may optionally be gzip-compressed. It's read in chunks of 10_000 rows.
+# The ETL process is as follows:
+# - Extract: each row is evaluated with JSON.parse. Next, the GeoJSON feature contained
+#   is passed to a Filter instance, which may return false to reject the row. The Filter class
+#   is specified in country_profile under open_address/filter.
+# - Transform: a Transformer instance receives the feature, and returns zero or more hashes
+#   that match the PostAddress table/schema. The Transformer class is also specified in
+#   country_profile, under open_address/filter.
+# - Transform: after the Transformer, addresses are passed to a Corrections::Corrector, which runs one or
+#   more dedicated Correctors. These modify the passed address inline, and may clear the address hash to reject it.
+#   The Corrector classes are specified in country_profile under ingestion/correctors/open_address.
+# - Load: addresses are upserted into the PostAddresses table. When two addresses have the same
+#   province + locale + city + street + zip (a conflict on index_post_addresses_on_pc_zp_st_ct_lc),
+#   the building_and_unit_ranges field is merged using JSON_MERGE and other fields (like lat, lon) are overwritten.
+module AtlasEngine
+  module AddressImporter
+    module OpenAddress
+      class GeoJsonImportJob < AddressImporter::ResumableImportJob
+        extend T::Sig
+        include HandlesInterruption
+        include PreparesGeoJsonFile
+        attr_reader :geojson_path, :country_import, :country_code, :loader, :transformer
+
+        CHUNK_SIZE = 10_000
+        REPORT_STEP = 5
+
+        around_perform :setup_and_download
+
+        # Setup boilerplate: JobIteration doesn't let us override #perform. Instead
+        # the around_perform callback is used for that.
+        def setup_and_download(&block)
+          @loader = Loader.new
+          @country_code = argument(:country_code)
+          @geojson_path = Pathname.new(argument(:geojson_file_path))
+          @locale = argument(:locale)&.downcase
+          @country_import = CountryImport.find(argument(:country_import_id))
+          @transformer = Transformer.new(country_import: country_import, locale: @locale)
+
+          import_log_info(
+            country_import: country_import,
+            message: "Downloading geojson file",
+            additional_params: { file_path: geojson_path.to_s },
+          )
+
+          download_geojson(&block)
+        end
+
+        StringProps = T.type_alias { T::Hash[String, T.untyped] }
+        BatchOfRows = T.type_alias { T::Array[StringProps] }
+
+        # Part of JobIteration: returns an Enumerator that yields batches of addresses, and the cursor position.
+        # If cursor is present, the enumerator starts at that position. This stage does extraction (parsing JSON)
+        # and filtering.
+        sig do
+          params(
+            params: T::Hash[Symbol, T.untyped],
+            cursor: T.untyped,
+          ).returns(T::Enumerator[[BatchOfRows, Integer]])
+        end
+        def build_enumerator(params, cursor:)
+          start_at = if cursor.nil?
+            import_log_info(country_import: country_import, message: "Importing whole file")
+            0
+          else
+            import_log_info(country_import: country_import, message: "Starting import at chunk #{cursor}")
+            cursor.to_i
+          end
+
+          io.each
+            # NOTE: The bigger the chunk size, the less rountrips to MySQL, and therefore faster.
+            .each_slice(CHUNK_SIZE)
+            .lazy
+            .drop(start_at) # Cursor is chunk number. When resuming, skip that many chunks.
+            .with_index(start_at) # Include skipped chunks in numbering
+            .map do |lines, chunk_num|
+              track_progress(chunk_num)
+              [lines.map { JSON.parse(_1) }, chunk_num]
+            end
+            .map do |features, chunk_num|
+              [features.select(&row_filter), chunk_num]
+            end
+        end
+
+        # Part of JobIteration: ran for each batch of rows. This stage does transformation
+        # (converting a GeoJSON feature into a hash, then applying correctors) and loading (upserting into PostAddress).
+        sig { params(batch: BatchOfRows, element_id: T.untyped).void }
+        def each_iteration(batch, element_id)
+          exit_if_interrupted!(country_import)
+
+          addresses = attributes_from_batch(batch)
+          return if addresses.blank?
+
+          condensed = condense_addresses(addresses)
+
+          loader.load(condensed)
+        end
+
+        sig do
+          params(addresses: T::Array[T::Hash[Symbol,
+            T.untyped]]).returns(T::Array[T.nilable(T::Hash[Symbol, T.untyped])])
+        end
+        def condense_addresses(addresses)
+          addresses
+            .group_by { |attrs| [attrs[:province_code], attrs[:locale], attrs[:city], attrs[:street], attrs[:zip]] }
+            .map do |(_province_code, _locale, _city, _street, _zip), matched_addresses|
+              matched_addresses.reduce do |acc, matched_address|
+                acc.merge(matched_address) do |key, oldval, newval|
+                  if key == :building_and_unit_ranges
+                    oldval.merge(newval)
+                  else
+                    newval
+                  end
+                end
+              end
+            end
+        end
+
+        sig { params(batch: BatchOfRows).returns(T::Array[T::Hash[Symbol, T.untyped]]) }
+        def attributes_from_batch(batch)
+          batch
+            .filter_map do |feature|
+              attrs = transformer.transform(feature)
+              if attrs.nil?
+                incr_invalid_lines
+                next
+              end
+
+              attrs
+            end
+        end
+
+        sig { params(chunk_num: Integer).void }
+        def track_progress(chunk_num)
+          return unless chunk_num % REPORT_STEP == 0
+
+          lines_parsed = chunk_num * CHUNK_SIZE
+          import_log_info(
+            country_import: country_import,
+            message: "Processing chunk #{chunk_num}, lines parsed so far: #{lines_parsed}",
+          )
+
+          if lines_parsed != invalid_lines
+            import_log_info(
+              country_import: country_import,
+              message: "Lines discarded: #{invalid_lines}",
+              category: :invalid_address,
+            )
+          end
+        end
+
+        sig { returns(CountryProfile) }
+        def country_profile
+          @country_profile ||= CountryProfile.for(country_code)
+        end
+
+        FilterType = T.type_alias { T.proc.params(arg0: StringProps).returns(T::Boolean) }
+        # Returns a callable that takes a row and returns true if it should be imported
+        sig { returns(FilterType) }
+        def row_filter
+          @row_filter ||= case country_profile.open_address[:filter]
+          in nil # Undefined: let everything through
+            ->(_row) { true }
+          in /\w+(::\w+)+/ => sym # Class name
+            cls = sym.constantize
+            inst = cls.new(country_import: country_import)
+            inst.method(:filter).to_proc
+          end
+        end
+
+        Corrector = AddressImporter::Corrections::Corrector
+        # Returns a Corrector instance, or nil if no correctors are defined for this country.
+        sig { returns(T.nilable(Corrector)) }
+        def corrector
+          @corrector ||= if country_profile.ingestion.correctors(source: "open_address").empty?
+            nil
+          else
+            Corrector.new(
+              country_code: country_code,
+              source: "open_address",
+            )
+          end
+        end
+
+        # Returns an IO-like object that reads the geojson file.
+        # Returns untyped because GzipReader is IO-like, but not a subclass of IO.
+        sig { returns(T.untyped) }
+        def io
+          Zlib::GzipReader.new(geojson_path.open("rb"))
+        end
+
+        sig { void }
+        def incr_invalid_lines
+          if @invalid_lines.nil?
+            @invalid_lines = 0
+          else
+            @invalid_lines += 1
+          end
+        end
+
+        sig { returns(Integer) }
+        def invalid_lines
+          @invalid_lines || 0
+        end
+      end
+    end
+  end
+end

data/app/jobs/atlas_engine/address_importer/open_address/geo_json_import_launcher_job.rb

@@ -0,0 +1,67 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module AddressImporter
+    module OpenAddress
+      class GeoJsonImportLauncherJob < ApplicationJob
+        extend T::Sig
+        include ImportLogHelper
+
+        sig { params(country_code: String, geojson_file_path: String, clear_records: T::Boolean, locale: String).void }
+        def perform(country_code:, geojson_file_path:, clear_records:, locale:)
+          import = CountryImport.create!(country_code: country_code)
+          import.start!
+
+          import_log_info(
+            country_import: import,
+            message: "Starting import",
+            additional_params: { file: geojson_file_path },
+            notify: true,
+          )
+
+          geojson_file_paths = geojson_file_path.split(",")
+
+          geojson_import_jobs = geojson_file_paths.map do |geojson_file_path|
+            geojson_job_args = {
+              country_code: country_code,
+              country_import_id: import.id,
+              geojson_file_path: geojson_file_path,
+              locale: locale,
+            }
+            { job_name: GeoJsonImportJob, job_args: geojson_job_args }
+          end
+
+          street_backfill_job = {
+            job_name: AddressImporter::StreetBackfillJob,
+            job_args: { country_code: country_code, country_import_id: import.id },
+          }
+
+          if clear_records
+            import_log_info(country_import: import, message: "Clearing records before import...")
+
+            AddressImporter::ClearRecordsJob.perform_later(
+              country_import_id: import.id,
+              country_code: country_code.upcase,
+              followed_by: geojson_import_jobs + [street_backfill_job],
+            )
+          else
+            import_log_info(country_import: import, message: "Importing without clearing records...")
+
+            GeoJsonImportJob.perform_later(
+              **T.must(geojson_import_jobs.first)[:job_args],
+              followed_by: geojson_import_jobs.drop(1) + [street_backfill_job],
+            )
+          end
+        rescue StandardError => e
+          import_log_error(
+            country_import: T.must(import),
+            message: "Import failed with #{e.class}",
+            additional_params: { error: e },
+          )
+          import&.interrupt
+        end
+      end
+    end
+  end
+end

data/app/jobs/atlas_engine/address_importer/open_address/prepares_geo_json_file.rb

@@ -0,0 +1,41 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module AddressImporter
+    module OpenAddress
+      module PreparesGeoJsonFile
+        extend T::Sig
+        include HandlesBlob
+
+        ROOT_FOLDER = "openaddress"
+
+        sig { params(block: T.proc.void).void }
+        def download_geojson(&block)
+          if @geojson_path.exist?
+            yield
+          else
+            download_from_activestorage do |local_path|
+              @geojson_path = local_path
+              yield
+            end
+          end
+        end
+
+        private
+
+        sig do
+          params(block: T.proc.params(arg0: Pathname).void).void
+        end
+        def download_from_activestorage(&block)
+          root = Pathname.new(ROOT_FOLDER)
+          key = root.join(@geojson_path.basename).to_s
+
+          download(key) do |fp|
+            yield Pathname.new(fp.path)
+          end
+        end
+      end
+    end
+  end
+end

data/app/jobs/atlas_engine/address_importer/resumable_import_job.rb

@@ -0,0 +1,49 @@
+# typed: false
+# frozen_string_literal: true
+
+module AtlasEngine
+  module AddressImporter
+    class ResumableImportJob < ApplicationJob
+      include JobIteration::Iteration
+      include ImportLogHelper
+      include Concerns::AddressImporter::HandlesErrors
+
+      on_complete do |job|
+        next_job = job.argument(:followed_by)&.shift
+        if next_job.present?
+          job_args_with_followed_by = next_job[:job_args].merge({ followed_by: job.argument(:followed_by) })
+          next_job[:job_name].perform_later(**job_args_with_followed_by)
+        elsif country_import.present?
+          country_import.complete!
+          log_final_stats
+        end
+      end
+
+      sig { void }
+      def log_final_stats
+        message = if country_import.detected_invalid_addresses?
+          "Invalid addresses detected"
+        else
+          "No invalid addresses detected"
+        end
+
+        import_log_info(
+          country_import: country_import,
+          message: message,
+          notify: true,
+        )
+
+        import_log_info(
+          country_import: country_import,
+          message: "Import complete!",
+          notify: true,
+        )
+      end
+
+      def country_import
+        country_import_id = argument(:country_import_id)
+        CountryImport.find(country_import_id) if country_import_id.present?
+      end
+    end
+  end
+end

data/app/jobs/atlas_engine/address_importer/street_backfill_job.rb

@@ -0,0 +1,63 @@
+# typed: false
+# frozen_string_literal: true
+
+module AtlasEngine
+  module AddressImporter
+    class StreetBackfillJob < AddressImporter::ResumableImportJob
+      include ImportLogHelper
+      include LogHelper
+
+      def build_enumerator(params, cursor:)
+        enumerator_builder.build_times_enumerator(1, cursor: cursor)
+      end
+
+      def each_iteration(batch, params)
+        @country_code = params[:country_code]
+        @locales = CountryProfile.for(@country_code).validation.index_locales
+        return if @locales.nil? || @locales.size < 2
+
+        import_log_info(
+          country_import: country_import,
+          message: "Backfilling street data for locales #{@locales}...",
+        )
+
+        ActiveRecord::Base.connection.execute(backfill_streets_sql)
+      end
+
+      private
+
+      def backfill_streets_sql
+        <<-SQL.squish
+          UPDATE #{PostAddress.table_name} AS target
+          INNER JOIN (
+            SELECT
+              #{@locales.first}.source_id AS source_id,
+              COALESCE(#{@locales.map { |loc| "#{loc}.street" }.join(", ")}) AS final_street,
+              #{@locales.map { |loc| "CASE WHEN #{loc}.source_id IS NOT NULL THEN 1 ELSE 0 END AS #{loc}_present" }.join(",")}
+            FROM (#{records_by_locale[@locales.first]}) AS #{@locales.first}
+              #{join_tables_on_statement}
+            HAVING (#{locale_presence_count}) > 1
+          ) AS effective ON target.source_id = effective.source_id AND target.country_code = '#{@country_code}'
+          SET target.street = effective.final_street
+          WHERE target.country_code = '#{@country_code}' AND (target.street = '' OR target.street IS NULL)
+        SQL
+      end
+
+      def records_by_locale
+        @records_by_locale ||= @locales.index_with do |locale|
+          PostAddress.where(country_code: @country_code, locale: locale).to_sql
+        end
+      end
+
+      def join_tables_on_statement
+        T.must(@locales[1..-1]).map do |loc|
+          "LEFT JOIN (#{records_by_locale[loc]}) AS #{loc} ON #{@locales.first}.source_id = #{loc}.source_id"
+        end.join("\n")
+      end
+
+      def locale_presence_count
+        @locales.map { |loc| "#{loc}_present" }.join(" + ")
+      end
+    end
+  end
+end

data/app/jobs/atlas_engine/application_job.rb

@@ -0,0 +1,10 @@
+# typed: false
+# frozen_string_literal: true
+
+module AtlasEngine
+  class ApplicationJob < ActiveJob::Base
+    def argument(key)
+      arguments.first&.fetch(key, nil)
+    end
+  end
+end

data/app/jobs/atlas_engine/concerns/address_importer/handles_errors.rb

@@ -0,0 +1,43 @@
+# typed: false
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Concerns
+    module AddressImporter
+      module HandlesErrors
+        extend ActiveSupport::Concern
+        include ::AtlasEngine::AddressImporter::ImportLogHelper
+
+        included do
+          discard_on(StandardError) do |job, exception|
+            country_import_id = job.arguments.first[:country_import_id]
+            country_import = CountryImport.find(country_import_id)
+
+            job.import_log_error(
+              country_import: country_import,
+              message: ":errors: Import failed with exception: #{exception.message}",
+              additional_params: { stack_trace: exception.backtrace.inspect },
+            )
+
+            country_import.interrupt! if country_import.present?
+          end
+
+          retry_on(
+            Mysql2::Error::ConnectionError,
+            wait: 10.seconds,
+            attempts: 5,
+          ) do |job, exception|
+            country_import_id = job.arguments.first[:country_import_id]
+            country_import = CountryImport.find(country_import_id)
+
+            job.import_log_error(country_import: country_import, message:
+            "Job failed after 5 retries with error: #{exception.message}")
+
+            country_import.interrupt! if country_import.present?
+            raise exception
+          end
+        end
+      end
+    end
+  end
+end

data/app/lib/atlas_engine/concern_formatter.rb

@@ -0,0 +1,40 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module ConcernFormatter
+    include Kernel # https://github.com/sorbet/sorbet/issues/1109
+    extend T::Sig
+
+    sig { returns(AddressValidation::AbstractAddress) }
+    def address
+      raise NotImplementedError
+    end
+
+    sig { returns(String) }
+    def country_name
+      return "" if address.country_code.blank?
+
+      country.country? && country.full_name ? country.full_name : address.country_code
+    end
+
+    sig { returns(String) }
+    def province_name
+      return "" if address.country_code.blank? || address.province_code.blank?
+
+      province.province? && province.full_name ? province.full_name : address.province_code
+    end
+
+    private
+
+    sig { returns(Worldwide::Region) }
+    def country
+      @country ||= Worldwide.region(code: address.country_code)
+    end
+
+    sig { returns(Worldwide::Region) }
+    def province
+      @province ||= country.zone(code: address.province_code)
+    end
+  end
+end

data/app/lib/atlas_engine/restrictions/base.rb

@@ -0,0 +1,20 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Restrictions
+    module Base
+      extend T::Sig
+      extend T::Helpers
+      interface!
+
+      sig do
+        abstract.params(
+          address: AtlasEngine::AddressValidation::AbstractAddress,
+          params: T.untyped,
+        ).returns(T::Boolean)
+      end
+      def apply?(address:, params: nil); end
+    end
+  end
+end

data/app/lib/atlas_engine/restrictions/unsupported_script.rb

@@ -0,0 +1,31 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Restrictions
+    class UnsupportedScript
+      class << self
+        extend T::Sig
+        include Base
+
+        sig do
+          override.params(
+            address: AtlasEngine::AddressValidation::AbstractAddress,
+            params: T.untyped,
+          ).returns(T::Boolean)
+        end
+        def apply?(address:, params: {})
+          supported_script = params[:supported_script]
+          return false if supported_script.nil?
+
+          scripts = Worldwide.scripts.identify(
+            text: address.address1.to_s + " " + address.address2.to_s + " " + address.city.to_s,
+          )
+          return false if scripts.empty?
+
+          scripts.any? { |script| script != supported_script }
+        end
+      end
+    end
+  end
+end