atlas_engine 0.8.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1bdadba3857324dae37bf4c22aa4078b4f565876c0f865a4d3b99db69d0eaa41
-  data.tar.gz: 4004ade11bd387145ed6b82cb8b5df9a99989b3d1869cc1d40fd7fb310e3fc7c
+  metadata.gz: dde673ac6063402f30381011574f018aee19ac9d6e9b961c04252cf76cfd49e5
+  data.tar.gz: b5c0adce91a39d744004b902e2b3a4a13544a50af756bb1ace003e391b0d3643
 SHA512:
-  metadata.gz: 846730be55864274ddd3d61e90ffd970102ea54d25221e411cee78c91daaa974a1159d0c976cde708ac6ea355815deb9c1d91230ef9c4eb49dcf38fe59e4be27
-  data.tar.gz: 791c4a0050adab0b39092b660c56968d7f6862a1387914f50341f940153963487f9fc8ec8533ee6519fd8569bd69022c8cb7ce181c3017a14313fd32a49879a7
+  metadata.gz: dfe8b1e2a4ebe506f55b62a541b57ab22c0cca456ed4ef071007d43e106d0ed0b904389fd22a0424661ca76cd2f7bdc4ed5887866663eaee361f9afd5e7d3ac0
+  data.tar.gz: a632f397142d60a6da29ae440ed686ff920cb15e62349c3abb0ef9a3ccaddd975936870d55aee45867851337f10d46982f6740bbcf3379f9d74aeeed15369577

data/README.md

@@ -7,6 +7,7 @@ Atlas Engine is a rails engine that provides a global end-to-end address validat
 * [Local Development Installation](#local-development-installation)
 * [Address Data Ingestion](#address-data-ingestion)
 * [Elasticsearch Matching Strategy](#elasticsearch-matching-strategy)
+* [Hosted Solution](#hosted-solution)
 
 ## Address Validation API
 
@@ -167,17 +168,23 @@ The validation scope excludes zip because the zip was not successfully validated
 ## Rails App Installation
 
 ### Initial setup
-Add the engine to your gemfile
+* Add the engine to your gemfile
 ```
 gem "atlas_engine"
 ```
 
-Run the following commands to install the engine in your rails app
-
+* Run the following commands to install the engine in your rails app
 ```
 bundle lock
-bin/rails generate atlas_engine:install
+rails atlas_engine:install:migrations
+rails db:migrate
 ```
+* In `config/routes` mount AtlasEngine
+  * Adding the line `mount AtlasEngine::Engine => "/atlas_engine"
+`
+* In `app/assets/config/manifest.js`
+  * Adding the line `//= link atlas_engine/application.css`
+* Install [maintenance_tasks](https://github.com/Shopify/maintenance_tasks?tab=readme-ov-file#installation) - a dependency for Atlas Engine that is used to ingest country data.
 
 ### Updating to a newer version of the engine
 
@@ -297,23 +304,25 @@ At the moment, `atlas_engine` supports advanced address validation for the follo
 
 | Country/territory | Two-letter code | Locales  | Street | City | Postal Code | Province/State |
 |-------------------|-----------------|----------|--------|------|-------------|----------------|
-| Australia         | AU              |          |        | x    | x           | x              |
-| Austria           | AT              |          |        | x    | x           | x              |
-| Belgium           | BE              | fr,nl,de |        | x    | x           |                |
+| Australia         | AU              |          | x      | x    | x           | x              |
+| Austria           | AT              |          | x      | x    | x           | x              |
+| Belgium           | BE              | fr,nl,de | x      | x    | x           |                |
 | Bermuda           | BM              |          |        | x    | x           | x              |
 | Czechia           | CZ              |          |        | x    | x           |                |
 | Denmark           | DK              |          |        | x    | x           |                |
 | Faroe Islands     | FO              |          |        | x    | x           |                |
-| France            | FR              |          |        | x    | x           |                |
+| France            | FR              |          | x      | x    | x           |                |
+| Gurnsey           | GG              |          |        | x    | x           |                |
 | Italy             | IT              |          |        |      | x           |                |
 | Liechtenstein     | LI              |          |        | x    | x           | x              |
 | Luxembourg        | LU              | fr,lb    |        | x    | x           |                |
-| Netherlands       | NL              | nl       |        | x    | x           | x              |
-| Poland            | PL              |          |        | x    | x           | x              |
+| Netherlands       | NL              | nl       | x      | x    | x           | x              |
+| Poland            | PL              |          | x      | x    | x           | x              |
 | Portugal          | PT              |          |        | x    | x           | x              |
 | Slovenia          | SI              |          |        | x    | x           | x              |
 | South Korea       | KR              |          |        | x    | x           | x              |
-| Switzerland       | CH              | de,fr,it |        | x    | x           |                |
+| Switzerland       | CH              | de,fr,it | x      | x    | x           |                |
+| United States     | US              | en       | x      | x    | x           | x              |
 
 ### Downloading and indexing instructions
 
@@ -353,8 +362,7 @@ In this example, the country code of Australia is `AU`.
 link for a more detailed view. Once the import status has updated from `in_progress` to `complete` we will have all of
 the raw open address data imported into our mysql database's `atlas_engine_post_addresses` table.
 
-6. Navigate back to `http://localhost:3000/maintenance_tasks` and click on the `Maintenance::AtlasEngine::ElasticsearchIndexCreateTask`. This task will ingest the data we have staged in mysql
-and use it to create documents in a new elasticsearch index which Atlas Engine will ultimately use for validation.
+6. Navigate back to `http://localhost:3000/maintenance_tasks` and click on the `Maintenance::AtlasEngine::ElasticsearchIndexCreateTask`. This task will ingest the data we have staged in mysql and use it to create documents in a new elasticsearch index which Atlas Engine will ultimately use for validation.
 
 7. The `ElasticsearchIndexCreateTask` includes the following parameters:
 
@@ -380,6 +388,26 @@ If unchecked, the created index will need to be activated manually.
 We may now use the `es` and `es_street` matching strategies with `AU` addresses. See [below](#elasticsearch-matching-strategy)
 for an example of its usage.
 
+#### Instructions for US import
+
+1. Go to the [open addresses](https://openaddresses.io/) download center and download the collection-us-{region}.zip
+files for each of the four regions (west, midwest, northeast, south).
+
+2. Run the US create state geojson script to create a statewide geojson.gz file for each state
+```
+bin/us_create_state_geojson execute /path/to/us_collection_zips /path/to/output_dir
+```
+
+3. Start your app with `rails s` and navigate to `http://localhost:3000/maintenance_tasks`. There is a task only used
+for the US import called `Maintenance::AtlasEngine::UsGeoJsonDirectoryImportTask`
+
+4. Parameterize the `UsGeoJsonDirectoryImportTask` with the output directory that contains all of the `{state}-statewide.geojson.gz` files created in step 2.
+
+5. Once properly parameterized, click run. The process will initialize a `country_import` and should succeed immediately.
+
+6. Navigate to `http://localhost:3000/country_imports` to track the progress of the country import. Once the import is complete
+and the US data is in mysql the rest of the process for creating the elasticsearch index and verifying should be the same as above.
+
 ## Elasticsearch Matching Strategy
 
 Once we have successfully created and activated an elasticsearch index using open address data, we may now use
@@ -627,3 +655,19 @@ will produce the response:
 ```
 
 This response has no concerns or suggestions, and the input address is therefore considered to be valid.
+
+## Hosted Solution
+
+If you wish to use the Shopify hosted version of the `atlas_engine` codebase in your applications, you will need to [register for an api key](https://address-validation.shopify.dev/#request-api-key) and agree to our [terms and conditions](https://address-validation.shopify.dev/terms-of-service).
+
+Once you have successfully redeemed an api key, you will be able to access the `/graphql` endpoint, which can be queried using this example curl request:
+
+```
+curl --request POST \
+  --url https://atlas-validation.shopifyapps.com/graphql \
+  --header 'Authorization: Bearer {your-api-key}' \
+  --header 'Content-Type: application/json' \
+  --data '{"query":"query validation { validation(address: { address1: \"233 S Wacker Dr\" address2: \"\" city: \"Chicago\" countryCode: US provinceCode: \"IL\" zip: \"60606\" } locale: \"EN\" ) { validationScope concerns { code fieldNames suggestionIds type typeLevel message } suggestions { id address1 address2 city province provinceCode zip } }}","operationName":"validation"}'
+```
+
+Any updates to the `atlas_engine` codebase that are merged into our `main` branch will be deployed to our hosted solution as well.

data/app/countries/atlas_engine/at/country_profile.yml

@@ -9,7 +9,7 @@ ingestion:
   data_mapper: AtlasEngine::AddressValidation::Es::DataMappers::DecompoundingDataMapper
 validation:
   enabled: true
-  default_matching_strategy: es
+  default_matching_strategy: es_street
   has_provinces: true
   address_parser: AtlasEngine::At::ValidationTranscriber::AddressParser
   normalized_components:

data/app/countries/atlas_engine/au/country_profile.yml

@@ -8,6 +8,6 @@ ingestion:
 validation:
   address_parser: AtlasEngine::Au::ValidationTranscriber::AddressParser
   enabled: true
-  default_matching_strategy: es
+  default_matching_strategy: es_street
 open_address:
   filter: AtlasEngine::Au::AddressImporter::OpenAddress::Filter

data/app/countries/atlas_engine/be/country_profile.yml

@@ -5,9 +5,9 @@ ingestion:
     max_zip_edge_ngram: "4"
 validation:
   enabled: true
-  default_matching_strategy: local
+  default_matching_strategy: es_street
   address_parser: AtlasEngine::Be::ValidationTranscriber::AddressParser
-  has_provinces: false 
+  has_provinces: false
   index_locales:
   - fr
   - nl

data/app/countries/atlas_engine/ch/country_profile.yml

@@ -1,7 +1,7 @@
 id: CH
 validation:
   enabled: true
-  default_matching_strategy: local
+  default_matching_strategy: es_street
   has_provinces: false
   address_parser: AtlasEngine::De::ValidationTranscriber::AddressParser
   index_locales:

data/app/countries/atlas_engine/fr/country_profile.yml

@@ -10,6 +10,6 @@ ingestion:
 validation:
   address_parser: AtlasEngine::Fr::ValidationTranscriber::AddressParser
   enabled: true
-  default_matching_strategy: es
+  default_matching_strategy: es_street
   has_provinces: false
   zip_prefix_length: 3

data/app/countries/atlas_engine/gb/address_validation/es/query_builder.rb

@@ -16,7 +16,7 @@ module AtlasEngine
             ).void
           end
           def initialize(address, parsings, profile)
-            super(address, parsings, profile)
+            super
 
             @parsings = T.let(
               AtlasEngine::Gb::ValidationTranscriber::FullAddressParser

data/app/countries/atlas_engine/nl/country_profile.yml

@@ -7,7 +7,7 @@ ingestion:
 validation:
   enabled: true
   has_provinces: true
-  default_matching_strategy: es
+  default_matching_strategy: es_street
   normalized_components:
   - street_decompounded
   address_parser: AtlasEngine::Nl::ValidationTranscriber::AddressParser

data/app/countries/atlas_engine/pl/country_profile.yml

@@ -7,7 +7,7 @@ ingestion:
 validation:
   address_parser: AtlasEngine::Pl::ValidationTranscriber::AddressParser
   enabled: true
-  default_matching_strategy: es
+  default_matching_strategy: es_street
   exclusions:
     city:
       - AtlasEngine::Pl::AddressValidation::Exclusions::RuralAddress

data/app/countries/atlas_engine/si/address_importer/open_address/mapper.rb

@@ -10,7 +10,7 @@ module AtlasEngine
             params(feature: AtlasEngine::AddressImporter::OpenAddress::Feature).returns(T::Hash[Symbol, T.untyped])
           end
           def map(feature)
-            super(feature).merge(region4: feature["properties"]["district"])
+            super.merge(region4: feature["properties"]["district"])
           end
         end
       end

data/app/countries/atlas_engine/us/address_importer/open_address/filter.rb

@@ -0,0 +1,28 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Us
+    module AddressImporter
+      module OpenAddress
+        class Filter
+          extend T::Sig
+          include AtlasEngine::AddressImporter::OpenAddress::Filter
+
+          def initialize(country_import:); end
+
+          sig { override.params(feature: AtlasEngine::AddressImporter::OpenAddress::Feature).returns(T::Boolean) }
+          def filter(feature)
+            # Only consider features with lat lon geometry
+            geometry = feature["geometry"]
+            if geometry.present? && geometry["type"] == "Point"
+              return false if geometry["coordinates"].size > 2
+            end
+
+            true
+          end
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/us/address_importer/open_address/mapper.rb

@@ -0,0 +1,66 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Us
+    module AddressImporter
+      module OpenAddress
+        class Mapper < AtlasEngine::AddressImporter::OpenAddress::DefaultMapper
+          ORDINAL_REGEX = /\b(\d+)(?:\s+)(st|nd|rd|th)\b/i
+
+          sig do
+            params(feature: AtlasEngine::AddressImporter::OpenAddress::Feature).returns(T::Hash[Symbol, T.untyped])
+          end
+          def map(feature)
+            city, street, number, unit, postcode = feature["properties"].values_at(
+              "city",
+              "street",
+              "number",
+              "unit",
+              "postcode",
+            )
+            {
+              source_id: openaddress_source_id(feature),
+              locale: @locale,
+              country_code: @country_code,
+              province_code: @province_code, # region is inconsistently set, override with passed in province_code
+              # Omitted: region1..4
+              city: sanitize_city(city),
+              suburb: nil,
+              zip: postcode.first(5), # truncate zip+4 data
+              street: sanitize_street(street),
+              building_and_unit_ranges: housenumber_and_unit(number, unit),
+              latitude: geometry(feature)&.at(1),
+              longitude: geometry(feature)&.at(0),
+            }
+          end
+
+          private
+
+          sig { params(street: T.nilable(String)).returns(T.nilable(String)) }
+          def sanitize_street(street)
+            combine_ordinal_string(strip_extra_spaces(street&.downcase))&.titleize
+          end
+
+          sig { params(city: T.nilable(String)).returns(T::Array[String]) }
+          def sanitize_city(city)
+            return [] if city.nil?
+
+            city.split("/").map { |c| c.titleize.strip }
+          end
+
+          sig { params(text: T.nilable(String)).returns(T.nilable(String)) }
+          def combine_ordinal_string(text)
+            text&.gsub!(ORDINAL_REGEX) { "#{::Regexp.last_match(1)}#{::Regexp.last_match(2)}" }
+            text
+          end
+
+          sig { params(text: T.nilable(String)).returns(T.nilable(String)) }
+          def strip_extra_spaces(text)
+            text&.strip&.gsub(/\s+/, " ")
+          end
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/us/country_profile.yml

@@ -1,10 +1,15 @@
 id: US
-validation:
-  enabled: true
-  default_matching_strategy: es_street
-  address_parser: AtlasEngine::ValidationTranscriber::AddressParserNorthAmerica
 ingestion:
   settings:
     number_of_shards: "7"
     min_zip_edge_ngram: "1"
     max_zip_edge_ngram: "10"
+  post_address_mapper:
+    open_address: AtlasEngine::Us::AddressImporter::OpenAddress::Mapper
+validation:
+  enabled: true
+  default_matching_strategy: es_street
+  unmatched_components_suggestion_threshold: 1
+  address_parser: AtlasEngine::ValidationTranscriber::AddressParserNorthAmerica
+open_address:
+  filter: AtlasEngine::Us::AddressImporter::OpenAddress::Filter

data/app/countries/atlas_engine/us/jobs/address_importer/combined_import_job.rb

@@ -0,0 +1,120 @@
+# typed: false
+# frozen_string_literal: true
+
+require "zip"
+
+module AtlasEngine
+  module Us
+    module Jobs
+      module AddressImporter
+        class CombinedImportJob < ApplicationJob
+          include ::AtlasEngine::AddressImporter::ImportLogHelper
+          discard_on Exception
+          rescue_from(ArgumentError) { raise }
+
+          COUNTRY_CODE = "US"
+          LOCALE = "en"
+          US_STATES = [
+            "AK",
+            "AL",
+            "AR",
+            "AZ",
+            "CA",
+            "CO",
+            "CT",
+            "DC",
+            "DE",
+            "FL",
+            "GA",
+            "HI",
+            "IA",
+            "ID",
+            "IL",
+            "IN",
+            "KS",
+            "KY",
+            "LA",
+            "MA",
+            "MD",
+            "ME",
+            "MI",
+            "MN",
+            "MO",
+            "MS",
+            "MT",
+            "NC",
+            "ND",
+            "NE",
+            "NH",
+            "NJ",
+            "NM",
+            "NV",
+            "NY",
+            "OH",
+            "OK",
+            "OR",
+            "PA",
+            "RI",
+            "SC",
+            "SD",
+            "TN",
+            "TX",
+            "UT",
+            "VA",
+            "VT",
+            "WA",
+            "WI",
+            "WV",
+            "WY",
+          ]
+
+          def perform(geojson_directory:)
+            country_import = AtlasEngine::CountryImport.create!(country_code: COUNTRY_CODE)
+            country_import.start!
+
+            import_log_info(
+              country_import: country_import,
+              message: "Starting import for #{COUNTRY_CODE} from #{geojson_directory}",
+              notify: true,
+            )
+
+            jobs_to_run = job_list(geojson_directory, country_import)
+            first_job = jobs_to_run.shift
+            first_job[:job_name].perform_later(**first_job[:job_args].merge(followed_by: jobs_to_run))
+          end
+
+          private
+
+          def job_list(geojson_directory, country_import)
+            job_list = [
+              clear_records_job(country_import),
+            ]
+
+            US_STATES.each do |us_state|
+              job_list <<
+                {
+                  job_name: ::AtlasEngine::AddressImporter::OpenAddress::GeoJsonImportJob,
+                  job_args: {
+                    country_code: COUNTRY_CODE,
+                    province_code: us_state.upcase,
+                    locale: LOCALE,
+                    geojson_file_path: File.join(geojson_directory, "us-#{us_state.downcase}-statewide.geojson.gz"),
+                    country_import_id: country_import.id,
+                  },
+                }
+            end
+
+            job_list
+          end
+
+          def clear_records_job(country_import)
+            {
+              job_name: AtlasEngine::AddressImporter::ClearRecordsJob,
+              job_args: { country_code: COUNTRY_CODE, country_import_id: country_import.id },
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/app/graphql/atlas_engine/schema.graphql

@@ -6,8 +6,12 @@ input AddressInput {
   address2: String
   city: String
   countryCode: ValidationSupportedCountry
+  line2: String
+  neighborhood: String
   phone: String
   provinceCode: String
+  streetName: String
+  streetNumber: String
   zip: String
 }
 

data/app/graphql/atlas_engine/types/address_validation/address_input.rb

@@ -12,7 +12,11 @@ module AtlasEngine
         description "Address fields used to fulfill a validation request"
 
         argument :address1, String, required: false
+        argument :street_name, String, required: false
+        argument :street_number, String, required: false
         argument :address2, String, required: false
+        argument :line2, String, required: false
+        argument :neighborhood, String, required: false
         argument :city, String, required: false
         argument :country_code, ValidationSupportedCountry, required: false
         argument :province_code, String, required: false

data/app/jobs/atlas_engine/address_importer/open_address/geo_json_import_job.rb

@@ -24,7 +24,7 @@ module AtlasEngine
         extend T::Sig
         include HandlesInterruption
         include PreparesGeoJsonFile
-        attr_reader :geojson_path, :country_import, :country_code, :loader, :transformer
+        attr_reader :geojson_path, :country_import, :country_code, :province_code, :loader, :transformer
 
         CHUNK_SIZE = 10_000
         REPORT_STEP = 5
@@ -36,10 +36,11 @@ module AtlasEngine
         def setup_and_download(&block)
           @loader = Loader.new
           @country_code = argument(:country_code)
+          @province_code = argument(:province_code)
           @geojson_path = Pathname.new(argument(:geojson_file_path))
           @locale = argument(:locale)&.downcase
           @country_import = CountryImport.find(argument(:country_import_id))
-          @transformer = Transformer.new(country_import: country_import, locale: @locale)
+          @transformer = Transformer.new(country_import: country_import, province_code: @province_code, locale: @locale)
 
           import_log_info(
             country_import: country_import,

data/app/lib/atlas_engine/validation_transcriber/formatter.rb

@@ -22,7 +22,11 @@ module AtlasEngine
       sig do
         params(
           address1: String,
+          street_name: String,
+          street_number: String,
           address2: String,
+          line2: String,
+          neighborhood: String,
           city: String,
           province_code: String,
           zip: String,
@@ -30,10 +34,15 @@ module AtlasEngine
           phone: String,
         ).returns(AddressValidation::Address)
       end
-      def build_address(address1: "", address2: "", city: "", province_code: "", zip: "", country_code: "", phone: "")
+      def build_address(address1: "", street_name: "", street_number: "", address2: "", line2: "", neighborhood: "",
+        city: "", province_code: "", zip: "", country_code: "", phone: "")
         AddressValidation::Address.new(
           address1: address1,
+          street_name: street_name,
+          street_number: street_number,
           address2: address2,
+          line2: line2,
+          neighborhood: neighborhood,
           city: city,
           province_code: province_code,
           zip: zip,

data/app/models/atlas_engine/address_importer/open_address/default_mapper.rb

@@ -7,9 +7,10 @@ module AtlasEngine
       class DefaultMapper
         extend T::Sig
         include FeatureHelper
-        sig { params(country_code: String, locale: T.nilable(String)).void }
-        def initialize(country_code:, locale: nil)
+        sig { params(country_code: String, province_code: T.nilable(String), locale: T.nilable(String)).void }
+        def initialize(country_code:, province_code: nil, locale: nil)
           @country_code = country_code
+          @province_code = province_code
           @locale = locale
         end
 
@@ -27,7 +28,7 @@ module AtlasEngine
             source_id: openaddress_source_id(feature),
             locale: @locale,
             country_code: @country_code,
-            province_code: nil,
+            province_code: @province_code,
             region1: region,
             # Don't titleize. The sources have proper capitalization, and it's a problem for cities like
             # 's-Graveland, which would get titleized to "'S Graveland" which is wrong.

data/app/models/atlas_engine/address_importer/open_address/transformer.rb

@@ -8,11 +8,12 @@ module AtlasEngine
         extend T::Sig
         extend T::Helpers
 
-        def initialize(country_import:, locale: nil)
+        def initialize(country_import:, province_code: nil, locale: nil)
           @country_code = country_import.country_code
           @locale = locale
+          @province_code = province_code
           @mapper = CountryProfile.for(@country_code).ingestion.post_address_mapper("open_address").new(
-            country_code: @country_code, locale: @locale,
+            country_code: @country_code, province_code: @province_code, locale: @locale,
           )
           @corrector = AddressImporter::Corrections::Corrector.new(country_code: @country_code, source: "open_address")
           @validator = AddressImporter::Validation::Wrapper.new(

data/app/models/atlas_engine/address_number_range.rb

@@ -164,7 +164,7 @@ module AtlasEngine
 
     sig { params(value: String).returns(T::Boolean) }
     def include_fractions?(value)
-      %r{^([0-9]+ )?([0-9]+/[0-9]+)?$}.match?(value)
+      %r{^([0-9]+ )?([0-9]+/[1-9][0-9]*)?$}.match?(value)
     end
 
     sig { params(str: String).returns(String) }

data/app/models/atlas_engine/address_validation/abstract_address.rb

@@ -13,9 +13,21 @@ module AtlasEngine
       sig { abstract.returns(ComponentType) }
       def address1; end
 
+      sig { returns(ComponentType) }
+      def street_name; end
+
+      sig { returns(ComponentType) }
+      def street_number; end
+
       sig { abstract.returns(ComponentType) }
       def address2; end
 
+      sig { returns(ComponentType) }
+      def line2; end
+
+      sig { returns(ComponentType) }
+      def neighborhood; end
+
       sig { abstract.returns(ComponentType) }
       def city; end
 

data/app/models/atlas_engine/address_validation/address.rb

@@ -13,7 +13,11 @@ module AtlasEngine
       AddressInput = T.type_alias { Types::AddressValidation::AddressInput }
 
       const :address1, ComponentType
+      const :street_name, ComponentType
+      const :street_number, ComponentType
       const :address2, ComponentType
+      const :line2, ComponentType
+      const :neighborhood, ComponentType
       const :city, ComponentType
       const :province_code, ComponentType
       const :phone, ComponentType
@@ -33,7 +37,11 @@ module AtlasEngine
         def from_address(address:)
           new(
             address1: address.address1,
+            street_name: address.street_name,
+            street_number: address.street_number,
             address2: address.address2,
+            line2: address.line2,
+            neighborhood: address.neighborhood,
             city: address.city,
             country_code: address.country_code,
             province_code: address.province_code,

data/app/models/atlas_engine/address_validation/concern_record.rb

@@ -9,9 +9,21 @@ module AtlasEngine
       sig { returns(String) }
       attr_reader :address1
 
+      sig { returns(String) }
+      attr_reader :street_name
+
+      sig { returns(String) }
+      attr_reader :street_number
+
       sig { returns(String) }
       attr_reader :address2
 
+      sig { returns(String) }
+      attr_reader :line2
+
+      sig { returns(String) }
+      attr_reader :neighborhood
+
       sig { returns(String) }
       attr_reader :city
 
@@ -73,7 +85,11 @@ module AtlasEngine
           timestamp: Time,
           origin: String,
           address1: String,
+          street_name: String,
+          street_number: String,
           address2: String,
+          line2: String,
+          neighborhood: String,
           city: String,
           province_code: String,
           country_code: String,
@@ -87,7 +103,11 @@ module AtlasEngine
         timestamp: Time.zone.now,
         origin: "",
         address1: "",
+        street_name: "",
+        street_number: "",
         address2: "",
+        line2: "",
+        neighborhood: "",
         city: "",
         province_code: "",
         country_code: "",
@@ -100,7 +120,11 @@ module AtlasEngine
         @timestamp = timestamp
         @origin = origin
         @address1 = address1
+        @street_name = street_name
+        @street_number = street_number
         @address2 = address2
+        @line2 = line2
+        @neighborhood = neighborhood
         @city = city
         @province_code = province_code
         @country_code = country_code
@@ -114,7 +138,11 @@ module AtlasEngine
       def address_attributes
         {
           address1: address1,
+          street_name: street_name,
+          street_number: street_number,
           address2: address2,
+          line2: line2,
+          neighborhood: neighborhood,
           city: city,
           province_code: province_code,
           zip: zip,

data/app/models/atlas_engine/address_validation/validator.rb

@@ -114,6 +114,8 @@ module AtlasEngine
         local_concerns = {}
         cache = Validators::Predicates::Cache.new(pipeline_address)
         @predicate_pipeline.pipeline.each do |config|
+          break if local_concerns[:country].present?
+
           local_concerns[config.field] = [] if local_concerns[config.field].nil?
           next if local_concerns[config.field].present?
 

data/app/models/atlas_engine/address_validation/validators/full_address/suggestion_builder.rb

@@ -12,7 +12,11 @@ module AtlasEngine
               params(
                 address: {
                   address1: T.nilable(String),
+                  street_name: T.nilable(String),
+                  street_number: T.nilable(String),
                   address2: T.nilable(String),
+                  line2: T.nilable(String),
+                  neighborhood: T.nilable(String),
                   city: T.nilable(String),
                   province_code: T.nilable(String),
                   country_code: T.nilable(String),

data/app/models/atlas_engine/address_validation/validators/full_address/unmatched_field_concern_builder.rb

@@ -8,7 +8,7 @@ module AtlasEngine
         class UnmatchedFieldConcernBuilder
           extend T::Sig
           include ConcernFormatter
-          attr_reader :address, :component, :matched_components, :unmatched_field
+          attr_reader :address, :unmatched_component, :matched_components, :unmatched_field
 
           COMPONENTS_TO_LABELS = {
             zip: "ZIP",
@@ -31,9 +31,9 @@ module AtlasEngine
           end
           def initialize(unmatched_component, matched_components, address, unmatched_field = nil)
             @address = address
-            @component = unmatched_component
+            @unmatched_component = unmatched_component
             @matched_components = matched_components
-            @unmatched_field = unmatched_field
+            @unmatched_field = unmatched_field || unmatched_component
           end
 
           sig do
@@ -61,7 +61,7 @@ module AtlasEngine
 
           sig { returns(Symbol) }
           def code
-            "#{shortened_component_name}_inconsistent".to_sym
+            "#{unmatched_component_name}_inconsistent".to_sym
           end
 
           sig { returns(T::Array[Symbol]) }
@@ -69,21 +69,14 @@ module AtlasEngine
             [field_name]
           end
 
-          sig { returns(T::Array[String]) }
-          def valid_address_component_values
-            matched_components.last(2).map do |component|
-              component == :province_code ? province_name : address[component]
-            end
-          end
-
           sig { returns(Symbol) }
-          def shortened_component_name
-            SHORTENED_COMPONENT_NAMES[component] || component
+          def unmatched_component_name
+            SHORTENED_COMPONENT_NAMES[unmatched_component] || unmatched_component
           end
 
           sig { returns(Symbol) }
           def field_name
-            unmatched_field || shortened_component_name
+            SHORTENED_COMPONENT_NAMES[unmatched_field] || unmatched_field
           end
         end
       end

data/app/models/atlas_engine/address_validation/validators/predicates/city/present.rb

@@ -9,7 +9,6 @@ module AtlasEngine
           class Present < Predicate
             sig { override.returns(T.nilable(Concern)) }
             def evaluate
-              return unless @cache.country.country?
               return if @cache.country.field(key: :city).autofill(locale: :en).present?
 
               build_concern if @address.city.blank?

data/app/models/atlas_engine/address_validation/validators/predicates/no_emojis.rb

@@ -8,8 +8,6 @@ module AtlasEngine
         class NoEmojis < Predicate
           sig { override.returns(T.nilable(Concern)) }
           def evaluate
-            return unless @cache.country.country?
-
             build_concern if contains_blocked_codepoints?(address.send(@field))
           end
 

data/app/models/atlas_engine/address_validation/validators/predicates/no_html_tags.rb

@@ -8,8 +8,6 @@ module AtlasEngine
         class NoHtmlTags < Predicate
           sig { override.returns(T.nilable(Concern)) }
           def evaluate
-            return unless @cache.country.country?
-
             build_concern if contains_html_tags(address.send(@field))
           end
 

data/app/models/atlas_engine/address_validation/validators/predicates/no_url.rb

@@ -8,8 +8,6 @@ module AtlasEngine
         class NoUrl < Predicate
           sig { override.returns(T.nilable(Concern)) }
           def evaluate
-            return unless @cache.country.country?
-
             build_concern if contains_url(address.send(@field))
           end
 

data/app/models/atlas_engine/address_validation/validators/predicates/not_exceed_max_length.rb

@@ -9,8 +9,6 @@ module AtlasEngine
           MAX_COMPONENT_LENGTH = 255
           sig { override.returns(T.nilable(Concern)) }
           def evaluate
-            return unless @cache.country.country?
-
             build_concern if address.send(@field).to_s.length > MAX_COMPONENT_LENGTH
           end
 

data/app/models/atlas_engine/address_validation/validators/predicates/phone/valid.rb

@@ -9,8 +9,6 @@ module AtlasEngine
           class Valid < Predicate
             sig { override.returns(T.nilable(Concern)) }
             def evaluate
-              return if @address.country_code.blank?
-              return unless @cache.country.country?
               return if @address.phone.blank?
 
               phone = Worldwide::Phone.new(number: @address.phone, country_code: @address.country_code)

data/app/models/atlas_engine/address_validation/validators/predicates/province/exists.rb

@@ -9,7 +9,6 @@ module AtlasEngine
           class Exists < Predicate
             sig { override.returns(T.nilable(Concern)) }
             def evaluate
-              return unless @cache.country.country?
               return if address.province_code.present? ||
                 country_has_no_provinces ||
                 @cache.country.province_optional?

data/app/models/atlas_engine/address_validation/validators/predicates/province/valid_for_country.rb

@@ -9,7 +9,6 @@ module AtlasEngine
           class ValidForCountry < Predicate
             sig { override.returns(T.nilable(Concern)) }
             def evaluate
-              return unless @cache.country.country?
               return if address.province_code.blank?
               return if @cache.country.zones.none?(&:province?)
               return if @cache.country.hide_provinces_from_addresses

data/app/models/atlas_engine/address_validation/validators/predicates/street/building_number_in_address1.rb

@@ -9,8 +9,6 @@ module AtlasEngine
           class BuildingNumberInAddress1 < Predicate
             sig { override.returns(T.nilable(Concern)) }
             def evaluate
-              return unless @cache.country.country?
-
               return unless @cache.country.building_number_required
               return if @cache.country.building_number_may_be_in_address2
 

data/app/models/atlas_engine/address_validation/validators/predicates/street/building_number_in_address1_or_address2.rb

@@ -9,7 +9,6 @@ module AtlasEngine
           class BuildingNumberInAddress1OrAddress2 < Predicate
             sig { override.returns(T.nilable(Concern)) }
             def evaluate
-              return unless @cache.country.country?
               return unless @cache.country.building_number_required
               return unless @cache.country.building_number_may_be_in_address2
               return if contains_number?(T.must(@address.address1)) || contains_number?(@address.address2)

data/app/models/atlas_engine/address_validation/validators/predicates/street/present.rb

@@ -9,8 +9,6 @@ module AtlasEngine
           class Present < Predicate
             sig { override.returns(T.nilable(Concern)) }
             def evaluate
-              return unless @cache.country.country?
-
               build_concern if @address.address1.blank?
             end
 

data/app/models/atlas_engine/address_validation/validators/predicates/zip/zip_base.rb

@@ -11,7 +11,6 @@ module AtlasEngine
 
             sig { returns(T.nilable(T::Boolean)) }
             def concerning?
-              return false unless @cache.country.country?
               return false unless @cache.country.has_zip?
               return false unless @cache.country.zip_required?
 

data/app/tasks/maintenance/atlas_engine/geo_json_import_task.rb

@@ -12,9 +12,6 @@ module Maintenance
       # ISO3166 two-letter country code.
       attribute :country_code, :string
       validates :country_code, presence: true
-      # Filename to import. When running in staging or production, the worker expects to find
-      # this file in the relevant GCS bucket, configured in `config/storage/{environment}.yml`
-      # It must be placed under `openaddress/` with the same filename.
       attribute :geojson_file_path, :string
       attribute :locale, :string
 

data/app/tasks/maintenance/atlas_engine/us_geo_json_directory_import_task.rb

@@ -0,0 +1,20 @@
+# typed: false
+# frozen_string_literal: true
+
+module Maintenance
+  module AtlasEngine
+    class UsGeoJsonDirectoryImportTask < MaintenanceTasks::Task
+      include ::AtlasEngine::HandlesBlob
+
+      no_collection
+
+      attribute :geojson_directory, :string
+
+      def process
+        ::AtlasEngine::Us::Jobs::AddressImporter::CombinedImportJob.perform_later(
+          geojson_directory: geojson_directory.strip,
+        )
+      end
+    end
+  end
+end

data/lib/atlas_engine/version.rb

@@ -2,5 +2,5 @@
 # frozen_string_literal: true
 
 module AtlasEngine
-  VERSION = "0.8.0"
+  VERSION = "1.1.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: atlas_engine
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Shopify
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-04-02 00:00:00.000000000 Z
+date: 2024-05-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: annex_29
@@ -178,6 +178,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: sprockets-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: state_machines-activerecord
   requirement: !ruby/object:Gem::Requirement
@@ -364,7 +378,10 @@ files:
 - app/countries/atlas_engine/si/validation_transcriber/address_parser.rb
 - app/countries/atlas_engine/tt/address_importer/open_address/mapper.rb
 - app/countries/atlas_engine/tt/country_profile.yml
+- app/countries/atlas_engine/us/address_importer/open_address/filter.rb
+- app/countries/atlas_engine/us/address_importer/open_address/mapper.rb
 - app/countries/atlas_engine/us/country_profile.yml
+- app/countries/atlas_engine/us/jobs/address_importer/combined_import_job.rb
 - app/countries/atlas_engine/us/synonyms.yml
 - app/graphql/atlas_engine/errors/locale_unsupported_error.rb
 - app/graphql/atlas_engine/schema.graphql
@@ -548,6 +565,7 @@ files:
 - app/models/atlas_engine/street.rb
 - app/tasks/maintenance/atlas_engine/elasticsearch_index_create_task.rb
 - app/tasks/maintenance/atlas_engine/geo_json_import_task.rb
+- app/tasks/maintenance/atlas_engine/us_geo_json_directory_import_task.rb
 - app/views/atlas_engine/connectivity/index.html.erb
 - app/views/atlas_engine/country_imports/index.html.erb
 - app/views/atlas_engine/country_imports/show.html.erb
@@ -597,7 +615,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.7
+rubygems_version: 3.5.10
 signing_key: 
 specification_version: 4
 summary: Address Validation API