overture_maps 0.1.0

data/lib/overture_maps/import/location_based_runner.rb

@@ -0,0 +1,176 @@
+# frozen_string_literal: true
+
+module OvertureMaps
+  module Import
+    # Orchestrates a location-based import: resolves the location (bbox
+    # string or division name) to a bounding box, downloads or reuses a
+    # bbox-filtered parquet extract per feature type, and streams it into the
+    # target model via Runner.
+    #
+    # Themes map one or more feature types to models:
+    #   models: { "segment" => OvertureSegment, "connector" => OvertureConnector }
+    # For single-model themes, model_class: alone is enough.
+    #
+    # This class never prompts or exits. Interactive behavior is injected by
+    # the rake layer through callbacks:
+    #   select_division: ->(results) { result or nil }   nil aborts
+    #   confirm_cached:  ->(path) { :use | :refresh | :abort }
+    # Without callbacks (library/job usage) it picks the largest matching
+    # division and reuses exact cache hits.
+    class LocationBasedRunner
+      attr_reader :theme, :location, :imported_count, :error_count, :errors, :bbox
+
+      def initialize(theme:, location:, model_class: nil, models: nil, categories: nil,
+                     batch_size: nil, release: nil, output_dir: nil,
+                     select_division: nil, confirm_cached: nil)
+        @theme = theme
+        @location = location
+        @models = resolve_models(model_class, models)
+        @categories = categories
+        @batch_size = batch_size
+        @release = Releases.validate!(release || Releases.current)
+        @output_dir = output_dir
+        @select_division = select_division
+        @confirm_cached = confirm_cached
+        @imported_count = 0
+        @error_count = 0
+        @errors = []
+      end
+
+      def run
+        @bbox = resolve_bbox
+        log "Bounding box: #{bbox} (release #{@release})"
+
+        @models.each { |type, model_class| import_type(type, model_class) }
+        self
+      end
+
+      def success?
+        @error_count.zero?
+      end
+
+      private
+
+      def resolve_models(model_class, models)
+        return models if models&.any?
+        raise ArgumentError, "provide model_class: or models:" unless model_class
+
+        types = Downloader.types_for_theme(theme)
+        raise Error, "no types for theme: #{theme}" if types.empty?
+
+        types.to_h { |t| [t, model_class] }
+      end
+
+      def resolve_bbox
+        return location if location.is_a?(BoundingBox)
+
+        parsed = BoundingBox.parse(location)
+        return parsed if parsed
+
+        results = DivisionSearch.search(query: location, release: @release)
+        raise Error, "no divisions found matching #{location.inspect}" if results.empty?
+
+        division = choose_division(results)
+        raise CancelledError, "import cancelled" unless division
+
+        log "Using #{division[:name]} (#{division[:subtype]}, #{[division[:country], division[:region]].compact.join(" / ")})"
+        division[:bbox]
+      end
+
+      def choose_division(results)
+        return results.first if results.length == 1 || @select_division.nil?
+
+        @select_division.call(results)
+      end
+
+      def import_type(type, model_class)
+        downloader = Downloader.new(theme: theme, type: type, release: @release, output_dir: @output_dir)
+        path = resolve_extract(downloader)
+        unless path
+          log "#{theme}/#{type}: no data found"
+          return
+        end
+
+        runner = Runner.new(model_class: model_class, theme: theme, type: type,
+                            batch_size: @batch_size, release: @release)
+        reader = ParquetReader.new(theme: theme)
+        log "#{theme}/#{type}: importing #{File.basename(path)}..."
+        runner.import_from_reader(reader, source: path, filter: category_filter)
+
+        @imported_count += runner.imported_count
+        @error_count += runner.error_count
+        @errors.concat(runner.errors)
+        log "#{theme}/#{type}: imported #{runner.imported_count}, errors #{runner.error_count}"
+
+        record_imported_area(type, model_class, runner)
+      end
+
+      # Bookkeeping for overture_maps:sync. Skipped silently when the
+      # tracking table hasn't been migrated yet.
+      def record_imported_area(type, model_class, runner)
+        return unless runner.imported_count.positive?
+        return unless Models::ImportedArea.table_exists?
+
+        Models::ImportedArea.record!(
+          theme: theme, feature_type: type, model_class: model_class,
+          bbox: bbox, release: @release,
+          records_count: runner.imported_count
+        )
+      rescue StandardError => e
+        log "sync tracking skipped: #{e.message}"
+      end
+
+      def resolve_extract(downloader)
+        cached = downloader.cached_extract(bbox)
+
+        if cached
+          case cached_decision(cached)
+          when :use
+            log "Reusing cached extract #{File.basename(cached)}"
+            return cached
+          when :abort
+            raise CancelledError, "import cancelled"
+          else
+            File.delete(cached)
+          end
+        end
+
+        downloader.extract_bbox(bbox)
+      end
+
+      def cached_decision(path)
+        return :use unless @confirm_cached
+
+        @confirm_cached.call(path)
+      end
+
+      # Matches leaf categories against categories.primary and
+      # categories.alternate. Taxonomy groups (e.g. "eat_and_drink",
+      # "restaurant") expand to their leaves when the categories table has
+      # been populated (rails overture_maps:categories:populate).
+      def category_filter
+        return nil unless @categories&.any?
+
+        wanted = expand_categories(@categories.map(&:to_s))
+        lambda do |record|
+          cats = record["categories"]
+          primary = cats.is_a?(Hash) ? cats["primary"] : record["basic_category"]
+          alternates = cats.is_a?(Hash) ? Array(cats["alternate"]) : []
+
+          wanted.any? { |w| primary == w || alternates.include?(w) }
+        end
+      end
+
+      def expand_categories(names)
+        Models::Category.expand(names)
+      rescue StandardError
+        names
+      end
+
+      def log(message)
+        logger = OvertureMaps.configuration.logger
+        logger ? logger.info(message) : puts(message)
+      end
+    end
+  end
+end

data/lib/overture_maps/import/parquet_reader.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require "parquet"
+
+module OvertureMaps
+  module Import
+    # Reads local Overture parquet files row by row. Remote access and
+    # spatial filtering live in Downloader/QueryEngine — by the time a file
+    # reaches this class it is already a local extract.
+    class ParquetReader
+      THEMES = Downloader::THEMES
+
+      attr_reader :theme
+
+      def initialize(theme: nil)
+        @theme = theme
+      end
+
+      def each_record(source:, &block)
+        raise ArgumentError, "source must be a local file path" unless source.is_a?(String) && File.exist?(source)
+
+        Parquet.each_row(source, &block)
+      end
+
+      def record_count(source:)
+        raise ArgumentError, "source must be a local file path" unless source.is_a?(String) && File.exist?(source)
+
+        Parquet.metadata(source)["num_rows"]
+      end
+    end
+  end
+end

data/lib/overture_maps/import/record_mapper.rb

@@ -0,0 +1,182 @@
+# frozen_string_literal: true
+
+module OvertureMaps
+  module Import
+    # Maps raw Overture records (nested structs from parquet) onto flat model
+    # attributes. Every record gets the same key set — heterogeneous keys make
+    # ActiveRecord's insert_all/upsert_all reject the whole batch.
+    class RecordMapper
+      def self.for(theme:, model_class:, type: nil, release: nil)
+        new(theme: theme, model_class: model_class, type: type, release: release)
+      end
+
+      attr_reader :theme, :type, :model_class, :release
+
+      def initialize(theme:, model_class:, type: nil, release: nil)
+        @theme = theme
+        @type = type
+        @model_class = model_class
+        @release = release
+        @columns = model_class.column_names
+      end
+
+      # Returns attributes with a stable key set, or nil if the record can't
+      # be mapped (caller records the error).
+      def call(record)
+        attrs = base_attributes(record)
+        attrs.merge!(theme_attributes(record))
+        normalize(attrs)
+      end
+
+      # The attribute keys every mapped record will contain.
+      def key_set
+        @key_set ||= begin
+          sample = base_attributes({}).merge(theme_attributes({}))
+          keys = sample.keys.select { |k| @columns.include?(k.to_s) }
+          keys | %i[id geometry]
+        end
+      end
+
+      private
+
+      def base_attributes(record)
+        {
+          id: record["id"],
+          geometry: record["geometry"],
+          names: record["names"],
+          name: record.dig("names", "primary"),
+          sources: record["sources"],
+          overture_release: release,
+          created_at: now,
+          updated_at: now
+        }
+      end
+
+      def theme_attributes(record)
+        case theme
+        when "places" then place_attributes(record)
+        when "buildings" then building_attributes(record)
+        when "addresses" then address_attributes(record)
+        when "divisions" then division_attributes(record)
+        when "transportation" then transportation_attributes(record)
+        when "base" then base_theme_attributes(record)
+        else {}
+        end
+      end
+
+      def place_attributes(record)
+        {
+          # categories is deprecated in favor of basic_category + taxonomy
+          # (~Sep 2026); read both shapes so imports survive the cutover.
+          categories: record["categories"] || record["taxonomy"],
+          primary_category: record.dig("categories", "primary") || record["basic_category"],
+          brands: record["brand"],
+          addresses: record["addresses"],
+          confidence: record["confidence"],
+          operating_status: record["operating_status"],
+          websites: record["websites"],
+          socials: record["socials"],
+          emails: record["emails"],
+          phones: record["phones"],
+          country: first_address_value(record, "country")
+        }
+      end
+
+      def building_attributes(record)
+        {
+          subtype: record["subtype"],
+          building_class: record["class"],
+          height: record["height"],
+          num_floors: record["num_floors"],
+          level: record["level"],
+          is_underground: record["is_underground"]
+        }
+      end
+
+      def address_attributes(record)
+        levels = Array(record["address_levels"])
+        {
+          number: record["number"],
+          street: record["street"],
+          unit: record["unit"],
+          postcode: record["postcode"],
+          country: record["country"],
+          postal_city: record["postal_city"],
+          address_levels: record["address_levels"],
+          # Best-effort convenience columns; ordering of address_levels is
+          # country-dependent, the authoritative data is the jsonb column.
+          region: levels.dig(0, "value"),
+          locality: record["postal_city"] || levels.dig(1, "value")
+        }
+      end
+
+      # division_area records: the geocodable territory of a division.
+      def division_attributes(record)
+        bbox = record["bbox"] || {}
+        {
+          division_id: record["division_id"],
+          subtype: record["subtype"],
+          division_class: record["class"],
+          country: record["country"],
+          region: record["region"],
+          is_land: record["is_land"],
+          is_territorial: record["is_territorial"],
+          bbox_xmin: bbox["xmin"],
+          bbox_xmax: bbox["xmax"],
+          bbox_ymin: bbox["ymin"],
+          bbox_ymax: bbox["ymax"]
+        }
+      end
+
+      def transportation_attributes(record)
+        return {} unless type == "segment" # connectors carry only id + geometry
+
+        {
+          subtype: record["subtype"],
+          segment_class: record["class"],
+          subclass: record["subclass"],
+          connectors: record["connectors"],
+          routes: record["routes"],
+          speed_limits: record["speed_limits"],
+          access_restrictions: record["access_restrictions"],
+          prohibited_transitions: record["prohibited_transitions"],
+          road_surface: record["road_surface"],
+          road_flags: record["road_flags"],
+          destinations: record["destinations"]
+        }
+      end
+
+      def base_theme_attributes(record)
+        {
+          feature_type: type,
+          subtype: record["subtype"],
+          feature_class: record["class"],
+          surface: record["surface"],
+          elevation: record["elevation"],
+          level: record["level"],
+          wikidata: record["wikidata"],
+          source_tags: record["source_tags"]
+        }
+      end
+
+      def first_address_value(record, key)
+        addresses = record["addresses"]
+        return nil unless addresses.is_a?(Array) && addresses.first.is_a?(Hash)
+
+        addresses.first[key]
+      end
+
+      # Keep only real columns, ensure every key from the key set is present
+      # so batches stay homogeneous.
+      def normalize(attrs)
+        normalized = {}
+        key_set.each { |key| normalized[key] = attrs[key] }
+        normalized
+      end
+
+      def now
+        @now ||= Time.now.utc
+      end
+    end
+  end
+end

data/lib/overture_maps/import/runner.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+require "rgeo"
+require "rgeo/geo_json"
+require "json"
+
+module OvertureMaps
+  module Import
+    # Batches records into idempotent upserts. Re-running an import updates
+    # existing rows (keyed on the GERS id primary key) instead of failing.
+    class Runner
+      MAX_STORED_ERRORS = 50
+      PROGRESS_INTERVAL = 10_000
+
+      attr_reader :model_class, :theme, :type, :batch_size, :imported_count, :error_count, :errors
+
+      def initialize(model_class:, theme: nil, type: nil, batch_size: nil, mapper: nil,
+                     release: nil, progress_every: PROGRESS_INTERVAL)
+        @model_class = model_class
+        @theme = theme
+        @type = type
+        @release = release
+        @batch_size = batch_size || OvertureMaps.configuration.batch_size
+        @mapper = mapper
+        @progress_every = progress_every
+        @imported_count = 0
+        @error_count = 0
+        @errors = []
+      end
+
+      # Imports from anything that yields raw Overture record hashes.
+      def import_from_records(records, transform: nil, filter: nil)
+        batch = []
+
+        records.each do |record|
+          next if filter && !filter.call(record)
+
+          attrs = transform_record(record, transform)
+          next unless attrs
+
+          batch << attrs
+          if batch.length >= batch_size
+            flush_records(batch)
+            batch = []
+          end
+        end
+
+        flush_records(batch) if batch.any?
+        self
+      end
+
+      def import_from_reader(reader, source:, transform: nil, filter: nil)
+        import_from_records(reader.enum_for(:each_record, source: source), transform: transform, filter: filter)
+      end
+
+      def import_from_file(path, theme: nil, transform: nil, filter: nil)
+        reader = ParquetReader.new(theme: theme)
+        import_from_reader(reader, source: path, transform: transform, filter: filter)
+      end
+
+      def success?
+        @error_count.zero?
+      end
+
+      private
+
+      def transform_record(record, transform)
+        attrs = transform ? transform.call(record) : default_mapper.call(record)
+        return nil unless attrs
+
+        if attrs[:geometry] || attrs["geometry"]
+          key = attrs.key?(:geometry) ? :geometry : "geometry"
+          attrs[key] = parse_geometry(attrs[key])
+        end
+        attrs
+      rescue StandardError => e
+        record_error("mapping failed: #{e.message}", record["id"])
+        nil
+      end
+
+      def default_mapper
+        @default_mapper ||= @mapper || RecordMapper.for(
+          theme: theme, type: type, model_class: model_class, release: @release
+        )
+      end
+
+      def flush_records(records)
+        return if records.empty?
+
+        model_class.upsert_all(records, unique_by: model_class.primary_key)
+        @imported_count += records.length
+        log_progress(records.length)
+      rescue StandardError
+        # Isolate bad rows without abandoning the batch.
+        records.each do |record|
+          model_class.upsert_all([record], unique_by: model_class.primary_key)
+          @imported_count += 1
+        rescue StandardError => e
+          record_error(e.message, record[:id] || record["id"])
+        end
+      end
+
+      def record_error(message, record_id = nil)
+        @error_count += 1
+        @errors << { error: message, record_id: record_id } if @errors.length < MAX_STORED_ERRORS
+      end
+
+      # A line every @progress_every rows so long imports show life.
+      def log_progress(batch_length)
+        return unless @progress_every&.positive?
+        return unless (@imported_count / @progress_every) > ((@imported_count - batch_length) / @progress_every)
+
+        logger = OvertureMaps.configuration.logger
+        message = "  #{@imported_count} rows..."
+        logger ? logger.info(message) : puts(message)
+      end
+
+      # A geometry that fails to parse skips that record — it must never
+      # abort the import loop (see transform_record's rescue).
+      def parse_geometry(geom)
+        GeometryParser.parse(geom)
+      end
+    end
+  end
+end

data/lib/overture_maps/import.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module OvertureMaps
+  module Import
+    class Error < OvertureMaps::Error; end
+  end
+end
+
+require "overture_maps/import/downloader"
+require "overture_maps/import/parquet_reader"
+require "overture_maps/import/record_mapper"
+require "overture_maps/import/runner"
+require "overture_maps/import/location_based_runner"
+
+module OvertureMaps
+  module Import
+    # Programmatic file import. The transform may be given as a keyword or a
+    # block; it receives each raw record and returns attributes (or nil to
+    # skip the record).
+    def self.run!(theme:, model_class:, file_path:, batch_size: nil, transform: nil, release: nil, &block)
+      transform ||= block
+      runner = Runner.new(model_class: model_class, theme: theme, batch_size: batch_size, release: release)
+      runner.import_from_file(file_path, theme: theme, transform: transform)
+      runner
+    end
+  end
+end