overture_maps 0.1.0

data/lib/overture_maps/query.rb

@@ -0,0 +1,168 @@
+# frozen_string_literal: true
+
+require "securerandom"
+require "fileutils"
+require "tmpdir"
+
+module OvertureMaps
+  # Ad-hoc queries against Overture GeoParquet — no PostGIS import needed.
+  #
+  #   OvertureMaps.query(theme: "places", location: "Seattle").limit(10).each { |r| ... }
+  #   OvertureMaps.query(theme: "buildings", bbox: [47.5, -122.4, 47.7, -122.2]).count
+  #   OvertureMaps.query(theme: "places", bbox: "47.5,-122.4,47.7,-122.2").export("places.geojson")
+  #
+  # Unlimited queries spool through the same cache files the import pipeline
+  # uses (so a query warms the cache for a later import and vice versa);
+  # limited queries use a throwaway temp file. Records are hashes with the
+  # geometry parsed to an RGeo feature.
+  class Query
+    include Enumerable
+
+    EXPORT_EXTENSIONS = {
+      ".parquet" => "parquet",
+      ".geojson" => "geojson",
+      ".geojsonseq" => "geojsonseq",
+      ".gpkg" => "gpkg"
+    }.freeze
+
+    attr_reader :theme, :type, :release
+
+    def initialize(theme:, type: nil, bbox: nil, location: nil, release: nil, limit: nil)
+      @theme = theme
+      @type = type || infer_type(theme)
+      @release = Releases.validate!(release || Releases.current)
+      @location = location
+      @bbox = coerce_bbox(bbox) if bbox
+      raise ArgumentError, "provide bbox: or location:" unless @bbox || @location
+
+      @limit = limit && Integer(limit)
+    end
+
+    def limit(count)
+      self.class.new(theme: theme, type: type, bbox: bbox, location: @location,
+                     release: release, limit: count)
+    end
+
+    # The resolved bounding box (resolves a location name on first use).
+    def bbox
+      @bbox ||= resolve_location
+    end
+
+    # Fast remote count via row-group pushdown — nothing is downloaded.
+    def count
+      sql, params = Import::Downloader.bbox_query(
+        theme: theme, type: type, release: release, bbox: bbox, limit: @limit
+      )
+      rows = QueryEngine.instance.query("SELECT count(*) AS n FROM (#{sql})", params)
+      Integer(rows.first["n"])
+    end
+
+    def each(&block)
+      return enum_for(:each) unless block
+
+      with_extract do |path|
+        reader = Import::ParquetReader.new(theme: theme)
+        reader.each_record(source: path) do |record|
+          record["geometry"] = GeometryParser.parse(record["geometry"])
+          block.call(record)
+        end
+      end
+      self
+    end
+
+    def each_batch(size: 1000)
+      return enum_for(:each_batch, size: size) unless block_given?
+
+      batch = []
+      each do |record|
+        batch << record
+        if batch.length >= size
+          yield batch
+          batch = []
+        end
+      end
+      yield batch if batch.any?
+      self
+    end
+
+    # Writes the query result straight to a file; format inferred from the
+    # extension unless given explicitly.
+    def export(path, format: nil)
+      format ||= EXPORT_EXTENSIONS[File.extname(path).downcase] or
+        raise ArgumentError, "cannot infer format from #{path}; pass format:"
+
+      downloader.extract_bbox(bbox, format: format, output_path: path, limit: @limit) or
+        raise Error, "query returned no data"
+    end
+
+    # A GeoJSON FeatureCollection hash. Materializes everything — use
+    # #export for large results.
+    def to_geojson
+      features = map do |record|
+        geometry = record["geometry"]
+        {
+          type: "Feature",
+          geometry: geometry && RGeo::GeoJSON.encode(geometry),
+          properties: record.except("geometry", "bbox")
+        }
+      end
+      { type: "FeatureCollection", features: features }
+    end
+
+    private
+
+    def infer_type(theme)
+      types = Import::Downloader.types_for_theme(theme)
+      raise ArgumentError, "unknown theme: #{theme}" if types.empty?
+      return types.first if types.length == 1
+
+      raise ArgumentError, "theme #{theme} has multiple types (#{types.join(", ")}); pass type:"
+    end
+
+    def coerce_bbox(value)
+      case value
+      when BoundingBox then value
+      when Array
+        raise ArgumentError, "bbox array must be [lat1, lng1, lat2, lng2]" unless value.length == 4
+
+        BoundingBox.new(lat1: value[0], lng1: value[1], lat2: value[2], lng2: value[3])
+      when String
+        BoundingBox.parse(value) or raise ArgumentError, "unparseable bbox: #{value.inspect}"
+      else
+        raise ArgumentError, "bbox must be a BoundingBox, array, or string"
+      end
+    end
+
+    def resolve_location
+      results = DivisionSearch.search(query: @location, release: release)
+      raise Error, "no divisions found matching #{@location.inspect}" if results.empty?
+
+      results.first[:bbox]
+    end
+
+    def downloader
+      Import::Downloader.new(theme: theme, type: type, release: release)
+    end
+
+    # Unlimited queries share the import pipeline's cache files; limited
+    # queries spool to a temp file that is removed afterwards.
+    def with_extract
+      if @limit
+        path = File.join(Dir.tmpdir, "overture_query_#{SecureRandom.hex(8)}.parquet")
+        begin
+          extracted = downloader.extract_bbox(bbox, output_path: path, limit: @limit)
+          yield extracted if extracted
+        ensure
+          FileUtils.rm_f(path)
+        end
+      else
+        path = downloader.cached_extract(bbox) || downloader.extract_bbox(bbox)
+        yield path if path
+      end
+    end
+  end
+
+  def self.query(**options)
+    Query.new(**options)
+  end
+end

data/lib/overture_maps/query_engine.rb

@@ -0,0 +1,268 @@
+# frozen_string_literal: true
+
+require "open3"
+require "json"
+require "fileutils"
+
+module OvertureMaps
+  # Runs spatial SQL against Overture GeoParquet via DuckDB.
+  #
+  # Prefers the ruby-duckdb gem (in-process, bound parameters, no shell).
+  # Falls back to the DuckDB CLI: PATH lookup first, then a verified download.
+  # All user-supplied values go through bound parameters (native) or strict
+  # literal quoting (CLI) — never raw string interpolation.
+  class QueryEngine
+    DUCKDB_CLI_VERSION = "1.5.3"
+
+    class Error < OvertureMaps::Error; end
+
+    class << self
+      def instance
+        @instance ||= new
+      end
+
+      def reset!
+        @instance = nil
+      end
+    end
+
+    # Returns an array of row hashes. Intended for modest result sets
+    # (division searches, lookups) — bulk data must use #copy_to.
+    def query(sql, params = [])
+      if native?
+        native_query(sql, params)
+      else
+        cli_query(sql, params)
+      end
+    end
+
+    # Exports a query's results to a local file (parquet by default). This is
+    # the bulk-data path: DuckDB streams row groups from S3 and writes the
+    # file without materializing results in Ruby.
+    def copy_to(sql, params: [], output_path:, format: "parquet")
+      copy_sql = build_copy_sql(sql, output_path, format)
+
+      if native?
+        native_execute(copy_sql, params)
+      else
+        cli_execute(interpolate(copy_sql, params))
+      end
+      output_path
+    end
+
+    def native?
+      return @native unless @native.nil?
+
+      @native =
+        begin
+          require "duckdb"
+          true
+        rescue LoadError
+          false
+        end
+    end
+
+    private
+
+    def build_copy_sql(sql, output_path, format)
+      path = quote_string(output_path.to_s)
+      case format.to_s.downcase
+      when "parquet"
+        "COPY (#{sql}) TO #{path} (FORMAT PARQUET)"
+      when "geojson"
+        "COPY (#{sql}) TO #{path} WITH (FORMAT GDAL, DRIVER 'GeoJSON')"
+      when "geojsonseq"
+        "COPY (#{sql}) TO #{path} WITH (FORMAT GDAL, DRIVER 'GeoJSONSeq')"
+      when "gpkg", "geopackage"
+        "COPY (#{sql}) TO #{path} WITH (FORMAT GDAL, DRIVER 'GPKG')"
+      else
+        raise ArgumentError, "unknown export format: #{format}"
+      end
+    end
+
+    def init_statements
+      region = OvertureMaps.configuration.s3_region
+      raise Error, "invalid s3_region: #{region.inspect}" unless region.match?(/\A[a-z0-9-]+\z/)
+
+      # Division areas and building footprints have large row groups; the
+      # httpfs default of 30s times out on slower links, so give bulk
+      # fetches at least two minutes and a few retries.
+      http_timeout_ms = [Integer(OvertureMaps.configuration.timeout) * 1000, 120_000].max
+
+      [
+        "INSTALL spatial",
+        "LOAD spatial",
+        "INSTALL httpfs",
+        "LOAD httpfs",
+        "SET s3_region='#{region}'",
+        "SET http_timeout=#{http_timeout_ms}",
+        "SET http_retries=5"
+      ]
+    end
+
+    # --- native (ruby-duckdb) backend ---
+
+    def native_connection
+      @native_db ||= ::DuckDB::Database.open
+      con = @native_db.connect
+      init_statements.each { |stmt| con.query(stmt) }
+      con
+    end
+
+    def native_query(sql, params)
+      con = native_connection
+      result = con.query(sql, *params)
+      columns = result.columns.map(&:name)
+      result.map { |row| columns.zip(row.map { |v| normalize_value(v) }).to_h }
+    ensure
+      con&.close
+    end
+
+    def native_execute(sql, params)
+      con = native_connection
+      con.query(sql, *params)
+      nil
+    ensure
+      con&.close
+    end
+
+    def normalize_value(value)
+      case value
+      when ::DuckDB::Interval then value.to_s
+      else value
+      end
+    rescue NameError
+      value
+    end
+
+    # --- CLI backend ---
+
+    def cli_query(sql, params)
+      output = run_cli(interpolate(sql, params), json: true)
+      output = output.strip
+      return [] if output.empty?
+
+      JSON.parse(output)
+    end
+
+    def cli_execute(sql)
+      run_cli(sql, json: false)
+      nil
+    end
+
+    def run_cli(sql, json:)
+      cli = self.class.cli_path
+      full_sql = (init_statements + [sql]).join(";\n") + ";"
+      argv = [cli]
+      argv << "-json" if json
+      argv += ["-batch", "-noheader"] unless json
+
+      stdout, stderr, status = Open3.capture3(*argv, stdin_data: full_sql)
+      unless status.success?
+        detail = stderr.strip.empty? ? stdout.strip : stderr.strip
+        raise Error, "DuckDB CLI failed (exit #{status.exitstatus}): #{detail}"
+      end
+
+      stdout
+    end
+
+    # Replaces `?` placeholders with safely quoted literals for the CLI
+    # backend. Question marks inside string literals are not supported in
+    # our SQL, which we control.
+    def interpolate(sql, params)
+      remaining = params.dup
+      result = sql.gsub("?") do
+        raise Error, "not enough bind params for SQL" if remaining.empty?
+
+        quote(remaining.shift)
+      end
+      raise Error, "too many bind params for SQL" unless remaining.empty?
+
+      result
+    end
+
+    def quote(value)
+      case value
+      when Integer, Float then value.to_s
+      when Numeric then Float(value).to_s
+      when nil then "NULL"
+      when true, false then value.to_s
+      else quote_string(value.to_s)
+      end
+    end
+
+    def quote_string(str)
+      "'#{str.gsub("'", "''")}'"
+    end
+
+    class << self
+      # Resolve the DuckDB CLI: configured path, PATH, cached download —
+      # downloading a pinned release if necessary.
+      def cli_path
+        configured = OvertureMaps.configuration.duckdb_cli_path
+        return configured if configured && File.executable?(configured)
+
+        found = find_in_path("duckdb")
+        return found if found
+
+        cached = cached_cli_path
+        return cached if File.executable?(cached)
+
+        download_cli(cached)
+        cached
+      end
+
+      def cli_available?
+        !!(OvertureMaps.configuration.duckdb_cli_path || find_in_path("duckdb") ||
+           File.executable?(cached_cli_path))
+      end
+
+      private
+
+      def find_in_path(name)
+        ENV.fetch("PATH", "").split(File::PATH_SEPARATOR).each do |dir|
+          candidate = File.join(dir, name)
+          return candidate if File.file?(candidate) && File.executable?(candidate)
+        end
+        nil
+      end
+
+      def cached_cli_path
+        File.join(cache_root, "duckdb-#{DUCKDB_CLI_VERSION}", "duckdb")
+      end
+
+      def cache_root
+        base = ENV["XDG_CACHE_HOME"] || File.join(Dir.home, ".cache")
+        File.join(base, "overture_maps")
+      end
+
+      def cli_archive_name
+        case RUBY_PLATFORM
+        when /darwin/ then "duckdb_cli-osx-universal.zip"
+        when /aarch64-linux|arm64-linux/ then "duckdb_cli-linux-arm64.zip"
+        when /linux/ then "duckdb_cli-linux-amd64.zip"
+        else
+          raise Error, "no DuckDB CLI build for #{RUBY_PLATFORM}; install duckdb " \
+                       "manually (https://duckdb.org) or add the duckdb gem"
+        end
+      end
+
+      def download_cli(target)
+        require "overture_maps/storage"
+
+        url = "https://github.com/duckdb/duckdb/releases/download/v#{DUCKDB_CLI_VERSION}/#{cli_archive_name}"
+        dir = File.dirname(target)
+        FileUtils.mkdir_p(dir)
+        zip_path = File.join(dir, "cli.zip")
+
+        OvertureMaps.configuration.logger&.info("Downloading DuckDB CLI #{DUCKDB_CLI_VERSION}...")
+        Storage.download_url(url, to: zip_path)
+
+        system("unzip", "-o", "-q", "-j", zip_path, "-d", dir, exception: true)
+        FileUtils.chmod("+x", target)
+      ensure
+        FileUtils.rm_f(zip_path) if zip_path
+      end
+    end
+  end
+end

data/lib/overture_maps/releases.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require "json"
+
+module OvertureMaps
+  # Discovers available Overture releases. Primary source is the official
+  # STAC catalog; falls back to listing the bucket's release/ prefixes.
+  # Results are memoized per process.
+  module Releases
+    STAC_CATALOG_URL = "https://stac.overturemaps.org/catalog.json"
+    RELEASE_FORMAT = /\A\d{4}-\d{2}-\d{2}(?:\.\d+)?\z/
+    RELEASE_PATTERN = /\d{4}-\d{2}-\d{2}(?:\.\d+)?/
+
+    class Error < OvertureMaps::Error; end
+
+    class << self
+      # All known releases, newest first.
+      def all
+        @all ||= (from_stac || from_bucket).sort.reverse.freeze
+      end
+
+      def latest
+        all.first or raise Error, "no Overture releases found"
+      end
+
+      # The release to use: explicit config wins, otherwise latest.
+      def current
+        configured = OvertureMaps.configuration.release
+        return validate!(configured) if configured
+
+        latest
+      end
+
+      def validate!(release)
+        unless release.to_s.match?(RELEASE_FORMAT)
+          raise Error, "invalid release #{release.inspect} (expected e.g. 2026-06-17.0)"
+        end
+
+        release.to_s
+      end
+
+      def reset!
+        @all = nil
+      end
+
+      private
+
+      def from_stac
+        require "overture_maps/storage"
+
+        body = Storage.get(STAC_CATALOG_URL)
+        catalog = JSON.parse(body)
+        releases = Array(catalog["links"])
+                   .select { |link| link["rel"] == "child" }
+                   .filter_map { |link| link["href"].to_s[RELEASE_PATTERN] }
+                   .uniq
+        releases.empty? ? nil : releases
+      rescue StandardError => e
+        OvertureMaps.configuration.logger&.warn("STAC catalog unavailable (#{e.message}); listing bucket instead")
+        nil
+      end
+
+      def from_bucket
+        require "overture_maps/storage"
+
+        listing = Storage.list(prefix: "release/", delimiter: "/")
+        releases = listing[:prefixes].filter_map { |p| p[RELEASE_PATTERN] }.uniq
+        raise Error, "could not discover Overture releases from STAC or the bucket" if releases.empty?
+
+        releases
+      end
+    end
+  end
+end

data/lib/overture_maps/storage.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "uri"
+require "rexml/document"
+require "fileutils"
+
+module OvertureMaps
+  # Anonymous HTTP access to the public Overture bucket (or a configured
+  # mirror): object listing via the S3 REST API's XML responses and streaming
+  # downloads. No AWS SDK, no credentials, no global state.
+  module Storage
+    class Error < OvertureMaps::Error; end
+
+    MAX_REDIRECTS = 5
+
+    class << self
+      # Lists objects under a prefix. Returns { objects: [{key:, size:}],
+      # prefixes: [String] }, following continuation tokens.
+      def list(prefix:, delimiter: nil)
+        objects = []
+        prefixes = []
+        token = nil
+
+        loop do
+          params = { "list-type" => "2", "prefix" => prefix }
+          params["delimiter"] = delimiter if delimiter
+          params["continuation-token"] = token if token
+
+          doc = REXML::Document.new(get("#{base_url}/?#{URI.encode_www_form(params)}"))
+          root = doc.root
+          raise Error, "unexpected listing response" unless root&.name == "ListBucketResult"
+
+          root.each_element("Contents") do |el|
+            objects << {
+              key: el.elements["Key"]&.text,
+              size: el.elements["Size"]&.text.to_i
+            }
+          end
+          root.each_element("CommonPrefixes/Prefix") { |el| prefixes << el.text }
+
+          token = root.elements["NextContinuationToken"]&.text
+          break unless token
+        end
+
+        { objects: objects, prefixes: prefixes }
+      end
+
+      # Streams an object to a local file. Skips the download when the local
+      # file already has the expected size.
+      def download(key, to:, expected_size: nil)
+        if expected_size && File.exist?(to) && File.size(to) == expected_size
+          return :skipped
+        end
+
+        download_url("#{base_url}/#{escape_key(key)}", to: to)
+        :downloaded
+      end
+
+      # Downloads any URL to a file, following redirects, writing through a
+      # temp file so failures never leave a truncated file at the target path.
+      def download_url(url, to:)
+        FileUtils.mkdir_p(File.dirname(to))
+        tmp = "#{to}.part"
+
+        fetch_streaming(url) do |response|
+          File.open(tmp, "wb") do |file|
+            response.read_body { |chunk| file.write(chunk) }
+          end
+        end
+
+        FileUtils.mv(tmp, to)
+        to
+      ensure
+        FileUtils.rm_f(tmp) if tmp && File.exist?(tmp)
+      end
+
+      # GET returning the body as a string, following redirects.
+      def get(url)
+        body = nil
+        fetch_streaming(url) { |response| body = response.body }
+        body
+      end
+
+      private
+
+      def base_url
+        OvertureMaps.configuration.s3_http_url.chomp("/")
+      end
+
+      def escape_key(key)
+        key.split("/").map { |part| URI.encode_uri_component(part) }.join("/")
+      end
+
+      def fetch_streaming(url, redirects_left = MAX_REDIRECTS, &block)
+        uri = URI(url)
+        timeout = OvertureMaps.configuration.timeout
+
+        Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https",
+                        open_timeout: timeout, read_timeout: timeout) do |http|
+          http.request(Net::HTTP::Get.new(uri)) do |response|
+            case response
+            when Net::HTTPRedirection
+              raise Error, "too many redirects for #{url}" if redirects_left.zero?
+
+              location = response["location"]
+              raise Error, "redirect without location from #{url}" unless location
+
+              response.read_body # drain before following
+              return fetch_streaming(URI.join(url, location).to_s, redirects_left - 1, &block)
+            when Net::HTTPSuccess
+              yield response
+            else
+              raise Error, "HTTP #{response.code} for #{url}"
+            end
+          end
+        end
+      end
+    end
+  end
+end