rb_snowflake_client 0.0.6

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 70963fe960bc2ab443a82480ede2636c6c76f2704f1b72f0ddf67c166b3539e5
+  data.tar.gz: 4475b0653841836532b6934ea30630e4f3f89f7923ec4031f99d78d08c43e8ca
+SHA512:
+  metadata.gz: ba3ed329ca17e395d89e8d10a5e20d00e5b0d00ecf8f6689ad064e8385042070d9821c2944c7a0b6321baa2492c5602e49a66743294c37372ac306e9ffcee238
+  data.tar.gz: 211534933b4a6767c7866b1b926cc54cf309877ac2199d535e6b31b42ccbfcb938533e3a9692e7f4e06cf8aa723a238aa0a763231324d7ec361d0679740b421b

data/.github/workflows/ci.yml

@@ -0,0 +1,37 @@
+name: CI
+
+on:
+  push:
+    branches: "*"
+  pull_request:
+    branches: "*"
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@904f3fef85a9c80a3750cbe7d5159268fd5caa9f
+        with:
+          ruby-version: '3.0.6'
+      - name: Install dependencies
+        run: bundle install
+      - name: Build gem
+        run: rake build
+      # Enable this section to allow debugging via SSH
+      #- name: Setup upterm session
+        #uses: lhotari/action-upterm@v1
+        #with:
+          ### limits ssh access and adds the ssh public key for the user which triggered the workflow
+          #limit-access-to-actor: true
+      - name: Install gem
+        run: cd pkg && gem install --local *.gem
+      - name: Run tests
+        run: ruby -rrb_snowflake_client spec/test.rb
+        env: # Or as an environment variable
+          SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}
+          SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }}
+          SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
+          SNOWFLAKE_WAREHOUSE: ${{ secrets.SNOWFLAKE_WAREHOUSE }}
+          SNOWFLAKE_PRIVATE_KEY: ${{ secrets.SNOWFLAKE_CLIENT_TEST_PRIVATE_KEY }}

data/.github/workflows/release-gh-packages.yml

@@ -0,0 +1,26 @@
+name: Release
+on:
+  push:
+    branches:
+      - "master"
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@904f3fef85a9c80a3750cbe7d5159268fd5caa9f
+        with:
+          ruby-version: '3.0.6'
+      - name: Install dependencies
+        run: bundle install
+      - name: Build gem
+        run: rake build
+      - name: Build and publish to GitHub Package
+        uses: actionshub/publish-gem-to-github@v1.0.6
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          owner: rinsed-org

data/.github/workflows/release-rubygems.yml

@@ -0,0 +1,26 @@
+name: Release
+on:
+  push:
+    branches:
+      - "master"
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@904f3fef85a9c80a3750cbe7d5159268fd5caa9f
+        with:
+          ruby-version: '3.0.6'
+      - name: Install dependencies
+        run: bundle install
+      - name: Publish to RubyGems
+        run: |
+          mkdir -p $HOME/.gem
+          touch $HOME/.gem/credentials
+          chmod 0600 $HOME/.gem/credentials
+          printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
+          gem build *.gemspec
+          gem push *.gem
+        env:
+          GEM_HOST_API_KEY: "${{secrets.RUBYGEMS_AUTH_TOKEN}}"

data/.gitignore

@@ -0,0 +1,9 @@
+/.project
+/.rakeTasks
+.idea/*
+
+# ruby gems
+*.gem
+/.DS_Store
+.env
+.ruby-version

data/Gemfile

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in ruby_snowflake_client.gemspec
+gemspec
+
+gem "bundler"
+gem "concurrent-ruby"
+gem "connection_pool"
+gem "dotenv"
+gem "jwt"
+gem "oj"
+gem "rake"
+
+group :development do
+  gem "parallel"
+  gem "pry"
+end
+
+group :test do
+  gem "rspec"
+end

data/Gemfile.lock

@@ -0,0 +1,55 @@
+PATH
+  remote: .
+  specs:
+    rb_snowflake_client (0.0.6)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    coderay (1.1.3)
+    concurrent-ruby (1.2.2)
+    connection_pool (2.4.1)
+    diff-lcs (1.5.0)
+    dotenv (2.8.1)
+    jwt (2.7.1)
+    method_source (1.0.0)
+    oj (3.16.1)
+    parallel (1.23.0)
+    pry (0.14.2)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    rake (13.1.0)
+    rspec (3.12.0)
+      rspec-core (~> 3.12.0)
+      rspec-expectations (~> 3.12.0)
+      rspec-mocks (~> 3.12.0)
+    rspec-core (3.12.2)
+      rspec-support (~> 3.12.0)
+    rspec-expectations (3.12.3)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.12.0)
+    rspec-mocks (3.12.6)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.12.0)
+    rspec-support (3.12.1)
+
+PLATFORMS
+  arm64-darwin-21
+  arm64-darwin-22
+  x86_64-linux
+
+DEPENDENCIES
+  bundler
+  concurrent-ruby
+  connection_pool
+  dotenv
+  jwt
+  oj
+  parallel
+  pry
+  rake
+  rb_snowflake_client!
+  rspec
+
+BUNDLED WITH
+   2.4.19

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Dotan Nahum
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,131 @@
+# Ruby snowflake client using the v2 HTTP API
+
+# Why this library?
+
+The available options for connecting from Ruby to Snowflake include:
+* ODBC - which works, but can be very slow, especially for a lot of data, which is probably why you're using Snowflake
+* The [ruby snowflake client][https://github.com/rinsed-org/ruby-snowflake-client] that wraps the go client. This is probably the fastest single threaded option, which we also created. However, that library takes the ruby GVL and so stops all other processing in your ruby process (threads).
+
+This library is implemented in ruby and while it leverages some libraries that have native extensions, doesn't currently include anything itself. Depending on network latency and the shape of the data this library can be faster or slower than the go wrapper. The big advantages are:
+* It uses about half the memory when you pull a full result set into memory
+* It does not hold onto the [ruby GVL][https://www.speedshop.co/2020/05/11/the-ruby-gvl-and-scaling.html] and so does not block other threads while waiting on IO like the go wrapper client.
+* It will comsume more resources for the same data, because it's using the HTTP v2 API and getting JSON back, there is just more work to as compared to the go or python clients that use Apache Arrow under the covers.
+
+# Usage
+
+Add to your Gemfile or use `gem install rb-snowflake-client`
+```ruby
+  gem "rb-snowflake-client"
+```
+
+Then require, create a client
+```ruby
+require "rb_snowflake_client"
+
+
+# uses env variables, you can also new one up
+# see: https://github.com/rinsed-org/pure-ruby-snowflake-client/blob/master/lib/ruby_snowflake/client.rb#L43
+client = RubySnowflake::Client.new(
+  "https://yourinstance.region.snowflakecomputing.com", # insert your URL here
+  File.read("secrets/my_key.pem"),                      # path to your private key
+  "snowflake-organization",                             # your account name (doesn't match your URL)
+  "snowflake-account",                                  # typically your subdomain
+  "snowflake-user",                                     # Your snowflake user
+  "some_warehouse",                                     # The name of your warehouse to use by default
+)
+
+# alternatively you can use the connect method, which will pull these values from the following environment variables. You can either provide the path to the PEM file, or it's contents in an ENV variable.
+# SNOWFLAKE_URI
+# SNOWFLAKE_PRIVATE_KEY_PATH
+#   or
+# SNOWFLAKE_PRIVATE_KEY
+# SNOWFLAKE_ORGANIZATION
+# SNOWFLAKE_ACCOUNT
+# SNOWFLAKE_USER
+# SNOWFLAKE_DEFAULT_WAREHOUSE
+RubySnowflake::Client.connect
+```
+
+Once you have a client, make queries
+```ruby
+# will get all data in memory
+result = client.query("SELECT ID, NAME FROM SOMETABLE")
+
+# result is Enumerable
+result.each do |row|
+  puts row[:id]    # row supports access with symbols
+  puts row["name"] # or case insensitive strings
+  puts row.to_h    # and can produce a hash with keys/values
+end
+
+# You can also stream results and not hold them all in memory.
+# The client will prefetch the next data partition only. If you
+# have some IO in your processing there should usually be data
+# available for you.
+result = client.query("SELECT * FROM HUGETABLE", streaming: true)
+result.each do |row|
+  puts row
+end
+```
+
+# Gotchas
+
+1. Does not yet support multiple statements (work around is to wrap in `BEGIN ... END`)
+2. Only supports key pair authentication
+3. Its faster to work directly with the row value and not call to_h if you don't need to
+
+# Setting up a user for key pair authentication
+
+This library uses JWT to authenticate with the API which relies on key-pair authentication to connect to Snowflake.
+
+1. Generate a private/public key pair for your user. Your private key will now be in a file `private_key.pem`. Keep this safe! Don't check it in to source control.
+```bash
+openssl genpkey -algorithm RSA -out private_key.pem -pkeyopt rsa_keygen_bits:2048
+```
+2. Generate a public key in the format that Snowflake likes (will produce `public_key.pem`)
+```bash
+openssl rsa -pubout -in private_key.pem -out public_key.pem
+```
+3. Your public_key.pem file should look something like this
+```text
+-----BEGIN PUBLIC KEY-----
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAx8FaPusz9X9MCvv0h3N3
+v1QaruyU1ivHs8jLjo6idzLSHJPGk7n3LSXerIw5/LkhfA27ibJj225/fKFnPy+X
+gidbhE4BlvSdoVgdMH7WB1ZC3PpAwwqHeMisIzarwOwUu6mLyG9VY55ciKJY8CwA
+5xt19pgVsXg/lcOa72jDjK+ExdSAN6K2TqSKqq77yzeI5creslny5VuAGTbZy3Bt
+Wk0zg1xz8+C4regIOlSoFrzn1e4wHqbFv2zFFvORC2LV3HXFRaHYClB7jWRN1bFj
+om6gRpiTO8bsCSPKi0anxMN8qt1Lw2d/+cwezxCwI6xPLC7JhZYdx6u+hC0g3PVK
+PQIDAQAB
+-----END PUBLIC KEY-----
+```
+Snowflake doesn't like it in that format, but openssl can remove the newlines and begining and ending for you:
+```bash
+openssl rsa -pubin -in public_key.pem -outform DER | openssl base64 -A
+```
+(if it spits out a % at the end, remove that).
+```text
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArOL5WQYaXSnVhQXQZQHVIzrNt08A+bnGjBb6DWFVRao3dlPG+HOf9Nv0nGlk8m5AMvvETUnN3tihuRHOJ9MOUzDp58IYIr5xvOENSunbRVyJL7DuCGwZz8z1pEnlBjZPONzEX8dCKxCU0neJrksFgwdhfhIUs7GnbTuJjYP9EqXPlbsYNYTVVnFNZ9DHFur9PggPJpPHTfFDz8MEB3Xb3AWV3pE752ed/PtRcTODvgoQSpP80cTgsKjsG009NY2ulEtV3r7yNJgawxmcMTNLhFlSS7Wm2NSEIS0aNo+DgSZI72MnAOw2klUzvdBl0i43gI+aX0Y6y/y18VL1o9KMQwIDAQAB
+```
+4. Now, in the snowflake web console or through your favorite client, log in as a user with permissions to edit users. For your particular user (`EXAMPLE_USER` below) update the user with the modified public key from above:
+```sql
+ALTER USER EXAMPLE_USER SET RSA_PUBLIC_KEY = 'MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArOL5WQYaXSnVhQXQZQHVIzrNt08A+bnGjBb6DWFVRao3dlPG+HOf9Nv0nGlk8m5AMvvETUnN3tihuRHOJ9MOUzDp58IYIr5xvOENSunbRVyJL7DuCGwZz8z1pEnlBjZPONzEX8dCKxCU0neJrksFgwdhfhIUs7GnbTuJjYP9EqXPlbsYNYTVVnFNZ9DHFur9PggPJpPHTfFDz8MEB3Xb3AWV3pE752ed/PtRcTODvgoQSpP80cTgsKjsG009NY2ulEtV3r7yNJgawxmcMTNLhFlSS7Wm2NSEIS0aNo+DgSZI72MnAOw2klUzvdBl0i43gI+aX0Y6y/y18VL1o9KMQwIDAQAB'
+```
+5. Verify your auth setup. If you have `snowsql` installed, that has an easy method (CTRL-d to exit)
+```bash
+snowsql -a <account_identifier>.<region>p -u <user> --private-key-path private_key.pem
+```
+or alternatively, use the client to verify:
+```ruby
+client = RubySnowflake::Client.new(
+  "https://yourinstance.region.snowflakecomputing.com", # insert your URL here
+  File.read("secrets/my_key.pem"),                      # path to your private key
+  "snowflake-organization",                             # your account name (doesn't match your URL)
+  "snowflake-account",                                  # typically your subdomain
+  "snowflake-user",                                     # Your snowflake user
+  "some_warehouse",                                     # The name of your warehouse to use by default
+)
+```
+
+# Links:
+- snowflake API reference https://docs.snowflake.com/en/developer-guide/sql-api/reference
+- snowflake authentication docs: https://docs.snowflake.com/en/developer-guide/sql-api/authenticating

data/Rakefile

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'

data/lib/rb_snowflake_client.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+require "ruby_snowflake/client"
+
+module RubySnowflake
+end

data/lib/ruby_snowflake/client/http_connection_wrapper.rb

@@ -0,0 +1,32 @@
+module RubySnowflake
+  class Client
+    class HttpConnectionWrapper
+      def initialize(hostname, port)
+        @hostname = hostname
+        @port = port
+      end
+
+      def start
+        @connection = Net::HTTP.start(@hostname, @port, use_ssl: true)
+        self
+      rescue StandardError
+        raise ConnectionError.new "Error connecting to server."
+      end
+
+      def request(request)
+        # connections can timeout and close, re-open them
+        # which is what the connection pool expects
+        start unless connection.active?
+
+        begin
+          connection.request(request)
+        rescue StandardError => error
+          raise RequestError.new "HTTP error requesting data", cause: error
+        end
+      end
+
+      private
+        attr_accessor :connection
+    end
+  end
+end

data/lib/ruby_snowflake/client/single_thread_in_memory_strategy.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module RubySnowflake
+  class Client
+    class SingleThreadInMemoryStrategy
+      def self.result(statement_json_body, retreive_proc)
+        partitions = statement_json_body["resultSetMetaData"]["partitionInfo"]
+        result = Result.new(partitions.size, statement_json_body["resultSetMetaData"]["rowType"])
+        result[0] = statement_json_body["data"]
+
+        partitions.each_with_index do |partition, index|
+          next if index == 0 # already have the first partition
+          result[index] = retreive_proc.call(index)
+        end
+
+        result
+      end
+    end
+  end
+end

data/lib/ruby_snowflake/client/streaming_result_strategy.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module RubySnowflake
+  class Client
+    class StreamingResultStrategy
+      def self.result(statement_json_body, retreive_proc)
+        partitions = statement_json_body["resultSetMetaData"]["partitionInfo"]
+
+        result = StreamingResult.new(
+          partitions.size,
+          statement_json_body["resultSetMetaData"]["rowType"],
+          retreive_proc
+        )
+        result[0] = statement_json_body["data"]
+
+        result
+      end
+    end
+  end
+end

data/lib/ruby_snowflake/client/threaded_in_memory_strategy.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module RubySnowflake
+  class Client
+    class ThreadedInMemoryStrategy
+      def self.result(statement_json_body, retreive_proc, num_threads)
+        partitions = statement_json_body["resultSetMetaData"]["partitionInfo"]
+        result = Result.new(partitions.size, statement_json_body["resultSetMetaData"]["rowType"])
+        result[0] = statement_json_body["data"]
+
+        thread_pool = Concurrent::FixedThreadPool.new(num_threads)
+        futures = []
+        partitions.each_with_index do |partition, index|
+          next if index == 0 # already have the first partition
+          futures << Concurrent::Future.execute(executor: thread_pool) do
+            [index, retreive_proc.call(index)]
+          end
+        end
+        futures.each do |future|
+          if future.rejected?
+            raise ConnectionStarvedError.new(
+                    "A partition request timed out. This is usually do to using the client in" \
+                    "multiple threads. The client uses a connection thread pool and if too many" \
+                    "requests are all done in threads at the same time, threads can get starved" \
+                    "of access to connections. The solution for this is to spin up a new client" \
+                    "instance with it's own connection pool to snowflake."
+                  )
+          end
+          index, partition_data = future.value
+          result[index] = partition_data
+        end
+        result
+      end
+    end
+  end
+end

data/lib/ruby_snowflake/client.rb

@@ -0,0 +1,202 @@
+# frozen_string_literal: true
+
+require "base64"
+require "benchmark"
+require "concurrent"
+require "connection_pool"
+require "json"
+require "jwt"
+require "net/http"
+require "oj"
+require "openssl"
+require "securerandom"
+require "uri"
+
+
+require_relative "result"
+require_relative "streaming_result"
+require_relative "client/http_connection_wrapper"
+require_relative "client/single_thread_in_memory_strategy"
+require_relative "client/streaming_result_strategy"
+require_relative "client/threaded_in_memory_strategy"
+
+module RubySnowflake
+  class Error < StandardError
+    # This will get pulled through to Sentry, see:
+    # https://github.com/getsentry/sentry-ruby/blob/11ecd254c0d2cae2b327f0348074e849095aa32d/sentry-ruby/lib/sentry/error_event.rb#L31-L33
+    attr_reader :sentry_context
+
+    def initialize(details)
+      @sentry_context = details
+    end
+  end
+  class BadResponseError < Error ; end
+  class ConnectionError < Error ; end
+  class ConnectionStarvedError < Error ; end
+  class RequestError < Error ; end
+
+
+  class Client
+    JWT_TOKEN_TTL = 3600 # seconds, this is the max supported by snowflake
+    CONNECTION_TIMEOUT = 60 # seconds, how long for a thread to wait for a connection b4 erroring
+    MAX_CONNECTIONS = 8
+    MAX_THREADS = 8
+    THREAD_SCALE_FACTOR = 4 # parition count factor for number of threads (i.e. 2 == once we have 4 partitions, spin up a second thread)
+
+    def self.connect
+      private_key = ENV["SNOWFLAKE_PRIVATE_KEY"] || File.read(ENV["SNOWFLAKE_PRIVATE_KEY_PATH"])
+
+      new(
+        ENV["SNOWFLAKE_URI"],
+        private_key,
+        ENV["SNOWFLAKE_ORGANIZATION"],
+        ENV["SNOWFLAKE_ACCOUNT"],
+        ENV["SNOWFLAKE_USER"],
+        ENV["SNOWFLAKE_DEFAULT_WAREHOUSE"],
+      )
+    end
+
+    def initialize(uri, private_key, organization, account, user, default_warehouse)
+      @base_uri = uri
+      @private_key_pem = private_key
+      @organization = organization
+      @account = account
+      @user = user
+      @default_warehouse = default_warehouse
+      @public_key_fingerprint = public_key_fingerprint(@private_key_pem)
+
+      # start with an expired value to force creation
+      @token_expires_at = Time.now.to_i - 1
+      @token_semaphore = Concurrent::Semaphore.new(1)
+    end
+
+    def query(query, warehouse: nil, streaming: false)
+      warehouse ||= @default_warehouse
+
+      response = nil
+      connection_pool.with do |connection|
+        request_body = { "statement" => query, "warehouse" => warehouse }
+
+        response = request_with_auth_and_headers(
+          connection,
+          Net::HTTP::Post,
+          "/api/v2/statements?requestId=#{SecureRandom.uuid}",
+          Oj.dump(request_body)
+        )
+      end
+      handle_errors(response)
+      retreive_result_set(response, streaming)
+    end
+
+    private
+      def connection_pool
+        @connection_pool ||= ConnectionPool.new(size: MAX_CONNECTIONS, timeout: CONNECTION_TIMEOUT) do
+          HttpConnectionWrapper.new(hostname, port).start
+        end
+      end
+
+      def hostname
+        @hostname ||= URI.parse(@base_uri).hostname
+      end
+
+      def port
+        @port ||= URI.parse(@base_uri).port
+      end
+
+      def jwt_token
+        return @token unless jwt_token_expired?
+
+        @token_semaphore.acquire do
+          now = Time.now.to_i
+          @token_expires_at = now + JWT_TOKEN_TTL
+
+          private_key = OpenSSL::PKey.read(@private_key_pem)
+
+          payload = {
+            :iss => "#{@organization.upcase}-#{@account.upcase}.#{@user}.#{@public_key_fingerprint}",
+            :sub => "#{@organization.upcase}-#{@account.upcase}.#{@user}",
+            :iat => now,
+            :exp => @token_expires_at
+          }
+
+          @token = JWT.encode payload, private_key, "RS256"
+        end
+      end
+
+      def jwt_token_expired?
+        Time.now.to_i > @token_expires_at
+      end
+
+      def handle_errors(response)
+        if response.code != "200"
+          raise BadResponseError.new({}),
+            "Bad response! Got code: #{response.code}, w/ message #{response.body}"
+        end
+      end
+
+      def request_with_auth_and_headers(connection, request_class, path, body=nil)
+        uri = URI.parse("#{@base_uri}#{path}")
+        request = request_class.new(uri)
+        request["Content-Type"] = "application/json"
+        request["Accept"] = "application/json"
+        request["Authorization"] = "Bearer #{jwt_token}"
+        request["X-Snowflake-Authorization-Token-Type"] = "KEYPAIR_JWT"
+        request.body = body unless body.nil?
+
+        response = nil
+        bm = Benchmark.measure { response = connection.request(request) }
+        puts "HTTP Request time: #{bm.real}"
+        handle_errors(response)
+        response
+      end
+
+      def retreive_result_set(response, streaming)
+        json_body = Oj.load(response.body, oj_options)
+        statement_handle = json_body["statementHandle"]
+        num_threads = number_of_threads_to_use(json_body["resultSetMetaData"]["partitionInfo"].size)
+        retreive_proc = ->(index) { retreive_partition_data(statement_handle, index) }
+
+        if streaming
+          StreamingResultStrategy.result(json_body, retreive_proc)
+        elsif num_threads == 1
+          SingleThreadInMemoryStrategy.result(json_body, retreive_proc)
+        else
+          ThreadedInMemoryStrategy.result(json_body, retreive_proc, num_threads)
+        end
+      end
+
+      def retreive_partition_data(statement_handle, partition_index)
+        partition_response = nil
+        connection_pool.with do |connection|
+          partition_response = request_with_auth_and_headers(
+            connection,
+            Net::HTTP::Get,
+            "/api/v2/statements/#{statement_handle}?partition=#{partition_index}&requestId=#{SecureRandom.uuid}",
+          )
+        end
+
+        partition_json = nil
+        bm = Benchmark.measure { partition_json = Oj.load(partition_response.body, oj_options) }
+        puts "JSON parsing took: #{bm.real}"
+        partition_data = partition_json["data"]
+
+        partition_data
+      end
+
+      def number_of_threads_to_use(partition_count)
+        [[1, (partition_count / THREAD_SCALE_FACTOR.to_f).ceil].max, MAX_THREADS].min
+      end
+
+      def oj_options
+        { :bigdecimal_load => :bigdecimal }
+      end
+
+      def public_key_fingerprint(private_key_pem_string)
+        public_key_der = OpenSSL::PKey::RSA.new(private_key_pem_string).public_key.to_der
+        digest = OpenSSL::Digest::SHA256.new.digest(public_key_der)
+        fingerprint = Base64.strict_encode64(digest)
+
+        "SHA256:#{fingerprint}"
+      end
+  end
+end

data/lib/ruby_snowflake/result.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require "concurrent"
+
+require_relative "row"
+
+module RubySnowflake
+  class Result
+    include Enumerable
+
+    attr_reader :data
+
+    def initialize(partition_count, row_type_data)
+      @data = Concurrent::Array.new(partition_count)
+      extract_row_metadata(row_type_data)
+    end
+
+    def []=(index, value)
+      data[index] = value
+    end
+
+    def get_all_rows
+      map(&:to_h)
+    end
+
+    def each
+      return to_enum(:each) unless block_given?
+
+      data.each do |partition|
+        partition.each do |row|
+          yield wrap_row(row)
+        end
+      end
+    end
+
+    def size
+      data.map(&:size).sum
+    end
+
+    alias length size
+
+    def first
+      wrap_row(data.first.first)
+    end
+
+    def last
+      wrap_row(data.last.last)
+    end
+
+    def columns
+      @row_types.map {|type| type[:name].downcase }
+    end
+
+    private
+      def wrap_row(row)
+        Row.new(@row_types, @column_to_index, row)
+      end
+
+      def extract_row_metadata(row_type_data)
+        @row_types = []
+        @column_to_index = {}
+
+        row_type_data.each_with_index do |type_data, index|
+          @row_types[index] = {
+            type: type_data["type"].downcase.to_sym,
+            scale: type_data["scale"].to_i,
+            precision: type_data["precision"].to_i,
+            name: type_data["name"].to_sym,
+          }
+          @column_to_index[type_data["name"].downcase] = index
+        end
+      end
+  end
+end

data/lib/ruby_snowflake/row.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require "date"
+require "time"
+
+module RubySnowflake
+  class Row
+    EPOCH_JULIAN_DAY_NUMBER = Date.new(1970,1,1).jd
+    TIME_FORMAT = "%s.%N".freeze
+
+    def initialize(row_types, column_to_index, data)
+      @row_types = row_types
+      @data = data
+      @column_to_index = column_to_index
+    end
+
+    # see: https://docs.snowflake.com/en/developer-guide/sql-api/handling-responses#getting-the-data-from-the-results
+    def [](column)
+      index = column.is_a?(Numeric) ? Integer(column) : @column_to_index[column]
+      return nil if index.nil?
+      return nil if @data[index].nil?
+
+      case @row_types[index][:type]
+      when :boolean
+        @data[index] == "true"
+      when :date
+        Date.jd(Integer(@data[index]) + EPOCH_JULIAN_DAY_NUMBER)
+      when :fixed
+        if @row_types[index][:scale] == 0
+          Integer(@data[index])
+        else
+          BigDecimal(@data[index]).round(@row_types[index][:scale])
+        end
+      when :float, :double, :"double precision", :real
+        # snowflake treats these all as 64 bit IEEE 754 floating point numbers, and will we too
+        Float(@data[index])
+      when :time, :datetime, :timestamp, :timestamp_ltz, :timestamp_ntz
+        Time.strptime(@data[index], TIME_FORMAT).utc
+      when :timestamp_tz
+        timestamp, offset_minutes = @data[index].split(" ")
+        Time.strptime(@data[index], TIME_FORMAT).utc - (Integer(offset_minutes) * 60)
+      else
+        @data[index]
+      end
+    end
+
+    def to_h
+      output = {}
+      @column_to_index.each_pair do |name, index|
+        output[name.downcase] = self[index]
+      end
+      output
+    end
+
+    def to_s
+      to_h.to_s
+    end
+  end
+end

data/lib/ruby_snowflake/streaming_result.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require "concurrent"
+
+require_relative "result"
+
+module RubySnowflake
+  class StreamingResult < Result
+    def initialize(partition_count, row_type_data, retreive_proc)
+      super(partition_count, row_type_data)
+      @retreive_proc = retreive_proc
+    end
+
+    def each
+      return to_enum(:each) unless block_given?
+
+      thread_pool = Concurrent::FixedThreadPool.new 1
+
+      data.each_with_index do |_partition, index|
+        next_index = [index+1, data.size-1].min
+        if data[next_index].nil? # prefetch
+          data[next_index] = Concurrent::Future.execute(executor: thread_pool) do
+            @retreive_proc.call(next_index)
+          end
+        end
+
+        if data[index].is_a? Concurrent::Future
+          data[index] = data[index].value # wait for it to finish
+        end
+        data[index].each do |row|
+          yield wrap_row(row)
+        end
+      end
+    end
+
+
+    def size
+      not_implemented
+    end
+
+    def last
+      not_implemented
+    end
+
+    private
+      def not_implemented
+        raise "not implemented on streaming result set"
+      end
+  end
+end

data/lib/ruby_snowflake/version.rb

@@ -0,0 +1,3 @@
+module RubySnowflake
+  VERSION = '0.0.6'
+end

data/private_key_pkcs8.pem

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDHwVo+6zP1f0wK
++/SHc3e/VBqu7JTWK8ezyMuOjqJ3MtIck8aTufctJd6sjDn8uSF8DbuJsmPbbn98
+oWc/L5eCJ1uETgGW9J2hWB0wftYHVkLc+kDDCod4yKwjNqvA7BS7qYvIb1VjnlyI
+oljwLADnG3X2mBWxeD+Vw5rvaMOMr4TF1IA3orZOpIqqrvvLN4jlyt6yWfLlW4AZ
+NtnLcG1aTTODXHPz4Lit6Ag6VKgWvOfV7jAepsW/bMUW85ELYtXcdcVFodgKUHuN
+ZE3VsWOibqBGmJM7xuwJI8qLRqfEw3yq3UvDZ3/5zB7PELAjrE8sLsmFlh3Hq76E
+LSDc9Uo9AgMBAAECggEADvKy5mjy7TDet6a1nIpBQIXSblshYyqJCfTheLRHR5y7
+uOw9DLJ6ovwFvwtKTpVrAk3DxjDdhJ7u9txcLdAJqjF4SX0N5OY+woNlwcqICSoF
+HGS0wnOwAhOgQA1YCnqQLZ/pKDFepjMEUSw29BgS5Dh7mh205+Ar//DQkBBaq9UX
+uZtw3dKKmDPcWdPfFjFJQrH9YrrDdt+rcOULIk4iyzepA7nLV9RBiuir256Pnmi8
++aXzPkl7PjMgCrDvqx47+Kjxcvccq/yt0zlNuEBDKOa9ChPd6FOV8O64IknBl/Eq
+xU/2+06XI5QEuk9k1MKyMO06fOWS/3XSpOBSa1pooQKBgQDuiFk1897Bsxz5wwvb
+vwRVoVHhNcmWLixMiWggRvnDWfDKzm7lGKTAzrgafNnw84NPD0nvcgDXivzQpEgX
+Ge/SSZzn3DDjEl+QhAP2kg1zE1+7XVR2+2NNa4lS7uksjuEmffHbIg8PXFbyZ3az
+Z/8p63q4bq5Lo5O0EKbRBjQLXQKBgQDWYhCbxhLVXcrz9PMWIFsOn9JqvfL/DJoh
+A7RcxEAcPCHG13fOHaqIo40QoiuOJj86MjdD9rHUl2opQedKmGOE4KsN1PFzwZiU
+fqYsgodgyuJe+gA9EzKabPiXvUG0R8m9T1iZocW4/n0UKrr7qwKco9JVfZQ3MpoJ
+3d6/qt0sYQKBgEfxC6cAFHErSlPyBcOTCwWgkAdyp4gKCQZUcvk5Lt9CYyNnu36R
+Pi+AqXojqb/IUhQoMk5rfeGEY9exlKSTCAfOd+peOMhjpDr79g4J4tCQSVFDRtRT
+4ZFaG6LNt7nepf1sAuPqk8w4N2kI134ooYbNJMg3aY4amdO3Kfjk/BFpAoGABAlg
+qq42GJJlieZo1VfSnxnfSmc9NhQ0qr55HO4NDvsK38Wa2eSxG1WJCHElRM3vjtb3
+N25jts8LDq/Q/RSVeyEH14DU8wsIJnOkCV6D0gy/6m3mpz+/Si0XGTRVEmHvqILf
+9jSXln4kOMb9MlPr/qsPANEWg1uOjvUAhrde5sECgYEArHqQfneC3dG+PaRIlwDz
+cKl6QWrCcfSVHL6e0XsRWT2UHrEhC5cLwd1mqMyKTZ/AK0idgyA9HRDZVzAEeina
+O1kTzlr5+ukTF3wEC4N2GKoaKkeWy5LBWSduRpa4NtjJsXVVylgYmm/jqXKwzAnP
+9J6e0aokrs0mnEu/Ij/pB0A=
+-----END PRIVATE KEY-----

data/rb_snowflake_client.gemspec

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require_relative "lib/ruby_snowflake/version"
+
+Gem::Specification.new do |s|
+  s.name    = "rb_snowflake_client"
+  s.version = RubySnowflake::VERSION
+  s.summary = "Snowflake connector for Ruby"
+  s.author  = "Rinsed"
+  s.email   = ["reid@rinsed.co", "alex@rinsed.co"]
+  s.description = <<~DESC
+  Using the HTTP V2 Api for Snowflake runs queries & creates native Ruby objects.
+  DESC
+  s.license = "MIT" # TODO: double check
+
+  s.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features|vendor)/}) }
+  end
+
+  s.require_paths = ["lib"]
+end

metadata

@@ -0,0 +1,68 @@
+--- !ruby/object:Gem::Specification
+name: rb_snowflake_client
+version: !ruby/object:Gem::Version
+  version: 0.0.6
+platform: ruby
+authors:
+- Rinsed
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2023-11-28 00:00:00.000000000 Z
+dependencies: []
+description: 'Using the HTTP V2 Api for Snowflake runs queries & creates native Ruby
+  objects.
+
+  '
+email:
+- reid@rinsed.co
+- alex@rinsed.co
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".github/workflows/ci.yml"
+- ".github/workflows/release-gh-packages.yml"
+- ".github/workflows/release-rubygems.yml"
+- ".gitignore"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/rb_snowflake_client.rb
+- lib/ruby_snowflake/client.rb
+- lib/ruby_snowflake/client/http_connection_wrapper.rb
+- lib/ruby_snowflake/client/single_thread_in_memory_strategy.rb
+- lib/ruby_snowflake/client/streaming_result_strategy.rb
+- lib/ruby_snowflake/client/threaded_in_memory_strategy.rb
+- lib/ruby_snowflake/result.rb
+- lib/ruby_snowflake/row.rb
+- lib/ruby_snowflake/streaming_result.rb
+- lib/ruby_snowflake/version.rb
+- private_key_pkcs8.pem
+- rb_snowflake_client.gemspec
+homepage: 
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.2.33
+signing_key: 
+specification_version: 4
+summary: Snowflake connector for Ruby
+test_files: []