forest_admin_datasource_snowflake 1.29.1

data/forest_admin_datasource_snowflake.gemspec

@@ -0,0 +1,37 @@
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift lib unless $LOAD_PATH.include?(lib)
+
+require_relative 'lib/forest_admin_datasource_snowflake/version'
+
+Gem::Specification.new do |spec|
+  spec.name        = 'forest_admin_datasource_snowflake'
+  spec.version     = ForestAdminDatasourceSnowflake::VERSION
+  spec.authors     = ['Forest Admin']
+  spec.email       = ['support@forestadmin.com']
+  spec.homepage    = 'https://www.forestadmin.com'
+  spec.summary     = 'Snowflake datasource for Forest Admin (read-only).'
+  spec.description = 'Exposes Snowflake tables and views as Forest Admin collections via ODBC. ' \
+                     'Queries are translated from Forest ConditionTrees to parameterized SQL; ' \
+                     'no ActiveRecord involvement on the Snowflake side.'
+  spec.license     = 'GPL-3.0'
+  spec.required_ruby_version = '>= 3.0.0'
+
+  spec.metadata['homepage_uri']         = spec.homepage
+  spec.metadata['source_code_uri']      = 'https://github.com/ForestAdmin/agent-ruby'
+  spec.metadata['changelog_uri']        = 'https://github.com/ForestAdmin/agent-ruby/blob/main/CHANGELOG.md'
+  spec.metadata['rubygems_mfa_required'] = 'false'
+
+  spec.files = Dir.chdir(__dir__) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      (File.expand_path(f) == __FILE__) ||
+        f.start_with?(*%w[bin/ test/ spec/ features/ .git .circleci appveyor Gemfile])
+    end
+  end
+  spec.bindir       = 'exe'
+  spec.executables  = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'connection_pool', '~> 2.4'
+  spec.add_dependency 'ruby-odbc',       '~> 0.99999'
+  spec.add_dependency 'zeitwerk',        '~> 2.3'
+end

data/lib/forest_admin_datasource_snowflake/collection.rb

@@ -0,0 +1,141 @@
+require 'odbc'
+require 'date'
+require 'json'
+
+module ForestAdminDatasourceSnowflake
+  class Collection < ForestAdminDatasourceToolkit::Collection
+    READ_ONLY_MESSAGE = 'forest_admin_datasource_snowflake is read-only.'.freeze
+    ReadOnlyError = ForestAdminDatasourceToolkit::Exceptions::ForestException
+
+    attr_reader :table_name
+
+    def initialize(datasource, table_name)
+      super
+      @table_name = table_name
+      @json_columns = []
+      fetch_fields
+      enable_count
+    end
+
+    def list(_caller, filter, projection)
+      sql, binds = Utils::Query.new(self, projection: projection, filter: filter).to_sql
+      execute_to_hashes(sql, binds, projection.to_a)
+    end
+
+    def aggregate(_caller, filter, aggregation, limit = nil)
+      sql, binds, group_columns = Utils::Query.new(
+        self,
+        filter: filter,
+        aggregation: aggregation,
+        limit: limit
+      ).to_aggregate_sql
+
+      raw_rows = execute_raw(sql, binds)
+      raw_rows.map do |row|
+        value = coerce_value(row.first)
+        group_hash = group_columns.each_with_index.to_h { |col, i| [col, coerce_for_column(col, row[i + 1])] }
+        { 'value' => value, 'group' => group_hash }
+      end
+    end
+
+    def create(_caller, _data) = raise(ReadOnlyError, READ_ONLY_MESSAGE)
+    def update(_caller, _filter, _data) = raise(ReadOnlyError, READ_ONLY_MESSAGE)
+    def delete(_caller, _filter) = raise(ReadOnlyError, READ_ONLY_MESSAGE)
+
+    private
+
+    def fetch_fields
+      rows = @datasource.snowflake_columns_for(@table_name)
+      pk_names = resolve_primary_keys(rows)
+
+      rows.each do |row|
+        column_name    = row[1]
+        snowflake_type = row[2]
+        nullable       = row[3].to_s.casecmp('YES').zero?
+
+        forest_type = Parser::Column.forest_type_for_snowflake_native(snowflake_type)
+        @json_columns << column_name if forest_type == 'Json'
+
+        field = ForestAdminDatasourceToolkit::Schema::ColumnSchema.new(
+          column_type: forest_type,
+          filter_operators: Parser::Column.operators_for_column_type(forest_type),
+          is_primary_key: pk_names.include?(column_name),
+          is_read_only: true,
+          is_sortable: true,
+          default_value: nil,
+          enum_values: [],
+          validation: nullable ? [] : [{ operator: 'Present' }]
+        )
+        add_field(column_name, field)
+      end
+    end
+
+    def resolve_primary_keys(rows)
+      column_names = rows.map { |r| r[1] }
+      override = @datasource.primary_keys_override_for(@table_name)
+      return resolve_override_primary_keys(override, column_names) if override
+
+      declared_pks = @datasource.primary_keys_for(@table_name)
+      matches = declared_pks.filter_map do |pk|
+        column_names.find { |name| name.to_s.casecmp(pk.to_s).zero? }
+      end
+      return matches if matches.any?
+
+      fallback = (rows.find { |r| r[1].to_s.casecmp('id').zero? } || rows.first)&.dig(1)
+      fallback ? [fallback] : []
+    end
+
+    def resolve_override_primary_keys(override, column_names)
+      override.map do |declared_pk|
+        column_names.find { |name| name.to_s.casecmp(declared_pk.to_s).zero? } ||
+          raise(ForestAdminDatasourceSnowflake::Error,
+                "primary_keys override '#{declared_pk}' does not match any column on table " \
+                "'#{@table_name}' (available: #{column_names.join(", ")})")
+      end
+    end
+
+    def execute_to_hashes(sql, binds, projected_columns)
+      rows = execute_raw(sql, binds)
+      rows.map { |row| projected_columns.each_with_index.to_h { |col, i| [col, coerce_for_column(col, row[i])] } }
+    end
+
+    def coerce_for_column(column_name, value)
+      return parse_json_value(value) if @json_columns.include?(column_name)
+
+      coerce_value(value)
+    end
+
+    def coerce_value(value)
+      case value
+      when ::ODBC::TimeStamp
+        ::Time.utc(value.year, value.month, value.day, value.hour, value.minute, value.second)
+      when ::ODBC::Date
+        ::Date.new(value.year, value.month, value.day)
+      when ::ODBC::Time
+        format('%<h>02d:%<m>02d:%<s>02d', h: value.hour, m: value.minute, s: value.second)
+      else
+        value
+      end
+    end
+
+    def parse_json_value(value)
+      return value unless value.is_a?(String)
+
+      JSON.parse(value)
+    rescue JSON::ParserError
+      value
+    end
+
+    def execute_raw(sql, binds)
+      @datasource.with_connection do |conn|
+        stmt = conn.prepare(sql)
+        begin
+          stmt.execute(*binds)
+          stmt.fetch_all || []
+        ensure
+          stmt.drop
+        end
+      end
+    end
+  end
+end

data/lib/forest_admin_datasource_snowflake/datasource.rb

@@ -0,0 +1,258 @@
+require 'odbc'
+require 'connection_pool'
+
+module ForestAdminDatasourceSnowflake
+  class Datasource < ForestAdminDatasourceToolkit::Datasource
+    DEFAULT_POOL_SIZE    = 5
+    DEFAULT_POOL_TIMEOUT = 5
+
+    CONNECTION_LOST_PATTERNS = [
+      /Communication link failure/i,
+      /Connection.*lost/i,
+      /Connection is closed/i,
+      /Session.*expired/i,
+      /Session.*timed out/i,
+      /Broken pipe/i,
+      /Not connected/i,
+      /timeout expired/i,
+      /Authentication token has expired/i,
+      /token.*expired/i
+    ].freeze
+
+    SYSTEM_SCHEMAS = %w[INFORMATION_SCHEMA].freeze
+
+    attr_reader :pool
+
+    def initialize(conn_str:,
+                   pool_size: DEFAULT_POOL_SIZE, pool_timeout: DEFAULT_POOL_TIMEOUT,
+                   statement_timeout: nil, primary_keys: nil)
+      super()
+      @schema_override        = extract_schema_from_conn_str(conn_str)
+      @statement_timeout      = statement_timeout
+      @primary_keys_override  = (primary_keys || {}).transform_keys { |k| k.to_s.upcase }
+      @pool                   = ConnectionPool.new(size: pool_size, timeout: pool_timeout) do
+        open_connection(conn_str)
+      end
+
+      @schema_override ||= resolve_default_schema!
+
+      generate_collections
+      discover_relations
+    end
+
+    def with_connection(&block)
+      retried = false
+      begin
+        @pool.with(&block)
+      rescue ::ODBC::Error => e
+        if !retried && connection_lost?(e)
+          retried = true
+          reset_pool!
+          retry
+        end
+        raise
+      end
+    end
+
+    def shutdown!
+      @pool.shutdown { |conn| safe_disconnect(conn) }
+    end
+
+    def primary_keys_for(table_name)
+      upper = table_name.to_s.upcase
+      return Array(@primary_keys_override[upper]) if @primary_keys_override.key?(upper)
+
+      (snowflake_primary_keys || {})[upper] || []
+    end
+
+    def primary_keys_override_for(table_name)
+      upper = table_name.to_s.upcase
+      return nil unless @primary_keys_override.key?(upper)
+
+      Array(@primary_keys_override[upper])
+    end
+
+    def snowflake_columns_for(table_name)
+      (snowflake_columns || {})[table_name.to_s.upcase] || []
+    end
+
+    private
+
+    def snowflake_columns
+      return @snowflake_columns if defined?(@snowflake_columns)
+
+      @snowflake_columns = fetch_snowflake_columns
+    end
+
+    def fetch_snowflake_columns
+      sql, binds = build_snowflake_columns_query
+      with_connection do |conn|
+        stmt = conn.prepare(sql)
+        begin
+          stmt.execute(*binds)
+          rows = stmt.fetch_all || []
+          rows.group_by { |r| r[0].to_s.upcase }
+        ensure
+          stmt.drop
+        end
+      end
+    rescue ::ODBC::Error => e
+      warn "[forest_admin_datasource_snowflake] column introspection failed: #{e.message}"
+      nil
+    end
+
+    def build_snowflake_columns_query
+      base   = 'SELECT TABLE_NAME, COLUMN_NAME, DATA_TYPE, IS_NULLABLE FROM INFORMATION_SCHEMA.COLUMNS '
+      filter = @schema_override ? 'WHERE TABLE_SCHEMA = ? ' : "WHERE TABLE_SCHEMA <> 'INFORMATION_SCHEMA' "
+      binds  = @schema_override ? [@schema_override] : []
+      ["#{base}#{filter}ORDER BY TABLE_NAME, ORDINAL_POSITION", binds]
+    end
+
+    def snowflake_primary_keys
+      return @snowflake_primary_keys if defined?(@snowflake_primary_keys)
+
+      @snowflake_primary_keys = fetch_snowflake_primary_keys
+    end
+
+    def fetch_snowflake_primary_keys
+      with_connection do |conn|
+        stmt = conn.prepare('SHOW PRIMARY KEYS IN SCHEMA')
+        rows = begin
+          stmt.execute
+          stmt.fetch_all || []
+        ensure
+          stmt.drop
+        end
+
+        rows.group_by { |r| r[3].to_s.upcase }
+            .transform_values { |table_rows| table_rows.sort_by { |r| r[5].to_i }.map { |r| r[4].to_s } }
+      end
+    rescue ::ODBC::Error => e
+      warn "[forest_admin_datasource_snowflake] primary-key introspection failed: #{e.message}"
+      nil
+    end
+
+    def open_connection(conn_str)
+      driver = ::ODBC::Driver.new
+      driver.name  = 'odbc'
+      driver.attrs = parse_conn_str(conn_str)
+      conn = ::ODBC::Database.new.drvconnect(driver)
+      apply_session_settings(conn)
+      conn
+    end
+
+    def parse_conn_str(conn_str)
+      conn_str.split(';').reject(&:empty?).to_h do |option|
+        pair = option.split('=', 2)
+        if pair.size != 2
+          raise ForestAdminDatasourceSnowflake::Error,
+                "Malformed connection string option '#{option}': expected 'key=value'."
+        end
+        pair
+      end
+    end
+
+    def extract_schema_from_conn_str(conn_str)
+      attrs = parse_conn_str(conn_str)
+      pair  = attrs.find { |k, _| k.to_s.casecmp('schema').zero? }
+      value = pair && pair[1]
+      value if value && !value.empty?
+    end
+
+    def resolve_default_schema!
+      resolved = with_connection do |conn|
+        stmt = conn.prepare('SELECT CURRENT_SCHEMA()')
+        begin
+          stmt.execute
+          (stmt.fetch_all || []).first&.first
+        ensure
+          stmt.drop
+        end
+      end
+
+      return resolved if resolved && !resolved.to_s.empty?
+
+      raise ForestAdminDatasourceSnowflake::Error,
+            'Snowflake session has no default schema; set Schema=<name> in the connection string ' \
+            'so the datasource can scope introspection to a single schema.'
+    rescue ::ODBC::Error => e
+      raise ForestAdminDatasourceSnowflake::Error,
+            "Could not resolve default Snowflake schema (#{e.message}); set Schema=<name> in the connection string."
+    end
+
+    def apply_session_settings(conn)
+      run_session_statement(conn, "ALTER SESSION SET TIMEZONE = 'UTC'")
+      run_session_statement(conn, "USE SCHEMA #{Utils::Identifier.quote(@schema_override)}") if @schema_override
+
+      return unless @statement_timeout
+
+      seconds = Integer(@statement_timeout)
+      run_session_statement(conn, "ALTER SESSION SET STATEMENT_TIMEOUT_IN_SECONDS = #{seconds}")
+    end
+
+    def run_session_statement(conn, sql)
+      stmt = conn.prepare(sql)
+      begin
+        stmt.execute
+      ensure
+        stmt.drop
+      end
+    end
+
+    def connection_lost?(error)
+      message = error.message.to_s
+      CONNECTION_LOST_PATTERNS.any? { |pattern| pattern.match?(message) }
+    end
+
+    def reset_pool!
+      @pool.reload { |conn| safe_disconnect(conn) }
+    end
+
+    def safe_disconnect(connection)
+      connection.disconnect if connection.respond_to?(:disconnect)
+    rescue StandardError
+      nil
+    end
+
+    def generate_collections
+      visible_tables.each do |table_name|
+        add_collection(Collection.new(self, table_name))
+      end
+    end
+
+    def discover_relations
+      Parser::Relation.discover(self).each do |fk|
+        source = collections[fk[:source_table]]
+        next if source.nil? || collections[fk[:target_table]].nil?
+
+        relation = ForestAdminDatasourceToolkit::Schema::Relations::ManyToOneSchema.new(
+          foreign_collection: fk[:target_table],
+          foreign_key: fk[:source_column],
+          foreign_key_target: fk[:target_column]
+        )
+        relation_name = "#{fk[:source_column]}_#{fk[:target_table]}".downcase
+        source.add_field(relation_name, relation)
+      end
+    rescue ::ODBC::Error => e
+      warn "[forest_admin_datasource_snowflake] FK introspection skipped: #{e.message}"
+    end
+
+    def visible_tables
+      with_connection do |conn|
+        stmt = conn.tables
+        rows = begin
+          stmt.fetch_all || []
+        ensure
+          stmt.drop
+        end
+
+        rows
+          .map { |row| { catalog: row[0], schema: row[1], name: row[2], type: row[3] } }
+          .reject { |t| SYSTEM_SCHEMAS.include?(t[:schema].to_s.upcase) }
+          .reject { |t| t[:type].to_s.upcase == 'SYSTEM TABLE' }
+          .select { |t| @schema_override.nil? || t[:schema].to_s.casecmp(@schema_override).zero? }
+          .map { |t| t[:name] }
+      end
+    end
+  end
+end

data/lib/forest_admin_datasource_snowflake/parser/column.rb

@@ -0,0 +1,85 @@
+module ForestAdminDatasourceSnowflake
+  module Parser
+    module Column
+      include ForestAdminDatasourceToolkit::Components::Query::ConditionTree
+
+      SNOWFLAKE_NATIVE_TYPE_TO_FOREST = {
+        'NUMBER' => 'Number',
+        'DECIMAL' => 'Number',
+        'NUMERIC' => 'Number',
+        'INT' => 'Number',
+        'INTEGER' => 'Number',
+        'BIGINT' => 'Number',
+        'SMALLINT' => 'Number',
+        'TINYINT' => 'Number',
+        'BYTEINT' => 'Number',
+        'FLOAT' => 'Number',
+        'FLOAT4' => 'Number',
+        'FLOAT8' => 'Number',
+        'DOUBLE' => 'Number',
+        'DOUBLE PRECISION' => 'Number',
+        'REAL' => 'Number',
+        'BOOLEAN' => 'Boolean',
+        'TEXT' => 'String',
+        'VARCHAR' => 'String',
+        'CHAR' => 'String',
+        'CHARACTER' => 'String',
+        'STRING' => 'String',
+        'DATE' => 'Dateonly',
+        'TIME' => 'Time',
+        'DATETIME' => 'Date',
+        'TIMESTAMP' => 'Date',
+        'TIMESTAMP_NTZ' => 'Date',
+        'TIMESTAMP_LTZ' => 'Date',
+        'TIMESTAMP_TZ' => 'Date',
+        'VARIANT' => 'Json',
+        'OBJECT' => 'Json',
+        'ARRAY' => 'Json',
+        'BINARY' => 'Binary',
+        'VARBINARY' => 'Binary',
+        'GEOGRAPHY' => 'String',
+        'GEOMETRY' => 'String',
+        'VECTOR' => 'String'
+      }.freeze
+
+      module_function
+
+      def forest_type_for_snowflake_native(snowflake_type)
+        SNOWFLAKE_NATIVE_TYPE_TO_FOREST[snowflake_type.to_s.upcase] || 'String'
+      end
+
+      def operators_for_column_type(type)
+        result = [Operators::PRESENT, Operators::BLANK, Operators::MISSING]
+        equality = [Operators::EQUAL, Operators::NOT_EQUAL, Operators::IN, Operators::NOT_IN]
+        orderables = [
+          Operators::LESS_THAN,
+          Operators::GREATER_THAN,
+          Operators::LESS_THAN_OR_EQUAL,
+          Operators::GREATER_THAN_OR_EQUAL
+        ]
+        strings = [
+          Operators::CONTAINS,
+          Operators::I_CONTAINS,
+          Operators::NOT_CONTAINS,
+          Operators::NOT_I_CONTAINS,
+          Operators::STARTS_WITH,
+          Operators::I_STARTS_WITH,
+          Operators::ENDS_WITH,
+          Operators::I_ENDS_WITH,
+          Operators::LIKE,
+          Operators::I_LIKE,
+          Operators::SHORTER_THAN,
+          Operators::LONGER_THAN
+        ]
+
+        return result unless type.is_a?(String)
+
+        result += equality if %w[Boolean Binary Enum Uuid Json].include?(type)
+        result += equality + orderables if %w[Date Dateonly Time Number].include?(type)
+        result += equality + orderables + strings if type == 'String'
+
+        result
+      end
+    end
+  end
+end

data/lib/forest_admin_datasource_snowflake/parser/relation.rb

@@ -0,0 +1,35 @@
+module ForestAdminDatasourceSnowflake
+  module Parser
+    module Relation
+      PK_TABLE_NAME_IDX  = 3
+      PK_COLUMN_NAME_IDX = 4
+      FK_TABLE_NAME_IDX  = 7
+      FK_COLUMN_NAME_IDX = 8
+
+      QUERY = 'SHOW IMPORTED KEYS IN SCHEMA'.freeze
+
+      module_function
+
+      def discover(datasource)
+        rows = datasource.with_connection do |conn|
+          stmt = conn.prepare(QUERY)
+          begin
+            stmt.execute
+            stmt.fetch_all || []
+          ensure
+            stmt.drop
+          end
+        end
+
+        rows.map do |row|
+          {
+            source_table: row[FK_TABLE_NAME_IDX],
+            source_column: row[FK_COLUMN_NAME_IDX],
+            target_table: row[PK_TABLE_NAME_IDX],
+            target_column: row[PK_COLUMN_NAME_IDX]
+          }
+        end
+      end
+    end
+  end
+end

data/lib/forest_admin_datasource_snowflake/utils/identifier.rb

@@ -0,0 +1,14 @@
+module ForestAdminDatasourceSnowflake
+  module Utils
+    module Identifier
+      module_function
+
+      def quote(name)
+        s = name.to_s
+        return s if s == '*'
+
+        %("#{s.gsub('"', '""')}")
+      end
+    end
+  end
+end