quill-sql 0.1.0

data/lib/quill-sql.rb

@@ -0,0 +1,370 @@
+# Ruby 2.3 Compatible Implementation
+require 'json'
+require 'net/http'
+require 'uri'
+require 'dotenv'
+require_relative './db/cached_connection'
+require_relative './db/db_helper'
+require_relative 'utils/tenants'
+require_relative 'models/filters'
+require_relative 'utils/schema'
+require_relative 'utils/run_query_processes'
+
+Dotenv.load(File.expand_path('../.env', __dir__))
+
+module DatabaseType
+  POSTGRESQL = 'postgresql'.freeze
+  SNOWFLAKE = 'snowflake'.freeze
+  BIGQUERY = 'bigquery'.freeze
+  MYSQL = 'mysql'.freeze
+  CLICKHOUSE = 'clickhouse'.freeze
+
+  def self.valid?(type)
+    constants.map { |c| const_get(c) }.include?(type)
+  end
+end
+
+SINGLE_TENANT = "QUILL_SINGLE_TENANT".freeze
+ALL_TENANTS = "QUILL_ALL_TENANTS".freeze
+
+HOST = (ENV['ENV'] == 'development' ? 
+    'http://localhost:8080' : 
+    'https://quill-344421.uc.r.appspot.com').freeze
+
+FLAG_TASKS = Set.new(['dashboard', 'report', 'item', 'report-info', 'filter-options']).freeze
+
+class QuillAPIError < StandardError; end
+
+class Quill
+  attr_reader :target_connection, :base_url, :config
+
+  def initialize(options = {})
+    validate_options(options)    
+    @base_url = options[:metadata_server_url] || HOST
+    @config = { headers: { 'Authorization' => "Bearer #{options[:private_key]}" } }
+    
+    credentials = options[:database_config]
+    if options[:database_connection_string]
+      credentials = DatabaseHelper.get_database_credentials(
+        options[:database_type],
+        options[:database_connection_string]
+      )
+    end
+
+    @target_connection = CachedConnection.new(
+      options[:database_type],
+      credentials,
+      options[:cache] || {}
+    )
+  end
+
+  def query(params = {})
+    validate_query_params(params)
+
+    tenants = params[:tenants]
+    flags = params[:flags]
+    metadata = params[:metadata]
+    metadata = metadata.transform_keys(&:to_sym)
+    filters = params[:filters]
+
+    # TODO: Add support for multiple tenants
+    if tenants&.any?
+      @target_connection.tenant_ids = TenantUtils.extract_tenant_ids(tenants)
+    end
+
+    response_metadata = {}
+
+    unless metadata[:task]
+      return { error: "Missing task.", status: "error", data: {} }
+    end
+
+    tenant_flags = nil
+
+    begin
+      # If the task requires flags to be synthesized from tenants
+      if FLAG_TASKS.include?(metadata[:task]) &&
+         tenants&.first != ALL_TENANTS &&
+         tenants&.first != SINGLE_TENANT &&
+         (metadata[:task] != 'filter-options' ||
+          !metadata[:reportId])
+         
+        response = post_quill('tenant-mapped-flags', {
+          reportId: metadata[:reportId] || metadata[:dashboardItemId],
+          dashboardName: metadata[:name],
+          tenants: tenants,
+          flags: flags
+        })
+
+        return {
+          status: "error",
+          error: response[:error],
+          data: response[:metadata] || {}
+        } if response[:error]
+
+        flag_query_results = run_queries(response[:queries], @target_connection.database_type)
+        tenant_flags = response[:metadata][:queryOrder].map.with_index do |tenant_field, index|
+          {
+            tenant_field: tenant_field,
+            flags: Set.new(
+              flag_query_results[:queryResults][index][:rows].map do |row|
+                row['quill_flag']
+              end
+            ).to_a
+          }
+        end
+      elsif tenants&.first == SINGLE_TENANT && flags
+        if flags.length && flags[0].is_a?(Hash)
+          tenant_flags = [{ tenant_field: SINGLE_TENANT, flags: flags }]
+        else
+          tenant_flags = flags.map do |flag|
+            { tenant_field: SINGLE_TENANT, flags: flag }
+          end
+        end
+      end
+
+      pre_query_results = if metadata[:preQueries]
+                            run_queries(
+                              metadata[:preQueries],
+                              @target_connection.database_type,
+                              metadata[:databaseType],
+                              metadata[:runQueryConfig]
+                            )
+                          else
+                            {}
+                          end
+
+      if metadata.dig(:runQueryConfig, "overridePost")
+        return {
+          data: { queryResults: pre_query_results },
+          status: "success"
+        }
+      end
+
+      response = post_quill(metadata[:task], {
+        **metadata,
+        sdk_filters: filters&.map { |filter| FilterUtils.convert_custom_filter(filter) },
+        **pre_query_results,
+        tenants: tenants,
+        flags: tenant_flags,
+        viewQuery: metadata[:preQueries]&.first
+      })
+
+      return {
+        status: "error",
+        error: response[:error],
+        data: response[:metadata] || {}
+      } if response[:error]
+
+      response_metadata = response[:metadata] if response[:metadata]
+
+      results = run_queries(
+        response[:queries],
+        @target_connection.database_type,
+        metadata[:database_type],
+        response_metadata.dig(:runQueryConfig)
+      )
+      if results[:mappedArray] && response_metadata.dig(:runQueryConfig, :arrayToMap)
+        array_to_map = response_metadata.dig(:runQueryConfig, :arrayToMap)
+        results[:mappedArray].each_with_index do |array, index|
+          target_item = response_metadata.dig(array_to_map[:arrayName].to_sym, index)
+          if target_item
+            target_item[array_to_map[:field]] = array
+          end
+        end
+        results.delete(:mappedArray)
+      end
+
+      if results[:queryResults]&.size == 1
+        query_results = results[:queryResults].first
+        response_metadata[:rows] = query_results[:rows] if query_results[:rows]
+        response_metadata[:fields] = query_results[:fields] if query_results[:fields]
+      end
+
+      {
+        data: response_metadata,
+        queries: results,
+        status: "success"
+      }
+    rescue StandardError => e
+      if metadata[:task] == "update-view"
+        post_quill("set-broken-view", {
+          table: metadata[:name],
+          clientId: metadata[:clientId],
+          error: e.message
+        })
+      end
+
+      {
+        status: "error",
+        error: e.message,
+        data: response_metadata || {}
+      }
+    end
+  end
+
+  private
+
+  def validate_options(options)
+    if !options[:private_key]
+      raise ArgumentError, "Private key is required"
+    end
+    if !options[:database_type]
+      raise ArgumentError, "Database type is required"
+    end
+    if !DatabaseType.valid?(options[:database_type])
+      raise ArgumentError, "Invalid database type"
+    end
+    if !options[:database_connection_string] && !options[:database_config]
+      raise ArgumentError, "Either database_connection_string or database_config is required"
+    end
+  end
+
+  def validate_query_params(params)
+    if !params[:tenants]
+      raise ArgumentError, "tenants is required"
+    end
+    if !params[:metadata]
+      raise ArgumentError, "metadata is required"
+    end
+  end
+
+  def run_queries(queries, pk_database_type, database_type = nil, run_query_config = nil)
+    return { queryResults: [] } unless queries
+  
+    if database_type && database_type.downcase != pk_database_type.downcase
+      return {
+        dbMismatched: true,
+        backendDatabaseType: pk_database_type,
+        queryResults: []
+      }
+    end
+  
+    results = {}
+
+    run_query_config = run_query_config.transform_keys(&:to_sym) if run_query_config
+  
+    if run_query_config&.dig(:arrayToMap)
+      mapped_array = RunQueryProcesses.map_queries(queries, @target_connection)
+      results[:queryResults] = []
+      results[:mappedArray] = mapped_array
+  
+    elsif run_query_config&.dig(:getColumns)
+      query_result = @target_connection.query("#{queries[0].gsub(/;/, '')} limit 1000")
+      columns = query_result[:fields].map do |field|
+        {
+          fieldType: Schema.convert_type_to_postgres(field[:dataTypeID]),
+          name: field[:name],
+          displayName: field[:name],
+          isVisible: true,
+          field: field[:name]
+        }
+      end
+      results[:columns] = columns  
+    elsif run_query_config&.dig(:getColumnsForSchema)
+      query_results = queries.map do |table|
+        if table[:viewQuery].nil? || (!table[:isSelectStar] && !table[:customFieldInfo])
+          table
+        else
+          limit = run_query_config[:limitBy] ? " limit #{run_query_config[:limitBy]}" : ""
+          begin
+            query_result = @target_connection.query("#{table[:viewQuery].gsub(/;/, '')} #{limit}")
+            columns = query_result[:fields].map do |field|
+              {
+                fieldType: Schema.convert_type_to_postgres(field[:dataTypeID]),
+                name: field[:name],
+                displayName: field[:name],
+                isVisible: true,
+                field: field[:name]
+              }
+            end
+            table.merge(columns: columns, rows: query_result[:rows])
+          rescue StandardError => e
+            table.merge(error: "Error fetching columns: #{e.message}")
+          end
+        end
+      end
+  
+      results[:queryResults] = query_results
+  
+      if run_query_config&.dig(:fieldsToRemove)
+        results[:queryResults] = query_results.map do |table|
+          removed_columns = table[:columns]&.reject { |column| run_query_config[:fieldsToRemove]&.include?(column[:name]) }
+          table.merge(columns: removed_columns)
+        end
+      end
+  
+    elsif run_query_config&.dig(:getTables)
+      schema_names = run_query_config[:schemaNames] || run_query_config[:schema]
+      tables_info = DatabaseHelper.get_tables_by_schema_by_database(@target_connection.database_type, @target_connection.pool, schema_names)
+      schema_info = DatabaseHelper.get_column_info_by_schema_by_database(@target_connection.database_type, @target_connection.pool, run_query_config[:schema], tables_info)
+      return schema_info
+  
+    else
+      modified_queries = queries
+      if run_query_config&.dig(:limitThousand)
+        modified_queries = queries.map { |q| "#{q.gsub(/;/, '')} limit 1000;" }
+      elsif run_query_config&.dig(:limitBy)
+        modified_queries = queries.map { |q| "#{q.gsub(/;/, '')} limit #{run_query_config[:limitBy]};" }
+      end
+  
+      query_results = modified_queries.map { |query| @target_connection.query(query) }
+      results[:queryResults] = query_results
+  
+      if run_query_config&.dig(:fieldsToRemove)
+        results[:queryResults] = query_results.map do |result|
+          RunQueryProcesses.remove_fields(result, run_query_config[:fieldsToRemove])
+        end
+      end
+
+      if run_query_config&.dig(:convertDatatypes)
+        results = query_results.map do |result|
+          {
+            fields: result[:fields].map do |field|
+              field.merge(
+                fieldType: Schema.convert_type_to_postgres(field[:dataTypeID]),
+                isVisible: true,
+                field: field[:name],
+                displayName: field[:name],
+                name: field[:name]
+              )
+            end,
+            rows: result[:rows]
+          }
+        end
+      end
+    end
+
+    results
+  end
+
+  def post_quill(path, payload)
+    uri = URI("#{@base_url}/sdk/#{path}")
+    http = Net::HTTP.new(uri.host, uri.port)
+    http.use_ssl = uri.scheme == 'https'
+    request = Net::HTTP::Post.new(uri)
+    request['Authorization'] = @config[:headers]["Authorization"]
+    request['Content-Type'] = 'application/json'
+    request.body = payload.to_json
+
+    response = http.request(request)
+    
+    if !response.is_a?(Net::HTTPSuccess)
+      body = JSON.parse(response.body, symbolize_names: true)
+      raise QuillAPIError.new("#{body[:error]}")
+    end
+    
+    JSON.parse(response.body, symbolize_names: true)
+  rescue JSON::ParserError => e
+    raise QuillAPIError.new("Invalid JSON response: #{e.message}")
+  rescue => e
+    raise QuillAPIError.new(e.message)
+  end
+  
+  def async_dispose
+    close
+  end
+
+  def close
+    @target_connection&.close
+  end
+end

data/lib/utils/run_query_processes.rb

@@ -0,0 +1,29 @@
+class TableSchemaInfo
+  attr_accessor :field_type, :name, :display_name, :is_visible
+
+  def initialize(field_type:, name:, display_name:, is_visible:)
+    @field_type = field_type
+    @name = name
+    @display_name = display_name
+    @is_visible = is_visible
+  end
+end
+
+module RunQueryProcesses
+  def self.remove_fields(query_results, fields_to_remove)
+    fields = query_results[:fields].reject { |field| fields_to_remove.include?(field[:name]) }
+    rows = query_results[:rows].map do |row|
+      fields_to_remove.each { |field| row.delete(field) }
+      row
+    end
+    { fields: fields, rows: rows }
+  end
+
+  def self.map_queries(queries, target_connection)
+    queries.map do |query|
+      result = target_connection.query(query)
+      result[:rows]
+    end
+  end
+  
+end

data/lib/utils/schema.rb

@@ -0,0 +1,8 @@
+require_relative '../assets/pg_types'
+
+module Schema
+  def self.convert_type_to_postgres(data_type_id)
+    type = PG_TYPES.find { |t| data_type_id == t[:oid] }&.dig(:typname)
+    type || 'varchar'
+  end
+end

data/lib/utils/tenants.rb

@@ -0,0 +1,21 @@
+module TenantUtils
+  def self.extract_tenant_ids(tenants)
+    if tenants[0].is_a?(String) || tenants[0].is_a?(Numeric)
+      tenants
+    elsif tenants[0].is_a?(Hash) && tenants[0].key?('tenant_ids')
+      tenants[0]['tenant_ids']
+    else
+      raise 'Invalid format for tenants'
+    end
+  end
+
+  def self.extract_tenant_field(tenants, dashboard_owner)
+    if tenants[0].is_a?(String) || tenants[0].is_a?(Numeric)
+      dashboard_owner
+    elsif tenants[0].is_a?(Hash) && tenants[0].key?('tenant_field')
+      tenants[0]['tenant_field']
+    else
+      raise 'Invalid format for tenants'
+    end
+  end
+end

metadata

@@ -0,0 +1,167 @@
+--- !ruby/object:Gem::Specification
+name: quill-sql
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Shawn Magee, Albert Yan
+- Sam Bishop
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2025-01-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 7.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 7.0.0
+- !ruby/object:Gem::Dependency
+  name: click_house
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.2
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: dotenv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sinatra
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sinatra-cors
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Quill sdk for Ruby. See quill.co for more information.
+email:
+- shawn@quill.co
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- lib/assets/pg_types.rb
+- lib/db/cached_connection.rb
+- lib/db/clickhouse.rb
+- lib/db/db_helper.rb
+- lib/models/filters.rb
+- lib/quill-sql.rb
+- lib/utils/run_query_processes.rb
+- lib/utils/schema.rb
+- lib/utils/tenants.rb
+homepage: https://github.com/quill-sql/quill-ruby
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.7.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3.1
+signing_key: 
+specification_version: 4
+summary: Quill
+test_files: []