activerecord-pinot-adapter 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 34d05b55c0db37ef45a48eb13874ea21fd328e0d68fbdf1cfda03371401ff123
-  data.tar.gz: 0732be5d8c9cfc27ab00017d8daee56e07b1e8924996919bd8e673553cc8a635
+  metadata.gz: c09ea0dd800fdfd76f57fae011fbe90ef3e42498ea51b58d86cfcd0b22611a89
+  data.tar.gz: e336a92e3d15be236bfbfea8cb775be28f9d8b459e8ff04a92fd92071d9915a5
 SHA512:
-  metadata.gz: 88cbf5c67cc7dcdf077aaa5ab65b9da04926861acbc588793017f07b06b7cf316fc6984f59e83b36f17ab16543a1521efaa73fa390af85b566ec49641962f2db
-  data.tar.gz: 104d88ad0b25397026ab01f5a8d6a9f4aa2373adde0de49ba5166b60eeeaccb3605ccfa0b7a1f86ae92a25137c03c58aa85d2cba480a8e3b9790ea8575105e2c
+  metadata.gz: 33d9d6d982ec4b36813addc7a4e9463cc1279befb2e1692d7a66aa634e306bb5d7ad7a330c71af8ca3cbb542caf689a02ad4a581b6905f3d4da8a8d36caf7cd5
+  data.tar.gz: 63a9ab22309ef70b77a9ef72d319a1311f288c1adb62f72821e2f258bf5465695e228d6e8589bcfcca51479914c415f43c73d451e5f31ccc4510c10591acb115

data/Guardfile

@@ -0,0 +1,42 @@
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+## Uncomment and set this to only include directories you want to watch
+# directories %w(app lib config test spec features) \
+#  .select{|d| Dir.exist?(d) ? d : UI.warning("Directory #{d} does not exist")}
+
+## Note: if you are using the `directories` clause above and you are not
+## watching the project directory ('.'), then you will want to move
+## the Guardfile to a watched dir and symlink it back, e.g.
+#
+#  $ mkdir config
+#  $ mv Guardfile config/
+#  $ ln -s config/Guardfile .
+#
+# and, you'll have to watch "config/Guardfile" instead of "Guardfile"
+
+guard :minitest do
+  # with Minitest::Unit
+  watch(%r{^test/(.*)/?test_(.*)\.rb$})
+  watch(%r{^lib/(.*/)?([^/]+)\.rb$}) { |m| "test/#{m[1]}test_#{m[2]}.rb" }
+  watch(%r{^test/test_helper\.rb$}) { "test" }
+
+  # with Minitest::Spec
+  # watch(%r{^spec/(.*)_spec\.rb$})
+  # watch(%r{^lib/(.+)\.rb$})         { |m| "spec/#{m[1]}_spec.rb" }
+  # watch(%r{^spec/spec_helper\.rb$}) { 'spec' }
+
+  # Rails 4
+  # watch(%r{^app/(.+)\.rb$})                               { |m| "test/#{m[1]}_test.rb" }
+  # watch(%r{^app/controllers/application_controller\.rb$}) { 'test/controllers' }
+  # watch(%r{^app/controllers/(.+)_controller\.rb$})        { |m| "test/integration/#{m[1]}_test.rb" }
+  # watch(%r{^app/views/(.+)_mailer/.+})                    { |m| "test/mailers/#{m[1]}_mailer_test.rb" }
+  # watch(%r{^lib/(.+)\.rb$})                               { |m| "test/lib/#{m[1]}_test.rb" }
+  # watch(%r{^test/.+_test\.rb$})
+  # watch(%r{^test/test_helper\.rb$}) { 'test' }
+
+  # Rails < 4
+  # watch(%r{^app/controllers/(.*)\.rb$}) { |m| "test/functional/#{m[1]}_test.rb" }
+  # watch(%r{^app/helpers/(.*)\.rb$})     { |m| "test/helpers/#{m[1]}_test.rb" }
+  # watch(%r{^app/models/(.*)\.rb$})      { |m| "test/unit/#{m[1]}_test.rb" }
+end

data/docker-compose.yml

@@ -0,0 +1,43 @@
+version: '3.7'
+services:
+  pinot-zookeeper:
+    image: zookeeper:3.5.6
+    container_name: pinot-zookeeper
+    ports:
+      - "2181:2181"
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_TICK_TIME: 2000
+  pinot-controller:
+    image: apachepinot/pinot:1.0.0
+    command: "StartController -zkAddress pinot-zookeeper:2181"
+    container_name: pinot-controller
+    restart: unless-stopped
+    ports:
+      - "9000:9000"
+    environment:
+      JAVA_OPTS: "-Dplugins.dir=/opt/pinot/plugins -Xms1G -Xmx4G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xloggc:gc-pinot-controller.log"
+    depends_on:
+      - pinot-zookeeper
+  pinot-broker:
+    image: apachepinot/pinot:1.0.0
+    command: "StartBroker -zkAddress pinot-zookeeper:2181"
+    restart: unless-stopped
+    container_name: "pinot-broker"
+    ports:
+      - "8099:8099"
+    environment:
+      JAVA_OPTS: "-Dplugins.dir=/opt/pinot/plugins -Xms4G -Xmx4G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xloggc:gc-pinot-broker.log"
+    depends_on:
+      - pinot-controller
+  pinot-server:
+    image: apachepinot/pinot:1.0.0
+    command: "StartServer -zkAddress pinot-zookeeper:2181"
+    restart: unless-stopped
+    container_name: "pinot-server"
+    ports:
+      - "8098:8098"
+    environment:
+      JAVA_OPTS: "-Dplugins.dir=/opt/pinot/plugins -Xms4G -Xmx16G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xloggc:gc-pinot-server.log"
+    depends_on:
+      - pinot-broker

data/lib/active_record/connection_adapters/pinot_adapter/table_structure.rb

@@ -0,0 +1,53 @@
+module ActiveRecord
+  module ConnectionAdapters
+    class PinotAdapter < AbstractAdapter
+      module TableStructure
+        def self.from_schema(schema)
+          fields = []
+          schema["dimensionFieldSpecs"].each do |f|
+            fields << {
+              "name" => f["name"],
+              "type" => f["dataType"],
+              "pinot_type" => "dimension"
+            }
+          end
+          schema["metricFieldSpecs"].each do |f|
+            fields << {
+              "name" => f["name"],
+              "type" => f["dataType"],
+              "pinot_type" => "metric"
+            }
+          end
+          schema["dateTimeFieldSpecs"].each do |f|
+            fields << {
+              "name" => f.delete("name"),
+              "type" => f.delete("dataType"),
+              "pinot_type" => "dateTime",
+              "metadata" => f
+            }
+          end
+          # normalize values
+          fields.each do |f|
+            f["type"] = normalize_type(f["type"])
+          end
+          # set primary keys
+          primary_key_columns = schema["primaryKeyColumns"] || []
+          primary_key_columns.each do |pk|
+            pk_fields = fields.select { |x| x["name"] == pk }
+            pk_fields.each { |field| field["pk"] = 1 }
+          end
+          fields
+        end
+
+        def self.normalize_type(type)
+          case type
+          when "INT" then "integer"
+          when "STRING" then "varchar"
+          else
+            type
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_record/connection_adapters/pinot_adapter.rb

@@ -0,0 +1,129 @@
+require_relative "pinot_adapter/table_structure"
+
+module ActiveRecord
+  module ConnectionHandling # :nodoc:
+    def pinot_adapter_class
+      ConnectionAdapters::PinotAdapter
+    end
+
+    def pinot_connection(config)
+      pinot_adapter_class.new(config)
+    end
+  end
+
+  module ConnectionAdapters
+    class PinotAdapter < AbstractAdapter
+      TYPES = {
+        "INT" => Type::Integer.new,
+        "TIMESTAMP" => Type::DateTime.new,
+        "FLOAT" => Type::Decimal.new,
+        "LONG" => Type::Decimal.new
+      }
+      def initialize(config = {})
+        @pinot_host = config.fetch(:host)
+        @pinot_port = config.fetch(:port)
+        @pinot_controller_port = config.fetch(:controller_port)
+        @pinot_controller_host = config.fetch(:controller_host) || @pinot_host
+        # TODO: does it need connection pooling?
+        @pinot_client = ::Pinot::Client.new(host: @pinot_host, port: @pinot_port, controller_host: @pinot_controller_host, controller_port: @pinot_controller_port)
+
+        super(config)
+      end
+
+      def default_prepared_statements
+        false
+      end
+
+      def table_structure(table_name)
+        schema = @pinot_client.schema(table_name)
+        @table_structure = TableStructure.from_schema(schema)
+        @table_structure.sort_by! { |x| x[:name] }
+      end
+
+      def new_column_from_field(table_name, field, definitions)
+        default = nil
+
+        type_metadata = fetch_type_metadata(field["type"])
+        default_value = extract_value_from_default(default)
+        default_function = extract_default_function(default_value, default)
+
+        Column.new(
+          field["name"],
+          default_value,
+          type_metadata,
+          field["notnull"].to_i == 0,
+          default_function,
+          collation: field["collation"]
+        )
+      end
+      alias_method :column_definitions, :table_structure
+
+      def extract_value_from_default(default)
+        case default
+        when /^null$/i
+          nil
+        # Quoted types
+        when /^'([^|]*)'$/m
+          $1.gsub("''", "'")
+        # Quoted types
+        when /^"([^|]*)"$/m
+          $1.gsub('""', '"')
+        # Numeric types
+        when /\A-?\d+(\.\d*)?\z/
+          $&
+        # Binary columns
+        when /x'(.*)'/
+          [$1].pack("H*")
+        else
+          # Anything else is blank or some function
+          # and we can't know the value of that, so return nil.
+          nil
+        end
+      end
+
+      def extract_default_function(default_value, default)
+        default if has_default_function?(default_value, default)
+      end
+
+      def has_default_function?(default_value, default)
+        !default_value && %r{\w+\(.*\)|CURRENT_TIME|CURRENT_DATE|CURRENT_TIMESTAMP|\|\|}.match?(default)
+      end
+
+      INTEGER_REGEX = /integer/i
+      def is_column_the_rowid?(field, column_definitions)
+        return false unless INTEGER_REGEX.match?(field["type"]) && field["pk"] == 1
+        # is the primary key a single column?
+        column_definitions.one? { |c|
+          col_pk = c["pk"] || 0
+          col_pk > 0
+        }
+      end
+
+      def internal_exec_query(sql, name = "SQL", binds = [], prepare: false, async: false) # :nodoc:
+        # rows = [
+        #   [Time.now, 1, 2.0],
+        #   [Time.now, 2, 2.0],
+        #   [Time.now, 2, 2.0]
+        # ]
+        response = @pinot_client.execute(sql)
+        rows = response.rows
+        columns = response.columns
+        columns.transform_values! { |value| TYPES.fetch(value, value) }
+        ActiveRecord::Result.new(
+          response.columns.keys,
+          rows.to_a,
+          columns
+        )
+      end
+
+      def data_source_sql(name = nil, type: nil)
+      end
+
+      def primary_keys(table_name)
+        []
+      end
+    end
+
+    ActiveSupport.run_load_hooks(:active_record_pinotadapter, PinotAdapter)
+  end
+end

data/lib/active_record/pinot/adapter.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require "pinot"
+require "active_record"
+require_relative "../connection_adapters/pinot_adapter"
+
+module ActiveRecord
+  module Pinot
+    module Adapter
+      class Error < StandardError; end
+      # Your code goes here...
+    end
+  end
+end

data/lib/activerecord/pinot/adapter/version.rb

@@ -3,7 +3,7 @@
 module Activerecord
   module Pinot
     module Adapter
-      VERSION = "0.1.0"
+      VERSION = "0.1.1"
     end
   end
 end

data/lib/activerecord/pinot/adapter.rb

@@ -1,12 +1,6 @@
 # frozen_string_literal: true
 
 require_relative "adapter/version"
-
-module Activerecord
-  module Pinot
-    module Adapter
-      class Error < StandardError; end
-      # Your code goes here...
-    end
-  end
-end
+require "pinot"
+require "active_record"
+require "active_record/pinot/adapter"

metadata

@@ -1,15 +1,105 @@
 --- !ruby/object:Gem::Specification
 name: activerecord-pinot-adapter
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Celso Fernandes
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-01-16 00:00:00.000000000 Z
-dependencies: []
+date: 2024-05-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.2.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.2.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7.2'
+- !ruby/object:Gem::Dependency
+  name: pinot
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest-reporters
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest-focus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: ActiveRecord Apache Pinot Adapter
 email:
 - celso.fernandes@clickfunnels.com
@@ -20,9 +110,13 @@ files:
 - ".standard.yml"
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
+- Guardfile
 - README.md
 - Rakefile
-- activerecord-pinot-adapter.gemspec
+- docker-compose.yml
+- lib/active_record/connection_adapters/pinot_adapter.rb
+- lib/active_record/connection_adapters/pinot_adapter/table_structure.rb
+- lib/active_record/pinot/adapter.rb
 - lib/activerecord/pinot/adapter.rb
 - lib/activerecord/pinot/adapter/version.rb
 - sig/activerecord/pinot/adapter.rbs
@@ -47,7 +141,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.21
+rubygems_version: 3.5.3
 signing_key:
 specification_version: 4
 summary: ActiveRecord Apache Pinot Adapter

data/activerecord-pinot-adapter.gemspec

@@ -1,37 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "lib/activerecord/pinot/adapter/version"
-
-Gem::Specification.new do |spec|
-  spec.name = "activerecord-pinot-adapter"
-  spec.version = Activerecord::Pinot::Adapter::VERSION
-  spec.authors = ["Celso Fernandes"]
-  spec.email = ["celso.fernandes@clickfunnels.com"]
-
-  spec.summary = "ActiveRecord Apache Pinot Adapter"
-  spec.description = "ActiveRecord Apache Pinot Adapter"
-  spec.homepage = "https://github.com/fernandes/activerecord-pinot-adapter"
-  spec.required_ruby_version = ">= 2.6.0"
-
-  spec.metadata["homepage_uri"] = spec.homepage
-  spec.metadata["source_code_uri"] = "https://github.com/fernandes/activerecord-pinot-adapter"
-  spec.metadata["changelog_uri"] = "https://github.com/fernandes/activerecord-pinot-adapter/blob/main/CHANGELOG.md"
-
-  # Specify which files should be added to the gem when it is released.
-  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
-  spec.files = Dir.chdir(__dir__) do
-    `git ls-files -z`.split("\x0").reject do |f|
-      (File.expand_path(f) == __FILE__) ||
-        f.start_with?(*%w[bin/ test/ spec/ features/ .git .circleci appveyor Gemfile])
-    end
-  end
-  spec.bindir = "exe"
-  spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
-  spec.require_paths = ["lib"]
-
-  # Uncomment to register a new dependency of your gem
-  # spec.add_dependency "example-gem", "~> 1.0"
-
-  # For more information and examples about making a new gem, check out our
-  # guide at: https://bundler.io/guides/creating_gem.html
-end