activerecord-pinot-adapter 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 34d05b55c0db37ef45a48eb13874ea21fd328e0d68fbdf1cfda03371401ff123
-  data.tar.gz: 0732be5d8c9cfc27ab00017d8daee56e07b1e8924996919bd8e673553cc8a635
+  metadata.gz: 28c4f9381e73ca7e68d549d47f4a7dff6b7740b33bd965833e0397f07e27c3da
+  data.tar.gz: 05220a45210bd4e03329891870bb428c4f613ea60a4d4367c7d9bccf1b1d66f2
 SHA512:
-  metadata.gz: 88cbf5c67cc7dcdf077aaa5ab65b9da04926861acbc588793017f07b06b7cf316fc6984f59e83b36f17ab16543a1521efaa73fa390af85b566ec49641962f2db
-  data.tar.gz: 104d88ad0b25397026ab01f5a8d6a9f4aa2373adde0de49ba5166b60eeeaccb3605ccfa0b7a1f86ae92a25137c03c58aa85d2cba480a8e3b9790ea8575105e2c
+  metadata.gz: a4d61494dcafd454bfac76959c8e77bab174cc9472187dd19bd0715cf8ba8786b610afb86852b19c55694709ca2e2b6ab40076ebd25be28f3ea4a4bb127281aa
+  data.tar.gz: 2b723803e42dd71eab40751e15c34246fd9eca74593fc87fc4318565c0a46eccf60f291bc7efcac1bd8a5cf36c79c39fc1f09dafd27e6e82a6945e0af79d013b

data/Guardfile

@@ -0,0 +1,42 @@
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+## Uncomment and set this to only include directories you want to watch
+# directories %w(app lib config test spec features) \
+#  .select{|d| Dir.exist?(d) ? d : UI.warning("Directory #{d} does not exist")}
+
+## Note: if you are using the `directories` clause above and you are not
+## watching the project directory ('.'), then you will want to move
+## the Guardfile to a watched dir and symlink it back, e.g.
+#
+#  $ mkdir config
+#  $ mv Guardfile config/
+#  $ ln -s config/Guardfile .
+#
+# and, you'll have to watch "config/Guardfile" instead of "Guardfile"
+
+guard :minitest do
+  # with Minitest::Unit
+  watch(%r{^test/(.*)/?test_(.*)\.rb$})
+  watch(%r{^lib/(.*/)?([^/]+)\.rb$}) { |m| "test/#{m[1]}test_#{m[2]}.rb" }
+  watch(%r{^test/test_helper\.rb$}) { "test" }
+
+  # with Minitest::Spec
+  # watch(%r{^spec/(.*)_spec\.rb$})
+  # watch(%r{^lib/(.+)\.rb$})         { |m| "spec/#{m[1]}_spec.rb" }
+  # watch(%r{^spec/spec_helper\.rb$}) { 'spec' }
+
+  # Rails 4
+  # watch(%r{^app/(.+)\.rb$})                               { |m| "test/#{m[1]}_test.rb" }
+  # watch(%r{^app/controllers/application_controller\.rb$}) { 'test/controllers' }
+  # watch(%r{^app/controllers/(.+)_controller\.rb$})        { |m| "test/integration/#{m[1]}_test.rb" }
+  # watch(%r{^app/views/(.+)_mailer/.+})                    { |m| "test/mailers/#{m[1]}_mailer_test.rb" }
+  # watch(%r{^lib/(.+)\.rb$})                               { |m| "test/lib/#{m[1]}_test.rb" }
+  # watch(%r{^test/.+_test\.rb$})
+  # watch(%r{^test/test_helper\.rb$}) { 'test' }
+
+  # Rails < 4
+  # watch(%r{^app/controllers/(.*)\.rb$}) { |m| "test/functional/#{m[1]}_test.rb" }
+  # watch(%r{^app/helpers/(.*)\.rb$})     { |m| "test/helpers/#{m[1]}_test.rb" }
+  # watch(%r{^app/models/(.*)\.rb$})      { |m| "test/unit/#{m[1]}_test.rb" }
+end

data/README.md

@@ -1,24 +1,63 @@
-# Activerecord::Pinot::Adapter
+# Pinot Adapter
 
-TODO: Delete this and the text below, and describe your gem
+Ruby on Rails Active Record database adapter for Apache Pinot, a client library for Pinot-compatible database servers.
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/activerecord/pinot/adapter`. To experiment with that code, run `bin/console` for an interactive prompt.
+Disclaimer: This is alpha stage software, please use with caution.
 
 ## Installation
 
-TODO: Replace `UPDATE_WITH_YOUR_GEM_NAME_PRIOR_TO_RELEASE_TO_RUBYGEMS_ORG` with your gem name right after releasing it to RubyGems.org. Please do not do it earlier due to security reasons. Alternatively, replace this section with instructions to install your gem from git if you don't plan to release to RubyGems.org.
-
 Install the gem and add to the application's Gemfile by executing:
 
-    $ bundle add UPDATE_WITH_YOUR_GEM_NAME_PRIOR_TO_RELEASE_TO_RUBYGEMS_ORG
+    $ bundle add activerecord-pinot-adapter
 
 If bundler is not being used to manage dependencies, install the gem by executing:
 
-    $ gem install UPDATE_WITH_YOUR_GEM_NAME_PRIOR_TO_RELEASE_TO_RUBYGEMS_ORG
+    $ gem install activerecord-pinot-adapter
+
+This adapter uses the [pinot](https://rubygems.org/gems/pinot) gem, that is the Apache Pinot client, you can install an specific version of the client, but it's highly recommended to let this gem manage its own dependency.
 
 ## Usage
 
 TODO: Write usage instructions here
+First step is to configure the `config/database.yml`
+
+```yaml
+pinot:
+  adapter: pinot
+  host: host.to.your.broker.com
+  port: 443
+  controller_host: host.to.your.controller.com
+  controller_port: 443
+  protocol: https
+```
+
+Adapter also supports:
+
+- `socks5_uri` in case you are using are using a jump host as your socks proxy. Please provide the full uri with protocol and port (eg. `socks5://127.0.0.1:8080`)
+- `bearer_token` in case your broker/host need a bearer token to authenticate the requests
+- `query_options` in case needs to specify pinot [query options](https://docs.pinot.apache.org/users/user-guide-query/query-options), it's a hash, and keys are on the _underscore_ format.
+
+By design we avoided trying to guess the port and protocol based on any provided information, so port is always required, and protocol by default is `http` to make easier to test local, and then must be specified as `https` for production.
+
+### Models
+
+To start using on your application, it's recommended to create an abstract class, so it's only needed to specify the connection database in one place.
+
+```ruby
+class PinotRecord < ApplicationRecord
+  self.abstract_class = true
+  connects_to database: {writing: :pinot, reading: :pinot}
+end
+```
+
+After abstract class, just use it on any class you want to use.
+
+```ruby
+class MyModel < PinotRecord
+end
+```
+
+If you are incrementally migrating from one database to Pinot, or want to use both in parallel, prefix your models on a namespace, and use `self.table_name=` to specify the correct table name for the model.
 
 ## Development
 

data/docker-compose.yml

@@ -0,0 +1,43 @@
+version: '3.7'
+services:
+  pinot-zookeeper:
+    image: zookeeper:3.5.6
+    container_name: pinot-zookeeper
+    ports:
+      - "2181:2181"
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_TICK_TIME: 2000
+  pinot-controller:
+    image: apachepinot/pinot:1.0.0
+    command: "StartController -zkAddress pinot-zookeeper:2181"
+    container_name: pinot-controller
+    restart: unless-stopped
+    ports:
+      - "9000:9000"
+    environment:
+      JAVA_OPTS: "-Dplugins.dir=/opt/pinot/plugins -Xms1G -Xmx4G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xloggc:gc-pinot-controller.log"
+    depends_on:
+      - pinot-zookeeper
+  pinot-broker:
+    image: apachepinot/pinot:1.0.0
+    command: "StartBroker -zkAddress pinot-zookeeper:2181"
+    restart: unless-stopped
+    container_name: "pinot-broker"
+    ports:
+      - "8099:8099"
+    environment:
+      JAVA_OPTS: "-Dplugins.dir=/opt/pinot/plugins -Xms4G -Xmx4G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xloggc:gc-pinot-broker.log"
+    depends_on:
+      - pinot-controller
+  pinot-server:
+    image: apachepinot/pinot:1.0.0
+    command: "StartServer -zkAddress pinot-zookeeper:2181"
+    restart: unless-stopped
+    container_name: "pinot-server"
+    ports:
+      - "8098:8098"
+    environment:
+      JAVA_OPTS: "-Dplugins.dir=/opt/pinot/plugins -Xms4G -Xmx16G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xloggc:gc-pinot-server.log"
+    depends_on:
+      - pinot-broker

data/lib/active_record/connection_adapters/pinot_adapter/table_structure.rb

@@ -0,0 +1,54 @@
+module ActiveRecord
+  module ConnectionAdapters
+    class PinotAdapter < AbstractAdapter
+      module TableStructure
+        def self.from_schema(schema)
+          fields = []
+          schema["dimensionFieldSpecs"].each do |f|
+            fields << {
+              "name" => f["name"],
+              "type" => f["dataType"],
+              "pinot_type" => "dimension"
+            }
+          end
+          schema["metricFieldSpecs"] ||= []
+          schema["metricFieldSpecs"].each do |f|
+            fields << {
+              "name" => f["name"],
+              "type" => f["dataType"],
+              "pinot_type" => "metric"
+            }
+          end
+          schema["dateTimeFieldSpecs"].each do |f|
+            fields << {
+              "name" => f.delete("name"),
+              "type" => f.delete("dataType"),
+              "pinot_type" => "dateTime",
+              "metadata" => f
+            }
+          end
+          # normalize values
+          fields.each do |f|
+            f["type"] = normalize_type(f["type"])
+          end
+          # set primary keys
+          primary_key_columns = schema["primaryKeyColumns"] || []
+          primary_key_columns.each do |pk|
+            pk_fields = fields.select { |x| x["name"] == pk }
+            pk_fields.each { |field| field["pk"] = 1 }
+          end
+          fields
+        end
+
+        def self.normalize_type(type)
+          case type
+          when "INT" then "integer"
+          when "STRING" then "varchar"
+          else
+            type
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_record/connection_adapters/pinot_adapter.rb

@@ -0,0 +1,155 @@
+require_relative "pinot_adapter/table_structure"
+
+module ActiveRecord
+  module ConnectionHandling # :nodoc:
+    def pinot_adapter_class
+      ConnectionAdapters::PinotAdapter
+    end
+
+    def pinot_connection(config)
+      pinot_adapter_class.new(config)
+    end
+  end
+
+  module ConnectionAdapters
+    class PinotAdapter < AbstractAdapter
+      TYPES = {
+        "INT" => Type::Integer.new,
+        "TIMESTAMP" => Type::DateTime.new,
+        "FLOAT" => Type::Decimal.new,
+        "LONG" => Type::Decimal.new,
+        "STRING" => Type::String.new,
+        "JSON" => ActiveRecord::Type::Json.new
+      }
+      def initialize(config = {})
+        @pinot_host = config.fetch(:host)
+        @pinot_port = config.fetch(:port)
+        @pinot_controller_port = config.fetch(:controller_port)
+        @pinot_controller_host = config.fetch(:controller_host) || @pinot_host
+        @pinot_socks5_uri = config.fetch(:socks5_uri, nil)
+        @pinot_bearer_token = config.fetch(:bearer_token, nil)
+        @pinot_protocol = config.fetch(:protocol, "http")
+        @pinot_query_options = config.fetch(:query_options, {})
+        # TODO: does it need connection pooling?
+
+        @pinot_client = ::Pinot::Client.new(
+          host: @pinot_host,
+          port: @pinot_port,
+          controller_host: @pinot_controller_host,
+          controller_port: @pinot_controller_port,
+          protocol: @pinot_protocol,
+          socks5_uri: @pinot_socks5_uri,
+          bearer_token: @pinot_bearer_token,
+          query_options: @pinot_query_options
+        )
+
+        super(config)
+      end
+
+      def default_prepared_statements
+        false
+      end
+
+      def prepared_statements
+        false
+      end
+
+      def table_structure(table_name)
+        schema = @pinot_client.schema(table_name)
+        @table_structure = TableStructure.from_schema(schema)
+        @table_structure.sort_by! { |x| x[:name] }
+      end
+
+      def data_sources
+        @pinot_data_sources ||= @pinot_client.tables["tables"]
+      end
+
+      def new_column_from_field(table_name, field, definitions = nil)
+        default = nil
+
+        type_metadata = fetch_type_metadata(field["type"])
+        default_value = extract_value_from_default(default)
+        default_function = extract_default_function(default_value, default)
+
+        Column.new(
+          field["name"],
+          default_value,
+          type_metadata,
+          field["notnull"].to_i == 0,
+          default_function,
+          collation: field["collation"]
+        )
+      end
+      alias_method :column_definitions, :table_structure
+
+      def extract_value_from_default(default)
+        case default
+        when /^null$/i
+          nil
+        # Quoted types
+        when /^'([^|]*)'$/m
+          $1.gsub("''", "'")
+        # Quoted types
+        when /^"([^|]*)"$/m
+          $1.gsub('""', '"')
+        # Numeric types
+        when /\A-?\d+(\.\d*)?\z/
+          $&
+        # Binary columns
+        when /x'(.*)'/
+          [$1].pack("H*")
+        else
+          # Anything else is blank or some function
+          # and we can't know the value of that, so return nil.
+          nil
+        end
+      end
+
+      def extract_default_function(default_value, default)
+        default if has_default_function?(default_value, default)
+      end
+
+      def has_default_function?(default_value, default)
+        !default_value && %r{\w+\(.*\)|CURRENT_TIME|CURRENT_DATE|CURRENT_TIMESTAMP|\|\|}.match?(default)
+      end
+
+      INTEGER_REGEX = /integer/i
+      def is_column_the_rowid?(field, column_definitions)
+        return false unless INTEGER_REGEX.match?(field["type"]) && field["pk"] == 1
+        # is the primary key a single column?
+        column_definitions.one? { |c|
+          col_pk = c["pk"] || 0
+          col_pk > 0
+        }
+      end
+
+      def internal_exec_query(sql, name = "SQL", binds = [], prepare: false, async: false) # :nodoc:
+        # rows = [
+        #   [Time.now, 1, 2.0],
+        #   [Time.now, 2, 2.0],
+        #   [Time.now, 2, 2.0]
+        # ]
+        response = @pinot_client.execute(sql)
+        rows = response.rows
+        columns = response.columns
+        columns.transform_values! { |value| TYPES.fetch(value, value) }
+        ActiveRecord::Result.new(
+          response.columns.keys,
+          rows.to_a,
+          columns
+        )
+      end
+
+      def exec_query(sql, name = "SQL", binds = [], prepare: false)
+        internal_exec_query(sql, name, binds, prepare: prepare, async: false)
+      end
+
+      def primary_keys(table_name)
+        # TODO: implement
+        [:id]
+      end
+    end
+
+    ActiveSupport.run_load_hooks(:active_record_pinotadapter, PinotAdapter)
+  end
+end

data/lib/active_record/pinot/adapter.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require "pinot"
+require "active_record"
+require_relative "../connection_adapters/pinot_adapter"
+
+module ActiveRecord
+  module Pinot
+    module Adapter
+      class Error < StandardError; end
+      # Your code goes here...
+    end
+  end
+end

data/lib/activerecord/pinot/adapter/version.rb

@@ -3,7 +3,7 @@
 module Activerecord
   module Pinot
     module Adapter
-      VERSION = "0.1.0"
+      VERSION = "0.2.0"
     end
   end
 end

data/lib/activerecord/pinot/adapter.rb

@@ -1,12 +1,6 @@
 # frozen_string_literal: true
 
 require_relative "adapter/version"
-
-module Activerecord
-  module Pinot
-    module Adapter
-      class Error < StandardError; end
-      # Your code goes here...
-    end
-  end
-end
+require "pinot"
+require "active_record"
+require "active_record/pinot/adapter"

metadata

@@ -1,15 +1,105 @@
 --- !ruby/object:Gem::Specification
 name: activerecord-pinot-adapter
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Celso Fernandes
-autorequire:
+autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-01-16 00:00:00.000000000 Z
-dependencies: []
+date: 2024-07-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.2.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.2.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7.2'
+- !ruby/object:Gem::Dependency
+  name: pinot
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+- !ruby/object:Gem::Dependency
+  name: guard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest-reporters
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest-focus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: ActiveRecord Apache Pinot Adapter
 email:
 - celso.fernandes@clickfunnels.com
@@ -20,9 +110,13 @@ files:
 - ".standard.yml"
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
+- Guardfile
 - README.md
 - Rakefile
-- activerecord-pinot-adapter.gemspec
+- docker-compose.yml
+- lib/active_record/connection_adapters/pinot_adapter.rb
+- lib/active_record/connection_adapters/pinot_adapter/table_structure.rb
+- lib/active_record/pinot/adapter.rb
 - lib/activerecord/pinot/adapter.rb
 - lib/activerecord/pinot/adapter/version.rb
 - sig/activerecord/pinot/adapter.rbs
@@ -32,7 +126,7 @@ metadata:
   homepage_uri: https://github.com/fernandes/activerecord-pinot-adapter
   source_code_uri: https://github.com/fernandes/activerecord-pinot-adapter
   changelog_uri: https://github.com/fernandes/activerecord-pinot-adapter/blob/main/CHANGELOG.md
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -47,8 +141,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.21
-signing_key:
+rubygems_version: 3.5.14
+signing_key: 
 specification_version: 4
 summary: ActiveRecord Apache Pinot Adapter
 test_files: []

data/activerecord-pinot-adapter.gemspec

@@ -1,37 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "lib/activerecord/pinot/adapter/version"
-
-Gem::Specification.new do |spec|
-  spec.name = "activerecord-pinot-adapter"
-  spec.version = Activerecord::Pinot::Adapter::VERSION
-  spec.authors = ["Celso Fernandes"]
-  spec.email = ["celso.fernandes@clickfunnels.com"]
-
-  spec.summary = "ActiveRecord Apache Pinot Adapter"
-  spec.description = "ActiveRecord Apache Pinot Adapter"
-  spec.homepage = "https://github.com/fernandes/activerecord-pinot-adapter"
-  spec.required_ruby_version = ">= 2.6.0"
-
-  spec.metadata["homepage_uri"] = spec.homepage
-  spec.metadata["source_code_uri"] = "https://github.com/fernandes/activerecord-pinot-adapter"
-  spec.metadata["changelog_uri"] = "https://github.com/fernandes/activerecord-pinot-adapter/blob/main/CHANGELOG.md"
-
-  # Specify which files should be added to the gem when it is released.
-  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
-  spec.files = Dir.chdir(__dir__) do
-    `git ls-files -z`.split("\x0").reject do |f|
-      (File.expand_path(f) == __FILE__) ||
-        f.start_with?(*%w[bin/ test/ spec/ features/ .git .circleci appveyor Gemfile])
-    end
-  end
-  spec.bindir = "exe"
-  spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
-  spec.require_paths = ["lib"]
-
-  # Uncomment to register a new dependency of your gem
-  # spec.add_dependency "example-gem", "~> 1.0"
-
-  # For more information and examples about making a new gem, check out our
-  # guide at: https://bundler.io/guides/creating_gem.html
-end