data_conduit 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 72aab70fe54dcf66a97433914a5ded7f8c05887b591f2a211b1dac3798e9f539
+  data.tar.gz: 1d5f6bead14baca0f3ecfd0cb33a708887aad75684d57e4eae4ea204c5f29169
+SHA512:
+  metadata.gz: 046cbe44b22de14283106fe214b9f42f2b5e5191222f2e1e195a985b824f8410195f81787557c5b6ae68949c24a2806f978d526c7713dd291cc0ab50ccc1334a
+  data.tar.gz: 2f461e3aa46d3441b4325fc316e0d197d8db7a7b6ffe22d2162f32d440ef405b059e4efbfdc4ee30d88ba2b6929a36c55727673f3c40eeec6ac8f04f9542f8ba

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright Power Home Remodeling Group, LLC
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/lib/data_conduit/adapters/trino_repository.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+require "rest-client"
+require "json"
+require "base64"
+require "sequel"
+
+module DataConduit
+  module Adapters
+    class TrinoRepository
+      include DataWarehouseRepository
+
+      attr_reader :server, :user, :password, :catalog, :schema, :table_name, :conditions, :config
+
+      def initialize(table_name, conditions = nil, config = {})
+        @table_name = table_name
+        @conditions = conditions
+        @config = default_config.merge(config)
+        @server = @config[:server]
+        @user = @config[:user]
+        @password = @config[:password]
+        @catalog = @config[:catalog]
+        @schema = @config[:schema]
+
+        validate_config!
+      end
+
+      def query(sql_query = nil)
+        sql_query ||= build_query
+        execute(sql_query)
+      end
+
+      def execute(sql_query)
+        response_data = process_response(send_query(sql_query))
+        transform_response(response_data[:result_data], response_data[:result_columns])
+      end
+
+    private
+
+      def default_config
+        {
+          server: ENV.fetch("TRINO_SERVER", "http://localhost:8090"),
+          user: ENV.fetch("TRINO_USER", "trino"),
+          password: ENV.fetch("TRINO_PASSWORD", nil),
+          catalog: ENV.fetch("TRINO_CATALOG", "default"),
+          schema: ENV.fetch("TRINO_SCHEMA", "default"),
+        }
+      end
+
+      def validate_config!
+        %i[server user catalog schema].each do |key|
+          value = instance_variable_get(:"@#{key}")
+          raise ArgumentError, "#{key} cannot be nil or empty" if value.nil? || value.empty?
+        end
+      end
+
+      # Build a SQL query using Sequel as a sanitizer and SQL builder.
+      # We use Sequel.mock so that no actual connection is made.
+      def build_query
+        db = Sequel.mock
+        dataset = db.from(Sequel.identifier(table_name)).select_all
+
+        if conditions
+          unless conditions.is_a?(Hash)
+            raise ArgumentError, "Conditions must be provided as a Hash for safe query building"
+          end
+
+          dataset = dataset.where(conditions)
+        end
+
+        dataset.sql
+      end
+
+      def process_response(initial_response)
+        result_data = []
+        result_columns = nil
+        response_data = initial_response
+
+        while response_data
+          result_data.concat(response_data["data"]) if response_data["data"]
+          result_columns ||= response_data["columns"]
+          next_uri = response_data["nextUri"]
+          response_data = next_uri ? fetch_next(next_uri) : nil
+        end
+
+        { result_data: result_data, result_columns: result_columns }
+      end
+
+      def send_query(sql)
+        JSON.parse(RestClient.post("#{server}/v1/statement", sql, headers).body)
+      rescue JSON::ParserError => e
+        raise DataConduit::Error, "Failed to parse JSON response: #{e.message}"
+      rescue RestClient::ExceptionWithResponse => e
+        raise DataConduit::Error, "Query failed: #{e.response&.body}"
+      end
+
+      def fetch_next(uri)
+        JSON.parse(RestClient.get(uri, headers).body)
+      rescue RestClient::ExceptionWithResponse => e
+        raise DataConduit::Error, "Failed to fetch next page: #{e.response&.body}"
+      end
+
+      def headers
+        headers = {
+          "X-Trino-Catalog" => catalog,
+          "X-Trino-Schema" => schema,
+        }
+
+        # Add basic auth if password is provided, otherwise use X-Trino-User header
+        if password && !password.empty?
+          headers["Authorization"] = "Basic #{Base64.strict_encode64("#{user}:#{password}")}"
+        else
+          headers["X-Trino-User"] = user
+        end
+
+        headers
+      end
+    end
+  end
+end

data/lib/data_conduit/data_warehouse_repository.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module DataConduit
+  module DataWarehouseRepository
+    DEFAULT_TRANSFORM_OPTIONS = {
+      keys: :string,        # :string or :symbol
+      transform_keys: nil,  # optional proc for key transformation
+      transform_values: nil, # optional proc for value transformation
+    }.freeze
+
+    def self.included(base)
+      base.include(InstanceMethods)
+    end
+
+    module InstanceMethods
+      def initialize(_table_name, _conditions = nil, _config = {})
+        validate_table_name(table_name)
+        raise NotImplementedError, "You must implement the initialize method"
+      end
+
+      def query(_sql_query = nil)
+        raise NotImplementedError, "You must implement the query method"
+      end
+
+      def execute(_sql_query)
+        raise NotImplementedError, "You must implement the execute method"
+      end
+
+    protected
+
+      def validate_table_name(table_name)
+        raise ArgumentError, "Table name cannot be blank" if table_name.nil? || table_name.empty?
+
+        return if table_name.to_s.match?(/^[a-zA-Z0-9_\.]+$/)
+
+        raise ArgumentError, "Invalid table name format. Table name must contain only letters, " \
+                             "numbers, underscores, and periods."
+      end
+
+      def transform_response(result_data, result_columns)
+        return [] if result_data.nil? || result_data.empty?
+
+        columns = extract_column_names(result_columns)
+        result_data.map do |row|
+          transform_row(columns.zip(row).to_h)
+        end
+      end
+
+      def transform_row(row)
+        row = row.transform_keys { |key| transform_key(key) }
+        row = row.transform_values(&transform_options[:transform_values]) if transform_options[:transform_values]
+        row
+      end
+
+      def transform_key(key)
+        key = transform_options[:transform_keys]&.call(key) || key
+        transform_options[:keys] == :symbol ? key.to_sym : key.to_s
+      end
+
+      def transform_options
+        @transform_options ||= DEFAULT_TRANSFORM_OPTIONS.merge(
+          config.fetch(:transform_options, {})
+        )
+      end
+
+      # Can be overridden by adapters if they have different column name structures
+      def extract_column_names(columns)
+        columns.map { |col| col["name"] }
+      end
+    end
+  end
+end

data/lib/data_conduit/repository_factory.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module DataConduit
+  class RepositoryFactory
+    class << self
+      def repositories
+        @repositories ||= {}
+      end
+
+      def register(type, repository_class)
+        repositories[type.to_sym] = repository_class
+      end
+
+      def create(table_name:, type: :trino, conditions: nil, config: {})
+        repository_class = repository_for(type)
+        repository_class.new(table_name, conditions, config)
+      end
+
+    private
+
+      def repository_for(type)
+        type = type.to_sym
+        repositories[type] || raise(
+          ArgumentError,
+          "Unsupported repository type: #{type}. Available types: #{repositories.keys.join(', ')}"
+        )
+      end
+    end
+  end
+end

data/lib/data_conduit/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module DataConduit
+  VERSION = "0.1.0"
+end

data/lib/data_conduit.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require_relative "data_conduit/version"
+require_relative "data_conduit/data_warehouse_repository"
+require_relative "data_conduit/repository_factory"
+require_relative "data_conduit/adapters/trino_repository"
+
+# Register default adapters
+DataConduit::RepositoryFactory.register(:trino, DataConduit::Adapters::TrinoRepository)
+
+module DataConduit
+  class Error < StandardError; end
+end

metadata

@@ -0,0 +1,191 @@
+--- !ruby/object:Gem::Specification
+name: data_conduit
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Vinicius Dittgen
+bindir: bin
+cert_chain: []
+date: 2025-03-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+- !ruby/object:Gem::Dependency
+  name: securerandom
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.2
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 7.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 7.1.0
+- !ruby/object:Gem::Dependency
+  name: appraisal
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.5.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.5.0
+- !ruby/object:Gem::Dependency
+  name: license_finder
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '7.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '7.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.21'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.21'
+- !ruby/object:Gem::Dependency
+  name: rubocop-powerhome
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.18'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.18'
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.90.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.90.0
+description: A flexible data warehouse connector with support for Trino and extensibility
+  for other engines
+email:
+- vinipd@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE.txt
+- lib/data_conduit.rb
+- lib/data_conduit/adapters/trino_repository.rb
+- lib/data_conduit/data_warehouse_repository.rb
+- lib/data_conduit/repository_factory.rb
+- lib/data_conduit/version.rb
+homepage: https://github.com/powerhome/power-tools
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/powerhome/power-tools
+  source_code_uri: https://github.com/powerhome/power-tools/tree/main/packages/data_conduit
+  changelog_uri: https://github.com/powerhome/power-tools/blob/main/packages/data_conduit/CHANGELOG.md
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.2
+specification_version: 4
+summary: A Ruby connector for data warehouses
+test_files: []