activerecord-dbt 0.1.0

data/Rakefile

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+
+require 'bundler/gem_tasks'

data/lib/active_record/dbt/column/column.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Dbt
+    module Column
+      class Column
+        attr_reader :table_name, :column, :column_test, :primary_keys
+
+        delegate :name, :comment, to: :column, prefix: true
+        delegate :source_config, to: :@config
+
+        def initialize(table_name, column, column_test, primary_keys: [])
+          @table_name = table_name
+          @column = column
+          @column_test = column_test
+          @primary_keys = primary_keys
+          @config = ActiveRecord::Dbt::Config.instance
+        end
+
+        def config
+          {
+            'name' => column_name,
+            'description' => description,
+            'meta' => { 'column_type' => column.type.to_s },
+            **column_overrides.except(:tests),
+            'tests' => column_test.config
+          }.compact
+        end
+
+        private
+
+        def description
+          @description ||=
+            column_description ||
+            translated_attribute_name ||
+            column_comment ||
+            key_column_name ||
+            default_column_description ||
+            "Write a description of the '#{table_name}.#{column_name}' column."
+        end
+
+        def column_description
+          source_config.dig(:table_descriptions, table_name, :columns, column_name)
+        end
+
+        def translated_attribute_name
+          translated_column_name || translated_default_column_name
+        end
+
+        def translated_column_name
+          I18n.t("activerecord.attributes.#{table_name.singularize}.#{column_name}", default: nil)
+        end
+
+        def translated_default_column_name
+          I18n.t("attributes.#{column_name}", default: nil)
+        end
+
+        def key_column_name
+          column_name if primary_key? || foreign_key?
+        end
+
+        def primary_key?
+          primary_keys.include?(column_name)
+        end
+
+        def foreign_key?
+          ActiveRecord::Base.connection.foreign_key_exists?(table_name, column: column_name)
+        end
+
+        def default_column_description
+          source_config.dig(:defaults, :table_descriptions, :columns, :description)
+                       &.gsub(/{{\s*table_name\s*}}/, table_name)
+                       &.gsub(/{{\s*column_name\s*}}/, column_name)
+        end
+
+        def column_overrides
+          @column_overrides ||=
+            source_config.dig(:table_overrides, table_name, :columns, column_name) ||
+            {}
+        end
+      end
+    end
+  end
+end

data/lib/active_record/dbt/column/test.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Dbt
+    module Column
+      class Test
+        include ActiveRecord::Dbt::Column::Testable::AcceptedValuesTestable
+        include ActiveRecord::Dbt::Column::Testable::NotNullTestable
+        include ActiveRecord::Dbt::Column::Testable::RelationshipsTestable
+        include ActiveRecord::Dbt::Column::Testable::UniqueTestable
+
+        attr_reader :table_name, :column, :primary_keys, :foreign_keys
+
+        delegate :name, to: :column, prefix: true
+        delegate :source_config, to: :@config
+
+        def initialize(table_name, column, primary_keys: [], foreign_keys: [{}])
+          @table_name = table_name
+          @column = column
+          @primary_keys = primary_keys
+          @foreign_keys = foreign_keys
+          @config = ActiveRecord::Dbt::Config.instance
+        end
+
+        def config
+          (tests.keys | tests_overrides_hash.keys).map do |key|
+            tests_overrides_hash[key] || tests[key]
+          end.presence
+        end
+
+        private
+
+        def tests
+          {
+            'unique_test' => unique_test,
+            'not_null_test' => not_null_test,
+            'relationships_test' => relationships_test,
+            'accepted_values_test' => accepted_values_test
+          }.compact
+        end
+
+        def tests_overrides_hash
+          @tests_overrides_hash ||=
+            tests_overrides.index_by do |tests_override|
+              "#{extract_key(tests_override)}_test"
+            end
+        end
+
+        def extract_key(item)
+          item.is_a?(Hash) ? item.keys.first : item
+        end
+
+        def tests_overrides
+          @tests_overrides ||=
+            source_config.dig(:table_overrides, table_name, :columns, column_name, :tests) ||
+            []
+        end
+      end
+    end
+  end
+end

data/lib/active_record/dbt/column/testable/accepted_values_testable.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Dbt
+    module Column
+      module Testable
+        module AcceptedValuesTestable
+          REQUIRED_ACCEPTED_VALUES_TESTABLE_METHODS = %i[@config column table_name column_name].freeze
+
+          delegate :type, to: :column, prefix: true
+          delegate :add_log, to: :@config
+
+          REQUIRED_ACCEPTED_VALUES_TESTABLE_METHODS.each do |method_name|
+            define_method(method_name) do
+              raise NotImplementedError, "You must implement #{self.class}##{__method__}"
+            end
+          end
+
+          def accepted_values_test
+            return nil unless column_type == :boolean || enum_values.present?
+
+            {
+              'accepted_values' => {
+                'values' => values,
+                'quote' => quote?
+              }
+            }
+          end
+
+          private
+
+          def values
+            column_type == :boolean ? [true, false] : enum_accepted_values
+          end
+
+          def enum_accepted_values
+            enum_values.map { |key| quote? ? key.to_s : key }
+          end
+
+          def enum_values
+            @enum_values ||= enums[column_name]&.values
+          end
+
+          def enums
+            table_name.singularize.classify.constantize.defined_enums
+          rescue NameError => e
+            add_log(self.class, e)
+
+            {}
+          end
+
+          def quote?
+            @quote ||= %i[integer boolean].exclude?(column_type)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_record/dbt/column/testable/not_null_testable.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Dbt
+    module Column
+      module Testable
+        module NotNullTestable
+          REQUIRED_NOT_NULL_TESTABLE_METHODS = %i[column].freeze
+
+          REQUIRED_NOT_NULL_TESTABLE_METHODS.each do |method_name|
+            define_method(method_name) do
+              raise NotImplementedError, "You must implement #{self.class}##{__method__}"
+            end
+          end
+
+          def not_null_test
+            column.null == true ? nil : 'not_null'
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_record/dbt/column/testable/relationships_testable.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Dbt
+    module Column
+      module Testable
+        module RelationshipsTestable
+          REQUIRED_RELATIONSHIPS_TESTABLE_METHODS = %i[@config foreign_keys column_name].freeze
+
+          include ActiveRecord::Dbt::DbtPackage::Dbterd::Column::Testable::RelationshipsMetaRelationshipType
+
+          delegate :source_name, :data_sync_delayed?, to: :@config
+          delegate :to_table, to: :foreign_key
+
+          REQUIRED_RELATIONSHIPS_TESTABLE_METHODS.each do |method_name|
+            define_method(method_name) do
+              raise NotImplementedError, "You must implement #{self.class}##{__method__}"
+            end
+          end
+
+          def relationships_test
+            return nil if foreign_key.blank?
+
+            {
+              'relationships' => {
+                'severity' => data_sync_delayed? ? 'warn' : nil,
+                'to' => "source('#{source_name}', '#{to_table}')",
+                'field' => primary_key,
+                'meta' => relationships_meta_relationship_type
+              }.compact
+            }
+          end
+
+          private
+
+          def primary_key
+            foreign_key.dig(:options, :primary_key)
+          end
+
+          def foreign_key
+            @foreign_key ||= foreign_keys.find do |fk|
+              fk.dig(:options, :column) == column_name
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_record/dbt/column/testable/unique_testable.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Dbt
+    module Column
+      module Testable
+        module UniqueTestable
+          REQUIRED_UNIQUE_TESTABLE_METHODS = %i[table_name column_name primary_keys].freeze
+
+          REQUIRED_UNIQUE_TESTABLE_METHODS.each do |method_name|
+            define_method(method_name) do
+              raise NotImplementedError, "You must implement #{self.class}##{__method__}"
+            end
+          end
+
+          def unique_test
+            unique? ? 'unique' : nil
+          end
+
+          private
+
+          def unique?
+            primary_keys.include?(column_name) || unique_columns.include?(column_name)
+          end
+
+          def unique_columns
+            ActiveRecord::Base.connection.indexes(table_name).each_with_object([]) do |index, array|
+              if index.unique == true && (unique_indexes = index.columns).size == 1
+                array << unique_indexes.first
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_record/dbt/config.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require 'singleton'
+
+module ActiveRecord
+  module Dbt
+    class Config
+      include Singleton
+
+      include ActiveRecord::Dbt::Configuration::DataSync
+      include ActiveRecord::Dbt::Configuration::Logger
+      include ActiveRecord::Dbt::Configuration::Parser
+      include ActiveRecord::Dbt::Configuration::UsedDbtPackage
+
+      DEFAULT_CONFIG_DIRECTORY_PATH = 'lib/dbt'
+      DEFAULT_EXPORT_DIRECTORY_PATH = 'doc/dbt'
+
+      attr_writer :config_directory_path, :export_directory_path
+
+      def source_config_path
+        @source_config_path ||= "#{config_directory_path}/source_config.yml"
+      end
+
+      def source_config
+        @source_config ||= parse_yaml(source_config_path)
+      end
+
+      def source_name
+        @source_name ||= source_config.dig(:sources, :name).tap do |source_name|
+          raise SourceNameIsNullError, "'sources.name' is required in '#{source_config_path}'." if source_name.nil?
+        end
+      end
+
+      def config_directory_path
+        @config_directory_path ||= DEFAULT_CONFIG_DIRECTORY_PATH
+      end
+
+      def export_directory_path
+        @export_directory_path ||= DEFAULT_EXPORT_DIRECTORY_PATH
+      end
+
+      class SourceNameIsNullError < StandardError; end
+    end
+  end
+end

data/lib/active_record/dbt/configuration/data_sync.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Dbt
+    module Configuration
+      module DataSync
+        attr_writer :data_sync_delayed
+
+        def data_sync_delayed?
+          @data_sync_delayed ||= false
+        end
+      end
+    end
+  end
+end

data/lib/active_record/dbt/configuration/logger.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Dbt
+    module Configuration
+      module Logger
+        DEFAULT_LOG_FILE_PATH = './log/active_record_dbt.log'
+        EXCLUDE_EXCEPTION_CLASS_NAMES = %w[ArInternalMetadatum SchemaMigration].freeze
+
+        attr_writer :logger
+
+        def logger
+          @logger ||= ::Logger.new(DEFAULT_LOG_FILE_PATH)
+        end
+
+        def add_log(class_name, exception)
+          return if include_exception_class_names?(exception)
+
+          logger.info(class_name) { format_log(exception) }
+        end
+
+        private
+
+        def include_exception_class_names?(exception)
+          exception.instance_of?(NameError) &&
+            EXCLUDE_EXCEPTION_CLASS_NAMES.include?(exception.name.to_s)
+        end
+
+        def format_log(exception)
+          {
+            exception: [
+              exception.class,
+              exception.message
+            ],
+            exception_backtrace: exception.backtrace.first(5)
+          }.to_json
+        end
+      end
+    end
+  end
+end

data/lib/active_record/dbt/configuration/parser.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Dbt
+    module Configuration
+      module Parser
+        def parse_yaml(path)
+          return {} if path.nil?
+
+          YAML.load_file(path).with_indifferent_access || {}
+        end
+      end
+    end
+  end
+end

data/lib/active_record/dbt/configuration/used_dbt_package.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Dbt
+    module Configuration
+      module UsedDbtPackage
+        attr_writer :used_dbt_package_names
+
+        def used_dbt_utils?
+          used_dbt_package_names.include?('dbt-labs/dbt_utils')
+        end
+
+        def used_dbterd?
+          used_dbt_package_names.include?('datnguye/dbterd')
+        end
+
+        private
+
+        def used_dbt_package_names
+          @used_dbt_package_names ||= []
+        end
+      end
+    end
+  end
+end

data/lib/active_record/dbt/dbt_package/dbt_utils/table/testable/unique_combination_of_columns_testable.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Dbt
+    module DbtPackage
+      module DbtUtils
+        module Table
+          module Testable
+            module UniqueCombinationOfColumnsTestable
+              REQUIRED_UNIQUE_COMBINATION_OF_COLUMNS_TESTABLE_METHODS = %i[table_name @config].freeze
+
+              delegate :used_dbt_utils?, to: :@config
+
+              REQUIRED_UNIQUE_COMBINATION_OF_COLUMNS_TESTABLE_METHODS.each do |method_name|
+                define_method(method_name) do
+                  raise NotImplementedError, "You must implement #{self.class}##{__method__}"
+                end
+              end
+
+              def unique_combination_of_columns_test
+                return nil unless used_dbt_utils?
+
+                ActiveRecord::Base.connection.indexes(table_name).each_with_object([]) do |index, array|
+                  next if index.unique == false
+                  next if (unique_indexes = index.columns).size == 1
+
+                  array.push(
+                    {
+                      'dbt_utils.unique_combination_of_columns' => {
+                        'combination_of_columns' => unique_indexes
+                      }
+                    }
+                  )
+                end.presence
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_record/dbt/dbt_package/dbterd/column/testable/relationships_meta_relationship_type.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Dbt
+    module DbtPackage
+      module Dbterd
+        module Column
+          module Testable
+            module RelationshipsMetaRelationshipType
+              REQUIRED_RELATIONSHIP_TYPE_TESTABLE_METHODS = %i[@config foreign_key].freeze
+
+              delegate :used_dbterd?, :add_log, to: :@config
+
+              REQUIRED_RELATIONSHIP_TYPE_TESTABLE_METHODS.each do |method_name|
+                define_method(method_name) do
+                  raise NotImplementedError, "You must implement #{self.class}##{__method__}"
+                end
+              end
+
+              def relationships_meta_relationship_type
+                return nil unless used_dbterd?
+                return nil if foreign_key.nil? || relationship_type.blank?
+
+                {
+                  'relationship_type' => relationship_type
+                }
+              rescue NotSpecifiedOrNotInvalidIdError, StandardError => e
+                add_log(self.class, e)
+
+                {
+                  'relationship_type' => 'many-to-one',
+                  'active_record_dbt_error' => {
+                    'class' => e.class.to_s,
+                    'message' => e.message.to_s
+                  }
+                }
+              end
+
+              private
+
+              # MEMO: It seems to be a good idea to set only 'many-to-one', 'one-to-one', and 'many-to-one'.
+              # * [Relationship Types - DaaC from dbt artifacts](https://dbterd.datnguyen.de/1.13/nav/metadata/relationship_type.html)
+              # * [Syntax | DBML](https://dbml.dbdiagram.io/docs/#relationships--foreign-key-definitions)
+              #   * DBML supports 'one-to-many', 'many-to-one', 'one-to-one', and 'many-to-many'.
+              def relationship_type
+                # if one_to_many?
+                #   'one-to-many'
+                # elsif zero_to_many?
+                #   'zero-to-many'
+                # elsif many_to_many?
+                #   'many-to-many'
+                # elsif one_to_one?
+                if one_to_one?
+                  'one-to-one'
+                elsif many_to_one?
+                  'many-to-one'
+                else
+                  raise NotSpecifiedOrNotInvalidIdError, 'Not specified/Invalid value'
+                end
+              end
+
+              # MEMO: If 'many-to-one' is specified, 'one-to-many' should not be necessary.
+              # def one_to_many?
+              # end
+
+              # MEMO:
+              # * It seems that `zero-to-many` cannot be specified in a dbt relationship.
+              #   * The reverse may be possible, but cannot be specified in dbterd.
+              # * It doesn't look like it can be configured with dbml.
+              #   * [Syntax | DBML](https://dbml.dbdiagram.io/docs/#relationships--foreign-key-definitions)
+              # def zero_to_many?
+              # end
+
+              # # TODO: Usually, there is always an intermediate table, so there is no `many-to-many`.
+              # def many_to_many?
+              #   from_model_find_association_to_model?(:has_and_belongs_to_many) &&
+              #     to_model_find_association_from_model?(:has_and_belongs_to_many)
+              # end
+
+              def one_to_one?
+                from_model_find_association_to_model?(:belongs_to) &&
+                  to_model_find_association_from_model?(:has_one)
+              end
+
+              def many_to_one?
+                from_model_find_association_to_model?(:belongs_to) &&
+                  to_model_find_association_from_model?(:has_many)
+              end
+
+              def to_model_find_association_from_model?(association_type)
+                to_model.reflect_on_all_associations(association_type).any? do |association|
+                  association_klass(association) == from_model &&
+                    association_foreign_key(association) == foreign_key_column
+                end
+              end
+
+              def from_model_find_association_to_model?(association_type)
+                from_model.reflect_on_all_associations(association_type).any? do |association|
+                  association_klass(association) == to_model &&
+                    association_foreign_key(association) == foreign_key_column
+                end
+              end
+
+              def association_klass(association)
+                association.klass
+              rescue NoMethodError
+                association.options.fetch(:through).to_s.classify.constantize
+              end
+
+              def association_foreign_key(association)
+                association.foreign_key
+              rescue NoMethodError
+                association.options.fetch(
+                  :foreign_key,
+                  "#{association.active_record.to_s.underscore}_id"
+                )
+              end
+
+              def foreign_key_column
+                foreign_key.dig(:options, :column)
+              end
+
+              def from_model
+                @from_model ||= foreign_key.from_table.classify.constantize
+              end
+
+              def to_model
+                @to_model ||= foreign_key.to_table.classify.constantize
+              end
+
+              class NotSpecifiedOrNotInvalidIdError < StandardError; end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_record/dbt/factory/columns_factory.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Dbt
+    module Factory
+      module ColumnsFactory
+        def self.build(
+          table_name,
+          primary_keys: ActiveRecord::Base.connection.primary_keys(table_name),
+          foreign_keys: ActiveRecord::Base.connection.foreign_keys(table_name)
+        )
+          ActiveRecord::Base.connection.columns(table_name).map do |column|
+            column_test = ActiveRecord::Dbt::Column::Test.new(
+              table_name,
+              column,
+              primary_keys: primary_keys,
+              foreign_keys: foreign_keys
+            )
+
+            ActiveRecord::Dbt::Column::Column.new(
+              table_name,
+              column,
+              column_test,
+              primary_keys: primary_keys
+            )
+          end
+        end
+      end
+    end
+  end
+end