activerecord-dbt 0.2.0 → 0.4.0

data/lib/active_record/dbt/column/{test.rb → data_test.rb}

@@ -3,11 +3,11 @@
 module ActiveRecord
   module Dbt
     module Column
-      class Test
-        include ActiveRecord::Dbt::Column::Testable::AcceptedValuesTestable
-        include ActiveRecord::Dbt::Column::Testable::NotNullTestable
-        include ActiveRecord::Dbt::Column::Testable::RelationshipsTestable
-        include ActiveRecord::Dbt::Column::Testable::UniqueTestable
+      class DataTest
+        include ActiveRecord::Dbt::Column::DataTestable::AcceptedValuesDataTestable
+        include ActiveRecord::Dbt::Column::DataTestable::NotNullDataTestable
+        include ActiveRecord::Dbt::Column::DataTestable::RelationshipsDataTestable
+        include ActiveRecord::Dbt::Column::DataTestable::UniqueDataTestable
 
         attr_reader :table_name, :column, :primary_keys, :foreign_keys
 
@@ -22,15 +22,15 @@ module ActiveRecord
           @config = ActiveRecord::Dbt::Config.instance
         end
 
-        def config
-          (tests.keys | tests_overrides_hash.keys).map do |key|
-            tests_overrides_hash[key] || tests[key]
+        def properties
+          (data_tests.keys | data_tests_overrides_hash.keys).map do |key|
+            data_tests_overrides_hash[key] || data_tests[key]
           end.presence
         end
 
         private
 
-        def tests
+        def data_tests
           {
             'unique_test' => unique_test,
             'not_null_test' => not_null_test,
@@ -39,10 +39,10 @@ module ActiveRecord
           }.compact
         end
 
-        def tests_overrides_hash
-          @tests_overrides_hash ||=
-            tests_overrides.index_by do |tests_override|
-              "#{extract_key(tests_override)}_test"
+        def data_tests_overrides_hash
+          @data_tests_overrides_hash ||=
+            data_tests_overrides.index_by do |data_tests_override|
+              "#{extract_key(data_tests_override)}_test"
             end
         end
 
@@ -50,9 +50,9 @@ module ActiveRecord
           item.is_a?(Hash) ? item.keys.first : item
         end
 
-        def tests_overrides
-          @tests_overrides ||=
-            source_config.dig(:table_overrides, table_name, :columns, column_name, :tests) ||
+        def data_tests_overrides
+          @data_tests_overrides ||=
+            source_config.dig(:table_overrides, table_name, :columns, column_name, :data_tests) ||
             []
         end
       end

data/lib/active_record/dbt/column/{testable/accepted_values_testable.rb → data_testable/accepted_values_data_testable.rb}

@@ -3,8 +3,8 @@
 module ActiveRecord
   module Dbt
     module Column
-      module Testable
-        module AcceptedValuesTestable
+      module DataTestable
+        module AcceptedValuesDataTestable
           extend ActiveRecord::Dbt::RequiredMethods
 
           define_required_methods :@config, :column, :table_name, :column_name

data/lib/active_record/dbt/column/{testable/not_null_testable.rb → data_testable/not_null_data_testable.rb}

@@ -3,14 +3,20 @@
 module ActiveRecord
   module Dbt
     module Column
-      module Testable
-        module NotNullTestable
+      module DataTestable
+        module NotNullDataTestable
           extend ActiveRecord::Dbt::RequiredMethods
 
           define_required_methods :column
 
           def not_null_test
-            column.null == true ? nil : 'not_null'
+            null? ? nil : 'not_null'
+          end
+
+          private
+
+          def null?
+            column.null == true
           end
         end
       end

data/lib/active_record/dbt/column/{testable/relationships_testable.rb → data_testable/relationships_data_testable.rb}

@@ -3,9 +3,9 @@
 module ActiveRecord
   module Dbt
     module Column
-      module Testable
-        module RelationshipsTestable
-          include ActiveRecord::Dbt::DbtPackage::Dbterd::Column::Testable::RelationshipsMetaRelationshipType
+      module DataTestable
+        module RelationshipsDataTestable
+          include ActiveRecord::Dbt::DbtPackage::Dbterd::Column::DataTestable::RelationshipsMetaRelationshipType
 
           extend ActiveRecord::Dbt::RequiredMethods
 

data/lib/active_record/dbt/column/{testable/unique_testable.rb → data_testable/unique_data_testable.rb}

@@ -3,8 +3,8 @@
 module ActiveRecord
   module Dbt
     module Column
-      module Testable
-        module UniqueTestable
+      module DataTestable
+        module UniqueDataTestable
           extend ActiveRecord::Dbt::RequiredMethods
 
           define_required_methods :table_name, :column_name, :primary_keys

data/lib/active_record/dbt/column/{column.rb → yml.rb}

@@ -3,29 +3,30 @@
 module ActiveRecord
   module Dbt
     module Column
-      class Column
+      class Yml
         include ActiveRecord::Dbt::DataType::Mapper
+        include ActiveRecord::Dbt::I18nWrapper::Translate
 
-        attr_reader :table_name, :column, :column_test, :primary_keys
+        attr_reader :table_name, :column, :column_data_test, :primary_keys
 
         delegate :name, :comment, to: :column, prefix: true
         delegate :source_config, to: :@config
 
-        def initialize(table_name, column, column_test, primary_keys: [])
+        def initialize(table_name, column, column_data_test, primary_keys: [])
           @table_name = table_name
           @column = column
-          @column_test = column_test
+          @column_data_test = column_data_test
           @primary_keys = primary_keys
           @config = ActiveRecord::Dbt::Config.instance
         end
 
-        def config
+        def properties
           {
             'name' => column_name,
             'description' => description,
-            'data_type' => data_type,
-            **column_overrides.except(:tests),
-            'tests' => column_test.config
+            'data_type' => data_type(column.type),
+            **column_overrides.except(:data_tests),
+            'data_tests' => column_data_test.properties
           }.compact
         end
 
@@ -45,18 +46,6 @@ module ActiveRecord
           source_config.dig(:table_descriptions, table_name, :columns, column_name)
         end
 
-        def translated_attribute_name
-          translated_column_name || translated_default_column_name
-        end
-
-        def translated_column_name
-          I18n.t("activerecord.attributes.#{table_name.singularize}.#{column_name}", default: nil)
-        end
-
-        def translated_default_column_name
-          I18n.t("attributes.#{column_name}", default: nil)
-        end
-
         def key_column_name
           column_name if primary_key? || foreign_key?
         end

data/lib/active_record/dbt/config.rb

@@ -9,38 +9,10 @@ module ActiveRecord
 
       include ActiveRecord::Dbt::Configuration::DataSync
       include ActiveRecord::Dbt::Configuration::DwhPlatform
+      include ActiveRecord::Dbt::Configuration::I18nConfiguration
       include ActiveRecord::Dbt::Configuration::Logger
-      include ActiveRecord::Dbt::Configuration::Parser
+      include ActiveRecord::Dbt::Configuration::Source
       include ActiveRecord::Dbt::Configuration::UsedDbtPackage
-
-      DEFAULT_CONFIG_DIRECTORY_PATH = 'lib/dbt'
-      DEFAULT_EXPORT_DIRECTORY_PATH = 'doc/dbt'
-
-      attr_writer :config_directory_path, :export_directory_path
-
-      def source_config_path
-        @source_config_path ||= "#{config_directory_path}/source_config.yml"
-      end
-
-      def source_config
-        @source_config ||= parse_yaml(source_config_path)
-      end
-
-      def source_name
-        @source_name ||= source_config.dig(:sources, :name).tap do |source_name|
-          raise SourceNameIsNullError, "'sources.name' is required in '#{source_config_path}'." if source_name.nil?
-        end
-      end
-
-      def config_directory_path
-        @config_directory_path ||= DEFAULT_CONFIG_DIRECTORY_PATH
-      end
-
-      def export_directory_path
-        @export_directory_path ||= DEFAULT_EXPORT_DIRECTORY_PATH
-      end
-
-      class SourceNameIsNullError < StandardError; end
     end
   end
 end

data/lib/active_record/dbt/configuration/i18n_configuration.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Dbt
+    module Configuration
+      module I18nConfiguration
+        def locale=(locale = I18n.locale)
+          I18n.load_path += Rails.root.glob('config/locales/**/*.{rb,yml}')
+          I18n.locale = locale
+        end
+      end
+    end
+  end
+end

data/lib/active_record/dbt/configuration/source.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Dbt
+    module Configuration
+      module Source
+        include ActiveRecord::Dbt::Configuration::Parser
+
+        DEFAULT_CONFIG_DIRECTORY_PATH = 'lib/dbt'
+        DEFAULT_EXPORT_DIRECTORY_PATH = 'doc/dbt'
+
+        attr_writer :config_directory_path, :export_directory_path
+
+        def config_directory_path
+          @config_directory_path ||= DEFAULT_CONFIG_DIRECTORY_PATH
+        end
+
+        def export_directory_path
+          @export_directory_path ||= DEFAULT_EXPORT_DIRECTORY_PATH
+        end
+
+        def source_config_path
+          @source_config_path ||= "#{config_directory_path}/source_config.yml"
+        end
+
+        def source_config
+          @source_config ||= parse_yaml(source_config_path)
+        end
+
+        def source_name
+          @source_name ||= source_config.dig(:sources, :name).tap do |source_name|
+            raise SourceNameIsNullError, "'sources.name' is required in '#{source_config_path}'." if source_name.nil?
+          end
+        end
+
+        class SourceNameIsNullError < StandardError; end
+      end
+    end
+  end
+end

data/lib/active_record/dbt/data_type/mapper.rb

@@ -4,31 +4,98 @@ module ActiveRecord
   module Dbt
     module DataType
       module Mapper
-        include ActiveRecord::Dbt::DataType::DwhPlatform::ApacheSpark
-        include ActiveRecord::Dbt::DataType::DwhPlatform::Bigquery
-        include ActiveRecord::Dbt::DataType::DwhPlatform::PostgreSql
-        include ActiveRecord::Dbt::DataType::DwhPlatform::Redshift
-        include ActiveRecord::Dbt::DataType::DwhPlatform::Snowflake
-
         extend ActiveRecord::Dbt::RequiredMethods
 
         # [Platform-specific data types | dbt Developer Hub](https://docs.getdbt.com/reference/resource-properties/data-types)
         RUBY_TO_DWH_PLATFORM_TYPE_MAP = {
-          'bigquery' => RUBY_TO_BIGQUERY_TYPES,
-          'postgres' => RUBY_TO_POSTGRES_TYPES,
-          'redshift' => RUBY_TO_REDSHIFT_TYPES,
-          'snowflake' => RUBY_TO_SNOWFLAKE_TYPES,
-          'spark' => RUBY_TO_SPARK_TYPES
+          # [Data types  |  BigQuery  |  Google Cloud](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#data_type_list)
+          'bigquery' => {
+            binary: 'bytes',
+            boolean: 'bool',
+            date: 'date',
+            datetime: 'datetime',
+            decimal: 'int64',
+            float: 'float64',
+            integer: 'int64',
+            json: 'json',
+            string: 'string',
+            text: 'string',
+            time: 'time'
+          },
+
+          # TODO: I have not tried it. I don't know if this is the correct data_type.
+          # [PostgreSQL: Documentation: 16: Chapter 8. Data Types](https://www.postgresql.org/docs/current/datatype.html)
+          'postgres' => {
+            binary: 'bytea',
+            boolean: 'boolean',
+            date: 'date',
+            datetime: 'timestamp',
+            decimal: 'bigint',
+            float: 'double precision',
+            integer: 'bigint',
+            json: 'json',
+            string: 'text',
+            text: 'text',
+            time: 'time'
+          },
+
+          # TODO: I have not tried it. I don't know if this is the correct data_type.
+          # [Data types - Amazon Redshift](https://docs.aws.amazon.com/redshift/latest/dg/c_Supported_data_types.html)
+          'redshift' => {
+            binary: 'varbyte',
+            boolean: 'bool',
+            date: 'date',
+            datetime: 'timestamp',
+            decimal: 'decimal',
+            float: 'double precision',
+            integer: 'integer',
+            json: 'super',
+            string: 'varchar',
+            text: 'varchar',
+            time: 'time'
+          },
+
+          # TODO: I have not tried it. I don't know if this is the correct data_type.
+          # [Summary of data types | Snowflake Documentation](https://docs.snowflake.com/en/sql-reference/intro-summary-data-types)
+          'snowflake' => {
+            binary: 'binary',
+            boolean: 'boolean',
+            date: 'date',
+            datetime: 'datetime',
+            decimal: 'decimal',
+            float: 'float',
+            integer: 'integer',
+            json: 'variant',
+            string: 'varchar',
+            text: 'varchar',
+            time: 'time'
+          },
+
+          # TODO: I have not tried it. I don't know if this is the correct data_type.
+          # [Data Types - Spark 3.5.1 Documentation](https://spark.apache.org/docs/latest/sql-ref-datatypes.html)
+          'spark' => {
+            binary: 'binary',
+            boolean: 'boolean',
+            date: 'date',
+            datetime: 'timestamp',
+            decimal: 'decimal',
+            float: 'float',
+            integer: 'integer',
+            json: 'string',
+            string: 'string',
+            text: 'string',
+            time: 'string'
+          }
         }.freeze
 
-        define_required_methods :column, :@config
+        define_required_methods :@config
 
         delegate :dwh_platform, to: :@config
 
         private
 
-        def data_type
-          RUBY_TO_DWH_PLATFORM_TYPE_MAP[dwh_platform].fetch(column.type, 'unknown')
+        def data_type(type)
+          RUBY_TO_DWH_PLATFORM_TYPE_MAP[dwh_platform].fetch(type, 'unknown')
         end
       end
     end

data/lib/active_record/dbt/dbt_package/dbt_utils/table/{testable/unique_combination_of_columns_testable.rb → data_testable/unique_combination_of_columns_data_testable.rb}

@@ -5,8 +5,8 @@ module ActiveRecord
     module DbtPackage
       module DbtUtils
         module Table
-          module Testable
-            module UniqueCombinationOfColumnsTestable
+          module DataTestable
+            module UniqueCombinationOfColumnsDataTestable
               extend ActiveRecord::Dbt::RequiredMethods
 
               define_required_methods :table_name, :@config

data/lib/active_record/dbt/dbt_package/dbterd/column/{testable → data_testable}/relationships_meta_relationship_type.rb

@@ -5,7 +5,7 @@ module ActiveRecord
     module DbtPackage
       module Dbterd
         module Column
-          module Testable
+          module DataTestable
             module RelationshipsMetaRelationshipType
               extend ActiveRecord::Dbt::RequiredMethods
 
@@ -14,13 +14,12 @@ module ActiveRecord
               delegate :used_dbterd?, :add_log, to: :@config
 
               def relationships_meta_relationship_type
-                return nil unless used_dbterd?
-                return nil if no_relationship?
+                return nil if !used_dbterd? || no_relationship?
 
                 {
                   'relationship_type' => relationship_type
                 }
-              rescue NotSpecifiedOrNotInvalidIdError, StandardError => e
+              rescue NotSpecifiedOrNotInvalidIdError, NameError => e
                 relationships_meta_relationship_type_with_active_record_dbt_error(e)
               end
 
@@ -90,7 +89,7 @@ module ActiveRecord
               end
 
               def association_klass(association)
-                association.klass
+                association.class_name.constantize
               rescue NoMethodError
                 association.options.fetch(:through).to_s.classify.constantize
               end

data/lib/active_record/dbt/factory/columns/yml_factory.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Dbt
+    module Factory
+      module Columns
+        module YmlFactory
+          def self.build(
+            table_name,
+            primary_keys: ActiveRecord::Base.connection.primary_keys(table_name),
+            foreign_keys: ActiveRecord::Base.connection.foreign_keys(table_name)
+          )
+            ActiveRecord::Base.connection.columns(table_name).map do |column|
+              column_data_test = ActiveRecord::Dbt::Column::DataTest.new(
+                table_name,
+                column,
+                primary_keys: primary_keys,
+                foreign_keys: foreign_keys
+              )
+
+              ActiveRecord::Dbt::Column::Yml.new(
+                table_name,
+                column,
+                column_data_test,
+                primary_keys: primary_keys
+              )
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_record/dbt/factory/model/staging/yml_factory.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Dbt
+    module Factory
+      module Model
+        module Staging
+          module YmlFactory
+            def self.build(table_name)
+              table_factory = ActiveRecord::Dbt::Factory::Table::YmlFactory.build(table_name)
+              yml = ActiveRecord::Dbt::Model::Staging::Yml.new(table_factory)
+              struct = Struct.new(:export_path, :dump, keyword_init: true)
+
+              struct.new(
+                export_path: yml.export_path,
+                dump: yml.dump
+              )
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_record/dbt/factory/source/yml_factory.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Dbt
+    module Factory
+      module Source
+        module YmlFactory
+          def self.build
+            tables_factory = ActiveRecord::Dbt::Factory::Tables::YmlFactory.build
+
+            ActiveRecord::Dbt::Source::Yml.new(tables_factory)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_record/dbt/factory/table/yml_factory.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Dbt
+    module Factory
+      module Table
+        module YmlFactory
+          def self.build(table_name)
+            table_data_test = ActiveRecord::Dbt::Table::DataTest.new(table_name)
+            columns = ActiveRecord::Dbt::Factory::Columns::YmlFactory.build(table_name)
+
+            ActiveRecord::Dbt::Table::Yml.new(table_name, table_data_test, columns)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_record/dbt/factory/tables/yml_factory.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Dbt
+    module Factory
+      module Tables
+        module YmlFactory
+          def self.build
+            ActiveRecord::Base.connection.tables.sort.map do |table_name|
+              ActiveRecord::Dbt::Factory::Table::YmlFactory.build(table_name)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_record/dbt/i18n_wrapper/translate.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Dbt
+    module I18nWrapper
+      module Translate
+        extend ActiveRecord::Dbt::RequiredMethods
+
+        define_required_methods :table_name
+
+        delegate :singularize, to: :table_name, prefix: true
+
+        def translated_table_name
+          I18n.t("activerecord.models.#{table_name_singularize}", default: nil)
+        end
+
+        def translated_attribute_name
+          @translated_attribute_name ||=
+            translated_column_name || translated_default_column_name
+        end
+
+        private
+
+        def translated_column_name
+          I18n.t("activerecord.attributes.#{table_name_singularize}.#{column_name}", default: nil)
+        end
+
+        def translated_default_column_name
+          I18n.t("attributes.#{column_name}", default: nil)
+        end
+      end
+    end
+  end
+end

data/lib/active_record/dbt/model/staging/base.rb

@@ -36,7 +36,7 @@ module ActiveRecord
           private
 
           def basename
-            "#{export_directory_path}/#{model_name}"
+            "#{export_directory_path}/models/staging/#{source_name}/#{model_name}"
           end
 
           def model_name

data/lib/active_record/dbt/model/staging/yml.rb

@@ -21,23 +21,27 @@ module ActiveRecord
             "#{basename}.yml"
           end
 
-          def config
+          def dump
+            YAML.dump(properties.deep_stringify_keys)
+          end
+
+          private
+
+          def properties
             {
               'version' => 2,
               'models' => [
                 {
                   'name' => model_name,
-                  **table.config.except('name', 'columns'),
+                  **table.properties.except('name', 'columns'),
                   'columns' => override_columns
                 }
               ]
             }
           end
 
-          private
-
           def columns
-            @columns ||= sort_columns(table.config['columns'])
+            @columns ||= sort_columns(table.properties['columns'])
           end
 
           def sort_columns(columns)
@@ -75,7 +79,7 @@ module ActiveRecord
           end
 
           def add_relationship_test(column)
-            column['tests'].push(relationships_test(column['name']))
+            column['data_tests'].push(relationships_test(column['name']))
           end
 
           def rename_primary_id_in_column(column)