activerecord-dbt 0.1.0

data/lib/active_record/dbt/factory/model/staging_factory.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Dbt
+    module Factory
+      module Model
+        module StagingFactory
+          def self.yml_build(table_name)
+            table_factory = ActiveRecord::Dbt::Factory::TableFactory.build(table_name)
+            yml = ActiveRecord::Dbt::Model::Staging::Yml.new(table_factory)
+            struct = Struct.new(:export_path, :yml_dump, keyword_init: true)
+
+            struct.new(
+              export_path: yml.export_path,
+              yml_dump: YAML.dump(yml.config.deep_stringify_keys)
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_record/dbt/factory/source_factory.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Dbt
+    module Factory
+      module SourceFactory
+        def self.build
+          tables_factory = ActiveRecord::Dbt::Factory::TablesFactory.build
+          config = ActiveRecord::Dbt::Source::Yml.new(tables_factory).config
+
+          YAML.dump(config.deep_stringify_keys)
+        end
+      end
+    end
+  end
+end

data/lib/active_record/dbt/factory/table_factory.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Dbt
+    module Factory
+      module TableFactory
+        def self.build(table_name)
+          table_test = ActiveRecord::Dbt::Table::Test.new(table_name)
+          columns = ActiveRecord::Dbt::Factory::ColumnsFactory.build(table_name)
+
+          ActiveRecord::Dbt::Table::Yml.new(table_name, table_test, columns)
+        end
+      end
+    end
+  end
+end

data/lib/active_record/dbt/factory/tables_factory.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Dbt
+    module Factory
+      module TablesFactory
+        def self.build
+          ActiveRecord::Base.connection.tables.sort.map do |table_name|
+            ActiveRecord::Dbt::Factory::TableFactory.build(table_name)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_record/dbt/model/staging/base.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Dbt
+    module Model
+      module Staging
+        module Base
+          SORT_COLUMN_TYPES = %w[
+            ids enums
+            strings texts
+            integers floats decimals
+            binaries booleans
+            dates times
+            datetimes timestamps
+          ].freeze
+
+          delegate :source_name, :export_directory_path, to: :@config
+
+          def initialize(table_name)
+            @table_name = table_name
+            @config = ActiveRecord::Dbt::Config.instance
+          end
+
+          def rename_primary_id
+            @rename_primary_id ||= primary_key_eql_id? ? "#{table_name.singularize}_id" : nil
+          end
+
+          def primary_key_eql_id?
+            single_column_primary_key? && primary_keys.first == 'id'
+          end
+
+          def primary_key?(column_name)
+            primary_keys.include?(column_name)
+          end
+
+          private
+
+          def basename
+            "#{export_directory_path}/#{model_name}"
+          end
+
+          def model_name
+            "stg_#{source_name}__#{table_name}"
+          end
+
+          def single_column_primary_key?
+            primary_keys.size == 1
+          end
+
+          def primary_keys
+            @primary_keys = ActiveRecord::Base.connection.primary_keys(table_name)
+          end
+
+          def id?(column_name)
+            primary_key?(column_name) || foreign_key?(column_name)
+          end
+
+          def enum?(column_name)
+            table_name.singularize.classify.constantize.defined_enums.include?(column_name)
+          rescue NameError => e
+            add_log(self.class, e)
+
+            false
+          end
+
+          def foreign_key?(column_name)
+            ActiveRecord::Base.connection.foreign_key_exists?(table_name, column: column_name)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_record/dbt/model/staging/sql.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Dbt
+    module Model
+      module Staging
+        class Sql
+          include ActiveRecord::Dbt::Model::Staging::Base
+
+          attr_reader :table_name
+
+          def export_path
+            "#{basename}.sql"
+          end
+
+          def select_column_names
+            columns_group_by_column_type.sort_by do |key, _|
+              SORT_COLUMN_TYPES.index(key)
+            end.to_h
+          end
+
+          private
+
+          def columns
+            ActiveRecord::Base.connection.columns(table_name)
+          end
+
+          def columns_group_by_column_type
+            columns.group_by do |column|
+              if id?(column.name)
+                'ids'
+              elsif enum?(column.name)
+                'enums'
+              else
+                column.type.to_s.pluralize
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_record/dbt/model/staging/yml.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Dbt
+    module Model
+      module Staging
+        class Yml
+          include ActiveRecord::Dbt::Model::Staging::Base
+
+          attr_reader :table
+
+          delegate :table_name, to: :table
+          delegate :source_name, :data_sync_delayed?, :used_dbterd?, to: :@config
+
+          def initialize(table)
+            @table = table
+            super(table_name)
+          end
+
+          def export_path
+            "#{basename}.yml"
+          end
+
+          def config
+            {
+              'version' => 2,
+              'models' => [
+                {
+                  'name' => model_name,
+                  **table.config.except('name', 'columns'),
+                  'columns' => override_columns
+                }
+              ]
+            }
+          end
+
+          private
+
+          def columns
+            @columns ||= sort_columns(table.config['columns'])
+          end
+
+          def sort_columns(columns)
+            columns.sort_by do |column|
+              [
+                SORT_COLUMN_TYPES.index(column_type(column)) || -1,
+                columns.index(column)
+              ]
+            end
+          end
+
+          def column_type(column)
+            if id?(column['name'])
+              'ids'
+            elsif enum?(column['name'])
+              'enums'
+            else
+              column.dig('meta', 'column_type').pluralize
+            end
+          end
+
+          def override_columns
+            return columns unless single_column_primary_key?
+
+            columns.map { |column| override_column(column) }
+          end
+
+          def override_column(column)
+            return column unless primary_key?(column['name'])
+
+            column.tap do |c|
+              add_relationship_test(c)
+              rename_primary_id_in_column(c) if primary_key_eql_id?
+            end
+          end
+
+          def add_relationship_test(column)
+            column['tests'].push(relationships_test(column['name']))
+          end
+
+          def rename_primary_id_in_column(column)
+            column['description'] = rename_primary_id if column['name'] == column['description']
+            column['name'] = rename_primary_id
+          end
+
+          def relationships_test(column_name)
+            {
+              'relationships' => {
+                'severity' => data_sync_delayed? ? 'warn' : nil,
+                'to' => "source('#{source_name}', '#{table_name}')",
+                'field' => column_name,
+                'meta' => relationships_meta_relationship_type
+              }.compact
+            }
+          end
+
+          def relationships_meta_relationship_type
+            return nil unless used_dbterd?
+
+            {
+              'relationship_type' => 'one-to-one'
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_record/dbt/railtie.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Dbt
+    class Railtie < ::Rails::Railtie
+    end
+  end
+end

data/lib/active_record/dbt/source/yml.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Dbt
+    module Source
+      class Yml
+        attr_reader :tables
+
+        delegate :source_config, to: :@config
+
+        def initialize(tables)
+          @tables = tables
+          @config = ActiveRecord::Dbt::Config.instance
+        end
+
+        def config
+          {
+            'version' => 2,
+            'sources' => [
+              source_properties.merge('tables' => tables_properties)
+            ]
+          }
+        end
+
+        private
+
+        def source_properties
+          source_config[:sources]
+        end
+
+        def tables_properties
+          tables.map(&:config)
+        end
+      end
+    end
+  end
+end

data/lib/active_record/dbt/table/base.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Dbt
+    module Table
+      module Base
+        attr_reader :table_name
+
+        def initialize(table_name)
+          @table_name = table_name
+          @config = ActiveRecord::Dbt::Config.instance
+        end
+      end
+    end
+  end
+end

data/lib/active_record/dbt/table/test.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Dbt
+    module Table
+      class Test
+        include ActiveRecord::Dbt::DbtPackage::DbtUtils::Table::Testable::UniqueCombinationOfColumnsTestable
+
+        include ActiveRecord::Dbt::Table::Base
+
+        def config
+          [
+            *unique_combination_of_columns_test
+          ].compact.presence
+        end
+      end
+    end
+  end
+end

data/lib/active_record/dbt/table/yml.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Dbt
+    module Table
+      class Yml
+        include ActiveRecord::Dbt::Table::Base
+
+        attr_reader :table_test, :columns
+
+        delegate :source_config, to: :@config
+
+        def initialize(table_name, table_test, columns)
+          super(table_name)
+          @table_test = table_test
+          @columns = columns
+        end
+
+        def config
+          {
+            **table_properties,
+            'columns' => columns.map(&:config)
+          }.compact
+        end
+
+        private
+
+        def table_properties
+          {
+            'name' => table_name,
+            'description' => description,
+            **table_overrides.except(:columns),
+            'tests' => table_test.config
+          }
+        end
+
+        def description
+          return logical_name if table_description.blank?
+
+          [
+            "# #{logical_name}",
+            table_description
+          ].join("\n")
+        end
+
+        def logical_name
+          @logical_name ||=
+            source_config.dig(:table_descriptions, table_name, :logical_name) ||
+            translated_table_name ||
+            default_logical_name ||
+            "Write a logical_name of the '#{table_name}' table."
+        end
+
+        def translated_table_name
+          I18n.t("activerecord.models.#{table_name.singularize}", default: nil)
+        end
+
+        def default_logical_name
+          source_config.dig(:defaults, :table_descriptions, :logical_name)
+                       &.gsub(/{{\s*table_name\s*}}/, table_name)
+        end
+
+        def table_description
+          @table_description ||= source_config.dig(:table_descriptions, table_name, :description)
+        end
+
+        def table_overrides
+          @table_overrides ||=
+            source_config.dig(:table_overrides, table_name) ||
+            {}
+        end
+      end
+    end
+  end
+end

data/lib/active_record/dbt/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Dbt
+    VERSION = '0.1.0'
+  end
+end

data/lib/active_record/dbt.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require 'zeitwerk'
+loader = Zeitwerk::Loader.for_gem_extension(ActiveRecord)
+loader.setup
+
+module ActiveRecord
+  module Dbt
+    def self.configure
+      yield config = ActiveRecord::Dbt::Config.instance
+
+      config
+    end
+
+    class Error < StandardError; end
+  end
+end

data/lib/generators/active_record/dbt/config/USAGE

@@ -0,0 +1,9 @@
+Description:
+    Create configuration files for dbt.
+
+Example:
+    bin/rails generate active_record:dbt:config
+
+    This will create:
+        #{config_directory_path}/source_config.yml
+        #{config_directory_path}/staging_model.sql.tt

data/lib/generators/active_record/dbt/config/config_generator.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Dbt
+    module Generators
+      class ConfigGenerator < Rails::Generators::Base
+        source_root File.expand_path('templates', __dir__)
+
+        def create_source_config_file
+          template 'source_config.yml.tt', config.source_config_path
+        end
+
+        def copy_staging_model_sql_tt_file
+          copy_file File.expand_path('../staging_model/templates/staging_model.sql.tt', __dir__),
+                    "#{config.config_directory_path}/staging_model.sql.tt"
+        end
+
+        private
+
+        def config
+          @config ||= ActiveRecord::Dbt::Config.instance
+        end
+
+        def application_name
+          Rails.application.class.name.sub(/::Application$/, '').downcase
+        end
+      end
+    end
+  end
+end

data/lib/generators/active_record/dbt/config/templates/source_config.yml.tt

@@ -0,0 +1,68 @@
+sources:
+  name: <%= application_name %>
+  meta:
+    generated_by: activerecord-dbt
+  description: |-
+    Write a description of the '<%= application_name %>' source.
+    You can write multiple lines.
+
+table_overrides:
+  <table_name>:
+    meta: {<dictionary>}
+    identifier: <table_name>
+    loaded_at_field: <column_name>
+    tests:
+      - <test>
+    tags: [<string>]
+    freshness:
+      warn_after:
+        count: <positive_integer>
+        period: minute | hour | day
+      error_after:
+        count: <positive_integer>
+        period: minute | hour | day
+      filter: <where-condition>
+    quoting:
+      database: true | false
+      schema: true | false
+      identifier: true | false
+    external: {<dictionary>}
+    columns:
+      <column_name>:
+        meta: {<dictionary>}
+        quote: true | false
+        tests:
+          - <test>
+        tags: [<string>]
+
+defaults:
+  table_descriptions:
+    logical_name: Write a logical_name of the '{{ table_name }}' table.
+    columns:
+      description: Write a description of the '{{ table_name }}.{{ column_name }}' column.
+
+table_descriptions:
+  ar_internal_metadata:
+    logical_name: Internal Metadata
+    description: |-
+      By default Rails will store information about your Rails environment and schema
+      in an internal table named `ar_internal_metadata`.
+    columns:
+      key: Key
+      value: Value
+      created_at: Created At
+      updated_at: Updated At
+  schema_migrations:
+    logical_name: Schema Migrations
+    description: |-
+      Rails keeps track of which migrations have been committed to the database and
+      stores them in a neighboring table in that same database called `schema_migrations`.
+    columns:
+      version: The version number of the migration.
+  <table_name>:
+    logical_name: Write a table logical name.
+    description: |-
+      Write a description of the table.
+      You can write multiple lines.
+    columns:
+      <colume_name>: Write a description of the column.

data/lib/generators/active_record/dbt/initializer/USAGE

@@ -0,0 +1,8 @@
+Description:
+    Create an initializer file for dbt:.
+
+Example:
+    bin/rails generate active_record:dbt:initializer
+
+    This will create:
+        config/initializers/dbt.rb

data/lib/generators/active_record/dbt/initializer/initializer_generator.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Dbt
+    module Generators
+      class InitializerGenerator < Rails::Generators::Base
+        source_root File.expand_path('templates', __dir__)
+
+        def copy_initializer_file
+          copy_file 'dbt.rb', 'config/initializers/dbt.rb'
+        end
+      end
+    end
+  end
+end

data/lib/generators/active_record/dbt/initializer/templates/dbt.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+require 'active_record/dbt'
+
+ActiveRecord::Dbt.configure do |c|
+  c.config_directory_path = 'lib/dbt'
+  c.export_directory_path = 'doc/dbt'
+  c.data_sync_delayed = false
+  c.used_dbt_package_names = []
+end

data/lib/generators/active_record/dbt/source/USAGE

@@ -0,0 +1,8 @@
+Description:
+    Generate a source file for dbt.
+
+Example:
+    bin/rails generate active_record:dbt:source
+
+    This will create:
+        #{export_directory_path}/src_#{source_name}.yml

data/lib/generators/active_record/dbt/source/source_generator.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Dbt
+    module Generators
+      class SourceGenerator < Rails::Generators::Base
+        source_root File.expand_path('templates', __dir__)
+
+        def create_source_yml_file
+          create_file "#{config.export_directory_path}/src_#{config.source_name}.yml",
+                      ActiveRecord::Dbt::Factory::SourceFactory.build
+        end
+
+        private
+
+        def config
+          @config ||= ActiveRecord::Dbt::Config.instance
+        end
+      end
+    end
+  end
+end

data/lib/generators/active_record/dbt/staging_model/USAGE

@@ -0,0 +1,9 @@
+Description:
+    Generate staging model files for dbt that reference the specified TABLE_NAME.
+
+Example:
+    bin/rails generate active_record:dbt:staging_model TABLE_NAME
+
+    This will create:
+        #{export_directory_path}/stg_#{source_name}__#{table_name}.sql
+        #{export_directory_path}/stg_#{source_name}__#{table_name}.yml