activerecord-dbt 0.1.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 897ffab9f05715634ea7eb0ef68be9c70e8d305458657696fe544255f17293c9
-  data.tar.gz: f82f9f7b02c592572077f3ba41bf0920ae0abb831de065b707d107124080353d
+  metadata.gz: affd8c9f9864d11cbaf5c35a759ea603c72784b9eae777987debad89fd8cd509
+  data.tar.gz: 6b8ca507c1e413143f3d0a7330451767190b3acb731e70c8b2beb87bff9df64b
 SHA512:
-  metadata.gz: cd87b1c4a5ba3e97aad5320f58eb9151e14ff8752b93519b0b14314b9ba872b48afee79e93e09de813898ccf83dc3d41a0d64d07277c1e8cf7413dabd93a7fd2
-  data.tar.gz: 829ff19c802bc18c7e721ede97beff805e26e9642382153a64ececb21e7f5b6512cab9f226e1ef3c7ca51399a9ec4604cf7b3b1a1f61e237e74a99327b12429e
+  metadata.gz: 2798392f8cdcb454a64957a005d7d16e3eeec7e1648234ba5c598cbacb6b683198493337619c9e0a9f17f67a231f6a3fc6f34658651671de3776c7caf28093c9
+  data.tar.gz: d417dfdd69d8a0c41baee03a41a0880ecba05abbbb61c86b80ed1ef48021bcdda16a4e2b43dd414a29d7f56ad9cf92a0e3afe0a693b5ae4aa3a38e0bbeb9f03d

data/README.md

@@ -1,6 +1,10 @@
 # ActiveRecord::Dbt
 
-`ActiveRecord::Dbt` generates dbt files from the information of the database connected via ActiveRecord.
+[![Gem Version](https://badge.fury.io/rb/activerecord-dbt.svg)](https://badge.fury.io/rb/activerecord-dbt)
+[![Maintainability](https://api.codeclimate.com/v1/badges/ef9a0a71c73dae7b8156/maintainability)](https://codeclimate.com/github/yamotech/activerecord-dbt/maintainability)
+[![Ruby CI](https://github.com/yamotech/activerecord-dbt/actions/workflows/ruby-ci.yml/badge.svg)](https://github.com/yamotech/activerecord-dbt/actions/workflows/ruby-ci.yml)
+
+`ActiveRecord::Dbt` generates [dbt](https://www.getdbt.com/) files from the information of the database connected via ActiveRecord.
 
 Currently, it can generate `yaml` files for `sources` and `models` files for `staging`.
 
@@ -50,9 +54,28 @@ Configuration | Description
 --------- | ---------
 config_directory_path | The path to the directory where files generated by `bin/rails generate active_record:dbt:*` are stored. The default is `lib/dbt`.
 export_directory_path | The path to the directory where configuration files are stored. The default is `doc/dbt`.
+dwh_platform | Specify the data warehouse platform to which dbt connects. The default is `bigquery`.
 data_sync_delayed | Indicates whether there is a data delay. If set to `true`, `severity: warn` is applied to the `relationships` test. The default is `false`.
 logger | The destination for log output. The default is `Logger.new('./log/active_record_dbt.log')`.
 used_dbt_package_names | An array of `dbt` package names to use.
+locale | I18n locale. The default is `I18n.locale`.
+
+List of platforms that can currently be set with `dwh_platform`.
+
+Data Warehouse Platform | Link
+--------- | ---------
+bigquery | [BigQuery enterprise data warehouse - Google Cloud](https://cloud.google.com/bigquery?hl=en)
+postgres | [PostgreSQL: The world's most advanced open source database](https://www.postgresql.org/)
+redshift | [Cloud Data Warehouse - Amazon Redshift - AWS](https://aws.amazon.com/redshift/)
+snowflake | [The Snowflake AI Data Cloud - Mobilize Data, Apps, and AI](https://www.snowflake.com/en/)
+spark | [Apache Spark™ - Unified Engine for large-scale data analytics](https://spark.apache.org/)
+
+List of packages that can currently be set with `used_dbt_package_names`.
+
+dbt Package Name | Link
+--------- | ---------
+dbt-labs/dbt-utils | [dbt-labs/dbt-utils: Utility functions for dbt projects.](https://github.com/dbt-labs/dbt-utils)
+datnguye/dbterd | [datnguye/dbterd: Generate the ERD as a code from dbt artifacts](https://github.com/datnguye/dbterd)
 
 Example:
 
@@ -143,10 +166,10 @@ table_overrides:
 
 ##### defaults
 
-Set default values for the `name` and `description` of `tables`.
+Set the default value for the `description`(`logical_name`, `description`) of `tables`.
 
-In `logical_name` and `description` of `table_descriptions`, you can refer to the table name with `{{ table_name }}`.
-In the `description` of `columns`, you can refer to the table name with `{{ table_name }}` and the column name with `{{ column_name }}`.
+In the `logical_name` and `description` of `table_descriptions`, you can refer to the table name with `{{ table_name }}`.
+In the `description` of `table_descriptions.columns`, you can refer to the table name with `{{ table_name }}` and the column name with `{{ column_name }}`.
 
 Example:
 
@@ -294,25 +317,21 @@ sources:
     columns:
     - name: key
       description: Key
-      meta:
-        column_type: string
+      data_type: string
       tests:
       - unique
       - not_null
     - name: value
       description: Value
-      meta:
-        column_type: string
+      data_type: string
     - name: created_at
       description: Created At
-      meta:
-        column_type: datetime
+      data_type: datetime
       tests:
       - not_null
     - name: updated_at
       description: Updated At
-      meta:
-        column_type: datetime
+      data_type: datetime
       tests:
       - not_null
   - name: companies
@@ -320,29 +339,24 @@ sources:
     columns:
     - name: id
       description: id
-      meta:
-        column_type: integer
+      data_type: int64
       tests:
       - unique
       - not_null
     - name: name
       description: Write a description of the 'companies.name' column.
-      meta:
-        column_type: string
+      data_type: string
       tests:
       - not_null
     - name: establishment_date
       description: Write a description of the 'companies.establishment_date' column.
-      meta:
-        column_type: string
+      data_type: string
     - name: average_age
       description: Write a description of the 'companies.average_age' column.
-      meta:
-        column_type: float
+      data_type: float64
     - name: published
       description: Write a description of the 'companies.published' column.
-      meta:
-        column_type: boolean
+      data_type: bool
       tests:
       - not_null
       - accepted_values:
@@ -352,14 +366,12 @@ sources:
           quote: false
     - name: created_at
       description: Created At
-      meta:
-        column_type: datetime
+      data_type: datetime
       tests:
       - not_null
     - name: updated_at
       description: Updated At
-      meta:
-        column_type: datetime
+      data_type: datetime
       tests:
       - not_null
   - name: posts
@@ -367,15 +379,13 @@ sources:
     columns:
     - name: id
       description: ID
-      meta:
-        column_type: integer
+      data_type: int64
       tests:
       - unique
       - not_null
     - name: user_id
       description: User
-      meta:
-        column_type: integer
+      data_type: int64
       tests:
       - not_null
       - relationships:
@@ -385,28 +395,23 @@ sources:
             relationship_type: many-to-one
     - name: title
       description: Title
-      meta:
-        column_type: string
+      data_type: string
     - name: content
       description: Content
-      meta:
-        column_type: text
+      data_type: string
     - name: created_at
       description: Post Created At
-      meta:
-        column_type: datetime
+      data_type: datetime
       tests:
       - not_null
     - name: updated_at
       description: Post Updated At
-      meta:
-        column_type: datetime
+      data_type: datetime
       tests:
       - not_null
     - name: status
       description: Write a description of the 'posts.status' column.
-      meta:
-        column_type: integer
+      data_type: int64
       tests:
       - accepted_values:
           values:
@@ -424,8 +429,7 @@ sources:
     columns:
     - name: post_id
       description: post_id
-      meta:
-        column_type: integer
+      data_type: int64
       tests:
       - not_null
       - relationships:
@@ -438,8 +442,7 @@ sources:
               message: uninitialized constant PostsTag
     - name: tag_id
       description: tag_id
-      meta:
-        column_type: integer
+      data_type: int64
       tests:
       - not_null
       - relationships:
@@ -455,15 +458,13 @@ sources:
     columns:
     - name: id
       description: id
-      meta:
-        column_type: integer
+      data_type: int64
       tests:
       - unique
       - not_null
     - name: user_id
       description: user_id
-      meta:
-        column_type: integer
+      data_type: int64
       tests:
       - unique
       - not_null
@@ -474,26 +475,22 @@ sources:
             relationship_type: one-to-one
     - name: first_name
       description: Write a description of the 'profiles.first_name' column.
-      meta:
-        column_type: string
+      data_type: string
       tests:
       - not_null
     - name: last_name
       description: Write a description of the 'profiles.last_name' column.
-      meta:
-        column_type: string
+      data_type: string
       tests:
       - not_null
     - name: created_at
       description: Created At
-      meta:
-        column_type: datetime
+      data_type: datetime
       tests:
       - not_null
     - name: updated_at
       description: Updated At
-      meta:
-        column_type: datetime
+      data_type: datetime
       tests:
       - not_null
   - name: relationships
@@ -506,15 +503,13 @@ sources:
     columns:
     - name: id
       description: id
-      meta:
-        column_type: integer
+      data_type: int64
       tests:
       - unique
       - not_null
     - name: follower_id
       description: follower_id
-      meta:
-        column_type: integer
+      data_type: int64
       tests:
       - not_null
       - relationships:
@@ -524,8 +519,7 @@ sources:
             relationship_type: many-to-one
     - name: followed_id
       description: followed_id
-      meta:
-        column_type: integer
+      data_type: int64
       tests:
       - not_null
       - relationships:
@@ -535,14 +529,12 @@ sources:
             relationship_type: many-to-one
     - name: created_at
       description: Created At
-      meta:
-        column_type: datetime
+      data_type: datetime
       tests:
       - not_null
     - name: updated_at
       description: Updated At
-      meta:
-        column_type: datetime
+      data_type: datetime
       tests:
       - not_null
   - name: schema_migrations
@@ -553,8 +545,7 @@ sources:
     columns:
     - name: version
       description: The version number of the migration.
-      meta:
-        column_type: string
+      data_type: string
       tests:
       - unique
       - not_null
@@ -563,28 +554,24 @@ sources:
     columns:
     - name: id
       description: id
-      meta:
-        column_type: integer
+      data_type: int64
       tests:
       - unique
       - not_null
     - name: name
       description: Write a description of the 'tags.name' column.
-      meta:
-        column_type: string
+      data_type: string
       tests:
       - unique
       - not_null
     - name: created_at
       description: Created At
-      meta:
-        column_type: datetime
+      data_type: datetime
       tests:
       - not_null
     - name: updated_at
       description: Updated At
-      meta:
-        column_type: datetime
+      data_type: datetime
       tests:
       - not_null
   - name: user_tags
@@ -597,15 +584,13 @@ sources:
     columns:
     - name: id
       description: id
-      meta:
-        column_type: integer
+      data_type: int64
       tests:
       - unique
       - not_null
     - name: user_id
       description: user_id
-      meta:
-        column_type: integer
+      data_type: int64
       tests:
       - not_null
       - relationships:
@@ -615,8 +600,7 @@ sources:
             relationship_type: many-to-one
     - name: tag_id
       description: tag_id
-      meta:
-        column_type: integer
+      data_type: int64
       tests:
       - not_null
       - relationships:
@@ -626,14 +610,12 @@ sources:
             relationship_type: many-to-one
     - name: created_at
       description: Created At
-      meta:
-        column_type: datetime
+      data_type: datetime
       tests:
       - not_null
     - name: updated_at
       description: Updated At
-      meta:
-        column_type: datetime
+      data_type: datetime
       tests:
       - not_null
   - name: users
@@ -649,28 +631,24 @@ sources:
     columns:
     - name: id
       description: ID
-      meta:
-        column_type: integer
+      data_type: int64
       tests:
       - unique
       - not_null
     - name: created_at
       description: User Created At
-      meta:
-        column_type: datetime
+      data_type: datetime
       tests:
       - not_null:
           where: id != 1
     - name: updated_at
       description: User Updated At
-      meta:
-        column_type: datetime
+      data_type: datetime
       tests:
       - not_null
     - name: company_id
       description: company_id
-      meta:
-        column_type: integer
+      data_type: int64
       tests:
       - relationships:
           to: source('dummy', 'companies')
@@ -854,6 +832,10 @@ from final
 
 Example:
 
+> [!NOTE]
+>
+> The output will be as shown below. It is recommended to indent the YAML file with a tool of your choice.
+
 ```yaml
 ---
 version: 2
@@ -863,8 +845,7 @@ models:
   columns:
   - name: profile_id
     description: profile_id
-    meta:
-      column_type: integer
+    data_type: int64
     tests:
     - unique
     - not_null
@@ -875,8 +856,7 @@ models:
           relationship_type: one-to-one
   - name: user_id
     description: user_id
-    meta:
-      column_type: integer
+    data_type: int64
     tests:
     - unique
     - not_null
@@ -887,31 +867,137 @@ models:
           relationship_type: one-to-one
   - name: first_name
     description: Write a description of the 'profiles.first_name' column.
-    meta:
-      column_type: string
+    data_type: string
     tests:
     - not_null
   - name: last_name
     description: Write a description of the 'profiles.last_name' column.
-    meta:
-      column_type: string
+    data_type: string
     tests:
     - not_null
   - name: created_at
     description: Created At
-    meta:
-      column_type: datetime
+    data_type: datetime
     tests:
     - not_null
   - name: updated_at
     description: Updated At
-    meta:
-      column_type: datetime
+    data_type: datetime
     tests:
     - not_null
 
 ```
 
+### Generated dbt Seed Files
+
+#### dbt Seed Configuration
+
+In the `#{config_directory_path}/source_config.yml` file, describe the properties you want to set for the seed enum.
+You can configure `defaults` in this file.
+
+##### defaults
+
+Set the default value for the `description` of the `seeds` enum.
+
+In the `description` of `seed_descriptions.enum`, you can refer to the source name with `{{ source_name }}`, the translated table name with `{{ translated_table_name }}`, and the translated column name with `{{ translated_attribute_name }}`.
+
+Example:
+
+```yml
+defaults:
+  seed_descriptions:
+    enum:
+      description: "{{ source_name }} {{ translated_table_name }} {{ translated_attribute_name }} enum"
+
+```
+
+If nothing is set, it defaults to the following:
+
+```yml
+defaults:
+  seed_descriptions:
+    enum:
+      description: "{{ source_name }} {{ translated_table_name }} {{ translated_attribute_name }} enum"
+
+```
+
+#### Generate dbt Seed Enum Files
+
+Generate seed enum files for dbt:
+
+```bash
+$ bin/rails generate active_record:dbt:enum TABLE_NAME ENUM_COLUMN_NAME
+```
+
+Generate seed enum files for dbt from the specified `TABLE_NAME` and `ENUM_COLUMN_NAME`.
+
+File | Description
+--------- | ---------
+`#{export_directory_path}/seed_#{source_name}__#{table_name_singularize}_enum_#{enum_pluralized}.csv` | Seed enum file for dbt.
+`#{export_directory_path}/seed_#{source_name}__#{table_name_singularize}_enum_#{enum_pluralized}.yml` | Seed enum documentation file for dbt.
+
+Example:
+
+```bash
+$ bin/rails generate active_record:dbt:enum posts status
+```
+
+##### Generate `#{export_directory_path}/seed_#{source_name}__#{table_name_singularize}_enum_#{enum_pluralized}.csv`
+
+Example:
+
+```csv
+status_before_type_of_cast,status_key,status_en,status_ja
+0,draft,Draft,下書き
+1,published,Published,公開
+2,deleted,Deleted,削除
+
+```
+
+##### Generate `#{export_directory_path}/seed_#{source_name}__#{table_name_singularize}_enum_#{enum_pluralized}.yml`
+
+Example:
+
+> [!NOTE]
+>
+> The output will be as shown below. It is recommended to indent the YAML file with a tool of your choice.
+
+```yaml
+---
+version: 2
+seeds:
+- name: seed_dummy__post_enum_statuses
+  description: dummy Post Status enum
+  config:
+    column_types:
+      status_before_type_of_cast: int64
+      status_key: string
+      status_en: string
+      status_ja: string
+columns:
+- name: status_before_type_of_cast
+  description: Status
+  tests:
+  - unique
+  - not_null
+- name: status_key
+  description: Status(key)
+  tests:
+  - unique
+  - not_null
+- name: status_en
+  description: Status(en)
+  tests:
+  - unique
+  - not_null
+- name: status_ja
+  description: Status(ja)
+  tests:
+  - unique
+  - not_null
+
+```
+
 ## Contributing
 
 Contribution directions go here.

data/lib/active_record/dbt/column/column.rb

@@ -4,6 +4,9 @@ module ActiveRecord
   module Dbt
     module Column
       class Column
+        include ActiveRecord::Dbt::DataType::Mapper
+        include ActiveRecord::Dbt::I18nWrapper::Translate
+
         attr_reader :table_name, :column, :column_test, :primary_keys
 
         delegate :name, :comment, to: :column, prefix: true
@@ -21,7 +24,7 @@ module ActiveRecord
           {
             'name' => column_name,
             'description' => description,
-            'meta' => { 'column_type' => column.type.to_s },
+            'data_type' => data_type(column.type),
             **column_overrides.except(:tests),
             'tests' => column_test.config
           }.compact
@@ -43,18 +46,6 @@ module ActiveRecord
           source_config.dig(:table_descriptions, table_name, :columns, column_name)
         end
 
-        def translated_attribute_name
-          translated_column_name || translated_default_column_name
-        end
-
-        def translated_column_name
-          I18n.t("activerecord.attributes.#{table_name.singularize}.#{column_name}", default: nil)
-        end
-
-        def translated_default_column_name
-          I18n.t("attributes.#{column_name}", default: nil)
-        end
-
         def key_column_name
           column_name if primary_key? || foreign_key?
         end

data/lib/active_record/dbt/column/testable/accepted_values_testable.rb

@@ -5,17 +5,13 @@ module ActiveRecord
     module Column
       module Testable
         module AcceptedValuesTestable
-          REQUIRED_ACCEPTED_VALUES_TESTABLE_METHODS = %i[@config column table_name column_name].freeze
+          extend ActiveRecord::Dbt::RequiredMethods
+
+          define_required_methods :@config, :column, :table_name, :column_name
 
           delegate :type, to: :column, prefix: true
           delegate :add_log, to: :@config
 
-          REQUIRED_ACCEPTED_VALUES_TESTABLE_METHODS.each do |method_name|
-            define_method(method_name) do
-              raise NotImplementedError, "You must implement #{self.class}##{__method__}"
-            end
-          end
-
           def accepted_values_test
             return nil unless column_type == :boolean || enum_values.present?
 

data/lib/active_record/dbt/column/testable/not_null_testable.rb

@@ -5,16 +5,18 @@ module ActiveRecord
     module Column
       module Testable
         module NotNullTestable
-          REQUIRED_NOT_NULL_TESTABLE_METHODS = %i[column].freeze
+          extend ActiveRecord::Dbt::RequiredMethods
 
-          REQUIRED_NOT_NULL_TESTABLE_METHODS.each do |method_name|
-            define_method(method_name) do
-              raise NotImplementedError, "You must implement #{self.class}##{__method__}"
-            end
-          end
+          define_required_methods :column
 
           def not_null_test
-            column.null == true ? nil : 'not_null'
+            null? ? nil : 'not_null'
+          end
+
+          private
+
+          def null?
+            column.null == true
           end
         end
       end