k_domain 0.0.2 → 0.0.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a69b94955a43f91d71e034b94a5caf0d3fdbb7b130c480d4eabc45818bbe2262
-  data.tar.gz: 0f4a126ee7c9bed5d99ac7449e5f97ec558030b126dbaa3c6194cdb5ce8430bd
+  metadata.gz: a50df0535cc32830a2035c2d2ddd328454efcef8aa97199485ae4db1dc7d64e7
+  data.tar.gz: 34824c4725142b76154a256d6466e7c83e8fc684178f6cb589552ea508d49dfa
 SHA512:
-  metadata.gz: 3a7a877e80c9f23cdb93ab7952e027bdddc89bd6e888212c6fc0622cc0d5ae6d9b73e3e9651949ecd1afa2f1323e6446045f579cb6e83c030c5d8d711979deaa
-  data.tar.gz: 434275e01ec2308d4a1407f5e8bd7ab9ec9708218510b6a719c144e2d9b6abb7e2caafa67efc617ac33295cd276c2326ccd961468ee4870b38266ef1b1295296
+  metadata.gz: 404ae56ada4ee46644d0c562b97dc86e9ce64164e25d1f66119399c4a053e41c183ff5c28184e5de493c76218bfb7f71ff548df02f9b374cb5d57b38c4ceaf3c
+  data.tar.gz: 27c472902e4ca2ea1dfb4803489cf5719356d02a2e64bfcee27ac0179bec5d42bb8c684cbf7196e77dd26a737f8c427bf4aa6809a3a7b930d19acaf269ab9247

data/.rubocop.yml

@@ -6,12 +6,37 @@ AllCops:
   NewCops: enable
   Exclude:
     - "_/**/*"
-    - "spec/samples/**/*"
+    - "spec/sample_input/**/*"
+    - "spec/sample_output/**/*"
+    - "lib/k_domain/raw_db_schema/template.rb"
+
+Metrics/PerceivedComplexity:
+  Exclude:
+    - "lib/k_domain/domain_model/transform_steps/*.rb"
+    - "lib/k_domain/domain_model/dtos/entity.rb"
+    - "lib/k_domain/domain_model/dtos/old/entity.rb"
+
+Metrics/AbcSize:
+  Exclude:
+    - "lib/k_domain/domain_model/transform_steps/*.rb"
+    - "lib/k_domain/domain_model/dtos/rails_model.rb"
+    - "lib/k_domain/domain_model/dtos/entity.rb"
+    - "lib/k_domain/domain_model/dtos/column_old.rb"
+    - "lib/k_domain/domain_model/dtos/old/column_old.rb"
+    - "lib/k_domain/domain_model/dtos/old/rails_model.rb"
+    - "lib/k_domain/domain_model/dtos/old/entity.rb"
+
+Metrics/CyclomaticComplexity:
+  Exclude:
+    - "lib/k_domain/domain_model/transform_steps/*.rb"
+    - "lib/k_domain/domain_model/dtos/entity.rb"
+    - "lib/k_domain/domain_model/dtos/old/entity.rb"
 
 Metrics/BlockLength:
   Exclude:
     - "**/spec/**/*"
     - "*.gemspec"
+    - 'lib/k_domain/domain_model/dtos/erd_file.rb'    
   IgnoredMethods:
     - configure
     - context
@@ -34,12 +59,19 @@ Metrics/BlockLength:
 
 Metrics/MethodLength:
   Max: 25
+  Exclude:
+    - "lib/k_domain/domain_model/transform_steps/*.rb"
+    - "lib/k_domain/domain_model/dtos/rails_model.rb"
+    - "lib/k_domain/domain_model/dtos/old/entity.rb"
+    - "lib/k_domain/domain_model/dtos/old/rails_model.rb"
 
 Layout/LineLength:
   Max: 200
   # Ignores annotate output
   IgnoredPatterns: ['\A# \*\*']
   IgnoreCopDirectives: true
+  Exclude:
+    - "lib/k_domain/domain_model/transform_steps/*.rb"
 
 Lint/UnusedMethodArgument:
   AllowUnusedKeywordArguments: true
@@ -77,6 +109,12 @@ Lint/AmbiguousBlockAssociation:
 Style/AccessorGrouping:
   Enabled: false
 
+Style/FormatStringToken:
+  Enabled: false
+
+Style/Documentation:
+  Enabled: false
+
 Layout/SpaceBeforeComma:
   Enabled: false
 # My Preferences - End

data/Gemfile

@@ -23,3 +23,13 @@ group :development, :test do
   gem 'rubocop-rake', require: false
   gem 'rubocop-rspec', require: false
 end
+
+# If local dependency
+if ENV['KLUE_LOCAL_GEMS']&.to_s&.downcase == 'true'
+  group :development, :test do
+    puts 'Using Local GEMs'
+    gem 'peeky'                   , path: '../peeky'
+    gem 'k_log'                   , path: '../k_log'
+    gem 'k_util'                  , path: '../k_util'
+  end
+end

data/Rakefile

@@ -26,7 +26,9 @@ task :publish do
   system 'gem build'
   system "gem push #{GEM_NAME}-#{KDomain::VERSION}.gem"
 end
-
+task :build do
+  system 'gem build'
+end
 desc 'Remove old *.gem files'
 task :clean do
   system 'rm *.gem'

data/STORIES.md

@@ -8,9 +8,28 @@ As an Application Developer, I need a rich and configurable ERD schema, so I can
 
 ### Stories next on list
 
-As a Developer, I can DO_SOMETHING, so that I QUALITY_OF_LIFE
+As a Developer, I can print any of the domain structures, so that I can visually my domain
 
-- Subtask
+- Hook up log.structure
+
+As a Developer, I can customize domain configuration, so that I can have opinions about names and types
+
+- Handle traits
+
+As a Developer, I can read native rails model data, so that I can leverage existing rails applications for ERD modeling
+
+- Use Meta Programming and re-implement ActiveRecord::Base
+
+### Tasks next on list
+
+Refactor / Simply
+
+- Replace complex objects an with structs for ancillary data structures such as investigate
+
+User acceptance testing
+
+- Provide sample printers for each data structure to visually check data is loading
+- Point raw_db_schema loader towards a complex ERD and check how it performs
 
 ## Stories and tasks
 

data/k_domain.gemspec

@@ -33,10 +33,14 @@ Gem::Specification.new do |spec|
       f.match(%r{^(test|spec|features)/})
     end
   end
+  
   spec.bindir        = 'exe'
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
   # spec.extensions    = ['ext/k_domain/extconf.rb']
 
+  spec.add_dependency 'activesupport'   , '~> 6'
+  spec.add_dependency 'dry-struct', '~> 1'
   spec.add_dependency 'k_log' , '~> 0.0.0'
+  spec.add_dependency 'peeky' , '~> 0.0.0'
 end

data/lib/k_domain/domain_model/load.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+# Annotates the original schema with methods that implement existing method calls
+# that are already in the schema so that we can build a hash.
+#
+# Writes a new annotated schema.rb file with a public method called load that
+# builds the hash
+
+module KDomain
+  module DomainModel
+    class Load
+      include KLog::Logging
+
+      attr_reader :source_file
+      attr_reader :data
+
+      def initialize(source_file)
+        @source_file = source_file
+      end
+
+      def call
+        json = File.read(source_file)
+        @raw_data = KUtil.data.json_parse(json, as: :hash_symbolized)
+
+        @data = KDomain::Schemas::DomainModel.new(@raw_data)
+      end
+
+      def to_h
+        return nil unless defined? @raw_data
+
+        @raw_data
+      end
+    end
+  end
+end

data/lib/k_domain/domain_model/transform.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+# Loads the db schema object and works through a series of enrichment steps to
+# that builds the domain modal
+
+module KDomain
+  module DomainModel
+    class Transform
+      include KLog::Logging
+
+      attr_reader :db_schema
+      attr_reader :target_step_file
+      attr_reader :target_file
+      attr_reader :erd_path
+
+      def initialize(db_schema: , target_file: , target_step_file: , erd_path:)
+        @db_schema        = db_schema
+        @target_step_file = target_step_file
+        @target_file      = target_file
+        @erd_path         = erd_path
+      end
+
+      def call
+        valid = true
+        valid &&= step1
+        valid &&= step2
+        valid &&= step3
+        valid &&= step4
+        valid &&= step5
+        valid &&= step8 # NOT SURE WHERE THIS BELONGS
+
+        raise 'DomainModal transform failed' unless valid
+
+        write
+
+        nil
+      end
+
+      def step1
+        Step1AttachDbSchema.run(domain_data, db_schema: db_schema)
+        write(step: '1-attach-db-schema')
+      end
+
+      def step2
+        Step2AttachModels.run(domain_data, erd_path: erd_path)
+        write(step: '2-attach-model')
+      end
+
+      def step3
+        Step3AttachColumns.run(domain_data)
+        write(step: '3-attach-columns')
+      end
+
+      def step4
+        Step4AttachErdFiles.run(domain_data, erd_path: erd_path)
+        write(step: '4-attach-erd-files')
+      end
+
+      def step5
+        Step5AttachDictionary.run(domain_data, erd_path: erd_path)
+        write(step: '5-attach-dictionary')
+      end
+
+      def step8
+        Step8LocateRailsModels.run(domain_data, erd_path: erd_path)
+        write(step: '8-rails-files-models')
+      end
+
+      def write(step: nil)
+        file = if step.nil?
+                 target_file
+               else
+                 format(target_step_file, step: step)
+               end
+        FileUtils.mkdir_p(File.dirname(file))
+        File.write(file, JSON.pretty_generate(domain_data))
+      end
+
+      def domain_data
+        # The initial domain model structure is created here, but populated during the workflows.
+        @domain_data ||= {
+          domain: {
+            models: [],
+            erd_files: [],
+          },
+          rails: {
+            models: [],
+            controllers: [],
+          },
+          rails_resource: {
+            models: [],
+            controllers: [],
+          },
+          dictionary: {
+            items: []
+          },
+          database: {
+            tables: [],
+            indexes: [],
+            foreign_keys: [],
+            meta: {}
+          },
+          investigate: {
+            issues: []
+          }
+        }
+      end
+    end
+  end
+end

data/lib/k_domain/domain_model/transform_steps/_.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+# The require order is important due to dependencies
+require_relative './step'
+require_relative './step1_attach_db_schema'
+require_relative './step2_attach_models'
+require_relative './step3_attach_columns'
+require_relative './step4_attach_erd_files'
+require_relative './step5_attach_dictionary'
+require_relative './step8_locate_rails_models'

data/lib/k_domain/domain_model/transform_steps/step.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+module KDomain
+  module DomainModel
+    class Step
+      include KLog::Logging
+
+      attr_reader :domain_data
+      attr_reader :opts
+      attr_reader :valid
+      alias valid? valid
+
+      def initialize(domain_data, **opts)
+        # Useful for debugging
+        # log.info "Initialize #{self.class.name}"
+
+        @domain_data  = domain_data
+        @opts         = opts
+        @valid        = true
+      end
+
+      def call; end
+
+      def self.run(domain_data, **opts)
+        step = new(domain_data, **opts)
+        step.call
+        step
+      end
+
+      def guard(message)
+        log.error message
+        @valid = false
+      end
+
+      # Domain Model Accessor/Helpers
+      def domain
+        guard('domain is missing') if domain_data[:domain].nil?
+
+        domain_data[:domain]
+      end
+
+      def domain_models
+        domain[:models]
+      end
+
+      # Rails File Accessor/Helpers
+      def rails_resource
+        guard('rails_resource is missing') if domain_data[:rails_resource].nil?
+
+        domain_data[:rails_resource]
+      end
+
+      def rails_resource_models
+        rails_resource[:models]
+      end
+
+      def rails_resource_models=(value)
+        rails_resource[:models] = value
+      end
+
+      def rails_resource_controllers
+        rails_resource[:controllers]
+      end
+
+      # Database Accessor/Helpers
+      def database=(value)
+        domain_data[:database] = value
+      end
+
+      def database
+        guard('database is missing') if domain_data[:database].nil?
+
+        domain_data[:database]
+      end
+
+      def database_tables
+        guard('database_tables is missing') if database[:tables].nil?
+
+        database[:tables]
+      end
+
+      def database_foreign_keys
+        guard('database_foreign_keys is missing') if database[:foreign_keys].nil?
+
+        database[:foreign_keys]
+      end
+
+      def find_table_for_model(model)
+        database_tables.find { |table| table[:name] == model[:table_name] }
+      end
+
+      def table_name_exist?(table_name)
+        if table_name.nil?
+          guard('table_name_exist? was provided with a table_name: nil')
+          return false
+        end
+        database_table_name_hash.key?(table_name)
+      end
+
+      def find_foreign_table(lhs_table_name, column_name)
+        fk = database_foreign_keys.find { |foreign_key| foreign_key[:left] == lhs_table_name && foreign_key[:column] == column_name }
+        return fk[:right] if fk
+
+        nil
+      end
+
+      def investigate(step:, location:, key:, message:)
+        unique_key = build_key(step, location, key)
+
+        return if issue_hash.key?(unique_key)
+
+        value = { step: step, location: location, key: key, message: message }
+
+        issues << value                   # list
+        issue_hash[unique_key] = value    # lookup
+      end
+
+      def issues
+        domain_data[:investigate][:issues]
+      end
+
+      private
+
+      def database_table_name_hash
+        @database_table_name_hash ||= database_tables.to_h { |table| [table[:name], table[:name]] }
+      end
+
+      def build_key(*values)
+        values.join('-')
+      end
+
+      def issue_hash
+        return @issue_hash if defined? @issue_hash
+
+        @issue_hash = issues.to_h do |issue|
+          unique_key = build_key(issue[:step], issue[:location], issue[:key])
+          [unique_key, issue]
+        end
+      end
+    end
+  end
+end

data/lib/k_domain/domain_model/transform_steps/step1_attach_db_schema.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module KDomain
+  module DomainModel
+    class Step1AttachDbSchema < KDomain::DomainModel::Step
+      # Map database schema to domain model
+      def call
+        raise 'Schema not supplied' if opts[:db_schema].nil?
+
+        self.database = opts[:db_schema].clone
+
+        guard('tables are missing')               if database[:tables].nil?
+        guard('indexes are missing')              if database[:indexes].nil?
+        guard('foreign keys are missing')         if database[:foreign_keys].nil?
+        guard('rails version is missing')         if database[:meta][:rails].nil?
+        guard('postgres extensions are missing')  if database[:meta][:db_info][:extensions].nil?
+        guard('unique keys are missing')          if database[:meta][:unique_keys].nil?
+      end
+    end
+  end
+end

data/lib/k_domain/domain_model/transform_steps/step2_attach_models.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+# Loop through the db_schema tables and build up a
+# basic model for each table
+class Step2AttachModels < KDomain::DomainModel::Step
+  # Map database schema to domain model
+  def call
+    raise 'ERD path not supplied' if opts[:erd_path].nil?
+
+    # Schema is re-shaped into a format designed for domain modeling
+    domain[:models] = database_tables.map { |table| model(table) }
+  end
+
+  def model(table)
+    table_name = table[:name].to_s
+    model_name = table_name.singularize
+
+    {
+      name: model_name,
+      name_plural: table_name, # need to check if this is correct as I know it is wrong for account_history_datum
+      table_name: table_name,
+      pk: primary_key(table),
+      erd_location: location(table_name, model_name),
+      statistics: {},                                   # Load in future step
+      columns: []                                       # Load in future step
+    }
+  end
+
+  def primary_key(table)
+    {
+      name: table[:primary_key],
+      type: table[:primary_key_type],
+      exist: !table[:primary_key].nil?
+    }
+  end
+
+  # Location of source code
+  def location(table_name, model_name)
+    file_normal = File.join(opts[:erd_path], "#{model_name}.rb")
+    file_custom = File.join(opts[:erd_path], "#{table_name}.rb")
+    file_exist  = true
+    state = []
+
+    if File.exist?(file_normal)
+      file = file_normal
+      state.push(:has_ruby_model)
+    elsif File.exist?(file_custom)
+      file = file_custom
+      state.push(:has_ruby_model)
+      state.push(:nonconventional_name)
+    else
+      file = ''
+      file_exist = false
+    end
+
+    {
+      file: file,
+      exist: file_exist,
+      state: state                      # display_state: state.join(' ')
+    }
+  end
+end

data/lib/k_domain/domain_model/transform_steps/step3_attach_columns.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+# Attach columns to models
+class Step3AttachColumns < KDomain::DomainModel::Step
+  attr_accessor :table
+  attr_reader :column_name
+  attr_reader :column_symbol
+
+  def call
+    build_columns
+  end
+
+  def build_columns
+    domain_models.each do |model|
+      @table = find_table_for_model(model)
+      columns = columns(table[:columns])
+      columns = insert_primary_key(model, columns)
+      model[:columns] = columns
+    end
+  end
+
+  def column_data(name)
+    @column_name   = name
+    @column_symbol = name.to_sym
+    {
+      name: name,
+      name_plural: name.pluralize,
+      type: nil,
+      precision: nil,
+      scale: nil,
+      default: nil,
+      null: nil,
+      limit: nil,
+      array: nil
+    }
+  end
+
+  def columns(db_columns)
+    db_columns.map do |db_column|
+      column = column_data(db_column[:name]).merge(
+        type: check_type(db_column[:type]),
+        precision: db_column[:precision],
+        scale: db_column[:scale],
+        default: db_column[:default],
+        null: db_column[:null],
+        limit: db_column[:limit],
+        array: db_column[:array]
+      )
+
+      expand_column(column)
+    end
+  end
+
+  def insert_primary_key(model, columns)
+    return columns unless model[:pk][:exist]
+
+    column = column_data('id').merge(
+      type: check_type(model[:pk][:type])
+    )
+
+    columns.unshift(expand_column(column))
+    columns
+  end
+
+  def expand_column(column)
+    foreign_table = lookup_foreign_table(column_name)
+    is_foreign = !foreign_table.nil?
+    # is_foreign = foreign_key?(column_name)
+    structure_type = structure_type(is_foreign)
+
+    column.merge({
+                   structure_type: structure_type,
+                   foreign_key: is_foreign,
+                   foreign_table: (foreign_table || '').singularize,
+                   foreign_table_plural: (foreign_table || '').pluralize
+                 })
+  end
+
+  def check_type(type)
+    type = type.to_sym if type.is_a?(String)
+
+    return type if %i[string integer bigint bigserial boolean float decimal datetime date hstore text jsonb].include?(type)
+
+    if type.nil?
+      guard('nil type detected for db_column[:type]')
+
+      return :string
+    end
+
+    guard("new type detected for db_column[:type] - #{type}")
+
+    camel.parse(type.to_s).downcase
+  end
+
+  def lookup_foreign_table(column_name)
+    foreign_table = find_foreign_table(table[:name], column_name)
+
+    return foreign_table if foreign_table
+
+    cn = column_name.to_s
+
+    if cn.ends_with?('_id')
+      table_name = column_name[0..-4]
+      table_name_plural = table_name.pluralize
+
+      if table_name_exist?(table_name_plural.to_s)
+        investigate(step: :step3_attach_columns,
+                    location: :lookup_foreign_table,
+                    key: column_name,
+                    message: "#{@table[:name]}.#{column_name} => #{table_name_plural} - Relationship not found in DB, so have inferred this relationship. You may want to check that this relation is correct")
+
+        return table_name
+      end
+
+      investigate(step: :step3_attach_columns,
+                  location: :lookup_foreign_table,
+                  key: column_name,
+                  message: "#{@table[:name]}.#{column_name} => #{table_name_plural} - Table not found for a column that looks like foreign_key")
+    end
+
+    nil
+  end
+
+  # Need some configurable data dictionary where by
+  # _token can be setup on a project by project basis
+  def structure_type(is_foreign)
+    return :foreign_key         if is_foreign
+    return :primary_key         if column_symbol == :id
+    return :timestamp           if column_symbol == :created_at || column_symbol == :updated_at
+    return :timestamp           if column_symbol == :created_at || column_symbol == :updated_at
+    return :deleted_at          if column_symbol == :deleted_at
+    return :encrypted_password  if column_symbol == :encrypted_password
+    return :token               if column_name.ends_with?('_token') || column_name.ends_with?('_token_iv')
+
+    :data
+  end
+end