k_domain 0.0.16 → 0.0.20

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 98e1d3eb1435f17ad91084519399234f0adc41e399d1350071147f0e58f694c6
-  data.tar.gz: 2797a698e87169816a6b612239b1c22b8d2fd0db2fd6df2fc8e078fbf8828e69
+  metadata.gz: de5fc8f220781026dd327f40da53eb3a424ec23f38785be8fddc8f1856970dc1
+  data.tar.gz: 5573e60dc8665ab146e0edeb291ebcd3db6b792180e05e24721df9d519038f86
 SHA512:
-  metadata.gz: 1e6cd0d3fa8354f83b5910dc961212acd0ae149b0b90c31338bf992b4f64500ae4fb05f63c605bf439d818bed8f6f4806939994c0cf8ee6fbd0c0a51d232b02f
-  data.tar.gz: 28d42c273b6fdae5db9b32e4cfcd4c1826e42f929efbc9713941b16bc4240cff22eed7e3260aec195beff43ae3ec20bf136bb6526988eccbd5f475c18ec2566a
+  metadata.gz: 4ecaa0fef7e6d622f18d2dfe53bdddf2c17fa91e1c2796ee7bdddf5044880f742003d8cb6f7bb4007eeab4bd2f0becba6f8f0fb329f58776f807cd200bd4ced9
+  data.tar.gz: 76af19179f670d7e4b18121f031b77e18c366259526530b6623f988c5b3e2985ca3a993441e6caacfb7cb71da6afb05c99803234aa85045facdb2c8f09b1194b

data/STORIES.md

@@ -16,25 +16,54 @@ As a Developer, I can customize domain configuration, so that I can have opinion
 
 - Handle traits
 
+### Tasks next on list
+
+BUGs in domain_model load
+
+- spec/k_domain/domain_model/load_spec.rb:135
+- FIXED: module_name should be empty
+- FIXED: name mismatch (attr_accessor, attr_reader, attr_writer) in behaviours vs in (attr_accessors, attr_readers, attr_writers) functions
+- functions - (attr_accessors, attr_readers, attr_writers) are all empty
+
+External ruby process - ShimLoading and ExtractModel
+
+- You need a clear memory foot print for shim loading and model extraction, best to run these from inside a new ruby process
+
+Log Warning to Investigate Issues
+
+- FIXED: All the logged warnings in k_domain need to turn up in on the investigate issues register
+- Investigate needs a debug flag that when turned on, will write the issues to the console
+
+Print progress dot
+
+- Make this configurable
+- Decide what steps this should run for
+- Show step label via configuration
+
+## Stories and tasks
+
+### Stories - completed
+
 As a Developer, I can read native rails model data, so that I can leverage existing rails applications for ERD modeling
 
+- Proof of concept
 - Use Meta Programming and re-implement ActiveRecord::Base
 
-### Tasks next on list
+### Tasks - completed
 
 Refactor / Simply
 
-- Replace complex objects an with structs for ancillary data structures such as investigate
+- Replace complex objects with structs for ancillary data structures such as investigate
+
+Steps to support write methods on base class
+
+- Simplify lib/k_domain/domain_model/transform.rb
 
 User acceptance testing
 
 - Provide sample printers for each data structure to visually check data is loading
 - Point raw_db_schema loader towards a complex ERD and check how it performs
 
-## Stories and tasks
-
-### Tasks - completed
-
 Setup RubyGems and RubyDoc
 
 - Build and deploy gem to [rubygems.org](https://rubygems.org/gems/k_domain)

data/lib/k_domain/domain_model/transform.rb

@@ -23,13 +23,12 @@ module KDomain
       # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
       def call
         valid = true
-        valid &&= step1
-        valid &&= step2
-        valid &&= step3
-        valid &&= step4
-        valid &&= step5
-        valid &&= step8
-        valid &&= step9
+        valid &&= Step1AttachDbSchema.run(domain_data, db_schema: db_schema, step_file: step_file('1-attach-db-schema'))
+        valid &&= Step2AttachModels.run(domain_data, erd_path: erd_path, step_file: step_file('2-attach-model'))
+        valid &&= Step3AttachColumns.run(domain_data, step_file: step_file('3-attach-columns'))
+        valid &&= Step4RailsResourceModels.run(domain_data, erd_path: erd_path, step_file: step_file('4-rails-resource-models'))
+        valid &&= Step5RailsModels.run(domain_data, erd_path: erd_path, step_file: step_file('5-rails-models'))
+        valid &&= Step6AttachDictionary.run(domain_data, erd_path: erd_path, step_file: step_file('6-attach-dictionary'))
 
         raise 'DomainModal transform failed' unless valid
 
@@ -39,49 +38,13 @@ module KDomain
       end
       # rubocop:enable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
 
-      def step1
-        Step1AttachDbSchema.run(domain_data, db_schema: db_schema)
-        write(step: '1-attach-db-schema')
+      def step_file(step_name)
+        target_step_file % { step: step_name }
       end
 
-      def step2
-        Step2AttachModels.run(domain_data, erd_path: erd_path)
-        write(step: '2-attach-model')
-      end
-
-      def step3
-        Step3AttachColumns.run(domain_data)
-        write(step: '3-attach-columns')
-      end
-
-      def step4
-        Step4AttachErdFiles.run(domain_data, erd_path: erd_path)
-        write(step: '4-attach-erd-files')
-      end
-
-      def step5
-        Step5AttachDictionary.run(domain_data, erd_path: erd_path)
-        write(step: '5-attach-dictionary')
-      end
-
-      def step8
-        Step8RailsResourceModels.run(domain_data, erd_path: erd_path)
-        write(step: '8-rails-resource-models')
-      end
-
-      def step9
-        Step9RailsStructureModels.run(domain_data, erd_path: erd_path)
-        write(step: '8-rails-structure-models')
-      end
-
-      def write(step: nil)
-        file = if step.nil?
-                 target_file
-               else
-                 format(target_step_file, step: step)
-               end
-        FileUtils.mkdir_p(File.dirname(file))
-        File.write(file, JSON.pretty_generate(domain_data))
+      def write
+        FileUtils.mkdir_p(File.dirname(target_file))
+        File.write(target_file, JSON.pretty_generate(domain_data))
       end
 
       # rubocop:disable Metrics/MethodLength
@@ -90,7 +53,6 @@ module KDomain
         @domain_data ||= {
           domain: {
             models: [],
-            erd_files: []
           },
           rails_resource: {
             models: [],

data/lib/k_domain/domain_model/transform_steps/_.rb

@@ -5,7 +5,6 @@ require_relative './step'
 require_relative './step1_attach_db_schema'
 require_relative './step2_attach_models'
 require_relative './step3_attach_columns'
-require_relative './step4_attach_erd_files'
-require_relative './step5_attach_dictionary'
-require_relative './step8_rails_resource_models'
-require_relative './step9_rails_structure_models'
+require_relative './step4_rails_resource_models'
+require_relative './step5_rails_models'
+require_relative './step6_attach_dictionary'

data/lib/k_domain/domain_model/transform_steps/step.rb

@@ -24,7 +24,8 @@ module KDomain
       def self.run(domain_data, **opts)
         step = new(domain_data, **opts)
         step.call
-        step
+        step.write(opts[:step_file])
+        step.valid?
       end
 
       def guard(message)
@@ -32,6 +33,11 @@ module KDomain
         @valid = false
       end
 
+      def write(file)
+        FileUtils.mkdir_p(File.dirname(file))
+        File.write(file, JSON.pretty_generate(domain_data))
+      end
+
       # Domain Model Accessor/Helpers
       def domain
         guard('domain is missing') if domain_data[:domain].nil?

data/lib/k_domain/domain_model/transform_steps/{step8_rails_resource_models.rb → step4_rails_resource_models.rb}

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 # Locate rails model files
-class Step8RailsResourceModels < KDomain::DomainModel::Step
+class Step4RailsResourceModels < KDomain::DomainModel::Step
   attr_accessor :ruby_code
 
   def call

data/lib/k_domain/domain_model/transform_steps/step5_rails_models.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+# Locate rails model files
+class Step5RailsModels < KDomain::DomainModel::Step
+  def call
+    raise 'ERD path not supplied' unless opts[:erd_path]
+
+    self.rails_structure_models = rails_resource_models.map do |resource|
+      process_resource(OpenStruct.new(resource))
+    end
+  end
+
+  private
+
+  def process_resource(resource)
+    @model = {
+      model_name: resource.model_name,
+      table_name: resource.table_name,
+      file: resource.file,
+      exist: resource.exist,
+      state: resource.state,
+      code: resource.exist ? File.read(resource.file) : '',
+      behaviours: {},
+      functions: {}
+    }
+
+    return @model unless  resource.exist
+
+    @model[:behaviours] = extract_model_behavior(resource.file)
+    @model[:functions] = extract_model_functions(resource.file)
+
+    @model
+  end
+
+  def extract_model_behavior(file)
+    extractor.extract(file)
+    extractor.model
+  end
+
+  def extract_model_functions(file)
+    klass_name = File.basename(file, File.extname(file))
+
+    klass = case klass_name
+            when 'clearbit_quota'
+              ClearbitQuota
+            when 'account_history_data'
+              AccountHistoryData
+            else
+              Module.const_get(klass_name.classify)
+            end
+
+    class_info = Peeky.api.build_class_info(klass.new)
+
+    class_info.to_h
+  rescue StandardError => e
+    log.exception(e)
+  end
+
+  def extractor
+    @extractor ||= KDomain::RailsCodeExtractor::ExtractModel.new(shim_loader)
+  end
+
+  def shim_loader
+    return opts[:shim_loader] if !opts[:shim_loader].nil? && opts[:shim_loader].is_a?(KDomain::RailsCodeExtractor::ShimLoader)
+
+    shim_loader = KDomain::RailsCodeExtractor::ShimLoader.new
+    shim_loader.register(:fake_module  , KDomain::Gem.resource('templates/fake_module_shims.rb'))
+    shim_loader.register(:active_record, KDomain::Gem.resource('templates/active_record_shims.rb'))
+    shim_loader
+  end
+end

data/lib/k_domain/domain_model/transform_steps/{step5_attach_dictionary.rb → step6_attach_dictionary.rb}

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 # Attach data dictionary
-class Step5AttachDictionary < KDomain::DomainModel::Step
+class Step6AttachDictionary < KDomain::DomainModel::Step
   attr_reader :dictionary
 
   def call
@@ -27,8 +27,11 @@ class Step5AttachDictionary < KDomain::DomainModel::Step
       entry[:model_count] = entry[:model_count] + 1
 
       unless entry[:types].include?(column_type)
-        # ADD to investigate
-        # log.warn("#{model_name} has a type mismatch for column name: #{column_name}")
+        investigate(step: :step5_attach_dictionary,
+          location: :process,
+          key: "#{model_name},#{column_name},#{column_type}",
+          message: "#{model_name} has a type mismatch for column name: #{column_name}")
+
         entry[:types] << column_type
         entry[:type_count] = entry[:type_count] + 1
       end

data/lib/k_domain/rails_code_extractor/extract_model.rb

@@ -6,16 +6,53 @@ module KDomain
     class ExtractModel
       include KLog::Logging
 
-      attr_reader :source_file
-      attr_reader :data
+      attr_reader :shims_loaded
+      attr_reader :models
+      attr_reader :model
 
-      def initialize(source_file)
-        @source_file = source_file
+      def initialize(load_shim)
+        @load_shim = load_shim
+        @shims_loaded = false
+        @models = []
       end
 
-      def call
-        log.kv 'source_file', source_file
+      def extract(file)
+        load_shims unless shims_loaded
+
+        ActiveRecord.current_class = nil
+
+        load_retry(file, 10)
+      rescue StandardError => e
+        log.exception(e)
+      end
+
+      private
+
+      def load_shims
+        @load_shim.call
+        @shims_loaded = true
+      end
+
+      # rubocop:disable Security/Eval,Style/EvalWithLocation,Style/DocumentDynamicEvalDefinition,Metrics/AbcSize
+      def load_retry(file, times)
+        return if times.negative?
+
+        load(file)
+
+        @model = ActiveRecord.current_class
+        @models << @model
+
+        # get_method_info(File.base_name(file))
+      rescue StandardError => e
+        puts e.message
+        if e.is_a?(NameError)
+          log.kv('add module', e.name)
+          eval("module #{e.name}; end")
+          return load_retry(path, times - 1)
+        end
+        log.exception(e)
       end
+      # rubocop:enable Security/Eval,Style/EvalWithLocation,Style/DocumentDynamicEvalDefinition,Metrics/AbcSize
     end
   end
 end

data/lib/k_domain/rails_code_extractor/{load_shim.rb → shim_loader.rb}

@@ -8,20 +8,17 @@
 # 2. Inject fake module/classes that would otherwise break code loading with various exceptions
 module KDomain
   module RailsCodeExtractor
-    class LoadShim
+    class ShimLoader
       include KLog::Logging
 
       attr_reader :shim_files
 
-      attr_reader :dsl_shim_file
-      attr_reader :fake_module_file
-
       def initialize
         @shim_files = []
       end
 
       def call
-        log.kv 'preload', preload
+        shim_files.select { |sf| sf[:exist] }.each { |sf| require sf[:file] }
       end
 
       def register(name, file)

data/lib/k_domain/schemas/_.rb

@@ -7,6 +7,7 @@ module Types
 end
 
 require_relative 'rails_resource'
+require_relative 'rails_structure'
 require_relative 'investigate'
 require_relative 'database/_'
 require_relative 'dictionary'

data/lib/k_domain/schemas/database/index.rb

@@ -8,7 +8,7 @@ module KDomain
       attribute :using                , Types::Nominal::String
       attribute :order?               , Types::Nominal::Hash
       attribute :where?               , Types::Nominal::Any.optional.default(nil)
-      attribute :unique? , Types::Nominal::Any.optional.default(nil)
+      attribute :unique?              , Types::Nominal::Any.optional.default(nil)
     end
   end
 end

data/lib/k_domain/schemas/database/table.rb

@@ -20,13 +20,13 @@ module KDomain
         attribute :limit?               , Types::Strict::Integer.optional.default(nil)
       end
 
-      attribute :name                 , Types::Strict::String
-      attribute :primary_key          , Types::Strict::String.optional.default(nil)
-      attribute :primary_key_type     , Types::Strict::String.optional.default(nil)
-      attribute :id?                  , Types::Nominal::Any.optional.default(nil)
-      attribute :columns              , Types::Strict::Array.of(KDomain::Database::Table::Column)
-      attribute :indexes              , Types::Strict::Array.of(KDomain::Database::Index) # May want to have a Table::Index, but for now this is a shared scheam
-      attribute :rails_schema         , KDomain::Database::Table::RailsSchema
+      attribute :name                   , Types::Strict::String
+      attribute :primary_key            , Types::Strict::String.optional.default(nil)
+      attribute :primary_key_type       , Types::Strict::String.optional.default(nil)
+      attribute :id?                    , Types::Nominal::Any.optional.default(nil)
+      attribute :columns                , Types::Strict::Array.of(KDomain::Database::Table::Column)
+      attribute :indexes                , Types::Strict::Array.of(KDomain::Database::Index) # May want to have a Table::Index, but for now this is a shared scheam
+      attribute :rails_schema           , KDomain::Database::Table::RailsSchema
     end
   end
 end

data/lib/k_domain/schemas/domain/domain.rb

@@ -5,7 +5,7 @@ module KDomain
   module DomainModel
     class Domain < Dry::Struct
       attribute :models               , Types::Strict::Array.of(KDomain::DomainModel::Model)
-      attribute :erd_files            , Types::Strict::Array.of(KDomain::DomainModel::ErdFile)
+      # attribute :erd_files            , Types::Strict::Array.of(KDomain::DomainModel::ErdFile)
     end
   end
 end

data/lib/k_domain/schemas/domain_model.rb

@@ -4,11 +4,12 @@
 module KDomain
   module Schemas
     class DomainModel < Dry::Struct
-      attribute :domain         , KDomain::DomainModel::Domain
-      attribute :database       , KDomain::Database::Schema
-      attribute :dictionary     , KDomain::Schemas::Dictionary
-      attribute :rails_resource , KDomain::Schemas::RailsResource
-      attribute :investigate    , KDomain::Schemas::Investigate
+      attribute :domain           , KDomain::DomainModel::Domain
+      attribute :database         , KDomain::Database::Schema
+      attribute :dictionary       , KDomain::Schemas::Dictionary
+      attribute :rails_resource   , KDomain::Schemas::RailsResource
+      attribute :rails_structure  , KDomain::Schemas::RailsStructure
+      attribute :investigate      , KDomain::Schemas::Investigate
     end
   end
 end

data/lib/k_domain/schemas/rails_structure.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+# Domain class holds a dictionary entry
+module KDomain
+  module Schemas
+    class RailsStructure < Dry::Struct
+      class DefaultScope < Dry::Struct
+        attribute :block?                     , Types::Strict::String
+      end
+
+      class BaseType < Dry::Struct
+        attribute :name?                      , Types::Strict::String
+        attribute :opts?                      , Types::Strict::Hash
+        attribute :block?                     , Types::Strict::String.optional.default(nil)
+      end
+
+      class Scope < KDomain::Schemas::RailsStructure::BaseType
+      end
+
+      class BelongsTo < KDomain::Schemas::RailsStructure::BaseType
+      end
+
+      class HasOne < KDomain::Schemas::RailsStructure::BaseType
+      end
+
+      class HasMany < KDomain::Schemas::RailsStructure::BaseType
+      end
+
+      class HasAndBelongsToMany < KDomain::Schemas::RailsStructure::BaseType
+      end
+
+      class Validate < Dry::Struct
+        attribute :names?                     , Types::Array.of(Types::Strict::String)
+        attribute :opts?                      , Types::Strict::Hash
+        attribute :block?                     , Types::Strict::String.optional.default(nil)
+      end
+
+      class Validates < KDomain::Schemas::RailsStructure::BaseType
+      end
+
+      class Behaviours < Dry::Struct
+        attribute :class_name?                , Types::Strict::String
+        attribute :default_scope?             , KDomain::Schemas::RailsStructure::DefaultScope
+        attribute :scopes?                    , Types::Strict::Array.of(KDomain::Schemas::RailsStructure::Scope)
+        attribute :belongs_to?                , Types::Strict::Array.of(KDomain::Schemas::RailsStructure::BelongsTo)
+        attribute :has_one?                   , Types::Strict::Array.of(KDomain::Schemas::RailsStructure::HasOne)
+        attribute :has_many?                  , Types::Strict::Array.of(KDomain::Schemas::RailsStructure::HasMany)
+        attribute :has_and_belongs_to_many?   , Types::Strict::Array.of(KDomain::Schemas::RailsStructure::HasAndBelongsToMany)
+        attribute :validate?                  , Types::Strict::Array.of(KDomain::Schemas::RailsStructure::Validate)
+        attribute :validates?                 , Types::Strict::Array.of(KDomain::Schemas::RailsStructure::Validates)
+        attribute :attr_accessor?             , Types::Array.of(Types::Strict::String)
+        attribute :attr_reader?               , Types::Array.of(Types::Strict::String)
+        attribute :attr_writer?               , Types::Array.of(Types::Strict::String)
+      end
+
+      class Method < Dry::Struct
+        attribute :name                       , Types::Strict::String
+      end
+
+      class Functions < Dry::Struct
+        attribute :class_name?                , Types::Strict::String
+        attribute :module_name?               , Types::Strict::String
+        attribute :class_full_name?           , Types::Strict::String
+        attribute :attr_accessor?             , Types::Array.of(Types::Strict::String)
+        attribute :attr_reader?               , Types::Array.of(Types::Strict::String)
+        attribute :attr_writer?               , Types::Array.of(Types::Strict::String)
+        attribute :klass?                     , Types::Strict::Array.of(KDomain::Schemas::RailsStructure::Method)
+        attribute :instance_public?           , Types::Strict::Array.of(KDomain::Schemas::RailsStructure::Method)
+        attribute :instance_private?          , Types::Strict::Array.of(KDomain::Schemas::RailsStructure::Method)
+      end
+
+      class Model < Dry::Struct
+        attribute :model_name                 , Types::Strict::String
+        attribute :table_name                 , Types::Strict::String
+        attribute :file                       , Types::Strict::String
+        attribute :exist                      , Types::Strict::Bool
+        attribute :state                      , Types::Strict::String
+        attribute :code                       , Types::Strict::String
+        attribute :behaviours?                , KDomain::Schemas::RailsStructure::Behaviours
+        attribute :functions?                 , KDomain::Schemas::RailsStructure::Functions
+      end
+
+      class Controller < Dry::Struct
+      end
+
+      attribute :models                       , Types::Strict::Array.of(KDomain::Schemas::RailsStructure::Model)
+      attribute :controllers                  , Types::Strict::Array.of(KDomain::Schemas::RailsStructure::Controller)
+    end
+  end
+end
+
+# attribute :domain           , KDomain::DomainModel::Domain

data/lib/k_domain/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module KDomain
-  VERSION = '0.0.16'
+  VERSION = '0.0.20'
 end

data/lib/k_domain.rb

@@ -11,8 +11,8 @@ require 'k_domain/raw_db_schema/load'
 require 'k_domain/domain_model/transform'
 require 'k_domain/domain_model/transform_steps/_'
 require 'k_domain/domain_model/load'
-require 'k_domain/rails_code_extractor/load_shim'
-# require 'k_domain/rails_code_extractor/extract_model'
+require 'k_domain/rails_code_extractor/shim_loader'
+require 'k_domain/rails_code_extractor/extract_model'
 
 # # This is useful if you want to initialize structures via Hash
 # class SymbolizeStruct < Dry::Struct

data/templates/active_record_shims.rb

@@ -1,10 +1,10 @@
 module ActiveRecord
-  def self.all_loaded_classes
-    @all_loaded_classes ||= []
+  def self.current_class
+    @current_class ||= nil
   end
 
-  def self.last_loaded_class
-    @last_loaded_class ||= nil
+  def self.current_class=(value)
+    @current_class = value
   end
 
   class Base
@@ -15,17 +15,11 @@ module ActiveRecord
     end
 
     def self.class_info
-      return @class_info if defined? @class_info
+      return ActiveRecord.current_class if ActiveRecord.current_class
 
-      ActiveRecord.last_loaded_class = name
-
-      @class_info = {
+      ActiveRecord.current_class = {
         class_name: name
       }
-
-      ActiveRecord.all_loaded_classes << class_info
-
-      @class_info
     end
 
     def self.set(key, value)
@@ -67,15 +61,36 @@ module ActiveRecord
       add(:enum, opts)
     end
 
+    # def self.attr_accessor(*args)
+    #   args.each do |arg|
+    #     self.class_eval("def #{arg};@#{arg};end")
+    #     self.class_eval("def #{arg}=(val);@#{arg}=val;end")
+    #   end
+    # end
+    # def self.attr_reader(*args)
+    #   args.each do |arg|
+    #     self.class_eval("def #{arg};@#{arg};end")
+    #   end
+    # end
+    # def self.attr_writer(*args)
+    #   args.each do |arg|
+    #     self.class_eval("def #{arg};@#{arg};end")
+    #     self.class_eval("def #{arg}=(val);@#{arg}=val;end")
+    #   end
+    # end
+  
     def self.attr_accessor(*names)
+      super(*names)
       add(:attr_accessor, names)
     end
 
     def self.attr_reader(*names)
+      super(*names)
       add(:attr_reader, names)
     end
 
     def self.attr_writer(*names)
+      super(*names)
       add(:attr_writer, names)
     end
 
@@ -159,8 +174,6 @@ module ActiveRecord
       block_source = nil
       block_source = lambda_source(block, 'validate') if block_given?
 
-      set(:default_scope, opts.merge(block: block_source))
-
       add(:validate, {
             names: names,
             opts: opts,

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: k_domain
 version: !ruby/object:Gem::Version
-  version: 0.0.16
+  version: 0.0.20
 platform: ruby
 authors:
 - David Cruwys
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-10-03 00:00:00.000000000 Z
+date: 2021-10-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -102,12 +102,11 @@ files:
 - lib/k_domain/domain_model/transform_steps/step1_attach_db_schema.rb
 - lib/k_domain/domain_model/transform_steps/step2_attach_models.rb
 - lib/k_domain/domain_model/transform_steps/step3_attach_columns.rb
-- lib/k_domain/domain_model/transform_steps/step4_attach_erd_files.rb
-- lib/k_domain/domain_model/transform_steps/step5_attach_dictionary.rb
-- lib/k_domain/domain_model/transform_steps/step8_rails_resource_models.rb
-- lib/k_domain/domain_model/transform_steps/step9_rails_structure_models.rb
+- lib/k_domain/domain_model/transform_steps/step4_rails_resource_models.rb
+- lib/k_domain/domain_model/transform_steps/step5_rails_models.rb
+- lib/k_domain/domain_model/transform_steps/step6_attach_dictionary.rb
 - lib/k_domain/rails_code_extractor/extract_model.rb
-- lib/k_domain/rails_code_extractor/load_shim.rb
+- lib/k_domain/rails_code_extractor/shim_loader.rb
 - lib/k_domain/raw_db_schema/load.rb
 - lib/k_domain/raw_db_schema/transform.rb
 - lib/k_domain/schemas/_.rb
@@ -141,6 +140,7 @@ files:
 - lib/k_domain/schemas/domain_model.rb
 - lib/k_domain/schemas/investigate.rb
 - lib/k_domain/schemas/rails_resource.rb
+- lib/k_domain/schemas/rails_structure.rb
 - lib/k_domain/version.rb
 - templates/active_record_shims.rb
 - templates/fake_module_shims.rb

data/lib/k_domain/domain_model/transform_steps/step4_attach_erd_files.rb

@@ -1,457 +0,0 @@
-# frozen_string_literal: true
-
-# Attach source code found in rails model definitions to models
-class Step4AttachErdFiles < KDomain::DomainModel::Step
-  attr_accessor :ruby_code
-
-  # NOTE: This code could be rewritten using monkey patched modules and peak
-  def call
-    domain[:erd_files] = domain_models.map { |model| load_dsl(model) }
-  end
-
-  private
-
-  def reset_dsl
-    @ruby_code = nil
-    @dsl = nil
-  end
-
-  def dsl
-    @dsl ||= {
-      name: '',
-      name_plural: ''
-    }
-  end
-
-  def load_dsl(model)
-    # this should be a configuration
-    # print '.'
-
-    reset_dsl
-
-    dsl[:name]        = model[:name]
-    dsl[:name_plural] = model[:name_plural]
-    dsl[:dsl_file]    = model[:erd_location][:exist] ? model[:erd_location][:file] : ''
-
-    return dsl unless File.exist?(dsl[:dsl_file])
-
-    @ruby_code = File.read(dsl[:dsl_file])
-
-    dsl[:source]  = read_dsl_source
-    dsl[:dsl]     = build_dsl
-    dsl[:todo]    = todo
-
-    dsl
-  end
-
-  def read_dsl_source
-    regex_split_private_public = /(?<public>.+?)(?=\bprivate\b)(?<private>.*)/m
-
-    split_code = regex_split_private_public.match(ruby_code)
-
-    public_code = nil
-    private_code = nil
-
-    if split_code
-      public_code = split_code[:public]
-      private_code = split_code[:private]
-    end
-
-    {
-      ruby: ruby_code,
-      public: public_code,
-      private: private_code,
-      all_methods: grab_methods(public_code, private_code)
-    }
-  end
-
-  def build_dsl
-    return if ruby_code.nil?
-
-    # need to support options as hash instead of options as string in the future
-    {
-      default_scope: grab_default_scope,
-      scopes: grab_scopes,
-      belongs_to: grab_belongs_to,
-      has_one: grab_has_one,
-      has_many: grab_has_many,
-      has_and_belongs_to_many: grab_has_and_belongs_to_many,
-      validate_on: grab_validate,
-      validates_on: grab_validates
-    }
-
-    # ^(?<spaces>\s*)(?<event_type>before_create|before_save|before_destroy|after_create|after_save|after_destroy) (:(?<name>\w*)[, ]?(?<scope>.*)|(?<scope>\{.*?\}.*$))
-  end
-
-  def grab_default_scope
-    regex = /default_scope \{(?<scope>.*?)\}/m
-
-    m = regex.match(ruby_code)
-
-    return "{ #{m[:scope].strip.gsub('\n', '')} }" if m
-
-    nil
-  end
-
-  def grab_scopes
-    entries = []
-    # Start from beginning of line and capture
-    # - number of spaces scope
-    # - name of scope
-    # - value of scope to end of line
-    regex = /^(?<spaces>\s*)scope :(?<name>\w*)[, ]?(?<scope>.*)/
-
-    # rubocop:disable Metrics/BlockLength
-    ruby_code.scan(regex) do
-      m = $LAST_MATCH_INFO
-      spaces = m[:spaces] # .delete("\n")
-      last_lf = spaces.rindex("\n")
-      spaces = last_lf ? spaces[spaces.rindex("\n") + 1..-1] : spaces
-      name = m[:name]
-      scope = m[:scope].strip
-
-      # Found a valid one liner
-      if scope.ends_with?('}') && (scope.scan(/{/).count == scope.scan(/}/).count)
-        scope = escape_single_quote(scope)
-        entries << { name: name, scope: scope }
-      else
-        # Have a multiline scope, lets see if it is cleanly formatted
-
-        start_anchor = "#{spaces}scope :#{name}"
-        end_anchor = "#{spaces}}"
-
-        # log.kv 'spaces', spaces.length
-        # log.kv 'name', name
-        # log.kv 'start_anchor', start_anchor
-        # log.kv 'end_anchor', end_anchor
-
-        start_index = ruby_code.index(/#{start_anchor}/)
-
-        if start_index.nil?
-          log.error("[#{@current_entity[:name]}] could not find [start] anchor index for [#{name}]")
-        else
-          ruby_section = ruby_code[start_index..-1]
-          end_index = ruby_section.index(/^#{end_anchor}/) # Add ^ start of line
-          if end_index.nil?
-            log.error("[#{@current_entity[:name]}] could not find [end] anchor index for [#{name}]")
-          else
-            scope = ruby_section[start_anchor.length + 1..end_index].strip
-            scope = escape_single_quote("#{scope}#{end_anchor}")
-            entries << { name: name, scope: scope }
-          end
-        end
-      end
-    end
-    entries
-  rescue StandardError => e
-    # bin ding.pry
-    puts e.message
-  end
-  # rubocop:enable Metrics/BlockLength
-
-  def grab_belongs_to
-    entries = []
-
-    # Start from beginning of line and capture
-    # - number of spaces before belongs_to
-    # - name of the belongs_to
-    # - value of belongs_to to end of line
-    regex = /^(?<spaces>\s*)belongs_to :(?<name>\w*)[, ]?(?<options>.*)/
-
-    ruby_code.scan(regex) do
-      m = $LAST_MATCH_INFO
-
-      # spaces = m[:spaces] # .delete("\n")
-      # last_lf = spaces.rindex("\n")
-      # spaces = last_lf ? spaces[spaces.rindex("\n") + 1..-1] : spaces
-      name = m[:name]
-
-      options = m[:options]
-                .gsub(':polymorphic => ', 'polymorphic: ')
-                .gsub(':class_name => ', 'class_name: ')
-                .gsub(':foreign_key => ', 'foreign_key: ')
-                .strip
-
-      options = clean_lambda(options)
-
-      entries << { name: name, options: extract_options(options), raw_options: options }
-    end
-    entries
-  rescue StandardError => e
-    # bin ding.pry
-    puts e.message
-  end
-
-  def grab_has_one
-    entries = []
-
-    # Start from beginning of line and capture
-    # - number of spaces before has_one
-    # - name of the has_one
-    # - value of has_one to end of line
-    regex = /^(?<spaces>\s*)has_one :(?<name>\w*)[, ]?(?<options>.*)/
-
-    ruby_code.scan(regex) do
-      m = $LAST_MATCH_INFO
-
-      # spaces = m[:spaces] # .delete("\n")
-      # last_lf = spaces.rindex("\n")
-      # spaces = last_lf ? spaces[spaces.rindex("\n") + 1..-1] : spaces
-      name = m[:name]
-      options = m[:options]
-                .strip
-      # .gsub(':polymorphic => ', 'polymorphic: ')
-      # .gsub(':class_name => ', 'class_name: ')
-      # .gsub(':foreign_key => ', 'foreign_key: ')
-
-      options = clean_lambda(options)
-
-      entries << { name: name, options: extract_options(options), raw_options: options }
-    end
-    entries
-  rescue StandardError => e
-    # bin ding.pry
-    puts e.message
-  end
-
-  def grab_has_many
-    entries = []
-    # Start from beginning of line and capture
-    # - number of spaces before has_many
-    # - name of the has_many
-    # - value of has_many to end of line
-    regex = /^(?<spaces>\s*)has_many :(?<name>\w*)[, ]?(?<options>.*)/
-
-    ruby_code.scan(regex) do
-      m = $LAST_MATCH_INFO
-
-      # spaces = m[:spaces] # .delete("\n")
-      # last_lf = spaces.rindex("\n")
-      # spaces = last_lf ? spaces[spaces.rindex("\n") + 1..-1] : spaces
-      name = m[:name]
-      options = m[:options]
-                .gsub(':dependent => ', 'dependent: ')
-                .gsub(':class_name => ', 'class_name: ')
-                .gsub(':foreign_key => ', 'foreign_key: ')
-                .gsub(':primary_key => ', 'primary_key: ')
-                .strip
-
-      options = clean_lambda(options)
-
-      entries << { name: name, options: extract_options(options), raw_options: options }
-    end
-    entries
-  rescue StandardError => e
-    # bin ding.pry
-    puts e.message
-  end
-
-  def grab_has_and_belongs_to_many
-    entries = []
-    # Start from beginning of line and capture
-    # - number of spaces before has_and_belongs_to_many
-    # - name of the has_and_belongs_to_many
-    # - value of has_and_belongs_to_many to end of line
-    regex = /^(?<spaces>\s*)has_and_belongs_to_many :(?<name>\w*)[, ]?(?<options>.*)/
-
-    ruby_code.scan(regex) do
-      m = $LAST_MATCH_INFO
-
-      # spaces = m[:spaces] # .delete("\n")
-      # last_lf = spaces.rindex("\n")
-      # spaces = last_lf ? spaces[spaces.rindex("\n") + 1..-1] : spaces
-      name = m[:name]
-      options = m[:options]
-                .gsub(':dependent => ', 'dependent: ')
-                .gsub(':class_name => ', 'class_name: ')
-                .gsub(':foreign_key => ', 'foreign_key: ')
-                .gsub(':primary_key => ', 'primary_key: ')
-                .strip
-
-      options = clean_lambda(options)
-
-      entries << { name: name, options: {}, raw_options: options }
-    end
-    entries
-  rescue StandardError => e
-    # bin ding.pry
-    puts e.message
-  end
-
-  def grab_validates
-    entries = []
-    # Start from beginning of line and capture
-    # - number of spaces before validates
-    # - name of the validates
-    # - value of validates to end of line
-    regex = /^(?<spaces>\s*)validates :(?<name>\w*)[, ]?(?<options>.*)/
-
-    ruby_code.scan(regex) do
-      m = $LAST_MATCH_INFO
-
-      # spaces = m[:spaces] # .delete("\n")
-      # last_lf = spaces.rindex("\n")
-      # spaces = last_lf ? spaces[spaces.rindex("\n") + 1..-1] : spaces
-      name = m[:name]
-
-      options = m[:options].strip
-
-      options = clean_lambda(options)
-
-      entries << { name: name, raw_options: options }
-    end
-    entries
-  rescue StandardError => e
-    # bin ding.pry
-    puts e.message
-  end
-
-  def grab_validate
-    entries = []
-    # Start from beginning of line and capture
-    # - number of spaces before validate
-    # - list of methods to call until to end of line
-    # regex = /^(?<spaces>\s*)validate :(?<name>\w*)[, ]?(?<options>.*)/
-    regex = /^(?<spaces>\s*)validate (?<line>:.*)/
-    # puts @current_entity[:name]
-
-    ruby_code.scan(regex) do
-      m = $LAST_MATCH_INFO
-
-      # spaces = m[:spaces] # .delete("\n")
-      # last_lf = spaces.rindex("\n")
-      # spaces = last_lf ? spaces[spaces.rindex("\n") + 1..-1] : spaces
-      line = m[:line]
-
-      entries << { line: line }
-      # puts @current_entity[:validate]
-    end
-    entries
-  rescue StandardError => e
-    # bin ding.pry
-    puts e.message
-  end
-
-  def grab_methods(public_code = ruby_code, private_code = nil)
-    # public_code = ruby_code_public.nil? ? ruby_code : ruby_code_public
-    # private_code = ruby_code_private
-
-    regex = /def (?<method>.*)/
-
-    # log.info(@current_entity[:name])
-
-    public_methods    = parse_methods(:public, public_code&.scan(regex)&.flatten || [])
-    private_methods   = parse_methods(:private, private_code&.scan(regex)&.flatten || [])
-    methods           = (public_methods + private_methods)
-
-    class_methods     = methods.select { |method| method[:class_method] == true }
-
-    all_instance      = methods.select { |method| method[:class_method] == false }
-    instance_public   = all_instance.select { |method| method[:scope] == :public }
-    instance_private  = all_instance.select { |method| method[:scope] == :private }
-
-    {
-      klass: class_methods,
-      instance: all_instance,
-      instance_public: instance_public,
-      instance_private: instance_private
-    }
-  end
-
-  def parse_methods(scope, methods)
-    methods.map do |value|
-      class_method = value.starts_with?('self.')
-      name = class_method ? value[5..-1] : value
-      arguments = nil
-      arguments_index = name.index('(')
-
-      if arguments_index
-        arguments = name[arguments_index..-1]
-        name = name[0..arguments_index - 1]
-      end
-
-      arguments = escape_single_quote(arguments)
-
-      {
-        name: name,
-        scope: scope,
-        class_method: class_method,
-        arguments: arguments&.strip.to_s
-      }
-    end
-  end
-
-  def todo
-    {
-      after_destroy: [], # to do
-      before_save: [], # to do
-      after_save: [], # to do
-      before_create: [], # to do
-      after_create: [], # to do
-      enum: [], # to do
-      attr_encrypted: [], # to do
-      validates_uniqueness_of: [], # to do
-      validates_confirmation_of: [], # to do
-      attr_accessor: [], # to do
-      attr_reader: [], # to do
-      attr_writer: [] # to do
-    }
-  end
-
-  def escape_single_quote(value)
-    return nil if value.nil?
-
-    value.gsub("'", "\\\\'")
-  end
-
-  # rubocop:disable Style/EvalWithLocation, Security/Eval, Style/DocumentDynamicEvalDefinition
-  def extract_options(options)
-    eval("{ #{options} }")
-  rescue StandardError => e
-    investigate(
-      step: :step4_attach_erd_files_models,
-      location: :extract_options,
-      key: nil,
-      message: e.message
-    )
-    {}
-  rescue SyntaxError => e
-    # may be the issue is from a comment at the off the line
-    comment_index = options.rindex('#') - 1
-
-    if comment_index.positive?
-      options_minus_comment = options[0..comment_index].squish
-      return extract_options(options_minus_comment)
-    end
-
-    investigate(
-      step: :step4_attach_erd_files_models,
-      location: :extract_options,
-      key: nil,
-      message: e.message
-    )
-    {}
-  end
-  # rubocop:enable Style/EvalWithLocation, Security/Eval, Style/DocumentDynamicEvalDefinition
-
-  def clean_lambda(options)
-    if /^->/.match?(options)
-      index = options.index(/}\s*,/)
-      if index.nil?
-        if options.count('{') == options.count('}')
-          index = options.rindex(/}/)
-          options = "a_lambda: '#{escape_single_quote(options[0..index])}'"
-        else
-          log.error(options)
-          options = "a_lambda: '#{escape_single_quote(options)}'"
-        end
-      else
-        options = "a_lambda: '#{escape_single_quote(options[0..index])}', #{options[index + 2..-1]}"
-      end
-    end
-    options
-  rescue StandardError => e
-    # bin ding.pry
-    puts e.message
-  end
-end

data/lib/k_domain/domain_model/transform_steps/step9_rails_structure_models.rb

@@ -1,33 +0,0 @@
-# frozen_string_literal: true
-
-# Locate rails model files
-class Step9RailsStructureModels < KDomain::DomainModel::Step
-  attr_accessor :ruby_code
-
-  def call
-    raise 'ERD path not supplied' if opts[:erd_path].nil?
-
-    self.rails_structure_models = rails_resource_models.map do |resource|
-      process_resource(OpenStruct.new(resource))
-    end
-  end
-
-  private
-
-  def process_resource(resource)
-    erd_path = opts[:erd_path]
-    puts erd_path
-    @model = {
-      model_name: resource.model_name,
-      table_name: resource.table_name,
-      file: resource.file,
-      exist: resource.exist,
-      state: resource.state,
-      code: resource.exist ? File.read(resource.file) : '',
-      behaviours: {},
-      functions: {}
-    }
-
-    @model
-  end
-end