k_domain 0.0.11 → 0.0.20

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e0641cc1145379d1ba83cad6765dbf85bd50bb40ce69105a977eb9ced5326046
-  data.tar.gz: ac7f352029bcea76171cf87ccf56212e11b3173256519394cc2802e21fd47981
+  metadata.gz: de5fc8f220781026dd327f40da53eb3a424ec23f38785be8fddc8f1856970dc1
+  data.tar.gz: 5573e60dc8665ab146e0edeb291ebcd3db6b792180e05e24721df9d519038f86
 SHA512:
-  metadata.gz: d2b9ce5ced5669eb620f9c3735fbbaf06e12b6c9f9e5eb6507d3526aefcfe5eb624ce5361a14bc39c2c8449eaa31b3e0caa082b6fca07fba23b166be550fc023
-  data.tar.gz: a22f3ee11a7b7ef9a9793d01237c5e14f0ed89972d91ede7ed87c29c260b57fca1445ea1bac92830d2299767335e49dc41f2f7b6917a7e08ffb5f8383bbf2ed4
+  metadata.gz: 4ecaa0fef7e6d622f18d2dfe53bdddf2c17fa91e1c2796ee7bdddf5044880f742003d8cb6f7bb4007eeab4bd2f0becba6f8f0fb329f58776f807cd200bd4ced9
+  data.tar.gz: 76af19179f670d7e4b18121f031b77e18c366259526530b6623f988c5b3e2985ca3a993441e6caacfb7cb71da6afb05c99803234aa85045facdb2c8f09b1194b

data/.rubocop.yml

@@ -8,7 +8,9 @@ AllCops:
     - "_/**/*"
     - "spec/sample_input/**/*"
     - "spec/sample_output/**/*"
-    - "lib/k_domain/raw_db_schema/template.rb"
+    - "templates/**/*"
+    - "lib/k_domain/schemas/domain/old/**/*"
+    - "spec/k_domain/ruby_loader/**/*"
 
 Metrics/PerceivedComplexity:
   Exclude:

data/Gemfile

@@ -28,8 +28,8 @@ end
 if ENV['KLUE_LOCAL_GEMS']&.to_s&.downcase == 'true'
   group :development, :test do
     puts 'Using Local GEMs'
-    gem 'peeky'                   , path: '../peeky'
     gem 'k_log'                   , path: '../k_log'
     gem 'k_util'                  , path: '../k_util'
+    gem 'peeky'                   , path: '../peeky'
   end
 end

data/Rakefile

@@ -26,7 +26,9 @@ task :publish do
   system 'gem build'
   system "gem push #{GEM_NAME}-#{KDomain::VERSION}.gem"
 end
-
+task :build do
+  system 'gem build'
+end
 desc 'Remove old *.gem files'
 task :clean do
   system 'rm *.gem'

data/STORIES.md

@@ -16,25 +16,54 @@ As a Developer, I can customize domain configuration, so that I can have opinion
 
 - Handle traits
 
+### Tasks next on list
+
+BUGs in domain_model load
+
+- spec/k_domain/domain_model/load_spec.rb:135
+- FIXED: module_name should be empty
+- FIXED: name mismatch (attr_accessor, attr_reader, attr_writer) in behaviours vs in (attr_accessors, attr_readers, attr_writers) functions
+- functions - (attr_accessors, attr_readers, attr_writers) are all empty
+
+External ruby process - ShimLoading and ExtractModel
+
+- You need a clear memory foot print for shim loading and model extraction, best to run these from inside a new ruby process
+
+Log Warning to Investigate Issues
+
+- FIXED: All the logged warnings in k_domain need to turn up in on the investigate issues register
+- Investigate needs a debug flag that when turned on, will write the issues to the console
+
+Print progress dot
+
+- Make this configurable
+- Decide what steps this should run for
+- Show step label via configuration
+
+## Stories and tasks
+
+### Stories - completed
+
 As a Developer, I can read native rails model data, so that I can leverage existing rails applications for ERD modeling
 
+- Proof of concept
 - Use Meta Programming and re-implement ActiveRecord::Base
 
-### Tasks next on list
+### Tasks - completed
 
 Refactor / Simply
 
-- Replace complex objects an with structs for ancillary data structures such as investigate
+- Replace complex objects with structs for ancillary data structures such as investigate
+
+Steps to support write methods on base class
+
+- Simplify lib/k_domain/domain_model/transform.rb
 
 User acceptance testing
 
 - Provide sample printers for each data structure to visually check data is loading
 - Point raw_db_schema loader towards a complex ERD and check how it performs
 
-## Stories and tasks
-
-### Tasks - completed
-
 Setup RubyGems and RubyDoc
 
 - Build and deploy gem to [rubygems.org](https://rubygems.org/gems/k_domain)

data/k_domain.gemspec

@@ -33,6 +33,7 @@ Gem::Specification.new do |spec|
       f.match(%r{^(test|spec|features)/})
     end
   end
+
   spec.bindir        = 'exe'
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']

data/lib/k_domain/domain_model/load.rb

@@ -20,9 +20,15 @@ module KDomain
 
       def call
         json = File.read(source_file)
-        data = KUtil.data.json_parse(json, as: :hash_symbolized)
+        @raw_data = KUtil.data.json_parse(json, as: :hash_symbolized)
 
-        @data = KDomain::DomainModel::Schema.new(data)
+        @data = KDomain::Schemas::DomainModel.new(@raw_data)
+      end
+
+      def to_h
+        return nil unless defined? @raw_data
+
+        @raw_data
       end
     end
   end

data/lib/k_domain/domain_model/transform.rb

@@ -13,21 +13,22 @@ module KDomain
       attr_reader :target_file
       attr_reader :erd_path
 
-      def initialize(db_schema, target_file, target_step_file, erd_path)
+      def initialize(db_schema: , target_file: , target_step_file: , erd_path:)
         @db_schema        = db_schema
         @target_step_file = target_step_file
         @target_file      = target_file
         @erd_path         = erd_path
       end
 
+      # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
       def call
         valid = true
-        valid &&= step1
-        valid &&= step2
-        valid &&= step3
-        valid &&= step4
-        valid &&= step5
-        valid &&= step8 # NOT SURE WHERE THIS BELONGS
+        valid &&= Step1AttachDbSchema.run(domain_data, db_schema: db_schema, step_file: step_file('1-attach-db-schema'))
+        valid &&= Step2AttachModels.run(domain_data, erd_path: erd_path, step_file: step_file('2-attach-model'))
+        valid &&= Step3AttachColumns.run(domain_data, step_file: step_file('3-attach-columns'))
+        valid &&= Step4RailsResourceModels.run(domain_data, erd_path: erd_path, step_file: step_file('4-rails-resource-models'))
+        valid &&= Step5RailsModels.run(domain_data, erd_path: erd_path, step_file: step_file('5-rails-models'))
+        valid &&= Step6AttachDictionary.run(domain_data, erd_path: erd_path, step_file: step_file('6-attach-dictionary'))
 
         raise 'DomainModal transform failed' unless valid
 
@@ -35,62 +36,34 @@ module KDomain
 
         nil
       end
+      # rubocop:enable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
 
-      def step1
-        Step1AttachDbSchema.run(domain_data, db_schema: db_schema)
-        write(step: '1-attach-db-schema')
+      def step_file(step_name)
+        target_step_file % { step: step_name }
       end
 
-      def step2
-        Step2AttachModels.run(domain_data, erd_path: erd_path)
-        write(step: '2-attach-model')
-      end
-
-      def step3
-        Step3AttachColumns.run(domain_data)
-        write(step: '3-attach-columns')
-      end
-
-      def step4
-        Step4AttachErdFiles.run(domain_data, erd_path: erd_path)
-        write(step: '4-attach-erd-files')
-      end
-
-      def step5
-        Step5AttachDictionary.run(domain_data, erd_path: erd_path)
-        write(step: '5-attach-dictionary')
-      end
-
-      def step8
-        Step8LocateRailsModels.run(domain_data, erd_path: erd_path)
-        write(step: '8-rails-files-models')
-      end
-
-      def write(step: nil)
-        file = if step.nil?
-                 target_file
-               else
-                 format(target_step_file, step: step)
-               end
-        FileUtils.mkdir_p(File.dirname(file))
-        File.write(file, JSON.pretty_generate(domain_data))
+      def write
+        FileUtils.mkdir_p(File.dirname(target_file))
+        File.write(target_file, JSON.pretty_generate(domain_data))
       end
 
+      # rubocop:disable Metrics/MethodLength
       def domain_data
         # The initial domain model structure is created here, but populated during the workflows.
         @domain_data ||= {
           domain: {
             models: [],
-            erd_files: [],
-            dictionary: []
           },
-          rails: {
+          rails_resource: {
             models: [],
-            controllers: [],
+            controllers: []
           },
-          rails_files: {
+          rails_structure: {
             models: [],
-            controllers: [],
+            controllers: []
+          },
+          dictionary: {
+            items: []
           },
           database: {
             tables: [],
@@ -103,6 +76,8 @@ module KDomain
           }
         }
       end
+
+      # rubocop:enable Metrics/MethodLength
     end
   end
 end

data/lib/k_domain/domain_model/transform_steps/_.rb

@@ -5,6 +5,6 @@ require_relative './step'
 require_relative './step1_attach_db_schema'
 require_relative './step2_attach_models'
 require_relative './step3_attach_columns'
-require_relative './step4_attach_erd_files'
-require_relative './step5_attach_dictionary'
-require_relative './step8_locate_rails_models'
+require_relative './step4_rails_resource_models'
+require_relative './step5_rails_models'
+require_relative './step6_attach_dictionary'

data/lib/k_domain/domain_model/transform_steps/step.rb

@@ -24,7 +24,8 @@ module KDomain
       def self.run(domain_data, **opts)
         step = new(domain_data, **opts)
         step.call
-        step
+        step.write(opts[:step_file])
+        step.valid?
       end
 
       def guard(message)
@@ -32,6 +33,11 @@ module KDomain
         @valid = false
       end
 
+      def write(file)
+        FileUtils.mkdir_p(File.dirname(file))
+        File.write(file, JSON.pretty_generate(domain_data))
+      end
+
       # Domain Model Accessor/Helpers
       def domain
         guard('domain is missing') if domain_data[:domain].nil?
@@ -43,23 +49,42 @@ module KDomain
         domain[:models]
       end
 
-      # Rails File Accessor/Helpers
-      def rails_files
-        guard('rails_files is missing') if domain_data[:rails_files].nil?
+      # Rails Resource File Accessor/Helpers
+      def rails_resource
+        guard('rails_resource is missing') if domain_data[:rails_resource].nil?
+
+        domain_data[:rails_resource]
+      end
+
+      def rails_resource_models
+        rails_resource[:models]
+      end
+
+      def rails_resource_models=(value)
+        rails_resource[:models] = value
+      end
+
+      def rails_resource_controllers
+        rails_resource[:controllers]
+      end
+
+      # Rails Structure File Accessor/Helpers
+      def rails_structure
+        guard('rails_structure is missing') if domain_data[:rails_structure].nil?
 
-        domain_data[:rails_files]
+        domain_data[:rails_structure]
       end
 
-      def rails_files_models
-        rails_files[:models]
+      def rails_structure_models
+        rails_structure[:models]
       end
 
-      def rails_files_models=(value)
-        rails_files[:models] = value
+      def rails_structure_models=(value)
+        rails_structure[:models] = value
       end
 
-      def rails_files_controllers
-        rails_files[:controllers]
+      def rails_structure_controllers
+        rails_structure[:controllers]
       end
 
       # Database Accessor/Helpers

data/lib/k_domain/domain_model/transform_steps/step1_attach_db_schema.rb

@@ -13,7 +13,7 @@ module KDomain
         guard('indexes are missing')              if database[:indexes].nil?
         guard('foreign keys are missing')         if database[:foreign_keys].nil?
         guard('rails version is missing')         if database[:meta][:rails].nil?
-        guard('postgres extensions are missing')  if database[:meta][:database][:extensions].nil?
+        guard('postgres extensions are missing')  if database[:meta][:db_info][:extensions].nil?
         guard('unique keys are missing')          if database[:meta][:unique_keys].nil?
       end
     end

data/lib/k_domain/domain_model/transform_steps/{step8_locate_rails_models.rb → step4_rails_resource_models.rb}

@@ -1,18 +1,20 @@
 # frozen_string_literal: true
 
 # Locate rails model files
-class Step8LocateRailsModels < KDomain::DomainModel::Step
+class Step4RailsResourceModels < KDomain::DomainModel::Step
   attr_accessor :ruby_code
 
   def call
     raise 'ERD path not supplied' if opts[:erd_path].nil?
 
-    self.rails_files_models = domain_models.map { |model| locate_rails_model(model[:name], model[:table_name]) }
+    self.rails_resource_models = domain_models.map do |model|
+      locate_rails_model(model[:name], model[:table_name])
+    end
   end
 
   private
 
-  def locate_rails_model(table_name, model_name)
+  def locate_rails_model(model_name, table_name)
     file_normal = File.join(opts[:erd_path], "#{model_name}.rb")
     file_custom = File.join(opts[:erd_path], "#{table_name}.rb")
     file_exist  = true

data/lib/k_domain/domain_model/transform_steps/step5_rails_models.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+# Locate rails model files
+class Step5RailsModels < KDomain::DomainModel::Step
+  def call
+    raise 'ERD path not supplied' unless opts[:erd_path]
+
+    self.rails_structure_models = rails_resource_models.map do |resource|
+      process_resource(OpenStruct.new(resource))
+    end
+  end
+
+  private
+
+  def process_resource(resource)
+    @model = {
+      model_name: resource.model_name,
+      table_name: resource.table_name,
+      file: resource.file,
+      exist: resource.exist,
+      state: resource.state,
+      code: resource.exist ? File.read(resource.file) : '',
+      behaviours: {},
+      functions: {}
+    }
+
+    return @model unless  resource.exist
+
+    @model[:behaviours] = extract_model_behavior(resource.file)
+    @model[:functions] = extract_model_functions(resource.file)
+
+    @model
+  end
+
+  def extract_model_behavior(file)
+    extractor.extract(file)
+    extractor.model
+  end
+
+  def extract_model_functions(file)
+    klass_name = File.basename(file, File.extname(file))
+
+    klass = case klass_name
+            when 'clearbit_quota'
+              ClearbitQuota
+            when 'account_history_data'
+              AccountHistoryData
+            else
+              Module.const_get(klass_name.classify)
+            end
+
+    class_info = Peeky.api.build_class_info(klass.new)
+
+    class_info.to_h
+  rescue StandardError => e
+    log.exception(e)
+  end
+
+  def extractor
+    @extractor ||= KDomain::RailsCodeExtractor::ExtractModel.new(shim_loader)
+  end
+
+  def shim_loader
+    return opts[:shim_loader] if !opts[:shim_loader].nil? && opts[:shim_loader].is_a?(KDomain::RailsCodeExtractor::ShimLoader)
+
+    shim_loader = KDomain::RailsCodeExtractor::ShimLoader.new
+    shim_loader.register(:fake_module  , KDomain::Gem.resource('templates/fake_module_shims.rb'))
+    shim_loader.register(:active_record, KDomain::Gem.resource('templates/active_record_shims.rb'))
+    shim_loader
+  end
+end

data/lib/k_domain/domain_model/transform_steps/{step5_attach_dictionary.rb → step6_attach_dictionary.rb}

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 # Attach data dictionary
-class Step5AttachDictionary < KDomain::DomainModel::Step
+class Step6AttachDictionary < KDomain::DomainModel::Step
   attr_reader :dictionary
 
   def call
@@ -13,7 +13,9 @@ class Step5AttachDictionary < KDomain::DomainModel::Step
       end
     end
 
-    domain[:dictionary] = dictionary.values
+    domain_data[:dictionary] = {
+      items: dictionary.values
+    }
   end
 
   private
@@ -25,7 +27,11 @@ class Step5AttachDictionary < KDomain::DomainModel::Step
       entry[:model_count] = entry[:model_count] + 1
 
       unless entry[:types].include?(column_type)
-        log.warn("#{model_name} has a type mismatch for column name: #{column_name}")
+        investigate(step: :step5_attach_dictionary,
+          location: :process,
+          key: "#{model_name},#{column_name},#{column_type}",
+          message: "#{model_name} has a type mismatch for column name: #{column_name}")
+
         entry[:types] << column_type
         entry[:type_count] = entry[:type_count] + 1
       end

data/lib/k_domain/rails_code_extractor/extract_model.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+# Takes a Rails Model and extracts DSL behaviours and custom functions (instance, class and private methods)
+module KDomain
+  module RailsCodeExtractor
+    class ExtractModel
+      include KLog::Logging
+
+      attr_reader :shims_loaded
+      attr_reader :models
+      attr_reader :model
+
+      def initialize(load_shim)
+        @load_shim = load_shim
+        @shims_loaded = false
+        @models = []
+      end
+
+      def extract(file)
+        load_shims unless shims_loaded
+
+        ActiveRecord.current_class = nil
+
+        load_retry(file, 10)
+      rescue StandardError => e
+        log.exception(e)
+      end
+
+      private
+
+      def load_shims
+        @load_shim.call
+        @shims_loaded = true
+      end
+
+      # rubocop:disable Security/Eval,Style/EvalWithLocation,Style/DocumentDynamicEvalDefinition,Metrics/AbcSize
+      def load_retry(file, times)
+        return if times.negative?
+
+        load(file)
+
+        @model = ActiveRecord.current_class
+        @models << @model
+
+        # get_method_info(File.base_name(file))
+      rescue StandardError => e
+        puts e.message
+        if e.is_a?(NameError)
+          log.kv('add module', e.name)
+          eval("module #{e.name}; end")
+          return load_retry(path, times - 1)
+        end
+        log.exception(e)
+      end
+      # rubocop:enable Security/Eval,Style/EvalWithLocation,Style/DocumentDynamicEvalDefinition,Metrics/AbcSize
+    end
+  end
+end

data/lib/k_domain/rails_code_extractor/shim_loader.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+# Code extraction requires shims to be loaded before extraction
+#
+# This shims server two purposes
+#
+# 1. Inject data capture methods calls that intercept DSL macros so that data can be extracted
+# 2. Inject fake module/classes that would otherwise break code loading with various exceptions
+module KDomain
+  module RailsCodeExtractor
+    class ShimLoader
+      include KLog::Logging
+
+      attr_reader :shim_files
+
+      def initialize
+        @shim_files = []
+      end
+
+      def call
+        shim_files.select { |sf| sf[:exist] }.each { |sf| require sf[:file] }
+      end
+
+      def register(name, file)
+        @shim_files << { name: name, file: file, exist: File.exist?(file) }
+      end
+    end
+  end
+end

data/lib/k_domain/raw_db_schema/load.rb

@@ -20,9 +20,15 @@ module KDomain
 
       def call
         json = File.read(source_file)
-        data = KUtil.data.json_parse(json, as: :hash_symbolized)
+        @raw_data = KUtil.data.json_parse(json, as: :hash_symbolized)
 
-        @data = KDomain::Database::Schema.new(data)
+        @data = KDomain::Database::Schema.new(@raw_data)
+      end
+
+      def to_h
+        return nil unless defined? @raw_data
+
+        @raw_data
       end
     end
   end

data/lib/k_domain/raw_db_schema/transform.rb

@@ -16,7 +16,7 @@ module KDomain
 
       def initialize(source_file)
         @source_file = source_file
-        @template_file = 'lib/k_domain/raw_db_schema/template.rb'
+        @template_file = KDomain::Gem.resource('templates/load_schema.rb')
       end
 
       def call
@@ -24,7 +24,6 @@ module KDomain
         # log.kv 'template_file', template_file
         # log.kv 'source_file?', File.exist?(source_file)
         # log.kv 'template_file?', File.exist?(template_file)
-
         log.error "Template not found: #{template_file}" unless File.exist?(template_file)
 
         content = File.read(source_file)
@@ -69,13 +68,13 @@ module KDomain
           return
         end
 
-        eval(schema_loader)#, __FILE__, __LINE__)
+        eval(schema_loader) # , __FILE__, __LINE__)
 
         loader = LoadSchema.new
         loader.load_schema
-        return loader.schema
-      rescue => ex
-        log.exception(ex)
+        loader.schema
+      rescue StandardError => e
+        log.exception(e)
       end
       # rubocop:enable Security/Eval
     end

data/lib/k_domain/schemas/_.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+# log.warn 'models->domain' if AppDebug.require?
+
+module Types
+  include Dry.Types()
+end
+
+require_relative 'rails_resource'
+require_relative 'rails_structure'
+require_relative 'investigate'
+require_relative 'database/_'
+require_relative 'dictionary'
+require_relative 'domain/_'
+
+require_relative './domain_model'

data/lib/k_domain/schemas/database/_.rb

@@ -1,14 +1,7 @@
 # frozen_string_literal: true
 
-module Types
-  include Dry.Types()
-end
-
 # The require order is important due to dependencies
-require_relative './column'
-require_relative './database'
 require_relative './index'
 require_relative './table'
 require_relative './foreign_key'
-require_relative './unique_key'
 require_relative './schema'

data/lib/k_domain/schemas/database/index.rb

@@ -8,7 +8,7 @@ module KDomain
       attribute :using                , Types::Nominal::String
       attribute :order?               , Types::Nominal::Hash
       attribute :where?               , Types::Nominal::Any.optional.default(nil)
-      attribute :unique? , Types::Nominal::Any.optional.default(nil)
+      attribute :unique?              , Types::Nominal::Any.optional.default(nil)
     end
   end
 end