cure 0.1.2 → 0.4.0

data/lib/cure/cli/templates/gitignore.erb

@@ -0,0 +1,181 @@
+# Created by https://www.toptal.com/developers/gitignore/api/rubymine,ruby
+# Edit at https://www.toptal.com/developers/gitignore?templates=rubymine,ruby
+
+.idea
+
+/sandpit
+
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status
+
+### Ruby ###
+*.gem
+*.rbc
+/.config
+/InstalledFiles
+/spec/examples.txt
+/test/tmp/
+/test/version_tmp/
+
+# Used by dotenv library to load environment variables.
+# .env
+
+# Ignore Byebug command history file.
+.byebug_history
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+*.bridgesupport
+build-iPhoneOS/
+build-iPhoneSimulator/
+
+## Specific to RubyMotion (use of CocoaPods):
+#
+# We recommend against adding the Pods directory to your .gitignore. However
+# you should judge for yourself, the pros and cons are mentioned at:
+# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
+# vendor/Pods/
+
+## Documentation cache and generated files:
+/.yardoc/
+/rdoc/
+
+## Environment normalization:
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc
+
+# Used by RuboCop. Remote config files pulled in from inherit_from directive.
+# .rubocop-https?--*
+
+### RubyMine ###
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# User-specific stuff
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf
+
+# AWS User-specific
+.idea/**/aws.xml
+
+# Generated files
+.idea/**/contentModel.xml
+
+# Sensitive or high-churn files
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+.idea/**/dbnavigator.xml
+
+# Gradle
+.idea/**/gradle.xml
+.idea/**/libraries
+
+# Gradle and Maven with auto-import
+# When using Gradle or Maven with auto-import, you should exclude module files,
+# since they will be recreated, and may cause churn.  Uncomment if using
+# auto-import.
+# .idea/artifacts
+# .idea/compiler.xml
+# .idea/jarRepositories.xml
+# .idea/modules.xml
+# .idea/*.iml
+# .idea/modules
+# *.iml
+# *.ipr
+
+# CMake
+cmake-build-*/
+
+# Mongo Explorer plugin
+.idea/**/mongoSettings.xml
+
+# File-based project format
+*.iws
+
+# IntelliJ
+out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Cursive Clojure plugin
+.idea/replstate.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+# Editor-based Rest Client
+.idea/httpRequests
+
+# Android studio 3.1+ serialized cache file
+.idea/caches/build_file_checksums.ser
+
+### RubyMine Patch ###
+# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721
+
+# *.iml
+# modules.xml
+# .idea/misc.xml
+# *.ipr
+
+# Sonarlint plugin
+# https://plugins.jetbrains.com/plugin/7973-sonarlint
+.idea/**/sonarlint/
+
+# SonarQube Plugin
+# https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin
+.idea/**/sonarIssues.xml
+
+# Markdown Navigator plugin
+# https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced
+.idea/**/markdown-navigator.xml
+.idea/**/markdown-navigator-enh.xml
+.idea/**/markdown-navigator/
+
+# Cache file creation bug
+# See https://youtrack.jetbrains.com/issue/JBR-2257
+.idea/$CACHE_FILE$
+
+# CodeStream plugin
+# https://plugins.jetbrains.com/plugin/12206-codestream
+.idea/codestream.xml
+
+# End of https://www.toptal.com/developers/gitignore/api/rubymine,ruby
+
+/input/*
+!/input/.gitkeep
+/output/*
+!/output/.gitkeep

data/lib/cure/cli/templates/new_template.rb.erb

@@ -0,0 +1,31 @@
+## Empty template, uncomment the sections that you need!
+#
+# metadata do
+#   name ""
+#   version ""
+#   comments ""
+#   additional data: {
+#     # Any additional data can go here. For example:
+#     key: "value"
+#   }
+# end
+#
+# extract do
+#   # ...
+# end
+#
+# build do
+#   # ...
+# end
+#
+# query do
+#   # ...
+# end
+#
+# transform do
+#   # ...
+# end
+#
+# export do
+#   # ...
+# end

data/lib/cure/cli/templates/tool-versions.erb

@@ -0,0 +1 @@
+ruby 3.2.1

data/lib/cure/config.rb

@@ -3,6 +3,7 @@
 require "cure"
 require "json"
 require "singleton"
+require "tempfile"
 
 module Cure
   module Configuration
@@ -14,45 +15,47 @@ module Cure
       conf
     end
 
+    # @param [Array<Cure::Configuration::CsvFileProxy>] source_files
+    # @param [Cure::Template] template
+    # @return [Config]
+    def create_config(source_files, template)
+      Config.new(source_files, template)
+    end
+
     # @param [Config] request_config
     def register_config(request_config)
       ConfigurationSource.instance.load_config(request_config)
     end
 
-    # @param [String] source_file_location
-    # @param [Cure::Template] template
-    # @param [String] output_dir
-    # @return [Config]
-    def create_config(source_file_location, template, output_dir)
-      Config.new(source_file_location, template, output_dir)
-    end
-
     # If we are overloading here as a "data store" and "config store", we
     # could break out variables and placeholders into their own singleton.
     #
     # This should be a kind of instance cache, which loads once per run,
     # and junk can be jammed in there?
     class Config
-      attr_accessor :source_file_location, :output_dir
 
-      # @return [Cure::Template] template
-      attr_accessor :template
+      # @return Array<Cure::Configuration::CsvFileProxy>
+      attr_accessor :source_files
 
-      # @return [Hash] variables
-      attr_accessor :variables
+      # @return [Cure::Template]
+      attr_accessor :template
 
-      # @param [String] source_file_location
+      # @param [Array<Cure::Configuration::CsvFileProxy>] source_files
       # @param [Cure::Template] template
-      # @param [String] output_dir
-      def initialize(source_file_location, template, output_dir)
-        @source_file_location = source_file_location
+      def initialize(source_files, template)
+        @source_files = source_files
         @template = template
-        @output_dir = output_dir
       end
 
       def placeholders
         @template.transformations.placeholders || {}
       end
+
+      def with_source_file(&block)
+        @source_files.each_with_index do |file, _idx|
+          file.with_file(&block)
+        end
+      end
     end
 
     class ConfigurationSource
@@ -66,5 +69,126 @@ module Cure
         @config = config
       end
     end
+
+    class CsvFileProxy
+
+      # @return [TrueClass,FalseClass]
+      attr_reader :valid
+      # @return [DefaultFileHandler]
+      attr_reader :csv_handler
+
+      # @return [CsvFileProxy]
+      def self.load_file(type, obj, ref_name)
+        handler =
+          case type
+          when :file
+            FileHandler.new(obj, ref_name)
+          when :file_contents
+            FileContentsHandler.new(obj, ref_name)
+          when :path, :pathname
+            PathnameHandler.new(obj, ref_name)
+          else
+            raise "Invalid file type handler [#{type}]. Supported: [:file, :file_contents, :path, :pathname]"
+          end
+
+        new(handler)
+      end
+
+      # @return [DefaultFileHandler]
+      def initialize(csv_handler)
+        @csv_handler = csv_handler
+      end
+
+      def description
+        @csv_handler.description
+      end
+
+      def with_file(&block)
+        @csv_handler.with_file(&block)
+      end
+    end
+
+    class DefaultFileHandler
+      attr_reader :type, :ref_name
+
+      def initialize(type, ref_name)
+        @type = type
+        @ref_name = ref_name
+      end
+
+      def with_file(&_block)
+        raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
+      end
+
+      def description
+        raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
+      end
+    end
+
+    class PathnameHandler < DefaultFileHandler
+      attr_accessor :pathname
+
+      # @param [Pathname] pathname
+      def initialize(pathname, ref_name)
+        super(:pathname, ref_name)
+        @pathname = pathname
+      end
+
+      def with_file(&_block)
+        yield @pathname, @ref_name
+      end
+
+      def description
+        @pathname.to_s
+      end
+    end
+
+    class FileHandler < DefaultFileHandler
+      # @return [File]
+      attr_accessor :file
+
+      # @param [File] file
+      def initialize(file, ref_name)
+        super(:file, ref_name)
+        @file = file
+      end
+
+      def with_file(&_block)
+        yield @file, @ref_name
+
+      ensure
+        @file&.close
+      end
+
+      def description
+        File.basename(file)
+      end
+    end
+
+    class FileContentsHandler < DefaultFileHandler
+      # @return [String]
+      attr_accessor :file_contents
+
+      # @param [String] file_contents
+      def initialize(file_contents, ref_name)
+        super(:file_contents, ref_name)
+        @file_contents = file_contents
+      end
+
+      def with_file(&_block)
+        tmp_file = Tempfile.new('cure-temp')
+        tmp_file.write(@file_contents)
+        tmp_file.open
+
+        yield tmp_file, @ref_name
+
+      ensure
+        tmp_file.unlink
+      end
+
+      def description
+        "<content provided>"
+      end
+    end
   end
 end

data/lib/cure/coordinator.rb

@@ -2,11 +2,13 @@
 
 require "cure/log"
 require "cure/config"
+require "cure/database"
 require "cure/helpers/file_helpers"
+require "cure/helpers/perf_helpers"
 
 require "cure/extract/extractor"
 require "cure/transformation/transform"
-require "cure/export/exporter"
+require "cure/export/manager"
 
 require "rcsv"
 
@@ -14,59 +16,93 @@ module Cure
   # Coordinates the entire process:
   # Extract -> Build -> Transform -> Export
   class Coordinator
-    include Configuration
     include Log
+    include Database
+    include Configuration
+    include Helpers::PerfHelpers
 
     def process
       # need to check config is init'd
+      result = nil
+      print_memory_usage do
+        print_time_spent do
+          # 1. Extract into SQLite
+          extract
+          # 2. Manipulate SQLite columns
+          build
+          # 3. Validate columns
+
+          # 4. Transform each row
+          with_tables.each do |table|
+            with_transformer(table) do |transformer|
+              with_exporters(table) do |exporters|
+                database_service.with_paged_result(table) do |row|
+                  transformed_row = transformer.transform(row)
+                  exporters.each do |exporter|
+                    # 4. Export
+                    exporter.process_row(transformed_row)
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
 
-      extracted_csv = extract
-      built_csv = build(extracted_csv)
-      transformed_csv = transform(built_csv)
-      export(transformed_csv)
+      result
     end
 
     private
 
-    # @return [Cure::Extract::WrappedCSV]
     def extract
       log_info "Beginning the extraction process..."
 
       extractor = Extract::Extractor.new({})
-      result = extractor.extract_from_file(config.source_file_location)
 
-      log_debug "Setting extracted variables to global conf for access downstream"
-      config.variables = result.variables
+      file_count = 0
+      config.with_source_file do |file, ref_name|
+        extractor.parse_csv(file, ref_name: ref_name)
+        file_count += 1
+      end
 
       log_info "...extraction complete"
-      result
     end
 
-    # @param [Cure::Extract::WrappedCSV] wrapped_csv
-    def build(wrapped_csv)
+    def build
       log_info "Beginning the building process..."
+      candidates = config.template.builder.candidates
+      candidates.each(&:perform)
+
       log_info "... building complete"
-      wrapped_csv
     end
 
-    # @return [Hash<String,Cure::Transformation::TransformResult>]
-    # @param [Cure::Extract::WrappedCSV] parsed_csv
-    def transform(parsed_csv)
+    # @yieldreturn [Cure::Transformation::Transform]
+    def with_transformer(named_range, &block)
+      raise "No block passed" unless block
+
       log_info "Beginning the transformation process..."
-      transformer = Cure::Transformation::Transform.new(config.template.transformations.candidates)
-      content = transformer.transform_content(parsed_csv)
+      candidates = config.template.transformations.candidates.select { |c| c.named_range == named_range.to_s }
+      transformer = Cure::Transformation::Transform.new(candidates)
+      yield transformer
 
       log_info "...transform complete"
-      content
     end
 
-    # @return [Hash<String,Cure::Transformation::TransformResult>]
-    def export(transformed_result)
+    # @yieldreturn [Cure::Export::Section]
+    def with_exporters(named_range, &block)
+      raise "No block passed" unless block
+
       log_info "Beginning export process..."
-      Cure::Export::Exporter.export_result(transformed_result, config.output_dir)
-      log_info "... export complete."
+      processors = config.template.exporters.processors.select { |c| c.named_range.to_s == named_range.to_s }
+      manager = Cure::Export::Manager.new(named_range, processors)
+
+      manager.with_processors(&block)
+
+      log_info "...export complete"
+    end
 
-      transformed_result
+    def with_tables
+      config.template.exporters.processors.map { |e| e.named_range.to_sym }.uniq
     end
   end
 end