cure 0.1.1 → 0.4.0

data/lib/cure/cli/templates/gitignore.erb

@@ -0,0 +1,181 @@
+# Created by https://www.toptal.com/developers/gitignore/api/rubymine,ruby
+# Edit at https://www.toptal.com/developers/gitignore?templates=rubymine,ruby
+
+.idea
+
+/sandpit
+
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status
+
+### Ruby ###
+*.gem
+*.rbc
+/.config
+/InstalledFiles
+/spec/examples.txt
+/test/tmp/
+/test/version_tmp/
+
+# Used by dotenv library to load environment variables.
+# .env
+
+# Ignore Byebug command history file.
+.byebug_history
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+*.bridgesupport
+build-iPhoneOS/
+build-iPhoneSimulator/
+
+## Specific to RubyMotion (use of CocoaPods):
+#
+# We recommend against adding the Pods directory to your .gitignore. However
+# you should judge for yourself, the pros and cons are mentioned at:
+# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
+# vendor/Pods/
+
+## Documentation cache and generated files:
+/.yardoc/
+/rdoc/
+
+## Environment normalization:
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc
+
+# Used by RuboCop. Remote config files pulled in from inherit_from directive.
+# .rubocop-https?--*
+
+### RubyMine ###
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# User-specific stuff
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf
+
+# AWS User-specific
+.idea/**/aws.xml
+
+# Generated files
+.idea/**/contentModel.xml
+
+# Sensitive or high-churn files
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+.idea/**/dbnavigator.xml
+
+# Gradle
+.idea/**/gradle.xml
+.idea/**/libraries
+
+# Gradle and Maven with auto-import
+# When using Gradle or Maven with auto-import, you should exclude module files,
+# since they will be recreated, and may cause churn.  Uncomment if using
+# auto-import.
+# .idea/artifacts
+# .idea/compiler.xml
+# .idea/jarRepositories.xml
+# .idea/modules.xml
+# .idea/*.iml
+# .idea/modules
+# *.iml
+# *.ipr
+
+# CMake
+cmake-build-*/
+
+# Mongo Explorer plugin
+.idea/**/mongoSettings.xml
+
+# File-based project format
+*.iws
+
+# IntelliJ
+out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Cursive Clojure plugin
+.idea/replstate.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+# Editor-based Rest Client
+.idea/httpRequests
+
+# Android studio 3.1+ serialized cache file
+.idea/caches/build_file_checksums.ser
+
+### RubyMine Patch ###
+# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721
+
+# *.iml
+# modules.xml
+# .idea/misc.xml
+# *.ipr
+
+# Sonarlint plugin
+# https://plugins.jetbrains.com/plugin/7973-sonarlint
+.idea/**/sonarlint/
+
+# SonarQube Plugin
+# https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin
+.idea/**/sonarIssues.xml
+
+# Markdown Navigator plugin
+# https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced
+.idea/**/markdown-navigator.xml
+.idea/**/markdown-navigator-enh.xml
+.idea/**/markdown-navigator/
+
+# Cache file creation bug
+# See https://youtrack.jetbrains.com/issue/JBR-2257
+.idea/$CACHE_FILE$
+
+# CodeStream plugin
+# https://plugins.jetbrains.com/plugin/12206-codestream
+.idea/codestream.xml
+
+# End of https://www.toptal.com/developers/gitignore/api/rubymine,ruby
+
+/input/*
+!/input/.gitkeep
+/output/*
+!/output/.gitkeep

data/lib/cure/cli/templates/new_template.rb.erb

@@ -0,0 +1,31 @@
+## Empty template, uncomment the sections that you need!
+#
+# metadata do
+#   name ""
+#   version ""
+#   comments ""
+#   additional data: {
+#     # Any additional data can go here. For example:
+#     key: "value"
+#   }
+# end
+#
+# extract do
+#   # ...
+# end
+#
+# build do
+#   # ...
+# end
+#
+# query do
+#   # ...
+# end
+#
+# transform do
+#   # ...
+# end
+#
+# export do
+#   # ...
+# end

data/lib/cure/cli/templates/tool-versions.erb

@@ -0,0 +1 @@
+ruby 3.2.1

data/lib/cure/config.rb

@@ -3,6 +3,7 @@
 require "cure"
 require "json"
 require "singleton"
+require "tempfile"
 
 module Cure
   module Configuration
@@ -14,30 +15,46 @@ module Cure
       conf
     end
 
+    # @param [Array<Cure::Configuration::CsvFileProxy>] source_files
+    # @param [Cure::Template] template
+    # @return [Config]
+    def create_config(source_files, template)
+      Config.new(source_files, template)
+    end
+
     # @param [Config] request_config
     def register_config(request_config)
       ConfigurationSource.instance.load_config(request_config)
     end
 
-    # @return [Config]
-    def create_config(source_file_location, template, output_dir)
-      Config.new(source_file_location, template, output_dir)
-    end
-
+    # If we are overloading here as a "data store" and "config store", we
+    # could break out variables and placeholders into their own singleton.
+    #
+    # This should be a kind of instance cache, which loads once per run,
+    # and junk can be jammed in there?
     class Config
-      attr_accessor :source_file_location, :template, :output_dir
 
-      # @param [String] source_file_location
-      # @param [Hash] template
-      # @param [String] output_dir
-      def initialize(source_file_location, template, output_dir)
-        @source_file_location = source_file_location
+      # @return Array<Cure::Configuration::CsvFileProxy>
+      attr_accessor :source_files
+
+      # @return [Cure::Template]
+      attr_accessor :template
+
+      # @param [Array<Cure::Configuration::CsvFileProxy>] source_files
+      # @param [Cure::Template] template
+      def initialize(source_files, template)
+        @source_files = source_files
         @template = template
-        @output_dir = output_dir
       end
 
       def placeholders
-        @template["placeholders"] || []
+        @template.transformations.placeholders || {}
+      end
+
+      def with_source_file(&block)
+        @source_files.each_with_index do |file, _idx|
+          file.with_file(&block)
+        end
       end
     end
 
@@ -52,5 +69,126 @@ module Cure
         @config = config
       end
     end
+
+    class CsvFileProxy
+
+      # @return [TrueClass,FalseClass]
+      attr_reader :valid
+      # @return [DefaultFileHandler]
+      attr_reader :csv_handler
+
+      # @return [CsvFileProxy]
+      def self.load_file(type, obj, ref_name)
+        handler =
+          case type
+          when :file
+            FileHandler.new(obj, ref_name)
+          when :file_contents
+            FileContentsHandler.new(obj, ref_name)
+          when :path, :pathname
+            PathnameHandler.new(obj, ref_name)
+          else
+            raise "Invalid file type handler [#{type}]. Supported: [:file, :file_contents, :path, :pathname]"
+          end
+
+        new(handler)
+      end
+
+      # @return [DefaultFileHandler]
+      def initialize(csv_handler)
+        @csv_handler = csv_handler
+      end
+
+      def description
+        @csv_handler.description
+      end
+
+      def with_file(&block)
+        @csv_handler.with_file(&block)
+      end
+    end
+
+    class DefaultFileHandler
+      attr_reader :type, :ref_name
+
+      def initialize(type, ref_name)
+        @type = type
+        @ref_name = ref_name
+      end
+
+      def with_file(&_block)
+        raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
+      end
+
+      def description
+        raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
+      end
+    end
+
+    class PathnameHandler < DefaultFileHandler
+      attr_accessor :pathname
+
+      # @param [Pathname] pathname
+      def initialize(pathname, ref_name)
+        super(:pathname, ref_name)
+        @pathname = pathname
+      end
+
+      def with_file(&_block)
+        yield @pathname, @ref_name
+      end
+
+      def description
+        @pathname.to_s
+      end
+    end
+
+    class FileHandler < DefaultFileHandler
+      # @return [File]
+      attr_accessor :file
+
+      # @param [File] file
+      def initialize(file, ref_name)
+        super(:file, ref_name)
+        @file = file
+      end
+
+      def with_file(&_block)
+        yield @file, @ref_name
+
+      ensure
+        @file&.close
+      end
+
+      def description
+        File.basename(file)
+      end
+    end
+
+    class FileContentsHandler < DefaultFileHandler
+      # @return [String]
+      attr_accessor :file_contents
+
+      # @param [String] file_contents
+      def initialize(file_contents, ref_name)
+        super(:file_contents, ref_name)
+        @file_contents = file_contents
+      end
+
+      def with_file(&_block)
+        tmp_file = Tempfile.new('cure-temp')
+        tmp_file.write(@file_contents)
+        tmp_file.open
+
+        yield tmp_file, @ref_name
+
+      ensure
+        tmp_file.unlink
+      end
+
+      def description
+        "<content provided>"
+      end
+    end
   end
 end

data/lib/cure/coordinator.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+require "cure/log"
+require "cure/config"
+require "cure/database"
+require "cure/helpers/file_helpers"
+require "cure/helpers/perf_helpers"
+
+require "cure/extract/extractor"
+require "cure/transformation/transform"
+require "cure/export/manager"
+
+require "rcsv"
+
+module Cure
+  # Coordinates the entire process:
+  # Extract -> Build -> Transform -> Export
+  class Coordinator
+    include Log
+    include Database
+    include Configuration
+    include Helpers::PerfHelpers
+
+    def process
+      # need to check config is init'd
+      result = nil
+      print_memory_usage do
+        print_time_spent do
+          # 1. Extract into SQLite
+          extract
+          # 2. Manipulate SQLite columns
+          build
+          # 3. Validate columns
+
+          # 4. Transform each row
+          with_tables.each do |table|
+            with_transformer(table) do |transformer|
+              with_exporters(table) do |exporters|
+                database_service.with_paged_result(table) do |row|
+                  transformed_row = transformer.transform(row)
+                  exporters.each do |exporter|
+                    # 4. Export
+                    exporter.process_row(transformed_row)
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+
+      result
+    end
+
+    private
+
+    def extract
+      log_info "Beginning the extraction process..."
+
+      extractor = Extract::Extractor.new({})
+
+      file_count = 0
+      config.with_source_file do |file, ref_name|
+        extractor.parse_csv(file, ref_name: ref_name)
+        file_count += 1
+      end
+
+      log_info "...extraction complete"
+    end
+
+    def build
+      log_info "Beginning the building process..."
+      candidates = config.template.builder.candidates
+      candidates.each(&:perform)
+
+      log_info "... building complete"
+    end
+
+    # @yieldreturn [Cure::Transformation::Transform]
+    def with_transformer(named_range, &block)
+      raise "No block passed" unless block
+
+      log_info "Beginning the transformation process..."
+      candidates = config.template.transformations.candidates.select { |c| c.named_range == named_range.to_s }
+      transformer = Cure::Transformation::Transform.new(candidates)
+      yield transformer
+
+      log_info "...transform complete"
+    end
+
+    # @yieldreturn [Cure::Export::Section]
+    def with_exporters(named_range, &block)
+      raise "No block passed" unless block
+
+      log_info "Beginning export process..."
+      processors = config.template.exporters.processors.select { |c| c.named_range.to_s == named_range.to_s }
+      manager = Cure::Export::Manager.new(named_range, processors)
+
+      manager.with_processors(&block)
+
+      log_info "...export complete"
+    end
+
+    def with_tables
+      config.template.exporters.processors.map { |e| e.named_range.to_sym }.uniq
+    end
+  end
+end

data/lib/cure/database.rb

@@ -0,0 +1,191 @@
+# frozen_string_literal: true
+
+require "sequel"
+require "sqlite3"
+require "singleton"
+
+require "cure/config"
+
+module Cure
+  module Database
+    def database_service
+      database = DatabaseSource.instance.database_service
+      return database if database
+
+      init_database
+
+      DatabaseSource.instance.database_service
+    end
+
+    def init_database
+      DatabaseSource.instance.init_database
+    end
+  end
+
+  class DatabaseSource
+    include Singleton
+
+    attr_reader :database_service
+
+    def init_database
+      @database_service = DatabaseService.new
+    end
+  end
+
+  class DatabaseService
+    include Cure::Configuration
+
+    # @return [Sequel::SQLite::Database]
+    attr_reader :database
+
+    def initialize
+      @database = init_database
+      setup_db
+    end
+
+    def setup_db
+      # Load this from config defined by user?
+      @database.execute <<-SQL
+        PRAGMA journal_mode = OFF;
+        PRAGMA synchronous = 0;
+        PRAGMA cache_size = 1000000;
+        PRAGMA locking_mode = EXCLUSIVE;
+        PRAGMA temp_store = MEMORY;
+      SQL
+    end
+
+    # App Service calls
+    def find_variable(property_name)
+      @database.from(:variables).where(name: property_name).get(:value)
+    end
+
+    def find_translation(source_value)
+      @database.from(:translations).where(source_value: source_value).get(:value)
+    end
+
+    def all_translations
+      @database.from(:translations).all
+    end
+
+    # @param [Symbol,String] tbl_name
+    # @param [Array] columns
+    def create_table(tbl_name, columns, auto_increment: true)
+      tbl_name = tbl_name.to_sym if tbl_name.class != Symbol
+
+      @database.create_table tbl_name do
+        primary_key :_id, auto_increment: auto_increment
+        columns.each do |col_name|
+          column col_name.to_sym, String
+        end
+      end
+    end
+
+    # @param [Symbol,String] tbl_name
+    def truncate_table(tbl_name)
+      @database[tbl_name.to_sym].truncate
+    end
+
+    # @param [Symbol,String] tbl_name
+    def table_count(tbl_name)
+      @database[tbl_name.to_sym].count
+    end
+
+    def with_transaction(&block)
+      @database.transaction({}, &block)
+    end
+
+    # @param [Symbol,String] tbl_name
+    # @return [TrueClass, FalseClass]
+    def table_exist?(tbl_name)
+      tbl_name = tbl_name.to_sym if tbl_name.class != Symbol
+
+      @database.table_exists?(tbl_name)
+    end
+    alias table_exists? table_exist?
+
+    # @param [Symbol,String] tbl_name
+    # @param [Array<String>] row
+    def insert_row(tbl_name, row)
+      @database[tbl_name.to_sym].insert(row)
+    end
+
+    # @param [Symbol,String] tbl_name
+    # @param [Array<String>] rows
+    def insert_batched_rows(tbl_name, rows)
+      @database[tbl_name.to_sym].import(@database[tbl_name.to_sym].columns, rows)
+    end
+
+    def add_column(tbl_name, new_column, default: "")
+      tbl_name = tbl_name.to_sym if tbl_name.class != Symbol
+      new_column = new_column.to_sym if new_column.class != Symbol
+
+      @database.add_column(tbl_name, new_column, String, default: default)
+    end
+
+    def remove_column(tbl_name, remove_column)
+      tbl_name = tbl_name.to_sym if tbl_name.class != Symbol
+      remove_column = remove_column.to_sym if remove_column.class != Symbol
+
+      @database.drop_column tbl_name, remove_column
+    end
+
+    def list_columns(tbl_name)
+      @database[tbl_name.to_sym].columns
+    end
+
+    def rename_column(tbl_name, old_column, new_column)
+      tbl_name = tbl_name.to_sym if tbl_name.class != Symbol
+      old_column = old_column.to_sym if old_column.class != Symbol
+      new_column = new_column.to_sym if new_column.class != Symbol
+
+      @database.rename_column tbl_name, old_column, new_column
+    end
+
+    def copy_column(tbl_name, from_column, to_column)
+      tbl_name = tbl_name.to_sym if tbl_name.class != Symbol
+      from_column = from_column.to_sym if from_column.class != Symbol
+      to_column = to_column.to_sym if to_column.class != Symbol
+
+      add_column tbl_name, to_column
+      run("UPDATE #{tbl_name} SET #{to_column} = #{from_column}")
+    end
+
+    def run(query, opts={})
+      @database.run(query, opts)
+    end
+
+    # Can we decouple query from named range? Probably more difficult
+    # than it seems. But would be nice to create two queries that doesn't
+    # require two tables (named ranges).
+    def with_paged_result(tbl_name, chunk_size: 100, &block)
+      raise "No block given" unless block
+
+      query = config.template.queries.find(tbl_name)
+      if query
+        @database[query.query].order(:_id).paged_each(rows_per_fetch: chunk_size, &block)
+      else
+        @database[tbl_name.to_sym].order(:_id).paged_each(rows_per_fetch: chunk_size, &block)
+      end
+    end
+
+    def list_tables
+      tbl_arr = @database.tables
+      tbl_arr.delete(:variables)
+
+      if tbl_arr.include?(:translations)
+        tbl_arr.delete(:translations)
+        tbl_arr.push(:translations)
+      end
+
+      tbl_arr
+    end
+
+    private
+
+    def init_database
+      # Build from config
+      # This must clean the database if its not in memory
+      Sequel.connect("sqlite:/")
+    end
+  end
+end