nando 1.0.6

data/lib/nando/baseline_templates/migration.rb

@@ -0,0 +1,9 @@
+class <%= migration_class_name %> < Nando::<%= migration_type %>
+  def up
+<%= migration_up_code %>
+  end
+
+  def down
+<%= migration_down_code %>
+  end
+end

data/lib/nando/errors.rb

@@ -0,0 +1,13 @@
+module Nando
+  class GenericError < StandardError
+    def initialize (error)
+      super
+    end
+  end
+
+  class InputError < StandardError
+    def initialize (error)
+      super
+    end
+  end
+end

data/lib/nando/generator.rb

@@ -0,0 +1,86 @@
+require 'erb'
+
+module MigrationGenerator
+
+  # creates the actual migration file
+  def self.create_migration_file (filepath, migration_name, migration_type)
+    dir = File.dirname(filepath)
+
+    if !File.directory?(dir)
+      raise Nando::GenericError.new("No directory '#{dir}' was found")
+    end
+
+    case migration_type
+    when Nando::Migration.name.demodulize
+      template_file_name = 'migration'
+    when Nando::MigrationWithoutTransaction.name.demodulize
+      template_file_name = 'migration_without_transaction'
+    end
+
+    migration_class_name = migration_name.camelize()
+    file = File.new(filepath, 'w')
+    # TODO: check if binding logic is correct, and if pathing changes when it's a gem
+    render_to_file(File.join(File.dirname(File.expand_path(__FILE__)), "templates/#{template_file_name}.rb"), binding, file)
+
+    puts "Creating a new migration: #{filepath}"
+  end
+
+  # based on the template renderer from the commercial engine
+  def self.render_to_file (template_file, context, output_file)
+    output_file.write render(template_file, context)
+  end
+
+  def self.render(template_file, context)
+    renderer = ERB.new(File.read(template_file), nil, nil)
+    renderer.result(context)
+  end
+
+  def self.create_baseline_file (filepath, migration_name)
+    dir = File.dirname(filepath)
+
+    if !File.directory?(dir)
+      raise Nando::GenericError.new("No directory '#{dir}' was found")
+    end
+
+    @db_connection = NandoMigrator.instance.get_database_connection();
+    results = @db_connection.exec("
+      SELECT n.nspname AS function_schema,
+             p.proname AS function_name,
+             l.lanname AS function_language,
+             CASE WHEN l.lanname = 'internal' THEN p.prosrc ELSE pg_get_functiondef(p.oid) END AS definition,
+             pg_get_function_arguments(p.oid) AS function_arguments,
+             t.typname AS return_type,
+             p.proowner AS p_owner
+        FROM pg_proc p
+        LEFT JOIN pg_namespace n ON p.pronamespace = n.oid
+        LEFT JOIN pg_language l ON p.prolang = l.oid
+        LEFT JOIN pg_type t ON t.oid = p.prorettype
+       WHERE n.nspname NOT IN ('pg_catalog', 'information_schema')
+       ORDER BY function_schema, function_name
+    ")
+
+    up_method = ''
+    number_of_functions = 0
+    indent = '    '
+
+    for row in results do
+      up_method += "\n" + indent + "update_function <<-'SQL'\n"
+      up_method += "#{row['definition']}"
+      up_method += "\n" + indent + "SQL\n"
+      number_of_functions += 1
+    end
+
+    new_file = File.new(filepath, 'w')
+
+    # binding
+    migration_class_name = migration_name.camelize
+    migration_type = Nando::Migration.name.demodulize # TODO: atm all baseline files are create as migrations with transactions, this might change later
+    migration_up_code = up_method
+    migration_down_code = indent + "# #{number_of_functions} functions have been added to this baseline"
+
+    render_to_file(File.join(File.dirname(File.expand_path(__FILE__)), 'baseline_templates/migration.rb'), binding, new_file)
+
+    puts "Creating a new baseline: #{filepath}"
+  end
+
+end

data/lib/nando/interface.rb

@@ -0,0 +1,87 @@
+module NandoInterface
+
+  # prints help message
+  def self.print_help_message
+    schema_mig_table = "schema_migrations" # left as a variable, might fill this value dynamically later
+
+    message = ''
+    message += "Usage: nando <command> [options]\n\n"
+
+    # commands
+    commands = [
+      ['up', "Executes all the migrations that are not yet on the #{schema_mig_table.white.bold} table"],
+      ['down', "Rollbacks the last migration in the #{schema_mig_table.white.bold} table"],
+      ['apply <version>', "Applies the migration with the specified version (even if it was already executed). Intended for development, not production"],
+      ['new <migration name>', "Creates a new migration with the specified name. Use the #{"-t/--type".white.bold} flag to specify the type of migration: #{"[Migration|MigrationWithoutTransaction]".white.bold}"],
+      ['parse <source> <destination>', "Parses all the #{"dbmate".white.bold} migrations in the source folder into #{"Nando".white.bold} migrations in the destination folder"],
+      ['baseline', "Creates a baseline Nando migration with all the functions currently in the database"],
+      ['update <path to migration>', "Updates the specified migration. Use #{"-f/--function".white.bold} to add one or more functions to the migration file"],
+      ['diff <source> <target>', "Compares 2 schemas in the database and suggests SQL commands to fix any changes found"]
+    ]
+
+    largest_command = commands.max { |a,b| a[0].length <=> b[0].length }
+    required_indent = "nando #{largest_command[0]}".length
+
+    message += "Commands:\n"
+    for command in commands do
+      message += build_command_message(command[0], command[1], required_indent)
+    end
+
+    # flags
+    flags = [
+      ['-t/--type', 'Used to specify the migration type'],
+      ['-f/--function', 'Used to specify which function files to add to a migration'],
+      ['-h/--help', 'Shows the help message']
+    ]
+
+    largest_flag = flags.max { |a,b| a[0].length <=> b[0].length }
+    required_indent = largest_flag[0].length
+
+    message += "\nFlags:\n"
+    for flag in flags do
+      message += build_flag_message(flag[0], flag[1], required_indent)
+    end
+
+    message += "Nando Version (#{Nando::VERSION})"
+
+    puts message
+  end
+
+  def self.build_command_message (command, description, required_indent)
+    command_message = "nando #{command}"
+    indent = " " * (required_indent - command_message.length)
+    return "#{command_message.white.bold}#{indent}  #{description}\n"
+  end
+
+  def self.build_flag_message (flag, description, required_indent)
+    indent = " " * (required_indent - flag.length)
+    return "#{flag.white.bold}#{indent}  #{description}\n"
+  end
+
+
+  def self.get_user_function_list
+    puts 'Enter the list of functions to add to the current migration: (Enter an empty line to exit)'.bold.magenta
+    input = multi_line_gets
+    return (input.split("\n").each { |line| line.strip! }.reject { |line| line == '' }) || []
+  end
+
+  def self.multi_line_gets (all_text = '')
+    until (text = gets) == "\n"
+      all_text << text
+    end
+    return all_text.chomp
+  end
+
+  # if input is Y/y return true, else return false
+  def self.get_user_input_boolean (message)
+    puts "\n#{message} (Y/N)".magenta.bold
+    # TODO: review error when not using $stdin (might need to use it above as well)
+    input = $stdin.gets.chomp.downcase.strip
+    if input == 'y'
+      return true
+    else
+      return false
+    end
+  end
+
+end

data/lib/nando/logger.rb

@@ -0,0 +1,30 @@
+# logger functions
+require 'colorize'
+
+def _warn (message, header = 'WARNING')
+  print "#{header}: ".yellow.bold
+  puts message
+end
+
+def _success (message, header = 'SUCCESS')
+  print "#{header}: ".green.bold
+  puts message
+end
+
+def _error (message, header = 'ERROR')
+  print "#{header}: ".red.bold
+  puts message
+end
+
+def _debug (message, header = 'DEBUG')
+  if ENV['DEBUG'] != 'true'
+    return
+  end
+
+  print "#{header}: ".light_cyan.bold
+  puts message
+end
+
+def _info (message)
+  puts message
+end

data/lib/nando/migration.rb

@@ -0,0 +1,347 @@
+module Nando
+
+  class Migration
+    def initialize (conn, version)
+      @conn = conn
+      @version = version
+      @migration_table = NandoMigrator.instance.migration_table
+      @migration_field = NandoMigrator.instance.migration_field
+    end
+
+    def execute (sql)
+      @conn.exec(sql)
+    end
+
+    def update_function (sql)
+      # TODO: add validations here
+      @conn.exec(sql)
+    end
+
+    def execute_migration (method)
+      # TODO: review this is the best way of creating a transaction (don't know if re-assigning connections has weird behaviours)
+      old_connection = @conn
+      @conn.transaction do |conn|
+        @conn = conn
+        self.send(method)
+      end
+      @conn = old_connection
+    end
+
+    def invalidate_postgresql_redis_cache!
+      # TODO: how to do this?
+      Rake::Task['cloudware:toconline:system:redis:postgresql_cache_invalidate'].invoke
+      Rake::Task['cloudware:toconline:system:redis:redis_current_company_invalidate'].invoke
+    end
+  end
+
+  class MigrationWithoutTransaction < Migration
+    def initialize (conn, version)
+      super(conn, version)
+      @conn.exec('DROP FUNCTION IF EXISTS sharding.create_company_shard(integer,text)')
+    end
+
+    def execute_migration (method)
+      self.send(method)
+    end
+
+
+    # custom CW methods
+
+    ###########################################################
+    # WORK OVER SHARDED COMPANIES, AND ALREADY SHARDED MODULES
+    ###########################################################
+    def migrate_companies (module_name = nil, options = {}, &block)
+      return if block.nil?
+
+      if module_name.nil?
+        work_on_schemas(get_sharded_company_schemas, 'global company', :up, options, &block)
+        # TODO: splited a "say_with_time", might need to review this
+        puts "[PUBLIC] Running migration on unsharded companies"
+        block.call('public', nil, nil, 'tablespace_000')
+      elsif module_name.to_sym == :all
+        work_on_schemas(get_companies_schemas, 'global company', :up, options, &block)
+      else
+        work_on_schemas(get_companies_schemas_from_module(module_name), 'global company', :up, options, &block)
+      end
+    end
+
+    def rollback_companies (module_name = nil, options = {}, &block)
+      if module_name.nil?
+        work_on_schemas(get_sharded_company_schemas, 'global company', :down, options, &block)
+        # TODO: splited a "say_with_time", might need to review this
+        puts "[PUBLIC] Rolling back migration on unsharded companies"
+        block.call('public', nil, nil, 'tablespace_000')
+      elsif module_name.to_sym == :all
+        work_on_schemas(get_companies_schemas, 'global company', :down, options, &block)
+      else
+        work_on_schemas(get_companies_schemas_from_module(module_name), 'global company', :down, options, &block)
+      end
+    end
+
+
+    #################################
+    # WORK OVER ACCOUNTING COMPANIES
+    #################################
+    def migrate_accounting_companies (options = {}, &block)
+      return if block.nil?
+
+      work_on_schemas(get_accounting_companies_schemas, 'accounting company', :up, options, &block)
+    end
+
+    def rollback_accounting_companies(options = {}, &block)
+      work_on_schemas(get_accounting_companies_schemas, 'accounting company', :down, options, &block)
+    end
+
+    def migrate_fiscal_years (options = {}, &block)
+      return if block.nil?
+      work_on_schemas get_accounting_companies_schemas, 'accounting company', :up, options do |schema, company_id, use_sharded_company, tablespace_name, company_schema|
+        each_fiscal_year(schema) do |fiscal_year|
+          block.call schema, fiscal_year['table_prefix'], fiscal_year, company_id, tablespace_name, company_schema, use_sharded_company
+        end
+      end
+    end
+
+    def rollback_fiscal_years (options = {}, &block)
+      work_on_schemas get_accounting_companies_schemas, 'accounting company', :down, options do |schema, company_id, use_sharded_company, tablespace_name, company_schema|
+        each_fiscal_year(schema) do |fiscal_year|
+          block.call schema, fiscal_year['table_prefix'], fiscal_year, company_id, tablespace_name, company_schema, use_sharded_company
+        end
+      end
+    end
+
+    def each_fiscal_year (schema, &block)
+      return if block.nil?
+      @conn.exec(%Q[SELECT * FROM "#{schema}"."fiscal_years"]).to_a.each(&block)
+    end
+
+
+    ###########################
+    # WORK OVER USER TEMPLATES
+    ###########################
+    def migrate_user_schemas (options = {}, &block)
+      return if block.nil?
+
+      work_on_schemas(get_user_schemas, 'user template', :up, options, &block)
+    end
+
+    def rollback_user_schemas (options = {}, &block)
+      work_on_schemas(get_user_schemas, 'user template', :down, options, &block)
+    end
+
+    def migrate_user_templates (options = {}, &block)
+      return if block.nil?
+
+      work_on_schemas get_user_schemas, 'user template', :up, options do |schema, id, use_sharded_company, tablespace_name|
+        each_user_template(schema) do |user_template|
+          block.call(schema, user_template['table_prefix'], user_template, tablespace_name)
+        end
+      end
+    end
+
+    def rollback_user_templates (options = {}, &block)
+      work_on_schemas get_user_schemas, 'user template', :down, options do |schema, id, use_sharded_company, tablespace_name|
+        each_user_template(schema) do |user_template|
+          block.call(schema, user_template['table_prefix'], user_template, tablespace_name)
+        end
+      end
+    end
+
+    def each_user_template (schema_name, &block)
+      return if block.nil?
+      @conn.exec(%Q[SELECT * FROM "accounting"."user_templates" WHERE schema_name='#{schema_name}' ORDER BY "id"]).to_a.each(&block)
+    end
+
+    def table_exists? (schema_name, table_name)
+      @conn.exec(%Q[SELECT 1 FROM "information_schema"."tables" WHERE "table_schema" = '#{schema_name}' AND "table_name" = '#{table_name}']).any?
+    end
+
+
+    # helper methods
+
+    def work_on_schemas (schemas, schema_type_description, direction, options, &block)
+      options ||= {}
+      options[:record_on_schema_migrations] = true unless options.has_key?(:record_on_schema_migrations)
+      options[:use_public_schema] = true unless options.has_key?(:use_public_schema)
+
+      schema_count = schemas.count.to_s
+      options[:max_schemas_per_conn] ||= 200 # min_queries_per_conn / max_queries_per_conn
+      options[:_internal_reset_counter] = 0
+
+      running_migration_version = get_migration_version
+
+      puts "#{direction == :up ? 'Migrating' : 'Rolling back'} on #{schema_count} #{schema_type_description} schema(s)"
+
+      schemas.each_with_index do |schema, index|
+        # create_schema_migrations_table_on_schema(schema) unless schema_migration_table_exists?(schema)
+        send :"#{direction}_on_schema", schema, running_migration_version, options, "[#{(index + 1).to_s.rjust(schema_count.length)}/#{schema_count}] ", &block
+      end
+    end
+
+    def up_on_schema (schema, running_migration_version, options, progress_feedback, &block)
+      with_new_connection(options) do
+        @conn.transaction do |conn|
+          # TODO: ensure this has the same behavior as "ActiveRecord::Base.transaction(requires_new: true)"
+          if migration_ran_on_schema?(schema, running_migration_version, conn)
+            puts "#{progress_feedback}Migration already ran on schema #{schema['schema_name']}, skipping"
+          else
+            # TODO: splited a "say_with_time", might need to review this
+            puts "#{progress_feedback}Running migration on schema #{schema['schema_name']}"
+            block.call(schema['schema_name'], schema['id'], schema['use_sharded_company'].to_b, schema['tablespace_name'], schema['company_schema']) unless block.nil?
+            migration_ran_on_schema!(schema, running_migration_version, conn) if options[:record_on_schema_migrations]
+          end
+        end
+      end
+    end
+
+    def down_on_schema (schema, running_migration_version, options, progress_feedback, &block)
+      with_new_connection(options) do
+        @conn.transaction do |conn|
+          # TODO: ensure this has the same behavior as "ActiveRecord::Base.transaction(requires_new: true)"
+          if migration_ran_on_schema?(schema, running_migration_version, conn)
+            # TODO: splited a "say_with_time", might need to review this
+            puts "#{progress_feedback}Rolling back migration on schema #{schema['schema_name']}"
+            block.call(schema['schema_name'], schema['id'], schema['use_sharded_company'].to_b, schema['tablespace_name'], schema['company_schema']) unless block.nil?
+            migration_rolled_back_on_schema!(schema, running_migration_version, conn) if options[:record_on_schema_migrations]
+          else
+            puts "#{progress_feedback}Migration didn't run on schema #{schema['schema_name']}, skipping"
+          end
+        end
+      end
+    end
+
+    def with_new_connection (options = {}, &block)
+      options[:_internal_reset_counter] ||= 0
+      if options[:_internal_reset_counter] == options[:max_schemas_per_conn] || ( 0 == options[:_internal_reset_counter] && !options[:statement_timeout].nil? )
+        # TODO: what was the objective of this? Is a reset enough?
+        # ActiveRecord::Base.connection.reset!()
+        # ActiveRecord::Base.connection.raw_connection.reset
+        # ActiveRecord::Base.connection.raw_connection.exec("SET statement_timeout TO #{0 == options[:statement_timeout]? "'48h'" : options[:statement_timeout]}") if options[:statement_timeout]
+        @conn.reset()
+        @conn.exec("SET statement_timeout TO #{0 == options[:statement_timeout]? "'48h'" : options[:statement_timeout]}") if options[:statement_timeout]
+        options[:_internal_reset_counter] = 0
+      end
+      options[:_internal_reset_counter] +=1
+      block.call unless block.nil?
+    end
+
+
+    # schema queries
+
+    def get_sharded_company_schemas
+      get_schemas %Q[
+        SELECT "id",
+               "schema_name",
+               "use_sharded_company",
+               common.get_tablespace_name("schema_name") AS "tablespace_name"
+          FROM "public"."companies"
+         WHERE "schema_name" IS NOT NULL
+           AND "use_sharded_company"
+           AND "is_deleted" IS DISTINCT FROM true
+           AND "companies"."cluster" = #{database_cluster}
+         ORDER BY "id"
+       ]
+    end
+
+    def get_companies_schemas_from_module (module_name)
+      get_schemas %Q[
+        SELECT "companies"."id",
+               "companies"."schema_name",
+               "companies"."use_sharded_company",
+               common.get_tablespace_name("companies"."schema_name") AS "tablespace_name"
+          FROM "public"."company_modules"
+          JOIN "public"."companies" ON "companies"."id" = "company_modules"."company_id"
+         WHERE "companies"."schema_name" IS NOT NULL
+           AND "company_modules"."name" = '#{module_name}'
+           AND "company_modules"."has_schema_structure"
+           AND "companies"."is_deleted" IS DISTINCT FROM true
+           AND "companies"."cluster" = #{database_cluster}
+         ORDER BY "id"
+      ]
+    end
+
+    def get_companies_schemas
+      get_schemas %Q[
+        SELECT "id",
+               "schema_name",
+               "use_sharded_company",
+               common.get_tablespace_name("schema_name") AS "tablespace_name"
+          FROM "public"."companies"
+         WHERE "schema_name" IS NOT NULL
+           AND "is_deleted" IS DISTINCT FROM true
+           AND "companies"."cluster" = #{database_cluster}
+         ORDER BY "id"
+      ]
+    end
+
+    def get_accounting_companies_schemas
+      get_schemas %Q[
+        SELECT "accounting_companies"."company_id" AS "id",
+               "accounting_companies"."schema_name",
+               common.get_tablespace_name("accounting_companies"."schema_name") AS "tablespace_name",
+               "companies"."schema_name" AS company_schema,
+               "companies"."use_sharded_company"
+          FROM "accounting"."accounting_companies"
+          JOIN "public"."companies" ON "companies"."id" = "accounting_companies"."company_id"
+         WHERE "companies"."is_deleted" IS DISTINCT FROM true
+           AND "companies"."cluster" = #{database_cluster}
+         ORDER BY "accounting_companies"."id"
+      ]
+    end
+
+    def get_user_schemas
+      get_schemas %Q[
+        SELECT DISTINCT "user_id" AS "id",
+               "schema_name",
+               false AS "use_sharded_company",
+               common.get_tablespace_name("schema_name") AS "tablespace_name"
+          FROM "accounting"."user_templates"
+         ORDER BY "id"
+      ]
+    end
+
+
+    # utils
+
+    def database_cluster
+      @conn.exec("SHOW cloudware.cluster").to_a[0]["cloudware.cluster"].to_i
+    end
+
+    def get_schemas (query)
+      schemas_rows = @conn.exec(query).to_a
+      return schemas_rows
+    end
+
+    def get_migration_version
+      return @version
+    end
+
+    def migration_ran_on_schema? (schema, migration_version, conn = nil)
+      query = %Q[SELECT 1 FROM "#{schema['schema_name']}"."#{@migration_table}" WHERE "#{@migration_field}" = '#{migration_version}']
+      if conn.nil?
+        @conn.exec(query).any?
+      else
+        conn.exec(query).any?
+      end
+    end
+
+    def migration_ran_on_schema! (schema, migration_version, conn = nil)
+      query = %Q[INSERT INTO "#{schema['schema_name']}"."#{@migration_table}" ("#{@migration_field}") VALUES ('#{migration_version}')]
+      if conn.nil?
+        @conn.exec(query)
+      else
+        conn.exec(query)
+      end
+    end
+
+    def migration_rolled_back_on_schema! (schema, migration_version, conn = nil)
+      query = %Q[DELETE FROM "#{schema['schema_name']}"."#{@migration_table}" WHERE "#{@migration_field}" = '#{migration_version}']
+      if conn.nil?
+        @conn.exec(query)
+      else
+        conn.exec(query)
+      end
+    end
+
+  end
+
+end