lhm-shopify 3.3.5

data/lib/lhm/migration.rb

@@ -0,0 +1,43 @@
+# Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require 'lhm/intersection'
+require 'lhm/timestamp'
+
+module Lhm
+  class Migration
+    attr_reader :origin, :destination, :conditions, :renames
+
+    def initialize(origin, destination, conditions = nil, renames = {}, time = Time.now)
+      @origin = origin
+      @destination = destination
+      @conditions = conditions
+      @renames = renames
+      @table_name = TableName.new(@origin.name, time)
+    end
+
+    def archive_name
+      @archive_name ||= @table_name.archived
+    end
+
+    def intersection
+      Intersection.new(@origin, @destination, @renames)
+    end
+
+    def origin_name
+      @table_name.original
+    end
+
+    def origin_columns
+      @origin_columns ||= intersection.origin.typed(origin_name)
+    end
+
+    def destination_name
+      @destination_name ||= destination.name
+    end
+
+    def destination_columns
+      @destination_columns ||= intersection.destination.joined
+    end
+  end
+end

data/lib/lhm/migrator.rb

@@ -0,0 +1,237 @@
+# Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require 'lhm/command'
+require 'lhm/migration'
+require 'lhm/sql_helper'
+require 'lhm/table'
+
+module Lhm
+  # Copies existing schema and applies changes using alter on the empty table.
+  # `run` returns a Migration which can be used for the remaining process.
+  class Migrator
+    include Command
+    include SqlHelper
+
+    attr_reader :name, :statements, :connection, :conditions, :renames, :origin
+
+    def initialize(table, connection = nil)
+      @connection = connection
+      @origin = table
+      @name = table.destination_name
+      @statements = []
+      @renames = {}
+    end
+
+    # Alter a table with a custom statement
+    #
+    # @example
+    #
+    #   Lhm.change_table(:users) do |m|
+    #     m.ddl("ALTER TABLE #{m.name} ADD COLUMN age INT(11)")
+    #   end
+    #
+    # @param [String] statement SQL alter statement
+    # @note
+    #
+    #   Don't write the table name directly into the statement. Use the #name
+    #   getter instead, because the alter statement will be executed against a
+    #   temporary table.
+    #
+    def ddl(statement)
+      statements << statement
+    end
+
+    # Add a column to a table
+    #
+    # @example
+    #
+    #   Lhm.change_table(:users) do |m|
+    #     m.add_column(:comment, "VARCHAR(12) DEFAULT '0'")
+    #   end
+    #
+    # @param [String] name Name of the column to add
+    # @param [String] definition Valid SQL column definition
+    def add_column(name, definition)
+      ddl('alter table `%s` add column `%s` %s' % [@name, name, definition])
+    end
+
+    # Change an existing column to a new definition
+    #
+    # @example
+    #
+    #   Lhm.change_table(:users) do |m|
+    #     m.change_column(:comment, "VARCHAR(12) DEFAULT '0' NOT NULL")
+    #   end
+    #
+    # @param [String] name Name of the column to change
+    # @param [String] definition Valid SQL column definition
+    def change_column(name, definition)
+      ddl('alter table `%s` modify column `%s` %s' % [@name, name, definition])
+    end
+
+    # Rename an existing column.
+    #
+    # @example
+    #
+    #   Lhm.change_table(:users) do |m|
+    #     m.rename_column(:login, :username)
+    #   end
+    #
+    # @param [String] old Name of the column to change
+    # @param [String] nu New name to use for the column
+    def rename_column(old, nu)
+      col = @origin.columns[old.to_s]
+
+      definition = col[:type]
+
+      definition += ' NOT NULL' unless col[:is_nullable] == "YES"
+      definition += " DEFAULT #{@connection.quote(col[:column_default])}" if col[:column_default]
+      definition += " COMMENT #{@connection.quote(col[:comment])}" if col[:comment]
+      definition += " COLLATE #{@connection.quote(col[:collate])}" if col[:collate]
+
+      ddl('alter table `%s` change column `%s` `%s` %s' % [@name, old, nu, definition])
+      @renames[old.to_s] = nu.to_s
+    end
+
+    # Remove a column from a table
+    #
+    # @example
+    #
+    #   Lhm.change_table(:users) do |m|
+    #     m.remove_column(:comment)
+    #   end
+    #
+    # @param [String] name Name of the column to delete
+    def remove_column(name)
+      ddl('alter table `%s` drop `%s`' % [@name, name])
+    end
+
+    # Add an index to a table
+    #
+    # @example
+    #
+    #   Lhm.change_table(:users) do |m|
+    #     m.add_index(:comment)
+    #     m.add_index([:username, :created_at])
+    #     m.add_index("comment(10)")
+    #   end
+    #
+    # @param [String, Symbol, Array<String, Symbol>] columns
+    #   A column name given as String or Symbol. An Array of Strings or Symbols
+    #   for compound indexes. It's possible to pass a length limit.
+    # @param [String, Symbol] index_name
+    #   Optional name of the index to be created
+    def add_index(columns, index_name = nil)
+      ddl(index_ddl(columns, false, index_name))
+    end
+
+    # Add a unique index to a table
+    #
+    # @example
+    #
+    #   Lhm.change_table(:users) do |m|
+    #     m.add_unique_index(:comment)
+    #     m.add_unique_index([:username, :created_at])
+    #     m.add_unique_index("comment(10)")
+    #   end
+    #
+    # @param [String, Symbol, Array<String, Symbol>] columns
+    #   A column name given as String or Symbol. An Array of Strings or Symbols
+    #   for compound indexes. It's possible to pass a length limit.
+    # @param [String, Symbol] index_name
+    #   Optional name of the index to be created
+    def add_unique_index(columns, index_name = nil)
+      ddl(index_ddl(columns, true, index_name))
+    end
+
+    # Remove an index from a table
+    #
+    # @example
+    #
+    #   Lhm.change_table(:users) do |m|
+    #     m.remove_index(:comment)
+    #     m.remove_index([:username, :created_at])
+    #   end
+    #
+    # @param [String, Symbol, Array<String, Symbol>] columns
+    #   A column name given as String or Symbol. An Array of Strings or Symbols
+    #   for compound indexes.
+    # @param [String, Symbol] index_name
+    #   Optional name of the index to be removed
+    def remove_index(columns, index_name = nil)
+      columns = [columns].flatten.map(&:to_sym)
+      from_origin = @origin.indices.find { |_, cols| cols.map(&:to_sym) == columns }
+      index_name ||= from_origin[0] unless from_origin.nil?
+      index_name ||= idx_name(@origin.name, columns)
+      ddl('drop index `%s` on `%s`' % [index_name, @name])
+    end
+
+    # Filter the data that is copied into the new table by the provided SQL.
+    # This SQL will be inserted into the copy directly after the "from"
+    # statement - so be sure to use inner/outer join syntax and not cross joins.
+    #
+    # @example Add a conditions filter to the migration.
+    #   Lhm.change_table(:sounds) do |m|
+    #     m.filter("inner join users on users.`id` = sounds.`user_id` and sounds.`public` = 1")
+    #   end
+    #
+    # @param [ String ] sql The sql filter.
+    #
+    # @return [ String ] The sql filter.
+    def filter(sql)
+      @conditions = sql
+    end
+
+    private
+
+    def validate
+      unless @connection.data_source_exists?(@origin.name)
+        error("could not find origin table #{ @origin.name }")
+      end
+
+      unless @origin.satisfies_id_column_requirement?
+        error('origin does not satisfy `id` key requirements')
+      end
+
+      dest = @origin.destination_name
+
+      if @connection.data_source_exists?(dest)
+        error("#{ dest } should not exist; not cleaned up from previous run?")
+      end
+    end
+
+    def execute
+      destination_create
+      @statements.each do |stmt|
+        @connection.execute(tagged(stmt))
+      end
+      Migration.new(@origin, destination_read, conditions, renames)
+    end
+
+    def destination_create
+      original    = %{CREATE TABLE `#{ @origin.name }`}
+      replacement = %{CREATE TABLE `#{ @origin.destination_name }`}
+      stmt = @origin.ddl.gsub(original, replacement)
+      @connection.execute(tagged(stmt))
+    end
+
+    def destination_read
+      Table.parse(@origin.destination_name, connection)
+    end
+
+    def index_ddl(cols, unique = nil, index_name = nil)
+      assert_valid_idx_name(index_name)
+      type = unique ? 'unique index' : 'index'
+      index_name ||= idx_name(@origin.name, cols)
+      parts = [type, index_name, @name, idx_spec(cols)]
+      'create %s `%s` on `%s` (%s)' % parts
+    end
+
+    def assert_valid_idx_name(index_name)
+      if index_name && !(index_name.is_a?(String) || index_name.is_a?(Symbol))
+        raise ArgumentError, 'index_name must be a string or symbol'
+      end
+    end
+  end
+end

data/lib/lhm/printer.rb

@@ -0,0 +1,59 @@
+module Lhm
+  module Printer
+    class Output
+      def write(message)
+        print message
+      end
+    end
+
+    class Base
+      def initialize
+        @output = Output.new
+      end
+    end
+
+    class Percentage < Base
+      def initialize
+        super
+        @max_length = 0
+      end
+
+      def notify(lowest, highest)
+        return if !highest || highest == 0
+        message = "%.2f%% (#{lowest}/#{highest}) complete" % (lowest.to_f / highest * 100.0)
+        write(message)
+      end
+
+      def end
+        write('100% complete')
+        @output.write "\n"
+      end
+
+      def exception(e)
+        write("failed: #{e}")
+        @output.write "\n"
+      end
+
+      private
+
+      def write(message)
+        if (extra = @max_length - message.length) < 0
+          @max_length = message.length
+          extra = 0
+        end
+
+        @output.write "\r#{message}" + (' ' * extra)
+      end
+    end
+
+    class Dot < Base
+      def notify(*)
+        @output.write '.'
+      end
+
+      def end
+        @output.write "\n"
+      end
+    end
+  end
+end

data/lib/lhm/railtie.rb

@@ -0,0 +1,9 @@
+module Lhm
+  class Railtie < Rails::Railtie
+    initializer "lhm.test_setup" do
+      if Rails.env.test? || Rails.env.development?
+        Lhm.execute_inline!
+      end
+    end
+  end
+end

data/lib/lhm/sql_helper.rb

@@ -0,0 +1,77 @@
+# Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+module Lhm
+  module SqlHelper
+    extend self
+
+    def annotation
+      '/* large hadron migration */'
+    end
+
+    def idx_name(table_name, cols)
+      column_names = column_definition(cols).map(&:first)
+      "index_#{ table_name }_on_#{ column_names.join('_and_') }"
+    end
+
+    def idx_spec(cols)
+      column_definition(cols).map do |name, length|
+        "`#{ name }`#{ length }"
+      end.join(', ')
+    end
+
+    def version_string
+      row = connection.select_one("show variables like 'version'")
+      value = struct_key(row, 'Value')
+      row[value]
+    end
+
+    def tagged(statement)
+      "#{ statement } #{ SqlHelper.annotation }"
+    end
+
+    private
+
+    def column_definition(cols)
+      Array(cols).map do |column|
+        column.to_s.match(/`?([^\(]+)`?(\([^\)]+\))?/).captures
+      end
+    end
+
+    # Older versions of MySQL contain an atomic rename bug affecting bin
+    # log order. Affected versions extracted from bug report:
+    #
+    #   http://bugs.mysql.com/bug.php?id=39675
+    #
+    # More Info: http://dev.mysql.com/doc/refman/5.5/en/metadata-locking.html
+    def supports_atomic_switch?
+      major, minor, tiny = version_string.split('.').map(&:to_i)
+
+      case major
+      when 4 then return false if minor and minor < 2
+      when 5
+        case minor
+        when 0 then return false if tiny and tiny < 52
+        when 1 then return false
+        when 4 then return false if tiny and tiny < 4
+        when 5 then return false if tiny and tiny < 3
+        end
+      when 6
+        case minor
+        when 0 then return false if tiny and tiny < 11
+        end
+      end
+      true
+    end
+
+    def struct_key(struct, key)
+      keys = if struct.is_a? Hash
+               struct.keys
+             else
+               struct.members
+             end
+
+      keys.find { |k| k.to_s.downcase == key.to_s.downcase }
+    end
+  end
+end

data/lib/lhm/sql_retry.rb

@@ -0,0 +1,61 @@
+require 'retriable'
+require 'lhm/sql_helper'
+
+module Lhm
+  # SqlRetry standardizes the interface for retry behavior in components like
+  # Entangler, AtomicSwitcher, ChunkerInsert.
+  #
+  # By default if an error includes the message "Lock wait timeout exceeded", or
+  # "Deadlock found when trying to get lock", SqlRetry will retry again
+  # once the MySQL client returns control to the caller, plus one second.
+  # It will retry a total of 10 times and output to the logger a description
+  # of the retry with error information, retry count, and elapsed time.
+  #
+  # This behavior can be modified by passing `options` that are documented in
+  # https://github.com/kamui/retriable. Additionally, a "log_prefix" option,
+  # which is unique to SqlRetry can be used to prefix log output.
+  class SqlRetry
+    def initialize(connection, options = {})
+      @connection = connection
+      @log_prefix = options.delete(:log_prefix)
+      @retry_config = default_retry_config.dup.merge!(options)
+    end
+
+    def with_retries
+      Retriable.retriable(retry_config) do
+        yield(@connection)
+      end
+    end
+
+    attr_reader :retry_config
+
+    private
+
+    # For a full list of configuration options see https://github.com/kamui/retriable
+    def default_retry_config
+      {
+        on: {
+          StandardError => [
+            /Lock wait timeout exceeded/,
+            /Timeout waiting for a response from the last query/,
+            /Deadlock found when trying to get lock/,
+            /Query execution was interrupted/,
+            /Lost connection to MySQL server during query/,
+            /Max connect timeout reached/,
+            /Unknown MySQL server host/,
+          ]
+        },
+        multiplier: 1, # each successive interval grows by this factor
+        base_interval: 1, # the initial interval in seconds between tries.
+        tries: 20, # Number of attempts to make at running your code block (includes initial attempt).
+        rand_factor: 0, # percentage to randomize the next retry interval time
+        max_elapsed_time: Float::INFINITY, # max total time in seconds that code is allowed to keep being retried
+        on_retry: Proc.new do |exception, try_number, total_elapsed_time, next_interval|
+          log = "#{exception.class}: '#{exception.message}' - #{try_number} tries in #{total_elapsed_time} seconds and #{next_interval} seconds until the next try."
+          log.prepend("[#{@log_prefix}] ") if @log_prefix
+          Lhm.logger.info(log)
+        end
+      }.freeze
+    end
+  end
+end