dynomite 1.2.6 → 2.0.0

data/lib/dynomite/migration/generator.rb

@@ -3,8 +3,6 @@ require "active_support/core_ext/string"
 class Dynomite::Migration
   # jets dynamodb:generate posts --partition-key id:string
   class Generator
-    include Dynomite::DbConfig
-
     attr_reader :migration_name, :table_name
     def initialize(migration_name, options)
       @migration_name = migration_name
@@ -12,21 +10,28 @@ class Dynomite::Migration
     end
 
     def generate
-      puts "Generating migration for #{@table_name}" unless @options[:quiet]
-      return if @options[:noop]
+      puts "Generating migration" unless @options[:quiet]
+      return if ENV['NOOP']
       create_migration
     end
 
     def create_migration
       FileUtils.mkdir_p(File.dirname(migration_path))
       IO.write(migration_path, migration_code)
-      puts "Migration file created: #{migration_path}. \nTo run:"
-      puts "  jets dynamodb:migrate #{migration_path}"
+      pretty_migration_path = migration_path.sub(/^\.\//,'') # remove leading ./
+      puts "Migration file created: #{pretty_migration_path}\nTo run:\n\n"
+      command = File.basename($0)
+      if command == "jets"
+        puts "    #{command} dynamodb:migrate"
+      else
+        puts "    #{command} migrate"
+      end
+      puts
     end
 
     def migration_code
-      path = File.expand_path("../templates/#{table_action}.rb", __FILE__)
-      result = Dynomite::Erb.result(path,
+      path = File.expand_path("../templates/#{action}.rb", __FILE__)
+      Dynomite::Erb.result(path,
         migration_class_name: migration_class_name,
         table_name: table_name,
         partition_key: @options[:partition_key],
@@ -35,30 +40,39 @@ class Dynomite::Migration
       )
     end
 
-    def table_action
-      @options[:table_action] || conventional_table_action
+    def action
+      # optoins[:table_action] is old and deprecated
+      action = @options[:action] || @options[:table_action] || conventional_action
+      case action
+      when /create/
+        "create_table"
+      when /delete/
+        "delete_table"
+      else
+        "update_table" # default and fallback
+      end
     end
 
-    def conventional_table_action
-      @migration_name.include?("update") ? "update_table" : "create_table"
+    def conventional_action
+      @migration_name.split("_").first
     end
 
     def table_name
-      @options[:table_name] || conventional_table_name
+      @options[:table_name] || conventional_table_name || "TABLE_NAME"
     end
 
     # create_posts => posts
     # update_posts => posts
     def conventional_table_name
-      @migration_name.sub(/^(create|update)_/, '')
+      @migration_name.sub(/^(create|update|delete)_/, '')
     end
 
     def migration_class_name
-      "#{@migration_name}_migration".classify # doesnt include timestamp
+      "#{@migration_name}".camelize # doesnt include timestamp
     end
 
     def migration_path
-      "#{Dynomite.app_root}dynamodb/migrate/#{timestamp}-#{@migration_name}_migration.rb"
+      "#{Dynomite.root}/dynamodb/migrate/#{timestamp}-#{@migration_name}.rb"
     end
 
     def timestamp

data/lib/dynomite/migration/helpers.rb

@@ -0,0 +1,7 @@
+class Dynomite::Migration
+  module Helpers
+    def table_name_with_namespace(table_name)
+      [Dynomite.config.namespace, table_name].reject {|s| s.nil? || s.empty?}.join(Dynomite.config.namespace_separator)
+    end
+  end
+end

data/lib/dynomite/migration/internal/migrate/create_schema_migrations.rb

@@ -0,0 +1,17 @@
+class CreateSchemaMigrations < Dynomite::Migration
+  include Dynomite::Client
+
+  def up
+    create_table :schema_migrations do |t|
+      t.partition_key "version:number" # required
+      t.billing_mode "PAY_PER_REQUEST"
+    end
+  end
+
+  def table_exist?(full_table_name)
+    client.describe_table(table_name: full_table_name)
+    true
+  rescue Aws::DynamoDB::Errors::ResourceNotFoundException
+    false
+  end
+end

data/lib/dynomite/migration/internal/models/schema_migration.rb

@@ -0,0 +1,6 @@
+class SchemaMigration < Dynomite::Item
+  disable_id!
+  field :version, type: :integer
+  field :time_took, type: :integer
+  fields :status, :path, :error_message
+end

data/lib/dynomite/migration/runner.rb

@@ -0,0 +1,178 @@
+require "timeout"
+require_relative "internal/migrate/create_schema_migrations"
+require_relative "internal/models/schema_migration"
+
+class Dynomite::Migration
+  class Runner
+    include Dynomite::Item::WaiterMethods
+
+    def initialize(options={})
+      @options = options
+    end
+
+    def run
+      puts "Running Dynomite migrations"
+      ensure_schema_migrations_exist!
+      clear_error_schema_migrations! if ENV['CLEAR_ERRORS']
+      check_for_migration_errors!
+      Dynomite::Migration::FileInfo.all_files.each do |path|
+        migrate(path)
+      end
+    end
+
+    def migrate(path)
+      # load migration class definition
+      #   CreatePosts#up (up is called below)
+      file_info = FileInfo.new(path)
+
+      schema_migration = SchemaMigration.find_by(version: file_info.version)
+      if schema_migration
+        teriminal_statuses = %w[complete error]
+        if teriminal_statuses.include?(schema_migration.status)
+          return
+        else
+          action = with_timeout(message: "Timed out. You must respond within 60s.") do
+            uncompleted_migration_prompt(file_info, schema_migration)
+          end
+        end
+      end
+
+      case action
+      when :skip
+        return
+      when :delete
+        schema_migration.delete
+      when :completed
+        schema_migration.status = "completed"
+        schema_migration.save
+        return
+      when :exit
+        puts "Exiting"
+        exit
+      end
+
+      puts "Running migration: #{file_info.pretty_path}"
+      load path
+
+      # INSERT schema_migration table - in_progress
+      unless schema_migration
+        schema_migration = SchemaMigration.new(version: file_info.version, status: "in_progress", path: file_info.pretty_path)
+        schema_migration.save
+      end
+      start_time = Time.now
+
+      # Run actual migration
+      migration_class = file_info.migration_class
+      error_message = nil
+      begin
+        # Runs migration up command. Example:
+        #   CreatePosts#up
+        #   Migration#create_table
+        #   Migration#execute (wait happens here)
+        migration_class.new.up # wait happens within create_table or update_table
+      rescue Aws::DynamoDB::Errors::ServiceError => error
+        puts "Unable to #{@method_name.to_s.gsub('_',' ')}: #{error.message}".color(:red)
+        error_message = error.message
+      end
+
+      # UPDATE schema_migrations table - complete status
+      if error_message
+        schema_migration.status = "error"
+        schema_migration.error_message = error_message
+      else
+        schema_migration.status = "complete"
+      end
+
+      schema_migration.time_took = (Time.now - start_time).to_i
+      schema_migration.save
+      # schema_migration.delete # HACK
+      puts "Time took: #{pretty_time_took(schema_migration.time_took)}"
+      exit 1 if error_message # otherwise continue to next migration file
+    end
+
+    def with_timeout(options={}, &block)
+      seconds = options[:seconds] || 60
+      message = options[:message] || "Timed out after #{seconds}s."
+      Timeout::timeout(seconds, &block)
+    rescue Timeout::Error => e
+      puts "#{e.class}: #{e.message}"
+      puts message
+      exit 1
+    end
+
+    def uncompleted_migration_prompt(file_info, schema_migration)
+      choice = nil
+      until %w[s d c e].include?(choice)
+        puts(<<~EOL)
+          The #{file_info.pretty_path} migration status is incomplete. Status: #{schema_migration.status}
+          This can happen and if it was interrupted by a CTRL-C.
+          Please check the migration to help determine what to do next.
+
+          Options:
+
+              s - skip and continue. leaves schema_migrations item as-is
+              d - delete and continue. deletes the schema_migrations item
+              c - mark as successful completed and continue. updates the schema_migrations item as completed.
+              e - exit
+
+          EOL
+          print "Choose an option (s/d/c/e): "
+        choice = $stdin.gets.strip
+      end
+
+      map = {
+        "s" => :skip,
+        "d" => :delete,
+        "c" => :completed,
+        "e" => :exit,
+      }
+      map[choice]
+    end
+
+    def ensure_schema_migrations_exist!
+      migration = CreateSchemaMigrations.new
+      return if migration.table_exist?(SchemaMigration.table_name)
+      puts "Creating #{SchemaMigration.table_name} table for the first time"
+      migration.up
+    end
+
+    def clear_error_schema_migrations!
+      SchemaMigration.warn_on_scan(false).where(status: "error").each do |schema_migration|
+        schema_migration.delete
+        puts "Deleted error schema_migration: #{schema_migration.path}"
+      end
+    end
+
+    def check_for_migration_errors!
+      errors = SchemaMigration.warn_on_scan(false).where(status: "error")
+      errors_info = SchemaMigration.warn_on_scan(false).where(status: "error").map do |schema_migration|
+        "    #{schema_migration.path} - #{schema_migration.error_message}"
+      end.join("\n")
+      if errors.count > 0
+        puts <<~EOL
+          Found error migrations. Please review the migration erors fix before continuing.
+          You can clear them out manually by deleting them from the #{SchemaMigration.table_name} table.
+
+          #{errors_info}
+
+          You can also clear them with the following command:
+
+              CLEAR_ERRORS=1 jets dynamodb:migrate
+
+        EOL
+        exit 1
+      end
+    end
+
+    # http://stackoverflow.com/questions/4175733/convert-duration-to-hoursminutesseconds-or-similar-in-rails-3-or-ruby
+    def pretty_time_took(total_seconds)
+      minutes = (total_seconds / 60) % 60
+      seconds = total_seconds % 60
+      if total_seconds < 60
+        "#{seconds.to_i}s"
+      else
+        "#{minutes.to_i}m #{seconds.to_i}s"
+      end
+    end
+  end
+end

data/lib/dynomite/migration/templates/create_table.rb

@@ -1,33 +1,17 @@
 class <%= @migration_class_name %> < Dynomite::Migration
   def up
     create_table :<%= @table_name %> do |t|
-      t.partition_key "<%= @partition_key %>" # required
+      t.partition_key :<%= @partition_key %>
 <% if @sort_key # so extra spaces are not added when generated -%>
       t.sort_key  "<%= @sort_key %>" # optional
 <% end -%>
-      t.billing_mode(:PROVISIONED)
-      t.provisioned_throughput(<%= @provisioned_throughput %>) # sets both read and write, defaults to 5 when not set
-
-      # Instead of using partition_key and sort_key you can set the
-      # key schema directly also
-      # t.key_schema([
-      #     {attribute_name: "id", :key_type=>"HASH"},
-      #     {attribute_name: "created_at", :key_type=>"RANGE"}
-      #   ])
-      # t.attribute_definitions([
-      #   {attribute_name: "id", attribute_type: "N"},
-      #   {attribute_name: "created_at", attribute_type: "S"}
-      # ])
-
-      # other ways to set provisioned_throughput
-      # t.provisioned_throughput(:read, 10)
-      # t.provisioned_throughput(:write, 10)
-      # t.provisioned_throughput(
-      #   read_capacity_units: 5,
-      #   write_capacity_units: 5
-      # )
+<% if %w[id id:string].include?(@partition_key) -%>
+      t.add_gsi :updated_at
+<% else %>
+      t.add_gsi :id
+<% end -%>
     end
   end
 end
 
-# More examples: https://github.com/tongueroo/dynomite/tree/master/docs
+# More examples: https://rubyonjets.com/docs/database/dynamodb/migration/

data/lib/dynomite/migration/templates/delete_table.rb

@@ -0,0 +1,7 @@
+class <%= @migration_class_name %> < Dynomite::Migration
+  def up
+    delete_table :<%= @table_name %>
+  end
+end
+
+# More examples: https://rubyonjets.com/docs/database/dynamodb/migration/

data/lib/dynomite/migration/templates/update_table.rb

@@ -1,26 +1,11 @@
 class <%= @migration_class_name %> < Dynomite::Migration
   def up
     update_table :<%= @table_name %> do |t|
-      t.gsi(:create) do |i|
-        i.partition_key "<%= @partition_key %>" # required
-<% if @sort_key # so extra spaces are not added when generated -%>
-        t.sort_key  "<%= @sort_key %>" # optional
-<% end -%>
+      t.add_gsi(partition_key: "<%= @partition_key %>", sort_key: "<%= @sort_key || 'updated_at' %>")
 
-        i.provisioned_throughput(5)
-      end
-
-      # Examples:
-      # t.gsi(:update, "update-me-index") do |i|
-      #   i.provisioned_throughput(10)
-      # end
-
-      # t.gsi(:delete, "delete-me-index")
-
-      # Must use :create, :update, :delete one at a time in separate migration files.
-      # DynamoDB imposes this.
+      # t.remove_gsi(partition_key: "<%= @partition_key %>", sort_key: "<%= @sort_key || 'updated_at' %>")
     end
   end
 end
 
-# More examples: https://github.com/tongueroo/dynomite/tree/master/docs
+# More examples: https://rubyonjets.com/docs/database/dynamodb/migration/

data/lib/dynomite/migration.rb

@@ -1,27 +1,70 @@
 module Dynomite
   class Migration
-    autoload :Dsl, "dynomite/migration/dsl"
-    autoload :Generator, "dynomite/migration/generator"
-    autoload :Executor, "dynomite/migration/executor"
+    include Item::WaiterMethods
+    include Dynomite::Client
+    include Helpers
 
     def up
       puts "Should defined an up method for your migration: #{self.class.name}"
     end
 
     def create_table(table_name, &block)
-      execute_with_dsl_params(table_name, :create_table, &block)
+      execute(table_name, :create_table, &block)
     end
 
     def update_table(table_name, &block)
-      execute_with_dsl_params(table_name, :update_table, &block)
+      execute(table_name, :update_table, &block)
     end
 
+    def delete_table(table_name, &block)
+      execute(table_name, :delete_table, &block)
+    end
+
+    def add_gsi(table_name, *args)
+      update_table(table_name) do |t|
+        t.add_gsi(*args)
+      end
+    end
+
+    def remove_gsi(table_name, *args)
+      update_table(table_name) do |t|
+        t.remove_gsi(*args)
+      end
+    end
+
+    def update_gsi(table_name, *args)
+      update_table(table_name) do |t|
+        t.update_gsi(*args)
+      end
+    end
+
+    def update_time_to_live(table_name, time_to_live_specification={})
+      table_name = table_name_with_namespace(table_name)
+      client.update_time_to_live(
+        table_name: table_name,
+        time_to_live_specification: time_to_live_specification
+      )
+      # fast enough of an operation, no need to wait
+    end
+    alias update_ttl update_time_to_live
+
   private
-    def execute_with_dsl_params(table_name, method_name, &block)
-      dsl = Dsl.new(method_name, table_name, &block)
-      params = dsl.params
-      executor = Executor.new(table_name, method_name, params)
-      executor.run
+    # execute with dsl params
+    # table name is short name and params[:table_name] is full namespaced name
+    def execute(table_name, method_name, &block)
+      params = Dsl.new(method_name, table_name, &block).params
+
+      puts "Running #{method_name} with:"
+      pp params
+      resp = client.send(method_name, params)
+      namespaced_table_name = params[:table_name] # full: demo-dev_posts short: posts
+      puts "DynamoDB Table: #{namespaced_table_name} Status: #{resp.table_description.table_status}"
+
+      if method_name.to_sym == :delete_table # already sym, to_sym just in case
+        waiter.wait_for_delete(params[:table_name])
+      else # create_table or update_table
+        waiter.wait(params[:table_name])
+      end
     end
   end
 end

data/lib/dynomite/query.rb

@@ -0,0 +1,48 @@
+module Dynomite
+  class Query
+    include Enumerable
+
+    def initialize(item, params)
+      @item = item
+      @params = params
+    end
+
+    def <<(item)
+      raise NotImplementedError
+    end
+
+    def inspect
+      "#<Dynomite::Query [#{first(2).map(&:inspect).join(', ')}, ...]>"
+    end
+
+    def each(&block)
+      run_query.each(&block)
+    end
+
+    def index_name(name)
+      self.class.new(@item, @params.merge(index_name: name))
+    end
+
+    def where(attributes)
+      raise "attributes.size == 1 only supported for now" if attributes.size != 1
+
+      attr_name = attributes.keys.first
+      attr_value = attributes[attr_name]
+
+      name_key, value_key = "##{attr_name}_name", ":#{attr_name}_value"
+      params = {
+        expression_attribute_names: { name_key => attr_name },
+        expression_attribute_values: { value_key => attr_value },
+        key_condition_expression: "#{name_key} = #{value_key}",
+      }
+
+      self.class.new(@item, @params.merge(params))
+    end
+
+    private
+
+    def run_query
+      @query ||= @item.query(@params)
+    end
+  end
+end

data/lib/dynomite/reserved_words.rb

@@ -1,18 +1,28 @@
 module Dynomite
   RESERVED_WORDS = %w[
     as_json
-    attrs
     attributes
-    delete
+    attrs
     columns
+    destroy
+    fields
     find
     getter
+    hash_key
+    hash_key_field
+    key_schema
     new_record
     param_name
     partition_key
+    partition_key_field
+    range_key
+    range_key_field
     replace
-    setter
+    save
     scan
+    setter
+    sort_key_field
+    sort_key_field_field
     table_name
   ].freeze
 end

data/lib/dynomite/seed.rb

@@ -0,0 +1,12 @@
+module Dynomite
+  class Seed
+    def initialize(options={})
+      @options = options
+    end
+
+    def run
+      file = "dynamodb/seeds.rb"
+      load(file) if File.exist?(file)
+    end
+  end
+end

data/lib/dynomite/types.rb

@@ -0,0 +1,22 @@
+module Dynomite
+  module Types
+    TYPE_MAP = {
+      string: 'S',
+      number: 'N',
+      binary: 'B',
+      boolean: 'BOOL',
+      null: 'NULL',
+      map: 'M',
+      list: 'L',
+      string_set: 'SS',
+      number_set: 'NS',
+      binary_set: 'BS',
+    }
+
+    # https://rubyonjets.com/docs/database/dynamodb/types/
+    # https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/HowItWorks.NamingRulesDataTypes.html#HowItWorks.DataTypeDescriptors
+    def type_map(attribute_type)
+      TYPE_MAP[attribute_type.to_s.downcase.to_sym] || attribute_type
+    end
+  end
+end

data/lib/dynomite/version.rb

@@ -1,3 +1,3 @@
 module Dynomite
-  VERSION = "1.2.6"
+  VERSION = "2.0.0"
 end

data/lib/dynomite/waiter.rb

@@ -0,0 +1,40 @@
+module Dynomite
+  class Waiter
+    include Client
+
+    def wait(table_name)
+      logger.info "Waiting for #{table_name} table to be ready."
+      statuses = [:initial]
+      until statuses.all? { |s| s == "ACTIVE" } do
+        resp = client.describe_table(table_name: table_name)
+
+        table = resp.table
+        # table_status: CREATING UPDATING DELETING ACTIVE INACCESSIBLE_ENCRYPTION_CREDENTIALS ARCHIVING ARCHIVED
+        table_status = table.table_status
+        statuses = [table_status]
+        # index_status: CREATING UPDATING DELETING ACTIVE
+        indexes = table.global_secondary_indexes || []
+        statuses += indexes.map { |i| i.index_status }
+        print '.'
+        sleep pause_time
+      end
+      puts
+    end
+
+    def wait_for_delete(table_name)
+      logger.info "Waiting for #{table_name} table to be deleted."
+      begin
+        client.describe_table(table_name: table_name)
+        print '.'
+        sleep pause_time
+      rescue Aws::DynamoDB::Errors::ResourceNotFoundException => e
+      end
+      puts
+    end
+
+  private
+    def pause_time
+      1
+    end
+  end
+end

data/lib/dynomite.rb

@@ -1,25 +1,19 @@
 $:.unshift(File.expand_path("../", __FILE__))
+require "active_support"
+require "active_support/concern"
+require "active_support/core_ext/hash"
+require "active_support/core_ext/string"
 require "dynomite/version"
+require "memoist"
 require "rainbow/ext/string"
 
-module Dynomite
-  ATTRIBUTE_TYPES = {
-    'string' => 'S',
-    'number' => 'N',
-    'binary' => 'B',
-    's' => 'S',
-    'n' => 'N',
-    'b' => 'B',
-  }
+require "dynomite/autoloader"
+Dynomite::Autoloader.setup
 
-  autoload :Migration, "dynomite/migration"
-  autoload :Dsl, "dynomite/dsl"
-  autoload :DbConfig, "dynomite/db_config"
-  autoload :Item, "dynomite/item"
-  autoload :Core, "dynomite/core"
-  autoload :Erb, "dynomite/erb"
-  autoload :Log, "dynomite/log"
-  autoload :Errors, "dynomite/errors"
+require "dynomite/reserved_words"
 
+module Dynomite
   extend Core
 end
+
+require "dynomite/engine" if defined?(Jets)