dynomite 1.2.6 → 2.0.0

data/lib/dynomite/cli.rb

@@ -0,0 +1,48 @@
+module Dynomite
+  class CLI < Command
+    desc "migrate", "Run migrations"
+    long_desc Help.text(:migrate)
+    def migrate
+      Migration::Runner.new(options).run
+    end
+
+    desc "generate [name]", "Creates a migration for a DynamoDB table"
+    long_desc Help.text('generate')
+    option :action, desc: "create_table or update_table. Defaults to convention based on the name of the migration."
+    option :partition_key, default: "id:string", desc: "table's partition key"
+    option :sort_key, default: nil, desc: "table's sort key"
+    option :table_name, desc: "override the the conventional table name"
+    def generate(name)
+      Migration::Generator.new(name, options).generate
+    end
+
+    desc "seed", "Seed data"
+    long_desc Help.text(:seed)
+    def seed
+      Seed.new.run
+    end
+
+    desc "install", "Install Dynomite"
+    long_desc Help.text(:install)
+    def install
+      Install.new.run
+    end
+
+    desc "completion *PARAMS", "Prints words for auto-completion."
+    long_desc Help.text(:completion)
+    def completion(*params)
+      Completer.new(CLI, *params).run
+    end
+
+    desc "completion_script", "Generates a script that can be eval to setup auto-completion."
+    long_desc Help.text(:completion_script)
+    def completion_script
+      Completer::Script.generate
+    end
+
+    desc "version", "prints version"
+    def version
+      puts VERSION
+    end
+  end
+end

data/lib/dynomite/client.rb

@@ -0,0 +1,118 @@
+require "aws-sdk-dynamodb"
+require "erb"
+require "fileutils"
+require "json"
+require "yaml"
+
+module Dynomite
+  module Client
+    extend ActiveSupport::Concern
+    delegate :client, :desc_table, :show_request, :show_response,
+             :warn_scan, :log_debug, :logger,
+             to: :class
+
+    class_methods do
+      extend Memoist
+
+      @@client = nil
+      def client
+        return @@client if @@client
+
+        endpoint = Dynomite.config.endpoint
+        check_dynamodb_local!(endpoint)
+
+        # Normally, do not set the endpoint to use the current configured region.
+        # Probably want to stay in the same region anyway for db connections.
+        #
+        # List of regional endpoints: https://docs.aws.amazon.com/general/latest/gr/rande.html#ddb_region
+        # Example:
+        #   endpoint: https://dynamodb.us-east-1.amazonaws.com
+        options = endpoint ? { endpoint: endpoint } : {}
+        log_level = Dynomite.config.log_level.to_s
+        # https://aws.amazon.com/blogs/developer/logging-requests/
+        # https://github.com/aws/aws-sdk-ruby/blob/249a0b34d0014dda50ecc8a09cd58e75e64b3ea4/gems/aws-sdk-core/lib/aws-sdk-core/log/formatter.rb#L212
+        if log_level == "debug"
+          options[:logger] = Dynomite.logger
+          options[:log_formatter] = Aws::Log::Formatter.colored # default short colored
+        end
+        @@client ||= Aws::DynamoDB::Client.new(options)
+      end
+
+      # useful for specs
+      def client=(client)
+        @@client = client
+      end
+
+      def desc_table(table_name)
+        client.describe_table(table_name: table_name).table
+      end
+      memoize :desc_table
+
+      def show_request(params)
+        logger.info("REQUEST: #{JSON.dump(params)}")
+      end
+
+      def show_response(resp)
+        logger.info("RESPONSE: #{JSON.dump(resp)}")
+      end
+
+      # When endoint has been configured to point at dynamodb local: localhost:8000
+      # check if port 8000 is listening and timeout quickly. Or else it takes a
+      # for DynamoDB local to time out, about 10 seconds...
+      # This wastes less of the users time.
+      def check_dynamodb_local!(endpoint)
+        return unless endpoint && endpoint.include?("8000")
+
+        open = port_open?("127.0.0.1", 8000, 0.2)
+        unless open
+          raise "You have configured your app to use DynamoDB local, but it is not running.  Please start DynamoDB local. Example: brew cask install dynamodb-local && dynamodb-local"
+        end
+      end
+
+      # Thanks: https://gist.github.com/ashrithr/5305786
+      def port_open?(ip, port, seconds=1)
+        # => checks if a port is open or not
+        Timeout::timeout(seconds) do
+          begin
+            TCPSocket.new(ip, port).close
+            true
+          rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH, SocketError
+            false
+          end
+        end
+      rescue Timeout::Error
+        false
+      end
+
+      def warn_scan(message, show: nil)
+        warn_on_scan = show.nil? ? Dynomite.config.warn_on_scan : show
+        return unless warn_on_scan
+        message += <<~EOL
+          You can disable this warning by setting Dynomite.config.warn_on_scan: false
+        EOL
+        logger.info(message)
+        logger.info("Called from: #{call_line}") if call_line
+      end
+
+      def call_line
+        caller.find { |l| l.include?(Dir.pwd) }
+      end
+
+      def log_debug(params)
+        return unless ENV['DYNOMITE_DEBUG']
+
+        call_location = caller_locations[1].to_s # IE: dynomite/item/query/relation.rb:62:in `block in raw_pages'
+        return if call_location.blank? # edge cases
+        method_name = call_location.split('`').last.split(" ").last # IE: raw_pages'
+        method_name.gsub!("'", "") # IE: raw_pages
+
+        logger.info "#{self}##{method_name}" # IE: Dynomite::Item::Query::Relation#raw_pages
+        pp params
+      end
+
+      def logger
+        Dynomite.logger
+      end
+    end
+  end
+end

data/lib/dynomite/command.rb

@@ -0,0 +1,89 @@
+require "thor"
+
+# Override thor's long_desc identation behavior
+# https://github.com/erikhuda/thor/issues/398
+class Thor
+  module Shell
+    class Basic
+      def print_wrapped(message, options = {})
+        message = "\n#{message}" unless message[0] == "\n"
+        stdout.puts message
+      end
+    end
+  end
+end
+
+module Dynomite
+  class Command < Thor
+    class << self
+      def dispatch(m, args, options, config)
+        # Allow calling for help via:
+        #   dynomite command help
+        #   dynomite command -h
+        #   dynomite command --help
+        #   dynomite command -D
+        #
+        # as well thor's normal way:
+        #
+        #   dynomite help command
+        help_flags = Thor::HELP_MAPPINGS + ["help"]
+        if args.length > 1 && !(args & help_flags).empty?
+          args -= help_flags
+          args.insert(-2, "help")
+        end
+
+        #   dynomite version
+        #   dynomite --version
+        #   dynomite -v
+        version_flags = ["--version", "-v"]
+        if args.length == 1 && !(args & version_flags).empty?
+          args = ["version"]
+        end
+
+        super
+      end
+
+      # Override command_help to include the description at the top of the
+      # long_description.
+      def command_help(shell, command_name)
+        meth = normalize_command_name(command_name)
+        command = all_commands[meth]
+        alter_command_description(command)
+        super
+      end
+
+      def alter_command_description(command)
+        return unless command
+
+        # Add description to beginning of long_description
+        long_desc = if command.long_description
+            "#{command.description}\n\n#{command.long_description}"
+          else
+            command.description
+          end
+
+        # add reference url to end of the long_description
+        unless website.empty?
+          full_command = [command.ancestor_name, command.name].compact.join('-')
+          url = "#{website}/reference/dynomite-#{full_command}"
+          long_desc += "\n\nHelp also available at: #{url}"
+        end
+
+        command.long_description = long_desc
+      end
+      private :alter_command_description
+
+      # meant to be overriden
+      def website
+        ""
+      end
+
+      # https://github.com/erikhuda/thor/issues/244
+      # Deprecation warning: Thor exit with status 0 on errors. To keep this behavior, you must define `exit_on_failure?` in `Lono::CLI`
+      # You can silence deprecations warning by setting the environment variable THOR_SILENCE_DEPRECATION.
+      def exit_on_failure?
+        true
+      end
+    end
+  end
+end

data/lib/dynomite/completer/script.rb

@@ -0,0 +1,6 @@
+class Dynomite::Completer::Script
+  def self.generate
+    bash_script = File.expand_path("script.sh", File.dirname(__FILE__))
+    puts "source #{bash_script}"
+  end
+end

data/lib/dynomite/completer/script.sh

@@ -0,0 +1,10 @@
+_dynomite() {
+  COMPREPLY=()
+  local word="${COMP_WORDS[COMP_CWORD]}"
+  local words=("${COMP_WORDS[@]}")
+  unset words[0]
+  local completion=$(dynomite completion ${words[@]})
+  COMPREPLY=( $(compgen -W "$completion" -- "$word") )
+}
+
+complete -F _dynomite dynomite

data/lib/dynomite/completer.rb

@@ -0,0 +1,159 @@
+=begin
+Code Explanation:
+
+There are 3 types of things to auto-complete:
+
+  1. command: the command itself
+  2. parameters: command parameters.
+  3. options: command options
+
+Here's an example:
+
+  mycli hello name --from me
+
+  * command: hello
+  * parameters: name
+  * option: --from
+
+When command parameters are done processing, the remaining completion words will be options.  We can tell that the command params are completed based on the method arity.
+
+## Arity
+
+For example, say you had a method for a CLI command with the following form:
+
+  ufo scale service count --cluster development
+
+It's equivalent ruby method:
+
+  scale(service, count) = has an arity of 2
+
+So typing:
+
+  ufo scale service count [TAB] # there are 3 parameters including the "scale" command according to Thor's CLI processing.
+
+So the completion should only show options, something like this:
+
+  --noop --verbose --cluster
+
+## Splat Arguments
+
+When the ruby method has a splat argument, it's arity is negative.  Here are some example methods and their arities.
+
+   ship(service) = 1
+   scale(service, count) = 2
+   ships(*services) = -1
+   foo(example, *rest) = -2
+
+Fortunately, negative and positive arity values are processed the same way. So we take simply take the absolute value of the arity and process it the same.
+
+Here are some test cases, hit TAB after typing the command:
+
+  dynomite completion
+  dynomite completion hello
+  dynomite completion hello name
+  dynomite completion hello name --
+  dynomite completion hello name --noop
+
+  dynomite completion
+  dynomite completion sub:goodbye
+  dynomite completion sub:goodbye name
+
+## Subcommands and Thor::Group Registered Commands
+
+Sometimes the commands are not simple thor commands but are subcommands or Thor::Group commands. A good specific example is the ufo tool.
+
+  * regular command: ufo ship
+  * subcommand: ufo docker
+  * Thor::Group command: ufo init
+
+Auto-completion accounts for each of these type of commands.
+=end
+module Dynomite
+  class Completer
+    def initialize(command_class, *params)
+      @params = params
+      @current_command = @params[0]
+      @command_class = command_class # CLI initiall
+    end
+
+    def run
+      if subcommand?(@current_command)
+        subcommand_class = @command_class.subcommand_classes[@current_command]
+        @params.shift # destructive
+        Completer.new(subcommand_class, *@params).run # recursively use subcommand
+        return
+      end
+
+      # full command has been found!
+      unless found?(@current_command)
+        puts all_commands
+        return
+      end
+
+      # will only get to here if command aws found (above)
+      arity = @command_class.instance_method(@current_command).arity.abs
+      if @params.size > arity or thor_group_command?
+        puts options_completion
+      else
+        puts params_completion
+      end
+    end
+
+    def subcommand?(command)
+      @command_class.subcommands.include?(command)
+    end
+
+    # hacky way to detect that command is a registered Thor::Group command
+    def thor_group_command?
+      command_params(raw=true) == [[:rest, :args]]
+    end
+
+    def found?(command)
+      public_methods = @command_class.public_instance_methods(false)
+      command && public_methods.include?(command.to_sym)
+    end
+
+    # all top-level commands
+    def all_commands
+      commands = @command_class.all_commands.reject do |k,v|
+        v.is_a?(Thor::HiddenCommand)
+      end
+      commands.keys
+    end
+
+    def command_params(raw=false)
+      params = @command_class.instance_method(@current_command).parameters
+      # Example:
+      # >> Sub.instance_method(:goodbye).parameters
+      # => [[:req, :name]]
+      # >>
+      raw ? params : params.map!(&:last)
+    end
+
+    def params_completion
+      offset = @params.size - 1
+      offset_params = command_params[offset..-1]
+      command_params[offset..-1].first
+    end
+
+    def options_completion
+      used = ARGV.select { |a| a.include?('--') } # so we can remove used options
+
+      method_options = @command_class.all_commands[@current_command].options.keys
+      class_options = @command_class.class_options.keys
+
+      all_options = method_options + class_options + ['help']
+
+      all_options.map! { |o| "--#{o.to_s.gsub('_','-')}" }
+      filtered_options = all_options - used
+      filtered_options.uniq
+    end
+
+    # Useful for debugging. Using puts messes up completion.
+    def log(msg)
+      File.open("/tmp/complete.log", "a") do |file|
+        file.puts(msg)
+      end
+    end
+  end
+end

data/lib/dynomite/config.rb

@@ -0,0 +1,39 @@
+module Dynomite
+  class Config
+    attr_accessor :logger, :log_level, :namespace_separator, :endpoint, :env,
+                  :default_count_method, :warn_on_scan, :default_namespace, :discover_fields,
+                  :migration, :undeclared_field_behavior, :default_field_type, :update_strategy
+    def initialize
+      @logger = Logger.new($stderr)
+      @logger.formatter = ActiveSupport::Logger::SimpleFormatter.new
+      @log_level = nil
+      @namespace_separator = "_"
+      @endpoint = ENV['DYNOMITE_ENDPOINT'] # allow to use local dynamodb
+      @env = ActiveSupport::StringInquirer.new(ENV['DYNOMITE_ENV'] || "development")
+      @default_count_method = :count # slow but accurate. :item_count is faster but can be stale by 6 hours
+      @warn_on_scan = true
+      @discover_fields = false
+      @migration = ActiveSupport::OrderedOptions.new
+      @undeclared_field_behavior = :warn # warn silent error allow
+      # Not implemented: :datetime, :date, :float, :array, :set, :map
+      # as we aws-sdk-dynamodb handles it via :infer
+      @default_field_type = :infer # :string, :integer, :boolean, :time, :infer
+      @update_strategy = :put_item # :put_item, :update_item
+    end
+
+    # User should use namespace. The default_namespace is only used internally so Jets can set it.
+    # Makes it easy to set the namespace from the Jets project namespace.
+    # Example:
+    #
+    # config/initializers/dynomite.rb
+    #
+    #     Dynomite.configure do |config|
+    #       config.namespace = Jets.project_namespace # IE: demo-dev
+    #     end
+    #
+    attr_writer :namespace
+    def namespace
+      ENV['DYNOMITE_NAMESPACE'] || @namespace || @default_namespace || 'dynomite'
+    end
+  end
+end

data/lib/dynomite/core.rb

@@ -1,25 +1,24 @@
 require 'logger'
 
-module Dynomite::Core
-  # Ensures trailing slash
-  # Useful for appending a './' in front of a path or leaving it alone.
-  # Returns: '/path/with/trailing/slash/' or './'
-  @@app_root = nil
-  def app_root
-    return @@app_root if @@app_root
-    @@app_root = ENV['APP_ROOT'] || ENV['JETS_ROOT'] || ENV['RAILS_ROOT']
-    @@app_root = '.' if @@app_root.nil? || @app_root == ''
-    @@app_root = "#{@@app_root}/" unless @@app_root.ends_with?('/')
-    @@app_root
-  end
+module Dynomite
+  module Core
+    @@root = nil
+    def root
+      return @@root if @@root
+      @@root = ENV['DYNOMITE_ROOT'] || ENV['JETS_ROOT'] || ENV['RAILS_ROOT'] || '.'
+    end
 
-  @@logger = nil
-  def logger
-    return @@logger if @@logger
-    @@logger = Logger.new($stderr)
-  end
+    @@config = nil
+    def config
+      @@config ||= Config.new
+    end
+
+    def logger
+      config.logger
+    end
 
-  def logger=(value)
-    @@logger = value
+    def configure
+      yield(config)
+    end
   end
 end

data/lib/dynomite/engine.rb

@@ -0,0 +1,45 @@
+module Dynomite
+  class Engine < ::Jets::Engine
+    config.after_initialize do
+      Dynomite.config.default_namespace = Jets.project_namespace # IE: demo-dev
+      Dynomite.config.migration.deletion_protection_enabled = Jets.env.production?
+
+      # Discover all the fields for all the models from attribute_definitions
+      # and create field methods. Has to be done after_initialize because
+      # need model names for the table_name.
+      quiet_dynamodb_logging do
+        Dynomite::Item.descendants.each do |klass|
+          klass.discover_fields!
+        end if Dynomite.config.discover_fields
+      end
+    end
+
+    def self.default_log_level
+      # Note: On AWS Lambda, ARGV[0] is nil
+      if ARGV[0]&.include?("dynamodb") # IE: dynamodb:migrate dynamodb:seed
+        :info
+      else
+        Jets.env.development? ? :debug : :info
+      end
+    end
+
+    def self.quiet_dynamodb_logging
+      if ENV['DYNOMITE_DEBUG']
+        # If in debug mode, then leave the log level alone which is debug in development
+        # This shows the describe_table calls on jets console bootup
+        Dynomite.config.log_level = default_log_level
+      else
+        # Otherwise, set the log level to info temporarily to quiet the describe_table calls
+        # Then reset the log level back to the user's configured log level.
+        user_log_level = Dynomite.config.log_level
+        Dynomite.config.log_level = :info
+      end
+
+      yield
+
+      Dynomite::Item.client = nil # reset client. Need to reset the client since it's cached
+      # Go back to the user's configured log level or the default log level if user did not set it.
+      Dynomite.config.log_level = user_log_level || default_log_level
+    end
+  end
+end

data/lib/dynomite/erb.rb

@@ -7,14 +7,12 @@ require 'erb'
 #   result = Dynomite::Erb.result(path, key1: "val1", key2: "val2")
 #
 class Dynomite::Erb
-  include Dynomite::Log
-
   class << self
     def result(path, variables={})
       set_template_variables(variables)
       template = IO.read(path)
       begin
-        ERB.new(template, nil, "-").result(binding)
+        ERB.new(template, trim_mode: '-').result(binding)
       rescue Exception => e
         log(e)
         log(e.backtrace) if ENV['DEBUG']
@@ -49,5 +47,9 @@ class Dynomite::Erb
         instance_variable_set(:"@#{key}", value)
       end
     end
+
+    def log(msg)
+      logger.info(msg)
+    end
   end
 end

data/lib/dynomite/error.rb

@@ -0,0 +1,12 @@
+module Dynomite
+  class Error < StandardError
+    class InvalidPut < Error; end
+    class PrimaryKeyChangedError < Error; end
+    class RecordNotFound < Error; end
+    class RecordNotUnique < Error; end
+    class ReservedWord < Error; end
+    class StaleObject < Error; end
+    class UndeclaredFields < Error; end
+    class Validation < Error; end
+  end
+end

data/lib/dynomite/help/completion.md

@@ -0,0 +1,20 @@
+## Examples
+
+    dynomite completion
+
+Prints words for TAB auto-completion.
+
+    dynomite completion
+    dynomite completion hello
+    dynomite completion hello name
+
+To enable, TAB auto-completion add the following to your profile:
+
+    eval $(dynomite completion_script)
+
+Auto-completion example usage:
+
+    dynomite [TAB]
+    dynomite hello [TAB]
+    dynomite hello name [TAB]
+    dynomite hello name --[TAB]

data/lib/dynomite/help/completion_script.md

@@ -0,0 +1,3 @@
+To use, add the following to your `~/.bashrc` or `~/.profile`
+
+    eval $(dynomite completion_script)

data/lib/dynomite/help/migrate.md

@@ -0,0 +1,3 @@
+## Examples
+
+    dynomite migrate

data/lib/dynomite/help.rb

@@ -0,0 +1,9 @@
+module Dynomite::Help
+  class << self
+    def text(namespaced_command)
+      path = namespaced_command.to_s.gsub(':','/')
+      path = File.expand_path("../help/#{path}.md", __FILE__)
+      IO.read(path) if File.exist?(path)
+    end
+  end
+end

data/lib/dynomite/install.rb

@@ -0,0 +1,4 @@
+module Dynomite
+  class Install
+  end
+end

data/lib/dynomite/item/abstract.rb

@@ -0,0 +1,15 @@
+class Dynomite::Item
+  module Abstract
+    extend ActiveSupport::Concern
+
+    class_methods do
+      def abstract?
+        !!@abstract
+      end
+
+      def abstract!
+        @abstract = true
+      end
+    end
+  end
+end