shred 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b2e054fbed6a3aea3fa024120dc4f04ab56b8d11
-  data.tar.gz: fb5a6578711711182bce19418638681f09336a31
+  metadata.gz: d94a83bf54dff1959be163040d2703457f1b6777
+  data.tar.gz: 99aa30b2a973d733eb52c258dfd18dd3ff80a752
 SHA512:
-  metadata.gz: 766546e3f1b2d13863dab3658da6809487569516f8c12319ede06c5e19473ee0dd8b57dee953fc2d327344f4b844882d5370dc6cd49dc02b35cc1ef40c4c477f
-  data.tar.gz: 84d5c56ea550317e12c2d772cbf1b7ea744858dfb7bba05746d20d40ead486b329b530fdad88fe74c97f805f847cd8049b755f20f40f7ebb57de135adc48a50d
+  metadata.gz: 675abe608dbbdd741e6ec746cbb44a8c5236af20a7138b176fedebd8c1fb4b5ab639c07fccbe4bf257b1264d9f012168045ad3b90af9383a47d982b234c99252
+  data.tar.gz: 965952a420c451921659a40dd8739e8edac1fe931344f97e7c16853180109fd5818eb26477a10cbac289bd47d1fc0890de09cff2a3b352bd80a1bf90c7f49e86

data/lib/shred/commands/base.rb

@@ -167,6 +167,25 @@ module Shred
           value
         end
 
+        def interpolate_value(value, context: {})
+          value.gsub(/{[^}]+}/) do |match|
+            ref = match.slice(1, match.length)
+            ref = ref.slice(0, ref.length - 1)
+            if ref =~ /^env\:(.+)$/
+              env_key = $1.upcase
+              if ENV.key?(env_key)
+                ENV[env_key]
+              else
+                raise "Unset environment variable '#{env_key}' referenced by value '#{value}'"
+              end
+            elsif context.key?(ref.to_sym)
+              context[ref.to_sym]
+            else
+              raise "Unknown interpolation variable '#{ref}' referenced by value '#{value}'"
+            end
+          end
+        end
+
         def run_shell_command(command)
           ShellCommandRunner.new(console: console).run(command)
         end

data/lib/shred/commands/db.rb

@@ -9,7 +9,7 @@ module Shred
       LONGDESC
       def init
         run_shell_command(ShellCommand.new(
-          command_lines: 'bin/rake db:create db:structure:load',
+          command_lines: 'bin/rake db:create:all db:structure:load',
           success_msg: 'Database initialized',
           error_msg: 'Database could not be initialized'
         ))
@@ -23,6 +23,27 @@ module Shred
           error_msg: 'Migrations could not be applied'
         ))
       end
+
+      desc 'dump', 'Dump the contents of the database to a file'
+      def dump
+        command_lines = Array(cfg('dump')).map { |v| interpolate_value(v) }
+        run_shell_command(ShellCommand.new(
+          command_lines: command_lines,
+          success_msg: 'Database dumped',
+          error_msg: 'Database could not be dumped'
+        ))
+      end
+
+      desc 'restore DUMPFILE', 'Load the contents of a dump file into the database'
+      def restore(dumpfile)
+        context = {dumpfile: dumpfile}
+        command_lines = Array(cfg('restore')).map { |v| interpolate_value(v, context: context) }
+        run_shell_command(ShellCommand.new(
+          command_lines: command_lines,
+          success_msg: 'Database restored',
+          error_msg: 'Database could not be restored'
+        ))
+      end
     end
   end
 end

data/lib/shred/commands/deploy.rb

@@ -1,6 +1,6 @@
-require 'shred/commands/base'
 require 'dotenv'
 require 'platform-api'
+require 'shred/commands/base'
 
 module Shred
   module Commands

data/lib/shred/commands/dotenv.rb

@@ -23,27 +23,32 @@ module Shred
 
         run_shell_command(ShellCommand.new(command_lines: 'heroku auth:whoami'))
 
-        io = StringIO.new
+        heroku = StringIO.new
         run_shell_command(ShellCommand.new(
           command_lines: "heroku config --app #{app_name} --shell",
-          output: io
+          output: heroku
         ))
 
-        File.open('.env', 'w') do |output|
-          io.string.split("\n").each do |line|
-            if line =~ /^([^=]+)=/ && vars.include?($1)
-              output.write("#{line}\n")
-            end
+        outvars = {}
+
+        heroku.string.split("\n").each do |line|
+          key, value = line.split('=', 2)
+          outvars[key] = value if vars.include?(key)
+        end
+
+        if custom
+          custom.each do |key, value|
+            outvars[key] = interpolate_value(value)
           end
-          console.say_ok("Heroku config written to .env")
+        end
 
-          if custom
-            custom.each do |key, value|
-              output.write("#{key}=#{value}\n")
-            end
-            console.say_ok("Custom config written to .env")
+        File.open('.env', 'w') do |dotenv|
+          outvars.sort_by(&:first).each do |(key, value)|
+            dotenv.write("#{key}=#{value}\n")
           end
         end
+
+        console.say_ok("Heroku config written to .env")
       end
     end
   end

data/lib/shred/commands/dynamo_db.rb

@@ -0,0 +1,67 @@
+require 'dotenv'
+require 'aws-sdk'
+require 'shred/commands/base'
+
+module Shred
+  module Commands
+    class DynamoDb < Base
+      desc 'mktable NAME REGION READ_UNITS WRITE_UNITS', 'Create a DynamoDB table'
+      long_desc <<-LONGDESC
+        Create a DynamoDB table with the given name in the given region with the given throughput capacity.
+
+        All values are used exactly as specified. They are *not* interpolated.
+      LONGDESC
+      option :pk, type: :string, default: 'id'
+      option :pk_type, type: :string, default: 'string'
+      def mktable(name, region, read_capacity_units, write_capacity_units)
+        ::Dotenv.load
+
+        create_table(name, region, read_capacity_units.to_i, write_capacity_units.to_i,
+                     hash_key: {options[:pk] => options[:pk_type]})
+      end
+
+      desc 'mktables', 'Create all configured DynamoDB tables'
+      long_desc <<-LONGDESC
+        Create a DynamoDB table for each element of the `commands.dynamodb.tables` config var.
+
+        If the `commands.dynamodb.table_prefix` config var is set, its value is prepended to each table name.
+
+        Prefixed table names and region names are interpolated.
+      LONGDESC
+      def mktables
+        ::Dotenv.load
+
+        prefix = cfg('table_prefix', required: false)
+
+        cfg('tables').each do |(name, table_cfg)|
+          name = "#{prefix}#{name}" if prefix
+          name = interpolate_value(name)
+          region = interpolate_value(table_cfg['region'])
+          read_capacity_units = table_cfg['read_capacity_units'].to_i
+          write_capacity_units = table_cfg['write_capacity_units'].to_i
+          pk = table_cfg['primary_key'].fetch('name', 'id')
+          pk_type = table_cfg['primary_key'].fetch('type', 'string')
+
+          create_table(name, region, read_capacity_units, write_capacity_units, hash_key: {pk => pk_type})
+        end
+      end
+
+      no_commands do
+        def create_table(name, region, read_capacity_units, write_capacity_units, hash_key: nil)
+          ddb = AWS::DynamoDB.new(region: region)
+          if ddb.tables[name].exists?
+            console.say_ok("Dynamo DB table #{name} already exists in region #{region}")
+          else
+            table = ddb.tables.create(name, read_capacity_units, write_capacity_units, hash_key)
+            sleep 1 while table.status == :creating
+            if table.status == :active
+              console.say_ok("Created Dynamo DB table #{name} in region #{region}")
+            else
+              console.say_err("Failed to create Dynamo DB table #{name} in region #{region}: status #{table.status}")
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/shred/commands/elasticsearch.rb

@@ -0,0 +1,119 @@
+require 'dotenv'
+require 'elasticsearch'
+require 'shred/commands/base'
+
+module Shred
+  module Commands
+    class Elasticsearch < Base
+      desc 'mkindex NAME', 'Create a search index'
+      long_desc <<-LONGDESC
+        Create the search index with the given name.
+
+        If the `commands.elasticsearch.indexes.<name>.create` config var is present, it is taken to specify a list
+        of shell commands to be used to create the index. This mode supports creating indices with custom mappings
+        or settings.
+
+        The index name is used exactly as specified; it is *not* interpolated. However, the shell commands *are*
+        interpolated.
+      LONGDESC
+      def mkindex(name)
+        ::Dotenv.load
+        create_index(name, cfg("indexes.#{name}", required: false))
+      end
+
+      desc 'mkindices', 'Create all configured search indexes'
+      long_desc <<-LONGDESC
+        Creates each search index listed for the `commands.elasticsearch.indexes` config var.
+
+        If the `commands.elasticsearch.indexes.<name>.create` config var is present, it is taken to specify a list
+        of shell commands to be used to create the index. This mode supports creating indices with custom mappings
+        or settings.
+
+        Each index name and shell command is interpolated.
+      LONGDESC
+      def mkindices
+        ::Dotenv.load
+        Array(cfg('indexes')).each do |name, index_cfg|
+          create_index(name, index_cfg)
+        end
+      end
+
+      desc 'rmindex NAME', 'Delete a search index'
+      long_desc <<-LONGDESC
+        Delete the search index with the given name.
+
+        The index name is used exactly as specified; it is *not* interpolated.
+      LONGDESC
+      def rmindex(name)
+        ::Dotenv.load
+        delete_index(name)
+      end
+
+      desc 'rmindices', 'Delete all configured search indexes'
+      long_desc <<-LONGDESC
+        Deletes each search index listed for the `commands.elasticsearch.indexes` config var.
+
+        Each index name is interpolated.
+      LONGDESC
+      def rmindices
+        ::Dotenv.load
+        Array(cfg('indexes').keys).each do |name|
+          delete_index(name)
+        end
+      end
+
+      desc 'import NAME', 'Import data into a search index'
+      long_desc <<-LONGDESC
+        Imports data into the search index with the given name.
+
+        Executes one or more shell commands taken from the `commands.elasticsearch.indexes.<name>.import` config var.
+
+        The index name is used exactly as specified; it is *not* interpolated. However, the shell commands *are*
+        interpolated.
+      LONGDESC
+      def import(name)
+        ::Dotenv.load
+        command_lines = Array(cfg("indexes.#{name}.import")).map { |l| interpolate_value(l) }
+        run_shell_command(ShellCommand.new(
+          command_lines: command_lines,
+          success_msg: "Data imported into index #{name}",
+          error_msg: "Failed to import data into index #{name}"
+        ))
+      end
+
+      no_commands do
+        def client
+          url = interpolate_value(cfg('url'))
+          @client ||= ::Elasticsearch::Client.new(url: url)
+        end
+
+        def create_index(name, index_cfg = nil)
+          if index_cfg && index_cfg['create']
+            command_lines = Array(index_cfg['create']).map { |l| interpolate_value(l) }
+            run_shell_command(ShellCommand.new(
+              command_lines: command_lines,
+              success_msg: "Created index #{name}",
+              error_msg: "Failed to create index #{name}"
+            ))
+          else
+            begin
+              client.indices.create(index: name)
+              console.say_ok("Created index #{name}")
+            rescue ::Elasticsearch::Transport::Transport::Errors::BadRequest => e
+              raise unless e.to_s =~ /IndexAlreadyExistsException/
+              console.say_err("Index #{name} already exists")
+            end
+          end
+        end
+
+        def delete_index(name)
+          client.indices.delete(index: name)
+          console.say_ok("Deleted index #{name}")
+        rescue ::Elasticsearch::Transport::Transport::Errors::NotFound => e
+          raise unless e.to_s =~ /IndexMissingException/
+          console.say_err("Index #{name} does not exist")
+        end
+      end
+    end
+  end
+end

data/lib/shred/commands/s3.rb

@@ -0,0 +1,50 @@
+require 'dotenv'
+require 'aws-sdk'
+require 'shred/commands/base'
+
+module Shred
+  module Commands
+    class S3 < Base
+      desc 'mkbucket NAME REGION', 'Create an S3 bucket'
+      long_desc <<-LONGDESC
+        Create an S3 bucket with the given name in the given region.
+
+        The bucket and region names are used exactly as specified. They are *not* interpolated.
+      LONGDESC
+      def mkbucket(name, region)
+        ::Dotenv.load
+
+        create_bucket(name, region)
+      end
+
+      desc 'mkbuckets', 'Create all configured S3 buckets'
+      long_desc <<-LONGDESC
+        Create an S3 bucket for each element of the `commands.s3.buckets` config var.
+
+        Bucket and region names are interpolated.
+      LONGDESC
+      def mkbuckets
+        ::Dotenv.load
+
+        cfg('buckets').each do |(key, bucket_cfg)|
+          name = interpolate_value(bucket_cfg['name'])
+          region = interpolate_value(bucket_cfg['region'])
+
+          create_bucket(name, region)
+        end
+      end
+
+      no_commands do
+        def create_bucket(name, region)
+          s3 = AWS::S3.new(region: region)
+          if s3.buckets[name].exists?
+            console.say_ok("S3 bucket #{name} already exists in region #{region}")
+          else
+            s3.buckets.create(name)
+            console.say_ok("Created S3 bucket #{name} in region #{region}")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/shred/version.rb

@@ -1,3 +1,3 @@
 module Shred
-  VERSION = '0.0.4'
+  VERSION = '0.0.5'
 end

data/lib/shred.rb

@@ -2,10 +2,13 @@ require 'shred/commands/app'
 require 'shred/commands/db'
 require 'shred/commands/deploy'
 require 'shred/commands/dotenv'
+require 'shred/commands/dynamo_db'
+require 'shred/commands/elasticsearch'
 require 'shred/commands/js_deps'
 require 'shred/commands/platform_deps'
 require 'shred/commands/ruby_deps'
 require 'shred/commands/services'
+require 'shred/commands/s3'
 require 'shred/commands/test'
 require 'shred/version'
 require 'thor'
@@ -43,13 +46,25 @@ module Shred
           subcommand 'js_deps', Commands::JsDeps
         end
         if commands.key?('db')
-          desc 'db SUBCOMMAND ...ARGS', 'Manage the database'
+          desc 'db SUBCOMMAND ...ARGS', 'Manage the relational database system'
           subcommand 'db', Commands::Db
         end
         if commands.key?('dotenv')
           desc 'dotenv SUBCOMMAND ...ARGS', 'Manage the environmental configuration'
           subcommand 'dotenv', Commands::Dotenv
         end
+        if commands.key?('s3')
+          desc 's3 SUBCOMMAND ...ARGS', 'Manage Amazon S3 buckets'
+          subcommand 's3', Commands::S3
+        end
+        if commands.key?('dynamo_db')
+          desc 'dynamo_db SUBCOMMAND ...ARGS', 'Manage Amazon Dynamo DB tables'
+          subcommand 'dynamo_db', Commands::DynamoDb
+        end
+        if commands.key?('elasticsearch')
+          desc 'elasticsearch subcommand ...ARGS', 'Manage Elasticsearch indexes'
+          subcommand 'elasticsearch', Commands::Elasticsearch
+        end
         if commands.key?('test')
           desc 'test SUBCOMMAND ...ARGS', 'Run tests'
           subcommand 'test', Commands::Test
@@ -68,6 +83,7 @@ module Shred
             self.class.config['commands']['setup'].each do |(cmd, subcmd)|
               invoke(cmd.to_sym, [subcmd.to_sym])
             end
+            console.say_ok("Setup complete!")
           end
         end
       end

data/lib/shred.yml.tt

@@ -3,6 +3,8 @@ commands:
     start:
       - foreman start
   db:
+    dump: pg_dump -Fc --no-acl --no-owner -h localhost <%= app_name %>_development > <%= app_name %>_development.dump
+    restore: pg_restore --verbose --clean --no-acl --no-owner -h localhost -d <%= app_name %>_development {dumpfile}
   deploy:
     default_environment: staging
     staging:
@@ -21,12 +23,42 @@ commands:
       app_name: <%= app_name %>-staging
       vars:
         - AIRBRAKE_API_KEY
+        - ELASTICSEARCH_URL
     custom:
       vars:
         RAILS_ENV: development
+        AWS_S3_BUCKET: <%= app_name %>-dev-{env:USER}
+        AWS_S3_REGION: us-west-2
+        AWS_DYNAMO_DB_REGION: us-west-2
+  dynamo_db:
+    table_prefix: 'dev-{env:USER}'
+    tables:
+      clicks:
+        region: '{env:AWS_DYNAMO_DB_REGION}'
+        primary_key:
+          name: uuid
+          type: string
+        read_capacity_units: 10
+        write_capacity_units: 10
+      impressions:
+        region: '{env:AWS_DYNAMO_DB_REGION}'
+        primary_key:
+          name: uuid
+          type: string
+        read_capacity_units: 10
+        write_capacity_units: 10
+  elasticsearch:
+    url: '{env:ELASTICSEARCH_URL}'
+    indexes:
+      stories-development:
+        create:
+          - thor stories:create
+        import:
+          - thor stories:index
   js_deps:
   platform_deps:
     homebrew:
+      - node
       - postgres
       - redis
       - heroku
@@ -53,6 +85,11 @@ commands:
       mailcatcher:
         start: mailcatcher
         stop: killall mailcatcher
+  s3:
+    buckets:
+      default:
+        name: '{env:AWS_S3_BUCKET}'
+        region: '{env:AWS_S3_REGION}'
   setup:
     platform_deps: install
     services: start
@@ -64,4 +101,4 @@ commands:
     server:
       - bin/rake spec
     client:
-      - bin/rake teaspoon:run
+      - bin/rake teaspoon

data/shred.gemspec

@@ -21,7 +21,9 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'bundler', '~> 1.7'
   spec.add_development_dependency 'rake', '~> 10.0'
 
+  spec.add_dependency 'aws-sdk'
   spec.add_dependency 'dotenv'
+  spec.add_dependency 'elasticsearch'
   spec.add_dependency 'platform-api'
   spec.add_dependency 'thor'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: shred
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - Brian
@@ -39,6 +39,20 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: dotenv
   requirement: !ruby/object:Gem::Requirement
@@ -53,6 +67,20 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: elasticsearch
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: platform-api
   requirement: !ruby/object:Gem::Requirement
@@ -105,9 +133,12 @@ files:
 - lib/shred/commands/db.rb
 - lib/shred/commands/deploy.rb
 - lib/shred/commands/dotenv.rb
+- lib/shred/commands/dynamo_db.rb
+- lib/shred/commands/elasticsearch.rb
 - lib/shred/commands/js_deps.rb
 - lib/shred/commands/platform_deps.rb
 - lib/shred/commands/ruby_deps.rb
+- lib/shred/commands/s3.rb
 - lib/shred/commands/services.rb
 - lib/shred/commands/test.rb
 - lib/shred/version.rb