clientside_aws 0.0.17

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: cc7c9075e9963aa0ea834be7548e3ebe0fe2bd72
+  data.tar.gz: fd64a23e57207414f358a7dd1f4455c4bdd5681d
+SHA512:
+  metadata.gz: 5ee2fae298c383b15e79ac83980ea96aaaec520051b87d89e510919b284f3ddae0e0f7ff72882d0220ca75252f1f94337628e35316de5b19235b1b3d76bc3ecd
+  data.tar.gz: 30e522b64c96cfe5423880c6a00957208e03e09e8491cb40c9e99df5c4e16f122b850ea7f88b1e8d883bfa3772a9caea581498453d585e7f7de2e7a5321344e3

data/.gitignore

@@ -0,0 +1,4 @@
+*~
+*#
+*.log
+.ruby-version

data/Dockerfile

@@ -0,0 +1,46 @@
+FROM phusion/baseimage:0.9.22
+MAINTAINER Perry Street Software
+
+# Set correct environment variables.
+ENV HOME /root
+
+# Use baseimage-docker's init system.
+CMD ["/sbin/my_init"]
+
+RUN mkdir /mnt/redis
+
+RUN apt-add-repository ppa:brightbox/ruby-ng -y
+RUN apt-get update && apt-get install -y ruby2.4 ruby2.4-dev git-core build-essential zlib1g-dev
+RUN apt-get install -y wget curl
+
+RUN cd /opt ; wget "http://download.redis.io/releases/redis-2.8.24.tar.gz"
+RUN cd /opt ; gunzip redis-2.8.24.tar.gz ; tar -xvf redis-2.8.24.tar
+RUN cd /opt/redis-2.8.24 ; ./configure ; make ; make install
+
+# Install for testing ffmpeg stuff
+RUN apt-get install -y libav-tools
+
+RUN mkdir /etc/service/redis-server
+ADD docker/redis-server-run /etc/service/redis-server/run
+RUN chmod 755 /etc/service/redis-server/run
+
+# Add redis conf file
+RUN mkdir /etc/redis
+RUN cd /opt/redis-2.8.24 ; cat redis.conf | sed "s/dir \.\//dir \/mnt\/redis\//" > /etc/redis/redis.conf
+
+# Now, fetch clientside aws
+RUN gem install bundler
+RUN cd /opt
+COPY . /opt/clientside_aws/
+RUN cd /opt/clientside_aws ; bundle install
+
+RUN mkdir /etc/service/clientside-aws
+ADD docker/clientside-aws-run /etc/service/clientside-aws/run
+RUN chmod 755 /etc/service/clientside-aws/run
+
+EXPOSE 4567
+
+# Clean up APT when done.
+RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+WORKDIR /opt/clientside_aws

data/Gemfile

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+# These are gem dependencies for our Docker image,
+# not for our Gemfile which points clients to our docker image
+gem 'aws-sdk', '2.10.26'
+gem 'aws-sdk-v1', '1.66.0'
+gem 'builder', '3.1.4'
+gem 'httparty', '0.15.6'
+gem 'json', '1.8.6'
+gem 'json_pure', '1.8.6'
+gem 'monkey-lib', '0.5.4'
+gem 'pry'
+gem 'rack', '1.5.2'
+gem 'rack-cors', '0.2.9', require: 'rack/cors'
+gem 'rack-test', '0.5.7'
+gem 'redis', '3.2.0'
+gem 'rspec', '2.14.1'
+gem 'sinatra', '1.4.8'
+gem 'sinatra-reloader', '0.5.0'
+gem 'uuid', '2.3.1'
+gem 'webmock', '3.1.0'

data/Gemfile.lock

@@ -0,0 +1,99 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.5.2)
+      public_suffix (>= 2.0.2, < 4.0)
+    aws-sdk (2.10.26)
+      aws-sdk-resources (= 2.10.26)
+    aws-sdk-core (2.10.26)
+      aws-sigv4 (~> 1.0)
+      jmespath (~> 1.0)
+    aws-sdk-resources (2.10.26)
+      aws-sdk-core (= 2.10.26)
+    aws-sdk-v1 (1.66.0)
+      json (~> 1.4)
+      nokogiri (>= 1.4.4)
+    aws-sigv4 (1.0.2)
+    backports (3.10.0)
+    builder (3.1.4)
+    coderay (1.1.2)
+    crack (0.4.3)
+      safe_yaml (~> 1.0.0)
+    diff-lcs (1.3)
+    hashdiff (0.3.7)
+    httparty (0.15.6)
+      multi_xml (>= 0.5.2)
+    jmespath (1.3.1)
+    json (1.8.6)
+    json_pure (1.8.6)
+    macaddr (1.7.1)
+      systemu (~> 2.6.2)
+    method_source (0.9.0)
+    mini_portile2 (2.3.0)
+    monkey-lib (0.5.4)
+      backports
+    multi_xml (0.6.0)
+    nokogiri (1.8.1)
+      mini_portile2 (~> 2.3.0)
+    pry (0.11.1)
+      coderay (~> 1.1.0)
+      method_source (~> 0.9.0)
+    public_suffix (3.0.0)
+    rack (1.5.2)
+    rack-cors (0.2.9)
+    rack-protection (1.5.3)
+      rack
+    rack-test (0.5.7)
+      rack (>= 1.0)
+    redis (3.2.0)
+    rspec (2.14.1)
+      rspec-core (~> 2.14.0)
+      rspec-expectations (~> 2.14.0)
+      rspec-mocks (~> 2.14.0)
+    rspec-core (2.14.8)
+    rspec-expectations (2.14.5)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.14.6)
+    safe_yaml (1.0.4)
+    sinatra (1.4.8)
+      rack (~> 1.5)
+      rack-protection (~> 1.4)
+      tilt (>= 1.3, < 3)
+    sinatra-advanced-routes (0.5.3)
+      sinatra (~> 1.0)
+    sinatra-reloader (0.5.0)
+      sinatra (~> 1.0)
+      sinatra-advanced-routes (~> 0.5.0)
+    systemu (2.6.5)
+    tilt (2.0.8)
+    uuid (2.3.1)
+      macaddr (~> 1.0)
+    webmock (3.1.0)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+      hashdiff
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  aws-sdk (= 2.10.26)
+  aws-sdk-v1 (= 1.66.0)
+  builder (= 3.1.4)
+  httparty (= 0.15.6)
+  json (= 1.8.6)
+  json_pure (= 1.8.6)
+  monkey-lib (= 0.5.4)
+  pry
+  rack (= 1.5.2)
+  rack-cors (= 0.2.9)
+  rack-test (= 0.5.7)
+  redis (= 3.2.0)
+  rspec (= 2.14.1)
+  sinatra (= 1.4.8)
+  sinatra-reloader (= 0.5.0)
+  uuid (= 2.3.1)
+  webmock (= 3.1.0)
+
+BUNDLED WITH
+   1.15.4

data/README.md

@@ -0,0 +1,105 @@
+clientside_aws
+===================
+
+This code is meant to be used by developers who are attempting to build web applications on AWS but wish to run client-side testing and validation. Presently, this project mocks DynamoDB and SQS.
+
+While creating and tearing down "free-tier" SQS and DynamoDB databases may be an acceptable solution for some, the time required (tens of seconds or minutes) quickly makes TDD (test-driven development) impractical. Just like we can use an in-memory sqlite3-based solution for mocking Mysql databases with ActiveRecord, we can now  mock SQS and DynamoDB databases in memory using Redis.
+
+To run this code, you will need ruby, sinatra, httparty, and the json and redis rubygems. I also use the sinatra/reloader gem to aid in development, but it is not necessary.
+
+You will also need redis-server installed locally
+
+Make sure redis-server is in your path
+
+Then, from the command line, run:
+
+    ruby spec/dynamodb_spec.rb
+or
+
+    ruby spec/sqs_spec.rb
+
+That will run the unit tests against this code.
+
+Overview
+--------
+
+This code works by overwriting the AWS service URLs in the aws-sdk gem, then monkeypatching the AWS::Core::Client request methods to use Rack's put, get, post and delete methods (see aws_mock.rb). This points to a Sinatra endpoint that processes the DynamoDB requests. Provided you are using the DynamoDB methods defined in aws-sdk when running tests and validations, the ruby client never knows it isn't talking to the real service.
+
+I have not packaged this up as a gem, because it needs to be a standalone sinatra project so you can launch a server from the command line (see below). I am open to suggestions about how to make it easier/cleaner to include dynamodb_mock into your actual project; right now you have to use a require statement that has knowledge of your directory structure.
+
+Adding to your project
+---------------------------
+
+First, if you plan on running any rspec unit tests, you should update the REDIS_PATH variable in spec_helper.rb to point to your redis binary.
+
+To start clientside_aws stand-alone, from the command line, run:
+
+    cd ~/clientside_aws/
+    ruby index.rb -p 4568
+
+This launches a Sinatra app, running on port 4568, that can respond to and support various services using the AWS protocol. You have your own, client-side SQS and DynamoDB server! If you are capable of mocking the requests in your language of choice to point to localhost:4568 you are ready to go. Included in this project is the code to mock in Ruby.
+
+For example, here's how I added clientside_aws to my Sinatra project:
+
+    configure :development do
+      require 'clientside_aws'  
+      DYNAMODB = AWS::DynamoDB.new(
+        :access_key_id => "...",
+        :secret_access_key => "...")
+      # more config
+    end
+
+I can then access the DynamoDB API from my code using the standard ruby aws-sdk DynamoDB class, discussed in more detail here:
+http://rubydoc.info/github/amazonwebservices/aws-sdk-for-ruby/master/AWS/DynamoDB
+
+Assuming you are including the 'aws_mock' file, you can call DynamoDB just as you normally would in your code. For example:
+
+    dynamo_db = AWS::DynamoDB.new(
+      :access_key_id => "...",
+      :secret_access_key => "...")
+
+    visitors_table = dynamo_db.tables.create("visitors", 10, 5,
+        :hash_key => { :creator_id => :number },
+        :range_key => {:date => :number})
+
+    visitors_table.hash_key = [:creator_id, :number]
+    visitors_table.range_key = [:date, :number]
+
+    (0..10).each do |idx|      
+      visitors_table.items.put(:creator_id => 1, :date => Time.now.to_f - (60 * idx), :target_id => 10 + idx)
+    end
+
+You can check out the dynamodb_spec.rb file for more unit tests and sample DynamoDB ruby code.
+
+If testing on a localhost(which you most likely are), you will need to add this line to your /etc/hosts file:
+
+127.0.0.1  test.localhost
+
+Amazon sticks the bucket to the front of the localhost to create a subdomain
+
+TODO
+--------------------
+
+I am developing this code for my own test purposes as I go along. There are certainly bugs and I have not yet implemented all the DynamoDB methods. Code lacks support at this time for the following:
+
+* Scan
+* UpdateTable
+* BatchGetItem
+
+I also have very a limited test suite; I will expand as I can. Feel free to fork, add, and submit a pull request.
+
+There are clearly many more AWS services one can mock up.
+
+* * *
+
+License
+=======
+MIT License (http://en.wikipedia.org/wiki/MIT_License). Some parts of this code were adapted from the aws-sdk project, which can be found at: https://github.com/amazonwebservices/aws-sdk-for-ruby and is itself licensed under the Apache 2.0 license.
+
+Copyright (C) 2012 Perry Street Software, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/bin/clientside_aws_build

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+path = File.dirname(__FILE__)
+dockerfile_path = File.join(path, '../')
+
+puts `docker build -t=clientside_aws:latest #{dockerfile_path}`

data/bin/clientside_aws_run

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+cmd = "docker run -d --rm -p 5567:4567 --name clientside_aws clientside_aws:latest"
+puts `#{cmd}`
+

data/bin/clientside_aws_test

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+
+cmd = "docker run -it --rm -p 6567:4567 -v ~/scruff/clientside_aws:/opt/clientside_aws --name clientside_aws_test clientside_aws bash"
+puts cmd

data/clientside_aws.gemspec

@@ -0,0 +1,31 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'clientside_aws/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'clientside_aws'
+  spec.version       = ClientsideAws::VERSION
+  spec.authors       = ['Perry Street Software, Inc.']
+  spec.email         = ['noreply@scruff.com']
+  spec.description   = 'This code is meant to be used by developers who are attempting to build web applications on AWS but wish to run client-side testing and validation.'
+  spec.summary       = 'Select AWS Services Replicated on Your Client'
+  spec.homepage      = 'https://github.com/perrystreetsoftware/clientside_aws'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files`.split($INPUT_RECORD_SEPARATOR)
+  spec.executables   = %w[clientside_aws_build clientside_aws_run]
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'rake', '10.1.0'
+  spec.add_development_dependency 'rack-test', '0.5.7'
+  spec.add_development_dependency 'rspec', '2.14.1'
+  spec.add_development_dependency 'bundler', '~> 1.3'
+  spec.add_dependency 'aws-sdk-v1', '1.59.0'
+  spec.add_dependency 'aws-sdk', '~> 2.0'
+  spec.add_dependency 'builder', '3.1.4'
+  spec.add_dependency 'httparty', '0.11.0'
+  spec.add_dependency 'json', '~> 1.8'
+  spec.add_dependency 'webmock', '3.1.0'
+end

data/clientside_aws/dynamodb.rb

@@ -0,0 +1,722 @@
+helpers do
+  def list_tables(_args)
+    tables = AWS_REDIS.smembers 'tables'
+    content_type 'application/x-amz-json-1.0'
+    { 'TableNames' => tables,
+      'LastEvaluatedTableName' => nil }.to_json
+  end
+
+  def delete_table(args)
+    halt 500 unless args['TableName']
+
+    table_name = args['TableName']
+    key_schema = get_key_schema(table_name)
+
+    keys = AWS_REDIS.keys "tables.#{args['TableName']}.*"
+    AWS_REDIS.del *keys unless keys.empty?
+    AWS_REDIS.srem 'tables', table_name
+
+    { Table:         { CreationDateTime: (Time.now.to_i * 1000),
+                       ItemCount: 0,
+                       KeySchema: key_schema,
+                       ProvisionedThroughput: {
+                         LastIncreaseDateTime: (Time.now.to_i * 1000),
+                         LastDecreaseDateTime: (Time.now.to_i * 1000),
+                         ReadCapacityUnits: 10,
+                         WriteCapacityUnits: 10
+                       },
+                       TableName: table_name,
+                       TableStatus: 'DELETING' } }.to_json
+  end
+
+  def clear_from_secondary_indices(table_name, record_id)
+    # Note the keys operation is a table scan that is brutally inefficient
+    secondary_indexes = AWS_REDIS.smembers "tables.#{table_name}.secondary_indexes"
+    secondary_indexes.each do |si_raw|
+      si = JSON.parse(si_raw)
+      keys = AWS_REDIS.keys "tables.#{table_name}.secondary_index.#{si['IndexName']}.*"
+      keys.each do |key|
+        AWS_REDIS.srem key, record_id
+      end
+    end
+  end
+
+  def get_key_schema(table)
+    hashkey_json = AWS_REDIS.get "tables.#{table}.hashkey"
+    rangekey_json = AWS_REDIS.get "tables.#{table}.rangekey"
+
+    result = {}
+    if hashkey_json
+      hashkey = JSON.parse(hashkey_json)
+      result[:HashKeyElement] =
+        { AttributeName: hashkey['AttributeName'], AttributeType: 'S' }
+    end
+    if rangekey_json
+      rangekey = JSON.parse(rangekey_json)
+      result[:RangeKeyElement] =
+        { AttributeName: rangekey['AttributeName'], AttributeType: 'N' }
+    end
+
+    result
+  end
+
+  def describe_table(args)
+    table_name = args['TableName']
+    key_schema = get_key_schema(table_name)
+    halt 404 unless key_schema && !key_schema.keys.empty?
+    { Table:         { CreationDateTime: (Time.now.to_i * 1000),
+                       ItemCount: 0,
+                       KeySchema: key_schema,
+                       ProvisionedThroughput: {
+                         LastIncreaseDateTime: (Time.now.to_i * 1000),
+                         LastDecreaseDateTime: (Time.now.to_i * 1000),
+                         ReadCapacityUnits: 10,
+                         WriteCapacityUnits: 10
+                       },
+                       TableName: table_name,
+                       TableSizeBytes: 1,
+                       TableStatus: 'ACTIVE' } }.to_json
+  end
+
+  def create_table(args)
+    halt 500, 'no table name' unless args['TableName']
+    halt 500, 'no key schema' unless args['KeySchema']
+    halt 500, 'no provisioned throughput' unless args['ProvisionedThroughput']
+    halt 500, 'already created' if AWS_REDIS.sismember('tables', args['TableName'])
+
+    AWS_REDIS.sadd 'tables', args['TableName']
+    AWS_REDIS.set "tables.#{args['TableName']}.auto_incr", 0
+
+    if args.key?('LocalSecondaryIndexes')
+      args['LocalSecondaryIndexes'].each do |si|
+        index_name = si['IndexName']
+        AWS_REDIS.sadd "tables.#{args['TableName']}.secondary_indexes", si.to_json
+      end
+    end
+
+    if args.key?('GlobalSecondaryIndexes')
+      args['GlobalSecondaryIndexes'].each do |si|
+        index_name = si['IndexName']
+        AWS_REDIS.sadd "tables.#{args['TableName']}.secondary_indexes", si.to_json
+      end
+    end
+
+    if args['KeySchema'].class == Array
+      args['KeySchema'].each do |ks|
+        key_defn = args['AttributeDefinitions'].select { |a| a['AttributeName'] == ks['AttributeName'] }.first
+        halt 500 unless key_defn
+
+        if ks['KeyType'] == 'HASH'
+          AWS_REDIS.set "tables.#{args['TableName']}.hashkey", key_defn.to_json
+        elsif ks['KeyType'] == 'RANGE'
+          AWS_REDIS.set "tables.#{args['TableName']}.rangekey", key_defn.to_json
+        end
+      end
+    else
+      args['KeySchema'].each do |k, v|
+        case k
+        when 'HashKeyElement'
+          AWS_REDIS.set "tables.#{args['TableName']}.hashkey", v.to_json
+        when 'RangeKeyElement'
+          AWS_REDIS.set "tables.#{args['TableName']}.rangekey", v.to_json
+        end
+      end
+    end
+
+    {
+      TableDescription: {
+        CreationDateTime: (Time.now.to_i * 1000),
+        KeySchema: args['KeySchema'],
+        ProvisionedThroughput: { ReadsPerSecond: args['ProvisionedThroughput']['ReadsPerSecond'],
+                                 WritesPerSecond: args['ProvisionedThroughput']['WritesPerSecond'] },
+        TableName: args['TableName'],
+        TableStatus: 'CREATING'
+      }
+    }.to_json
+  end
+
+  def update_item(args)
+    halt 500, 'no table name' unless args['TableName']
+
+    record_value = nil
+    record_id = get_record_id(args)
+
+    # No record, probably doing an add
+    if record_id.nil?
+
+      # Figure out the range key
+      rangekey_json = AWS_REDIS.get "tables.#{args['TableName']}.rangekey"
+      rangekey = JSON.parse(rangekey_json) if rangekey_json
+
+      # Add the range key and give a default value of zero
+      attribute_name = rangekey['AttributeName']
+      attribute_type = rangekey['AttributeType']
+      item_key = args['Key'].clone
+      item_key[attribute_name] = { attribute_type.to_s => 0 }
+      put_item('TableName' => args['TableName'], 'Item' => item_key)
+
+      record_id = get_record_id(args)
+    end
+
+    if record_id
+      record_value = JSON.parse(AWS_REDIS.get("tables.#{args['TableName']}.#{record_id}"))
+      args['AttributeUpdates'].each do |key, update|
+        if update['Action'] == 'ADD'
+          if update['Value'].key?('N')
+            increment_amount = update['Value']['N'].to_i
+
+            if record_value.key?(key)
+              halt 500, 'Incorrect type' unless record_value[key].key?('N')
+              record_value[key]['N'] = record_value[key]['N'].to_i + increment_amount
+            else # it's new, so add it
+              record_value[key] = update['Value']
+            end # record_value
+          elsif update['Value'].key?('S')
+            record_value[key] = update['Value']
+          end
+        elsif update['Action'] == 'DELETE'
+          record_value.delete(key)
+        end
+      end
+
+      AWS_REDIS.set "tables.#{args['TableName']}.#{record_id}",
+                    record_value.to_json
+    end
+
+    { Attributes: record_value, ConsumedCapacityUnits: 1 }.to_json
+  end
+
+  def put_item(args)
+    halt 500, 'no table name' unless args['TableName']
+    halt 500, 'no item' unless args['Item']
+
+    hashkey = rangekey = nil
+    hashkey_json = AWS_REDIS.get "tables.#{args['TableName']}.hashkey"
+    rangekey_json = AWS_REDIS.get "tables.#{args['TableName']}.rangekey"
+
+    hashkey = JSON.parse(hashkey_json) if hashkey_json
+    rangekey = JSON.parse(rangekey_json) if rangekey_json
+
+    halt 500 if hashkey.nil?
+
+    if args['Item'][hashkey['AttributeName']].key?('S')
+      hashkey_value = args['Item'][hashkey['AttributeName']]['S']
+    else
+      hashkey_value = BigDecimal(args['Item'][hashkey['AttributeName']]['N'])
+    end
+
+    if args.key?('Expected') && args['Expected'].key?('Name') &&
+       args['Expected']['Name'].key?('Exists') && args['Expected']['Name']['Exists'] == false
+      if AWS_REDIS.hexists "tables.#{args['TableName']}.hashkey_index", hashkey_value
+        halt 400, { '__type' => 'com.amazonaws.dynamodb.v20111205#ConditionalCheckFailedException', :message => 'The conditional request failed' }.to_json
+      end
+    end
+
+    halt 500 unless hashkey_value
+
+    if rangekey
+      rangekey_value = get_rangekey_value(args['Item'][rangekey['AttributeName']])
+
+      if AWS_REDIS.hexists("tables.#{args['TableName']}.hashkey_index.#{hashkey_value}", rangekey_value)
+        record_id = AWS_REDIS.hget "tables.#{args['TableName']}.hashkey_index.#{hashkey_value}", rangekey_value
+        clear_from_secondary_indices(args['TableName'], record_id)
+      else
+        record_id = AWS_REDIS.incr "tables.#{args['TableName']}.auto_incr"
+        AWS_REDIS.hset "tables.#{args['TableName']}.hashkey_index.#{hashkey_value}", rangekey_value, record_id
+        AWS_REDIS.lpush "tables.#{args['TableName']}.items", record_id
+        AWS_REDIS.hset "tables.#{args['TableName']}.hashkey_index", hashkey_value, record_id
+      end
+    else
+      record_id = AWS_REDIS.incr "tables.#{args['TableName']}.auto_incr"
+      AWS_REDIS.lpush "tables.#{args['TableName']}.items", record_id
+      AWS_REDIS.hset "tables.#{args['TableName']}.hashkey_index", hashkey_value, record_id
+    end
+
+    AWS_REDIS.set "tables.#{args['TableName']}.#{record_id}", args['Item'].to_json
+
+    # setup secondary indexes
+    secondary_indexes = AWS_REDIS.smembers "tables.#{args['TableName']}.secondary_indexes"
+    secondary_indexes.each do |raw|
+      lsi = JSON.parse(raw)
+      index_name = lsi['IndexName']
+      hashkey_value = nil
+      rangekey_value = nil
+
+      lsi['KeySchema'].each do |attrs|
+        attr_name = attrs['AttributeName']
+        key_type = attrs['KeyType']
+
+        if key_type == 'HASH'
+          if args['Item'][attrs['AttributeName']].key?('S')
+            hashkey_value = args['Item'][attrs['AttributeName']]['S']
+          else
+            hashkey_value = BigDecimal(args['Item'][attrs['AttributeName']]['N'])
+          end
+        else
+          if args['Item'][attrs['AttributeName']].key?('S')
+            rangekey_value = args['Item'][attrs['AttributeName']]['S']
+          else
+            rangekey_value = BigDecimal(args['Item'][attrs['AttributeName']]['N'])
+          end
+        end
+      end
+
+      # Secondary indexes store sets, not hmaps
+      # H => 1, R => 2, TS => 3
+      # H => 1, R => 3, TS => 3
+      # This means the secondary index on TS should have two records, {H => 1, R => 2} and {H => 1, R => 3}
+      # Storing as a hmap on H, TS would overwrite
+
+      AWS_REDIS.sadd "tables.#{args['TableName']}.secondary_index.#{index_name}.#{hashkey_value}/#{rangekey_value}", record_id
+    end
+
+    {
+      Attributes: args['Item'],
+      WritesUsed: 1
+    }.to_json
+  end
+
+  def get_record_id(args)
+    hashkey_value = nil
+    rangekey_value = nil
+
+    if !args['Key'].key?('HashKeyElement')
+      hashkey_json = AWS_REDIS.get "tables.#{args['TableName']}.hashkey"
+      rangekey_json = AWS_REDIS.get "tables.#{args['TableName']}.rangekey"
+
+      return nil unless hashkey_json
+
+      hashkey = JSON.parse(hashkey_json) if hashkey_json
+      rangekey = JSON.parse(rangekey_json) if rangekey_json
+
+      if args['Key'][hashkey['AttributeName']].key?('S')
+        hashkey_value = args['Key'][hashkey['AttributeName']]['S']
+      else
+        hashkey_value = BigDecimal(args['Key'][hashkey['AttributeName']]['N'])
+      end
+
+      if rangekey
+        if args['Key'][rangekey['AttributeName']].nil?
+          rangekey_value = nil
+        elsif args['Key'][rangekey['AttributeName']].key?('S')
+          rangekey_value = args['Key'][rangekey['AttributeName']]['S']
+        else
+          rangekey_value = BigDecimal(args['Key'][rangekey['AttributeName']]['N'])
+        end
+      end
+    else
+      halt 500 unless args['Key'].key?('HashKeyElement')
+
+      if args['Key']['HashKeyElement'].key?('S')
+        hashkey_value = args['Key']['HashKeyElement']['S']
+      else
+        hashkey_value = BigDecimal(args['Key']['HashKeyElement']['N'])
+      end
+
+      if args['Key'].key?('RangeKeyElement')
+        if args['Key']['RangeKeyElement'].key?('S')
+          rangekey_value = args['Key']['RangeKeyElement']['S']
+        else
+          rangekey_value = BigDecimal(args['Key']['RangeKeyElement']['N'])
+        end
+      end
+    end
+
+    raise 'no hashkey value' unless hashkey_value
+
+    record_id = nil
+    if rangekey_value.nil?
+      record_id = AWS_REDIS.hget("tables.#{args['TableName']}.hashkey_index", hashkey_value)
+    else
+      record_id = AWS_REDIS.hget("tables.#{args['TableName']}.hashkey_index.#{hashkey_value}", rangekey_value)
+    end
+
+    record_id
+  end
+
+  def get_item(args)
+    halt 500, 'no table name' unless args['TableName']
+    halt 500, 'no key' unless args['Key']
+
+    record_id = get_record_id(args)
+    record_value = record_id ? JSON.parse(AWS_REDIS.get("tables.#{args['TableName']}.#{record_id}")) : nil
+
+    record_value ? { Item: record_value, ReadsUsed: 1 }.to_json : {}.to_json
+  end
+
+  def convert_rangekey_value(rangekey_value, rangekey_type)
+    if rangekey_type == 'N'
+      return BigDecimal(rangekey_value)
+    else
+      return rangekey_value
+    end
+  end
+
+  def get_rangekey_value(rangekey)
+    rangekey_value = if rangekey.key?('N')
+                       BigDecimal(rangekey['N'])
+                     else
+                       rangekey['S']
+                     end
+
+    rangekey_value
+  end
+
+  def query(args)
+    halt 500, 'no table name' \
+      unless args['TableName']
+    halt 500, 'no hash key value' \
+      unless args['HashKeyValue'] ||
+             args['IndexName'] ||
+             args['KeyConditions']
+
+    key_conditions = args['KeyConditions']
+    limit = args.key?('Limit') ? args['Limit'] : nil
+    scan_index_forward = if args.key?('ScanIndexForward')
+                           args['ScanIndexForward']
+                         else
+                           true
+                         end
+
+    # exclusive_start_key = nil
+    last_evaluated_key = nil
+
+    hashkey_obj = \
+      JSON.parse((AWS_REDIS.get "tables.#{args['TableName']}.hashkey"))
+    hashkey_name = hashkey_obj['AttributeName']
+    # hashkey_type = hashkey_obj['AttributeType']
+
+    # V1 api sends HashKeyValue directly
+    if args.key?('HashKeyValue') && args['HashKeyValue'].key?('N')
+      hashkey_value = BigDecimal(args['HashKeyValue']['N'])
+    elsif !key_conditions.nil?
+      # V2 api sends hash key value inside a keyconditions hash
+      hashkey_value = \
+        get_rangekey_value( \
+          key_conditions[hashkey_name]['AttributeValueList'].first
+        )
+    else
+      raise 'Unknown hash key value'
+    end
+
+    rangekey_obj = \
+      JSON.parse(AWS_REDIS.get("tables.#{args['TableName']}.rangekey"))
+    rangekey_name = rangekey_obj['AttributeName']
+    rangekey_type = rangekey_obj['AttributeType']
+
+    exclusive_start_hashkey_value = nil
+    exclusive_start_rangekey_value = nil
+
+    unless args['ExclusiveStartKey'].nil?
+      exclusive_start_hashkey_value = \
+        args['ExclusiveStartKey']['HashKeyElement'].values.last
+      exclusive_start_rangekey_value = \
+        args['ExclusiveStartKey']['RangeKeyElement'].values.last
+    end
+
+    if args.key?('IndexName') # we are doing a new-style query
+      # remove the hash-key from the conditions,
+      # leaving only the key on which we are querying
+      rangekey_name = key_conditions.keys.select { |k| k != hashkey_name }.first
+      rangekey = key_conditions[rangekey_name]
+
+      rangekeys = AWS_REDIS.keys "tables.#{args['TableName']}.secondary_index.#{args['IndexName']}.#{hashkey_value}/*"
+
+      if rangekey['ComparisonOperator'] == 'GE'
+        rangekey_value = get_rangekey_value(rangekey['AttributeValueList'].first)
+        rangekey_type = rangekey['AttributeValueList'].first.keys.first # "N" or "S"
+
+        valid_rangekeys = rangekeys.select do |rk|
+                            (convert_rangekey_value(rk.split('/').last, rangekey_type) <=> rangekey_value) >= 0
+                          end.sort do |a, b|
+          convert_rangekey_value(a.split('/').last, rangekey_type) <=> convert_rangekey_value(b.split('/').last, rangekey_type)
+        end
+      elsif rangekey['ComparisonOperator'] == 'LE' || rangekey['ComparisonOperator'] == 'LT'
+        rangekey_value = get_rangekey_value(rangekey['AttributeValueList'].first)
+        rangekey_type = rangekey['AttributeValueList'].first.keys.first # "N" or "S"
+
+        valid_rangekeys = rangekeys.select do |rk|
+                            if rangekey['ComparisonOperator'] == 'LE'
+                              (convert_rangekey_value(rk.split('/').last, rangekey_type) <=> rangekey_value) <= 0
+                            else
+                              (convert_rangekey_value(rk.split('/').last, rangekey_type) <=> rangekey_value) < 0
+                            end
+                          end.sort do |a, b|
+          convert_rangekey_value(a.split('/').last, rangekey_type) <=> convert_rangekey_value(b.split('/').last, rangekey_type)
+        end
+      end
+
+      if !valid_rangekeys.empty?
+        record_ids = []
+        valid_rangekeys.each do |rk|
+          record_ids += AWS_REDIS.smembers(rk)
+        end
+        record_keys = record_ids.map { |record_id| "tables.#{args['TableName']}.#{record_id}" }
+        items = !record_keys.empty? ? (AWS_REDIS.mget *record_keys).map { |i| JSON.parse(i) } : []
+      else
+        items = []
+      end
+
+    elsif (args.key?('KeyConditions') && args['KeyConditions'][rangekey_name]) ||
+          args.key?('RangeKeyCondition')
+
+      if args['KeyConditions']
+        # New API v2: comes in as { table_name: {KeyCondtions}}
+        key_conditions = args['KeyConditions']
+      elsif args['RangeKeyCondition']
+        # Old API v1: comes in as { KeyConditions }
+        # So lets map it to the API v2
+        key_conditions = {}
+        key_conditions[rangekey_name] = args['RangeKeyCondition']
+      end
+
+      rangekeys = AWS_REDIS.hkeys "tables.#{args['TableName']}.hashkey_index.#{hashkey_value}"
+
+      rangekey_value = \
+        get_rangekey_value( \
+          key_conditions[rangekey_name]['AttributeValueList'].first
+        )
+      last_rangekey_value = \
+        get_rangekey_value( \
+          key_conditions[rangekey_name]['AttributeValueList'].last
+        )
+
+      case key_conditions[rangekey_name]['ComparisonOperator']
+      when 'LT'
+        comparator = lambda do |rk|
+          (rk <=> rangekey_value) == -1
+        end
+      when 'GT'
+        comparator = lambda do |rk|
+          (rk <=> rangekey_value) == +1
+        end
+      when 'GE'
+        comparator = lambda do |rk|
+          (rk <=> rangekey_value) >= 0
+        end
+      when 'LE'
+        comparator = lambda do |rk|
+          (rk <=> rangekey_value) <= 0
+        end
+      when 'EQ'
+        comparator = lambda do |rk|
+          (rk <=> rangekey_value).zero?
+        end
+      when 'BETWEEN'
+        comparator = lambda do |rk|
+          (rk <=> rangekey_value) >= 0 && (rk <=> last_rangekey_value) <= 0
+        end
+      end
+
+      valid_rangekeys = \
+        rangekeys.map do |rk|
+          convert_rangekey_value(rk, rangekey_type)
+        end.select(&comparator).sort
+
+      record_ids = []
+
+      unless valid_rangekeys.empty?
+        record_ids = \
+          AWS_REDIS.hmget("tables.#{args['TableName']}.hashkey_index." \
+                          "#{hashkey_value}", *valid_rangekeys).map do |rid|
+            "tables.#{args['TableName']}.#{rid}"
+          end
+      end
+
+      items = []
+      unless record_ids.empty?
+        items = (AWS_REDIS.mget record_ids).map do |i|
+          JSON.parse(i)
+        end
+      end
+    else
+      record_ids = AWS_REDIS.hvals("tables.#{args['TableName']}." \
+                                   "hashkey_index.#{hashkey_value}")
+      keys = record_ids.map do |item|
+        "tables.#{args['TableName']}.#{item}"
+      end
+      items = !keys.empty? ? AWS_REDIS.mget(*keys).map { |i| JSON.parse(i) } : []
+    end
+
+    if exclusive_start_hashkey_value && exclusive_start_rangekey_value
+
+      # So we move through it correctly depending on asc or desc
+      items.reverse! if scan_index_forward
+
+      idx = 0
+      items.each do |item|
+        idx += 1
+        hashkey_value_dict = item[hashkey_name]
+        rangekey_value_dict = item[rangekey_name]
+        hashkey_value_type = hashkey_value_dict.keys.first
+        rangekey_value_type = rangekey_value_dict.keys.first
+
+        hashkey_value = hashkey_value_dict.values.first
+        rangekey_value = rangekey_value_dict.values.first
+
+        if exclusive_start_hashkey_value == hashkey_value &&
+           exclusive_start_rangekey_value == rangekey_value
+          break
+        end
+      end
+
+      items = items[idx..-1]
+    end
+
+    if scan_index_forward && rangekey_name
+      items.sort! do |a, b|
+        get_rangekey_value(a[rangekey_name]) <=> get_rangekey_value(b[rangekey_name])
+      end
+    elsif !scan_index_forward && rangekey_name
+      items.sort! do |a, b|
+        get_rangekey_value(b[rangekey_name]) <=> get_rangekey_value(a[rangekey_name])
+      end
+    end
+
+    if items && items.count > 0 && limit && limit < items.count
+      hashkey_value_dict = items[limit][hashkey_name]
+      rangekey_value_dict = items[limit][rangekey_name]
+      hashkey_value_type = hashkey_value_dict.keys.first
+      rangekey_value_type = rangekey_value_dict.keys.first
+
+      hashkey_value = hashkey_value_dict.values.first
+      rangekey_value = rangekey_value_dict.values.first
+
+      last_evaluated_key = {
+        HashKeyElement: { hashkey_value_type => hashkey_value },
+        RangeKeyElement: { rangekey_value_type => rangekey_value }
+      }
+      items = items[0...limit] # apply limit
+    end
+
+    result = { Count: items.length, Items: items, ReadsUsed: 1 }
+
+    # This should not be the last key returned, but instead the next key you would
+    # have returned but didn't.
+
+    result[:LastEvaluatedKey] = last_evaluated_key if last_evaluated_key
+
+    result.to_json
+  end
+
+  def delete_item(args)
+    halt 500, 'no table name' unless args['TableName']
+    halt 500, 'no key' unless args['Key']
+
+    if args['Key'].key?('HashKeyElement')
+      if args['Key']['HashKeyElement'].key?('N')
+        hashkey_value = BigDecimal(args['Key']['HashKeyElement']['N'])
+      else
+        hashkey_value = args['Key']['HashKeyElement']['S']
+      end
+    else
+      hashkey_raw = AWS_REDIS.get "tables.#{args['TableName']}.hashkey"
+      hashkey = JSON.parse(hashkey_raw)
+      if hashkey['AttributeType'] == 'N'
+        hashkey_value = BigDecimal(args['Key'][hashkey['AttributeName']][hashkey['AttributeType']])
+      else
+        hashkey_value = args['Key'][hashkey['AttributeName']][hashkey['AttributeType']]
+      end
+    end
+
+    rangekey_value = nil
+    if args['Key'].key?('RangeKeyElement')
+      rangekey_value = get_rangekey_value(args['Key']['RangeKeyElement'])
+    else
+      rangekey_raw = AWS_REDIS.get "tables.#{args['TableName']}.rangekey"
+      if rangekey_raw
+        rangekey = JSON.parse(rangekey_raw)
+        if rangekey['AttributeType'] == 'N'
+          rangekey_value = BigDecimal(args['Key'][rangekey['AttributeName']][rangekey['AttributeType']])
+        else
+          rangekey_value = args['Key'][rangekey['AttributeName']][rangekey['AttributeType']]
+        end
+      end
+    end
+
+    if hashkey_value && rangekey_value
+      record_id = AWS_REDIS.hget "tables.#{args['TableName']}.hashkey_index.#{hashkey_value}", rangekey_value
+    else
+      record_id = AWS_REDIS.hget "tables.#{args['TableName']}.hashkey_index", hashkey_value
+    end
+
+    item = nil
+    if record_id
+      AWS_REDIS.hdel "tables.#{args['TableName']}.hashkey_index", hashkey_value
+      if rangekey_value
+        AWS_REDIS.hdel "tables.#{args['TableName']}.hashkey_index.#{hashkey_value}", rangekey_value
+      end
+      item = JSON.parse(AWS_REDIS.get("tables.#{args['TableName']}.#{record_id}"))
+      AWS_REDIS.del "tables.#{args['TableName']}.#{record_id}"
+
+      clear_from_secondary_indices(args['TableName'], record_id)
+    end
+
+    { 'Item' => item, 'ReadsUsed' => 1 }.to_json
+  end
+
+  def batch_write_item(args)
+    items = []
+    responses = {}
+
+    args['RequestItems'].each do |k, v|
+      table_name = k
+      requests = v
+      requests.each do |request|
+        request.each do |k2, v2|
+          case k2
+          when 'DeleteRequest'
+            delete_item('TableName' => table_name, 'Key' => v2['Key'])
+            responses[table_name] = { 'ConsumedCapacityUnits' => 1 }
+          end
+        end
+      end
+    end
+
+    { Responses: responses, UnprocessedItems: [] }.to_json
+  end
+end
+
+post %r{/dynamodb\.([\w-]+?)\.amazonaws\.com/?} do
+  req = Rack::Request.new(env)
+
+  amz_target = nil
+
+  %w(HTTP_X_AMZ_TARGET x-amz-target X-Amz-Target).each do |key|
+    next unless env.key?(key)
+    amz_target = env[key].split('.').last
+    break
+  end
+
+  args = if env['REQUEST_METHOD'] == 'POST'
+           JSON.parse(env['rack.input'].read)
+         else
+           env['rack.request.form_hash']
+         end
+
+  content_type 'application/x-amz-json-1.0'
+  case amz_target
+  when 'CreateTable'
+    return create_table(args)
+  when 'DeleteTable'
+    return delete_table(args)
+  when 'DescribeTable'
+    return describe_table(args)
+  when 'PutItem'
+    return put_item(args)
+  when 'GetItem'
+    return get_item(args)
+  when 'DeleteItem'
+    return delete_item(args)
+  when 'UpdateItem'
+    return update_item(args)
+  when 'Query'
+    return query(args)
+  when 'ListTables'
+    return list_tables(args)
+  when 'BatchWriteItem'
+    return batch_write_item(args)
+  else
+    halt 500, "unknown command #{req.inspect}"
+  end
+end