clientside_aws 0.0.24 → 0.0.25

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fa4605924db8e40740bbe014617f533b9a0ddc50
-  data.tar.gz: 4c9c9f6e90281a6dbff7e03d9d1851f5d950d501
+  metadata.gz: 8269c596909ba636737c30f536e8bf9b140360c7
+  data.tar.gz: df95be118f5e808665fba5d17b6357667f3ff254
 SHA512:
-  metadata.gz: 186e49b50aea5a13d9819213d82942c07d5dcbe6bb6203e22b982515503a557580f7675397760fb157f6e903c29a7cbec756fa4a51cdb5103a6f0a18a994108b
-  data.tar.gz: 52f690874266a4bec8892b33e8a4099c8213a6872ababbd54041f9eecf6bab39d70b4c369cb0176e3e91562034925c3da1dd5f2219b0487780d597bf369fabf3
+  metadata.gz: f36b7bcf69a94c70d341af1d20c165810962ee192f72e9ab354efd62d1fe223da1a290053178d7fd578bd814ca79bac0f8a3210e55f9cf4cf8f498deab1b3a48
+  data.tar.gz: f9ddeca0dbe59eba1cb861a7ce7f35c0a1db480bd484fd9823936230cc4009b6d2341e5e0050bc9167853ffb1bccca82608964724a1636a6a5ae40064e8cf8ae

data/Dockerfile

@@ -11,7 +11,7 @@ RUN mkdir /mnt/redis
 
 RUN apt-add-repository ppa:brightbox/ruby-ng -y
 RUN apt-get update && apt-get install -y ruby2.4 ruby2.4-dev git-core build-essential zlib1g-dev
-RUN apt-get install -y wget curl
+RUN apt-get install -y wget curl telnet
 
 RUN cd /opt ; wget "http://download.redis.io/releases/redis-2.8.24.tar.gz"
 RUN cd /opt ; gunzip redis-2.8.24.tar.gz ; tar -xvf redis-2.8.24.tar

data/Gemfile

@@ -4,19 +4,19 @@ source 'https://rubygems.org'
 
 # These are gem dependencies for our Docker image,
 # not for our Gemfile which points clients to our docker image
-gem 'aws-sdk', '2.10.26'
-gem 'aws-sdk-v1', '1.66.0'
+gem 'aws-sdk', '~> 2'
+gem 'aws-sdk-v1'
 gem 'builder', '~> 3.2'
 gem 'httparty', '0.15.6'
 gem 'json', '1.8.6'
 gem 'json_pure', '1.8.6'
 gem 'monkey-lib', '0.5.4'
 gem 'pry'
-gem 'rack', '1.5.2'
+gem 'rack', '~> 1.5'
 gem 'rack-cors', '0.2.9', require: 'rack/cors'
 gem 'rack-test', '0.5.7'
 gem 'redis', '~> 4.0'
-gem 'rspec', '~> 3.0'
+gem 'rspec', '~> 3'
 gem 'sinatra', '1.4.8'
 gem 'sinatra-reloader', '0.5.0'
 gem 'webmock', '3.1.0'

data/README.md

@@ -1,120 +1,125 @@
-clientside_aws
-===================
+# Benefits
 
-Run selected AWS services via a docker container locally, for more effective development and testing.
+If you require the Internet to develop and test your next ruby application, then **your dev environment is broken**.
 
-## Install
+With Clientside AWS, you can:
 
-In your gemfile:
+- Write code on a plane: Interact with AWS services like S3, SQS and DynamoDb, without Internet access
+- Eliminate tedious configuration of WebMock-powered mock AWS responses in your unit tests
+- Save time and money by avoiding the need for a test account in AWS where you upload S3 files and interact with DynamoDb databases along with other developers on your team
+- Instantly restore your development and test environments to Known Good States
 
-    gem 'clientside_aws', require: 'clientside_aws'
+# Alternatives
 
-## Features
+### [Fake S3](https://github.com/jubos/fake-s3)
 
-- Write code on a plane: Interact with AWS services like S3, SQS and DynamoDb, without internet access
-- No more tedious configuration of mocked responses in your unit tests
-- Save time and money by avoiding the need for a 'test' account in AWS where you upload S3 files and interact with DynamoDb databases along with other developers on your team
-- Instantly restore your development and test environments to known good states
+Pro: Very complete implementation of the AWS S3 protocol.
+Cons: Nonstandard license; no support for running docker; only mocks S3 (not other services)
 
-Think about how 
+### [DynamoDB local](https://aws.amazon.com/blogs/aws/dynamodb-local-for-desktop-development/)
 
+Pros: Complete implementation of the DynamoDB protocol
+Cons: Requires java installation locally; no docker support
 
-This code is meant to be used by developers who are attempting to build web applications on AWS but wish to run client-side testing and validation. Presently, this project mocks DynamoDB and SQS.
+### [LocalStack](https://github.com/localstack/localstack)
 
-While creating and tearing down "free-tier" SQS and DynamoDB databases may be an acceptable solution for some, the time required (tens of seconds or minutes) quickly makes TDD (test-driven development) impractical. Just like we can use an in-memory sqlite3-based solution for mocking Mysql databases with ActiveRecord, we can now  mock SQS and DynamoDB databases in memory using Redis.
+Pros: Implementations of a large number of AWS services; uses docker; supported by Atlassian.
+Cons: Unclear licensing; probably not free in the long-run. Does not offer ruby bindings.
 
-To run this code, you will need ruby, sinatra, httparty, and the json and redis rubygems. I also use the sinatra/reloader gem to aid in development, but it is not necessary.
+# Installation
 
-You will also need redis-server installed locally
+There are two parts to installation: running a docker container that mocks the AWS services, and including the `clientside_aws` gem in your application.
 
-Make sure redis-server is in your path
+## Container configuration
 
-Then, from the command line, run:
+From OSX, type:
 
-    ruby spec/dynamodb_spec.rb
-or
+    gem install clientside_aws
 
-    ruby spec/sqs_spec.rb
+This will install a gem and a script that lets you build the Clientside AWS docker image. Once the `clientside_aws` gem is installed locally on OSX, you will need to build the Clientside AWS docker image. Run:
 
-That will run the unit tests against this code.
+    clientside_aws_build
 
-Overview
---------
+Once that docker image is built, it will be available for you to manually launch or include in `docker-compose.yml` files.
 
-This code works by overwriting the AWS service URLs in the aws-sdk gem, then monkeypatching the AWS::Core::Client request methods to use Rack's put, get, post and delete methods (see aws_mock.rb). This points to a Sinatra endpoint that processes the DynamoDB requests. Provided you are using the DynamoDB methods defined in aws-sdk when running tests and validations, the ruby client never knows it isn't talking to the real service.
+    $ docker images
+    REPOSITORY                           TAG                 IMAGE ID            CREATED             SIZE
+    clientside_aws                       latest              fec78caf81be        just now            881MB
+    mysql                                5.7                 e27c7a14671f        20 months ago       361MB
+    redis                                latest              93a08017b97e        22 months ago       151MB
+    memcached                            latest              355142d48ea3        22 months ago       132MB
 
-I have not packaged this up as a gem, because it needs to be a standalone sinatra project so you can launch a server from the command line (see below). I am open to suggestions about how to make it easier/cleaner to include dynamodb_mock into your actual project; right now you have to use a require statement that has knowledge of your directory structure.
+To manually launch a docker container running Clientside AWS, run:
 
-Adding to your project
----------------------------
+    docker run -d --rm -p <YOUR_PREFERRED_PORT>:4567 --name clientside_aws clientside_aws:latest
 
-First, if you plan on running any rspec unit tests, you should update the REDIS_PATH variable in spec_helper.rb to point to your redis binary.
+Or, run:
 
-To start clientside_aws stand-alone, from the command line, run:
+    clientside_aws_run
 
-    cd ~/clientside_aws/
-    ruby index.rb -p 4568
+To include in a `docker-compose.yml` file, see [the example](/examples/dockerized/docker-compose.yml).
 
-This launches a Sinatra app, running on port 4568, that can respond to and support various services using the AWS protocol. You have your own, client-side SQS and DynamoDB server! If you are capable of mocking the requests in your language of choice to point to localhost:4568 you are ready to go. Included in this project is the code to mock in Ruby.
+## Application configuration
 
-For example, here's how I added clientside_aws to my Sinatra project:
+In your gemfile:
 
-    configure :development do
-      require 'clientside_aws'  
-      DYNAMODB = AWS::DynamoDB.new(
-        :access_key_id => "...",
-        :secret_access_key => "...")
-      # more config
-    end
+    gem 'clientside_aws', require: 'clientside_aws'
 
-I can then access the DynamoDB API from my code using the standard ruby aws-sdk DynamoDB class, discussed in more detail here:
-http://rubydoc.info/github/amazonwebservices/aws-sdk-for-ruby/master/AWS/DynamoDB
+Then, at the top of your application where you are configuring global services, type:
 
-Assuming you are including the 'aws_mock' file, you can call DynamoDB just as you normally would in your code. For example:
+    config = { region: 'us-mockregion-1',
+               access_key_id: '...',
+               secret_access_key: '...' }
 
-    dynamo_db = AWS::DynamoDB.new(
-      :access_key_id => "...",
-      :secret_access_key => "...")
+    Aws.config.update(config)
+    AWS.config(config)
 
-    visitors_table = dynamo_db.tables.create("visitors", 10, 5,
-        :hash_key => { :creator_id => :number },
-        :range_key => {:date => :number})
+You can see that you do not need to specify a valid access_key_id nor secret_access_key, but you must set the region to be `us-mockregion-1` -- this is how Clientside AWS identifies and redirects requests to AWS services.
 
-    visitors_table.hash_key = [:creator_id, :number]
-    visitors_table.range_key = [:date, :number]
+Clientside AWS is also capable of mocking requests from the v1 and v2 versions of the [aws-sdk-ruby](https://github.com/aws/aws-sdk-ruby) gem. It is not yet tested with v3.
 
-    (0..10).each do |idx|      
-      visitors_table.items.put(:creator_id => 1, :date => Time.now.to_f - (60 * idx), :target_id => 10 + idx)
-    end
+See [the example](/examples/dockerized/app/index.rb) for how to configure Clientside AWS in your application.
 
-You can check out the dynamodb_spec.rb file for more unit tests and sample DynamoDB ruby code.
+# Classic development approach
 
-If testing on a localhost(which you most likely are), you will need to add this line to your /etc/hosts file:
+![Classic approach diagram](/examples/documentation/classic_model.png)
 
-127.0.0.1  test.localhost
+In the classic approach to development, you would install application services like mysql, redis and memcached locally on OSX. You would then install the ruby runtime, with tools like [rvm](https://rvm.io/). With luck, you wouldn't encounter OSX-specific bugs, [like this one](https://blog.phusion.nl/2017/10/13/why-ruby-app-servers-break-on-macos-high-sierra-and-what-can-be-done-about-it/).
 
-Amazon sticks the bucket to the front of the localhost to create a subdomain
+When you needed to interact with AWS services, you would either use [WebMock](https://github.com/bblimke/webmock) to provide a known response to outbound requests (probably in your rspec tests), and for development you would create a test account on AWS that had empty S3 buckets and DynamoDb databases. This account might have been shared with other developers, and could (over time) get polluted with development data. It was flushed infrequently, if ever. It also cost money, even if a nominal amount.
 
-TODO
---------------------
+# Clientside AWS approach
+
+With Clientside AWS, you have two approaches to configuring your application services: Partial or Full Docker.
+
+## Partial docker
 
-I am developing this code for my own test purposes as I go along. There are certainly bugs and I have not yet implemented all the DynamoDB methods. Code lacks support at this time for the following:
+![Partial docker diagram](/examples/documentation/partial_docker.png)
 
-* Scan
-* UpdateTable
-* BatchGetItem
+With the partial docker approach, you continue to install runtime environments locally on OSX like [rvm](https://rvm.io/), but you also install [Docker for Mac](https://www.docker.com/docker-mac), on which you launch a container running Clientside AWS. You then `require clientside_aws` in your application which redirects AWS requests to the container. You also may consider running other services, like mysql, memcached and redis, inside a docker container.
 
-I also have very a limited test suite; I will expand as I can. Feel free to fork, add, and submit a pull request.
+See [this example](/examples/local/) for how to configure a partial docker development approach.
 
-There are clearly many more AWS services one can mock up.
+## Full docker
+
+![Full docker diagram](/examples/documentation/full_docker.png)
+
+With the full docker approach, you do 100% of your web application development from within Docker. Your OSX laptop only need to have a web browser, a text editor, and docker installed (and if you are doing mobile development, Xcode/Android Studio).
+
+When developing with the full docker approach, your application runtime it itself running in a container. However, because your application code will be changing and churning constantly, it doesn't make sense to run a COPY command and seal it up, as one normally does with docker containers that you are shipping to other places. Instead, you mount your local OSX filesystem into an Ubuntu container, on which you have also installed the ruby runtime. You also execute an interactive bash script on that container, so you can easily do things like stop and restart your web server. This container creates network links between itself and your other services, including Clientside AWS, mysql, redis, memcached, postgres, etc.
+
+See [this example](/examples/dockerized/) for how to figure a full docker development approach.
+
+TODO
+=======
+* Add support for V3 of the aws-sdk gem
 
-* * *
 
 License
 =======
-MIT License (http://en.wikipedia.org/wiki/MIT_License). Some parts of this code were adapted from the aws-sdk project, which can be found at: https://github.com/amazonwebservices/aws-sdk-for-ruby and is itself licensed under the Apache 2.0 license.
+[MIT License](http://en.wikipedia.org/wiki/MIT_License). Some parts of this code were adapted from the [aws-sdk project](https://github.com/aws/aws-sdk-core-ruby), which is itself licensed under the Apache 2.0 license.
 
-Copyright (C) 2012 Perry Street Software, Inc.
+Copyright (C) 2012-2017 Perry Street Software, Inc.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
 

data/clientside_aws.gemspec

@@ -21,11 +21,11 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rake', '10.1.0'
   spec.add_development_dependency 'rack-test', '0.5.7'
   spec.add_development_dependency 'bundler', '~> 1.3'
-  spec.add_dependency 'aws-sdk-v1', '1.66.0'
-  spec.add_dependency 'aws-sdk', '~> 2.0'
+  spec.add_dependency 'aws-sdk-v1', '1.67.0'
+  spec.add_dependency 'aws-sdk', '~> 2'
   spec.add_dependency 'builder', '~> 3.1'
   spec.add_dependency 'httparty', '~> 0.15'
   spec.add_dependency 'json', '~> 1.8'
   spec.add_dependency 'webmock', '~> 3.1'
-  spec.add_dependency 'rspec', '~> 3.0'
+  spec.add_dependency 'rspec', '~> 3'
 end

data/clientside_aws/elastic_transcoder.rb

@@ -1,17 +1,36 @@
 helpers do
+  def read_s3_input(bucket:, path:)
+    bucket = Aws::S3::Resource.new(S3_CONFIG).bucket(bucket)
+    object = bucket.object(path)
+
+    object.get.body.read if object.exists?
+  end
+
+  def write_s3_output(bucket:, path:, data:, content_type:)
+    bucket_obj = Aws::S3::Resource.new(S3_CONFIG).bucket(bucket)
+    object = bucket_obj.object(path)
+
+    object.put(body: data,
+               content_type: content_type,
+               acl: 'private')
+    puts "JUST WROTE #{bucket} path #{path} data #{data.length}"
+  end
+
   def encode_video(source_key, dest_key, pipeline_id)
     pipeline = JSON.parse(AWS_REDIS.get("pipeline:#{pipeline_id}"))
-    bucket = pipeline['InputBucket']
+    input_bucket = pipeline['InputBucket']
     input_obj_name = source_key
-    input_obj_key = "s3:bucket:#{bucket}:#{input_obj_name}"
 
-    bucket = pipeline['OutputBucket']
+    input_data = read_s3_input(bucket: input_bucket,
+                               path: input_obj_name)
+
+    output_bucket = pipeline['OutputBucket']
     output_obj_name = dest_key
-    output_obj_key = "s3:bucket:#{bucket}:#{output_obj_name}"
-    input_obj_body = AWS_REDIS.hget input_obj_key, 'body'
+    # output_obj_key = "s3:bucket:#{bucket}:#{output_obj_name}"
+    # input_obj_body = AWS_REDIS.hget input_obj_key, 'body'
 
     tmp = Tempfile.new(source_key)
-    tmp.write(input_obj_body)
+    tmp.write(input_data)
     tmp.close
 
     # Setup paths
@@ -37,8 +56,10 @@ helpers do
       video = file.read
       file.close
 
-      AWS_REDIS.hset output_obj_key, 'body', video
-      AWS_REDIS.hset output_obj_key, 'content-type', 'video/mp4'
+      write_s3_output(bucket: output_bucket,
+                      path: output_obj_name,
+                      data: video,
+                      content_type: 'video/mp4')
 
       begin
         File.delete(final_path) if File.exist?(final_path)

data/examples/dockerized/README.md

@@ -0,0 +1,21 @@
+# Installation
+
+To run this example, type
+
+    docker-compose up -d
+
+Once it has launched, run
+
+    docker ps
+
+This should show you something like this:
+
+    CONTAINER ID        IMAGE                     COMMAND                  CREATED             STATUS              PORTS                                              NAMES
+    ec9815828e64        clientside_aws_test_app   "/sbin/my_init"          3 seconds ago       Up 1 second         0.0.0.0:32847->4567/tcp                            clientside_aws_test_app
+    b19233b502f8        clientside_aws:latest     "/sbin/my_init"          5 seconds ago       Up 2 seconds        0.0.0.0:32846->4567/tcp                            clientside_aws_test_aws
+
+You can see that the `clientside_aws_test_app` container is running on port `32847`. Thus, from your local web browser, type in:
+
+    http://localhost:32847/
+
+And you should see the example running there.

data/examples/local/README.md

@@ -0,0 +1,11 @@
+# Installation
+
+This example assumes you have a dockerized clientside_aws container already running. If you do not, after you have installed clientside_aws locally on OSX and built its image, run
+
+    clientside_aws_run
+
+This will launch a container running on the correct port for this example to work. Then, simply run:
+
+    ruby test.rb
+
+And you should see this example run successfully.

data/index.rb

@@ -11,6 +11,7 @@ require 'base64'
 require 'rack'
 require 'rack/cors'
 require 'rack/protection'
+require 'aws-sdk'
 
 ENV['RACK_ENV'] = 'development' unless ENV['RACK_ENV']
 
@@ -51,6 +52,14 @@ end
 
 DYNAMODB_PREFIX = 'DynamoDBv20110924'.freeze
 
+S3_CONFIG = {
+  region: 'us-mockregion-1',
+  access_key_id: '...',
+  secret_access_key: '...',
+  force_path_style: true,
+  endpoint: 'http://app_rspec_localstack:4572'
+}.freeze
+
 get '/' do
   'hello'
 end

data/lib/clientside_aws/version.rb

@@ -1,3 +1,3 @@
 module ClientsideAws
-  VERSION = '0.0.24'.freeze
+  VERSION = '0.0.25'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: clientside_aws
 version: !ruby/object:Gem::Version
-  version: 0.0.24
+  version: 0.0.25
 platform: ruby
 authors:
 - Perry Street Software, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-10-18 00:00:00.000000000 Z
+date: 2018-03-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -58,28 +58,28 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.66.0
+        version: 1.67.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.66.0
+        version: 1.67.0
 - !ruby/object:Gem::Dependency
   name: aws-sdk
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: builder
   requirement: !ruby/object:Gem::Requirement
@@ -142,14 +142,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '3'
 description: This code is meant to be used by developers who are attempting to build
   web applications on AWS but wish to run client-side testing and validation.
 email:
@@ -188,11 +188,16 @@ files:
 - docker/clientside-aws-run
 - docker/redis-server-run
 - examples/dockerized/Dockerfile
+- examples/dockerized/README.md
 - examples/dockerized/app-run
 - examples/dockerized/app/Gemfile
 - examples/dockerized/app/index.rb
 - examples/dockerized/app/views/index.erb
 - examples/dockerized/docker-compose.yml
+- examples/documentation/classic_model.png
+- examples/documentation/full_docker.png
+- examples/documentation/partial_docker.png
+- examples/local/README.md
 - examples/local/test.rb
 - index.rb
 - lib/clientside_aws.rb