moe 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: bc6f10b8fa23a7f93c206d40edffd642321b389d
+  data.tar.gz: 83e093fe27c117a65e63de5fb85797c0bea852d2
+SHA512:
+  metadata.gz: 160f2712f8ecf52074e43c7573562962fedc8927db12afae9320632a52f675e133b28d2164dc23e625f2ce175d7f497f3ed0ae6d1f3415e2eda59735b38d8b72
+  data.tar.gz: d0157eda063695b14915b1f1867739c3c1cef272983102adb4378750f7b4a69dcdad8d646ceb84021e3ed0d7804e4fc244bed292202386e62ad59e175c8235a8

data/.gitignore

@@ -0,0 +1,19 @@
+.DS_Store
+*.gem
+*.rbc
+*.sw*
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.ruby-version

@@ -0,0 +1 @@
+2.1.0

data/.travis.yml

@@ -0,0 +1,5 @@
+language: ruby
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - 2.1.0

data/Gemfile

@@ -0,0 +1,4 @@
+source "https://rubygems.org"
+
+# gem dependencies in moe.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Geezeo
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.markdown

@@ -0,0 +1,66 @@
+[![Build Status](https://travis-ci.org/fuzz/moe.png?branch=master)](https://travis-ci.org/fuzz/moe)
+[![Code Climate](https://codeclimate.com/github/fuzz/moe.png)](https://codeclimate.com/github/fuzz/moe)
+[![Coverage Status](https://coveralls.io/repos/fuzz/moe/badge.png)](https://coveralls.io/r/fuzz/moe)
+
+# Moe
+
+A toolkit for working with DynamoDB at scale
+
+## Features
+
+### Multitable support
+
+Moe provides a table manager to create and rotate tables. Tables may be
+mirrored (each write goes to two or more tables) and/or split (reads happen
+across multiple tables).
+
+Currently mirroring uses a table with the same capacity as the primary table.
+In the future we may add Kinesis support for the mirror table so the mirrored
+writes can go to Kinesis first and then trickle into the mirror table so we can
+minimize the capacity required. This mill also allow us do Big Data type
+realtime procesing as it goes through Kinesis should that be desired. 
+
+There are different approaches to splitting based on the use case. The most
+straightforward mechanism is to shut everything down, push out a table change,
+start everything back up. This ensures that a set of data will not be written
+across multiple tables thus providing the most safety when expiring old tables.
+At the other end of the spectrum we can simply write the metadata item as the
+first in the sequence so knowledge of the sequence will disappear when the
+table its metadata is written on does. At this time the splitting strategy is
+determined by the persistence strategy though it would be good to be able to
+mix and match.
+
+### Persistence strategies
+
+DynamoDB presents a unique and interesting set of contraints. By designing
+persistence strategies around these contraints we can take advantage of useful
+patterns that fall out of them. Thus unlike a generalized ORM moe provides a
+number of different persistence strategies depending on your use cases and
+patterns. Cannot find a pattern that is a good match for your workload? Add
+one!
+
+The first persistence strategy available is the one described at
+http://fuzz.github.io -- it provides a model where an "owner" has many things
+that each have many things using a single DynamoDB table with no indexes. By
+comparison a more ruby-typical-but-dynamo-naive implementation might use two
+tables plus a GSI at ~3x the cost. Not to mention the additional complexity as
+capacity and other operational concerns have to be managed on each table and
+the GSI.
+
+### The Future: Partitioning
+
+DynamoDB capacity is evenly divided among a number of partitions that
+increase/decrease along with capacity. We do not know the number of partitions;
+we only know (if we pay attention) when our primary hash key strategy is
+insufficently random to spread evenly across all partitions. Moe will provide a
+number of partitioning strategies and instrumentation so you can compare your
+actual overall throughput to what AWS reports in order to determine the
+effectiveness of and tune your key randomization strategy.
+
+## JRuby
+
+To run the tests you will need to start fake_dynamo manually and
+
+```
+bundle exec rake rspec  # use rspec instead of spec to bypass auto fake_dynamo
+```

data/Rakefile

@@ -0,0 +1,69 @@
+require "aws-sdk-core"
+require "bundler/setup"
+require "rspec/core/rake_task"
+
+Aws.config = {
+    access_key_id: "xxx",
+    secret_access_key: "xxx",
+    dynamodb:  {
+                  api_version: "2012-08-10",
+                  endpoint: "http://localhost:4567"
+                },
+    region: "us-east-1"
+  }
+
+desc "Start fake_dynamo, run tests, cleanup"
+task :spec do |t|
+  require "tmpdir"
+  require "socket"
+
+  dir = Dir.mktmpdir
+  data_file = File.join(dir, "data.fdb")
+
+  # Launch fake_dynamo
+  pid = Process.spawn"fake_dynamo", "-d", data_file, err: "/dev/null", out: "/dev/null"
+
+  # Cleanup
+  at_exit {
+    Process.kill("TERM", pid)
+    FileUtils.rmtree(dir)
+  }
+
+  # Wait for fake_dynamo to start taking requests
+  40.downto(0) do |count| #Wait up to 2 seconds
+    begin
+      s = TCPSocket.new "localhost", 4567
+      s.close
+      break
+    rescue Errno::ECONNREFUSED
+      raise if(count == 0)
+      sleep 0.1
+    end
+  end
+
+  # Clean out old data
+  Rake::Task["dbdrop"].invoke
+
+  # Run specs
+  Rake::Task["rspec"].invoke
+end
+
+spec_tasks = Dir["spec/*/"].map { |d| File.basename(d) }
+
+spec_tasks.each do |folder|
+  RSpec::Core::RakeTask.new("spec:#{folder}") do |t|
+    t.pattern = "./spec/#{folder}/**/*_spec.rb"
+    t.rspec_opts = %w(-fs --color)
+  end
+end
+
+desc "Drop fake_dynamo"
+task :dbdrop do
+  sh "curl -X DELETE http://localhost:4567"
+end
+
+desc "Run specs"
+task rspec: spec_tasks.map { |f| "spec:#{f}" }
+
+desc "Default task"
+task :default => [:spec]

data/lib/moe.rb

@@ -0,0 +1,29 @@
+require "aws-sdk-core"
+require "multi_json"
+require "securerandom"
+
+require "moe/config"
+require "moe/dyna"
+require "moe/sequence"
+require "moe/sequence/collection"
+require "moe/sequence/collector"
+require "moe/sequence/item_fetcher"
+require "moe/sequence/locksmith"
+require "moe/sequence/metadata_item"
+require "moe/table_manager"
+require "moe/version"
+
+module Moe
+
+  module ModuleFunctions
+    def collection(name, owner_id)
+      Sequence::Collection.new name, owner_id
+    end
+
+    def collector(name, owner_id)
+      Sequence::Collector.new name, owner_id
+    end
+  end
+  extend ModuleFunctions
+
+end

data/lib/moe/config.rb

@@ -0,0 +1,21 @@
+module Moe
+
+  module ModuleFunctions
+    attr_accessor :config
+
+    def configure
+      self.config ||= Config.new
+      yield(config)
+    end
+  end
+  extend ModuleFunctions
+
+  class Config
+    attr_accessor :batch_limit, :tables
+
+    def initialize
+      @batch_limit = 15
+      @tables  = {}
+    end
+  end
+end

data/lib/moe/dyna.rb

@@ -0,0 +1,107 @@
+module Moe
+  class Dyna
+
+    attr_accessor :dynamodb
+
+    def initialize
+      @dynamodb = Aws.dynamodb
+    end
+
+    def batch_write_item(write_tables, items)
+      explosion = items.map { |item| explode item }
+
+      write_tables.each do |table_name|
+        dynamodb.batch_write_item request_items: {
+          table_name => batchify(explosion)
+        }
+      end
+    end
+
+    def create_table(name, copies=1, hash_key="hash", range_key=nil, read_capacity=5, write_capacity=10)
+      [].tap do |tables|
+        1.upto(copies).each do |copy|
+          schema = table_schema "#{name}_#{copy}",
+                                hash_key,
+                                range_key,
+                                read_capacity,
+                                write_capacity
+
+          table  = dynamodb.create_table schema
+
+          tables << "#{name}_#{copy}"
+        end
+      end
+    end
+
+    def explode(item)
+      clone = item.clone
+
+      clone.each do |key, value|
+        clone[key] = { s: value }
+      end
+    end
+
+    def get_item(read_tables, key)
+      item = nil
+      read_tables.each do |table_name|
+        item = dynamodb.get_item(table_name: table_name, key: key).item
+        break if item
+      end
+      item
+    end
+
+    def implode(item)
+      item.each_key { |key| item[key] = item[key][:s] }
+    end
+
+    def put_item(write_tables, item)
+      write_tables.each do |table_name|
+        dynamodb.put_item table_name: table_name, item: explode(item)
+      end
+    end
+
+    def find(name)
+      if dynamodb.list_tables.table_names.include? name
+        dynamodb.describe_table table_name: name
+      else
+        false
+      end
+    end
+
+    private
+
+    def batchify(items)
+      items.map do |item|
+        { put_request:
+          { item: item }
+        }
+      end
+    end
+
+    def table_schema(name, hash_key, range_key, read_capacity, write_capacity)
+      { table_name: name,
+        key_schema: [
+          attribute_name: hash_key,
+          key_type: "HASH"
+        ],
+        attribute_definitions: [
+          {
+            attribute_name: hash_key,
+            attribute_type: "S"
+          }
+        ],
+        provisioned_throughput: {
+          read_capacity_units: read_capacity,
+          write_capacity_units: write_capacity
+        }
+      }.tap do |table|
+        if range_key
+          table[:key_schema]            << { attribute_name: range_key,
+                                             key_type: "RANGE" }
+          table[:attribute_definitions] << { attribute_name: range_key,
+                                             attribute_type: "S" }
+        end
+      end
+    end
+  end
+end

data/lib/moe/sequence.rb

@@ -0,0 +1,23 @@
+module Moe
+  module Sequence
+
+    module ModuleFunctions
+      def setup(name, copies, read_capacity, write_capacity)
+        return "#{name} already exists in config" if Moe.config.tables[name]
+
+        table_manager = TableManager.new
+
+        tables = table_manager.build name,
+                                     copies,
+                                     "hash",
+                                     "range",
+                                     read_capacity,
+                                     write_capacity
+
+        Moe.config.tables[name] = tables
+      end
+    end
+    extend ModuleFunctions
+    
+  end
+end

data/lib/moe/sequence/collection.rb

@@ -0,0 +1,56 @@
+module Moe
+  module Sequence
+    class Collection
+      attr_accessor :dyna
+      attr_reader   :owner_id, :read_tables
+
+      def initialize(name, owner_id)
+        @dyna         = Dyna.new
+        @owner_id     = owner_id
+        @read_tables  = Moe.config.tables[name].first
+      end
+
+      def metadata_items
+        [].tap do |results|
+          read_tables.each do |table_name|
+
+            dyna.dynamodb.query(request table_name).items.each do |item|
+              results << process(table_name, item)
+            end
+
+          end
+        end
+      end
+
+      private
+
+      def process(table_name, item)
+        MetadataItem.new( table_name,
+                          owner_id,
+                          item["range"].s.gsub(/0\./, ""),
+                          item["count"].s.to_i,
+           MultiJson.load(item["payload"].s) )
+      end
+
+      def request(table_name)
+        {
+          table_name: table_name,
+          key_conditions: {
+            hash: {
+              attribute_value_list: [
+                { s: owner_id }
+              ],
+              comparison_operator: "EQ"
+            },
+            range: {
+              attribute_value_list: [
+                { s: "0" }
+              ],
+              comparison_operator: "BEGINS_WITH"
+            }
+          }
+        }
+      end
+    end
+  end
+end