moe 0.0.1

data/lib/moe/sequence/collector.rb

@@ -0,0 +1,58 @@
+module Moe
+  module Sequence
+    class Collector
+      attr_accessor :dyna, :flushed_count, :payloads
+      attr_reader   :owner_id, :write_tables, :uuid
+
+      def initialize(name, owner_id)
+        @dyna          = Dyna.new
+        @flushed_count = 0
+        @payloads      = []
+        @owner_id      = owner_id
+        @uuid          = SecureRandom.uuid
+        @write_tables  = Moe.config.tables[name].last
+      end
+
+      def add(payload={})
+        payloads << payload
+
+        if payloads.size >= Moe.config.batch_limit
+          items = keyify payloads
+          flush items
+
+          self.payloads = []
+        end
+      end
+
+      def save(payload={})
+        metadata_item = {
+          "count"    => (payloads.size + flushed_count).to_s,
+          "saved_at" => Time.now.to_s,
+          "payload"  => MultiJson.dump(payload)
+        }.merge Locksmith.itemize owner_id, payload, 0, uuid
+
+        items = keyify payloads
+
+        items << metadata_item
+
+        flush items
+      end
+
+      private
+
+      def flush(items)
+        result = dyna.batch_write_item write_tables, items
+
+        self.flushed_count += items.size
+      end
+
+      def keyify(items, uid=uuid)
+        count = flushed_count
+        items.each do |item|
+          count += 1
+          item.update Locksmith.key owner_id, count, uid
+        end
+      end
+    end
+  end
+end

data/lib/moe/sequence/item_fetcher.rb

@@ -0,0 +1,44 @@
+module Moe
+  module Sequence
+    class ItemFetcher
+      attr_accessor :dyna, :items, :sequence_id
+      attr_reader   :owner_id, :table_name, :uid
+
+      def initialize(table_name, owner_id, uid)
+        @dyna = Dyna.new
+        @items = []
+        @owner_id = owner_id
+        @sequence_id = 1
+        @table_name = table_name
+        @uid = uid
+      end
+
+      def fetch(limit)
+        request = {
+          request_items: {
+            table_name => { keys: [] }
+          }
+        }
+        keys = request[:request_items][table_name][:keys]
+
+        limit.times do
+          keys << dyna.explode( Locksmith.key(owner_id, sequence_id, uid) )
+
+          self.sequence_id += 1
+        end
+
+        munge dyna.dynamodb.batch_get_item(request)
+      end
+
+      private
+
+      def munge(results)
+        results.responses.each_value do |item|
+          item.each do |i|
+            items << dyna.implode(i)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/moe/sequence/locksmith.rb

@@ -0,0 +1,25 @@
+module Moe
+  module Sequence
+    module Locksmith
+
+      module ModuleFunctions
+        def itemize(owner_id, payload, sequence_id, uid)
+          {
+            "hash"    => owner_id,
+            "range"   => "#{sequence_id}.#{uid}",
+            "payload" => MultiJson.dump(payload)
+          }
+        end
+
+        def key(owner_id, sequence_id, uid)
+          {
+            "hash"    => owner_id,
+            "range"   => "#{sequence_id}.#{uid}"
+          }
+        end
+      end
+      extend ModuleFunctions
+
+    end
+  end
+end

data/lib/moe/sequence/metadata_item.rb

@@ -0,0 +1,21 @@
+module Moe
+  module Sequence
+    MetadataItem = Struct.new(:table_name, :owner_id, :uid, :count, :payload) do
+
+      def items
+        fetcher   = ItemFetcher.new table_name, owner_id, uid
+        remaining = count
+
+        while remaining > Moe.config.batch_limit
+          fetcher.fetch Moe.config.batch_limit
+
+          remaining -= Moe.config.batch_limit
+        end
+
+        fetcher.fetch remaining
+
+        fetcher.items
+      end
+    end
+  end
+end

data/lib/moe/table_manager.rb

@@ -0,0 +1,104 @@
+module Moe
+  class TableManager
+    attr_reader   :date
+    attr_accessor :dyna, :meta
+
+    def initialize
+      @date = Time.now.strftime("%F")
+      @dyna = Dyna.new
+      @meta = dyna.find(meta_table_names.first) || dyna.create_table(meta_table_name, 2)
+    end
+
+    def build(model, copies=1, hash_key="hash", range_key=nil, read_capacity=5, write_capacity=10, read_tables=[])
+      write_tables =  dyna.create_table table_name(model),
+                      copies,
+                      hash_key,
+                      range_key,
+                      read_capacity,
+                      write_capacity
+
+      metadata = {
+        read_tables:  read_tables << write_tables.first,
+        write_tables: write_tables
+      }
+
+      update_metadata model, metadata
+
+      [ read_tables, write_tables ]
+    end
+
+    def increment(model)
+      metadata = load_metadata model
+      table    = load_table metadata[:write_tables].first
+
+      if table[:table_name].include? date
+        raise "Moe sez: Cannot increment twice on the same day!"
+      end
+
+      build model,
+            metadata[:write_tables].size,
+            table[:key][:hash],
+            table[:key][:range],
+            table[:read_capacity],
+            table[:write_capacity],
+            metadata[:read_tables]
+    end
+
+    def load_metadata(model)
+      metadata = dyna.get_item meta_table_names,
+                                { "hash" => { s: munged_model(model) } }
+
+      MultiJson.load metadata["payload"]["s"], symbolize_keys: true
+    end
+
+    def load_table(table_name)
+      table = dyna.find table_name
+
+      {
+        table_name:     table.table.table_name,
+        key:            get_key(table.table.key_schema),
+        read_capacity:  table.table.provisioned_throughput.read_capacity_units,
+        write_capacity: table.table.provisioned_throughput.write_capacity_units
+      }
+    end
+
+    def meta_table_name
+      "moe_#{ENV['RAILS_ENV']}_manager"
+    end
+
+    def meta_table_names
+      ["#{meta_table_name}_1", "#{meta_table_name}_2"]
+    end
+
+    def table_name(model)
+      "moe_#{ENV['RAILS_ENV']}_#{date}_#{munged_model(model)}".downcase
+    end
+
+    def update_metadata(model, payload)
+      item = { 
+        "hash"    => munged_model(model),
+        "payload" => MultiJson.dump(payload)
+      }
+
+      dyna.put_item meta_table_names, item
+    end
+
+    private
+
+    def get_key(key_schema)
+      {}.tap do |key|
+        key_schema.each do |k|
+          if k.key_type == "HASH"
+            key[:hash]  = k.attribute_name
+          else
+            key[:range] = k.attribute_name
+          end
+        end
+      end
+    end
+
+    def munged_model(model)
+      model.gsub(/::/, "_")
+    end
+  end
+end

data/lib/moe/version.rb

@@ -0,0 +1,3 @@
+module Moe
+  VERSION = "0.0.1"
+end

data/moe.gemspec

@@ -0,0 +1,31 @@
+# coding: utf-8
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "moe/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "moe"
+  spec.version       = Moe::VERSION
+  spec.authors       = ["Fuzz Leonard"]
+  spec.email         = ["fuzz@fuzzleonard.com"]
+  spec.description   = %q{A toolkit for working with DynamoDB at scale}
+  spec.summary       = %q{A toolkit for working with DynamoDB}
+  spec.homepage      = "https://github.com/Geezeo/moe"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "aws-sdk-core"
+  spec.add_dependency "multi_json", "~> 1.8"
+
+  spec.add_development_dependency "bundler",     "~> 1.3"
+  spec.add_development_dependency "coveralls",   "~> 0.7"
+  spec.add_development_dependency "fake_dynamo", "0.2.5"
+  spec.add_development_dependency "pry",         "~> 0.9"
+  spec.add_development_dependency "rake",        "~> 10"
+  spec.add_development_dependency "rspec",       "~> 2"
+  spec.add_development_dependency "timecop",     "~> 0.7"
+end

data/spec/lib/config_spec.rb

@@ -0,0 +1,26 @@
+require "spec_helper"
+
+describe Moe::Config do
+
+  describe "config block" do
+    it "accepts a configuration block" do
+      Moe.configure do |c|
+        c.tables = { "foo" => "bar" }
+      end
+
+      expect( Moe.config.tables["foo"] ).to eq("bar")
+    end
+
+    it "allows configuration changes" do
+      Moe.configure do |c|
+        c.tables = { "foo" => "bar" }
+      end
+
+      Moe.configure do |c|
+        c.tables = { "foo" => "star" }
+      end
+
+      expect( Moe.config.tables["foo"] ).to eq("star")
+    end
+  end
+end

data/spec/lib/dyna_spec.rb

@@ -0,0 +1,113 @@
+require "spec_helper"
+
+$count = 0
+
+describe Moe::Dyna do
+  let(:count)           { $count += 1 }
+  let(:dyna)            { Moe::Dyna.new }
+  let(:dynamodb)        { Aws.dynamodb }
+  let(:item)            { { "hash" => "test#{count}" } }
+  let(:dynamo_item)     { dyna.explode(item) }
+  let(:created_tables)  { dyna.create_table("Testy#{count}") }
+  let(:table)           { dyna.find(created_tables.first).table }
+
+  describe "#batch_write_item" do
+    it "writes a batch of items" do
+
+      items = dyna.batch_write_item [table.table_name], [item, { "hash" => "zoo" }]
+      result = dyna.get_item [table.table_name], { "hash" => { s: "zoo" } }
+
+      expect( result["hash"]["s"] ).to eq("zoo")
+    end
+  end
+
+  describe "#create_table" do
+    it "creates a new table" do
+      new_table = dyna.create_table "Testy#{count}"
+
+      expect(
+        dynamodb.list_tables.table_names.include? "Testy#{count}_1"
+      ).to be_true
+    end
+
+    it "creates as many copies of a table as requested" do
+      new_tables = dyna.create_table "Testie#{count}", 5
+
+      expect(
+        dynamodb.list_tables.table_names.include? "Testie#{count}_1"
+      ).to be_true
+
+      expect(
+        dynamodb.list_tables.table_names.include? "Testie#{count}_5"
+      ).to be_true
+    end
+  end
+
+  describe "#explode" do
+    it "returns a DynamoDB-flavored hash given a plain hash" do
+      expect(
+        dyna.explode({ "foo" => "bar" })
+      ).to eq({"foo" => { s: "bar" } })
+    end
+  end
+
+  describe "#get_item" do
+    it "gets an item" do
+      dynamodb.put_item table_name: table.table_name, item: dynamo_item
+      result = dyna.get_item [table.table_name], dynamo_item
+
+      expect( result["hash"]["s"] ).to eq("test#{count}")
+    end
+
+    it "gets an item across multiple tables" do
+      dynamodb.put_item table_name: table.table_name, item: dynamo_item
+      empty_table = dyna.create_table "Testy#{count}_empty"
+      result      = dyna.get_item [table.table_name, "Testy#{count}_empty_1"], dynamo_item
+
+      expect( result["hash"]["s"] ).to eq("test#{count}")
+    end
+  end
+
+  describe "#implode" do
+    it "returns a plain hash given a DynamoDB-flavored hash" do
+      expect(
+        dyna.implode({"foo" => { s: "bar" } })
+      ).to eq({ "foo" => "bar" })
+    end
+  end
+
+  describe "#find" do
+    it "finds a table" do
+      dyna.create_table "Testy#{count}"
+
+      expect(
+        dyna.find("Testy#{count}_1").table.table_name
+      ).to match("Testy#{count}")
+    end
+
+    it "returns false if it does not find a table" do
+      expect(
+        dyna.find "nope"
+      ).to be_false
+    end
+  end
+
+  describe "#put_item" do
+    it "puts an item" do
+      dyna.put_item [table.table_name], item
+
+      expect(
+        dynamodb.get_item(table_name: table.table_name, key: dynamo_item).item["hash"]["s"]
+      ).to eq("test#{count}")
+    end
+
+    it "puts an item to multiple tables" do
+      mirror_tables = dyna.create_table "Testie#{count}", 2
+      dyna.put_item ["Testie#{count}_1", "Testie#{count}_2"], item
+
+      expect(
+        dynamodb.get_item(table_name: "Testie#{count}_1", key: dynamo_item).item["hash"]["s"]
+      ).to eq(dynamodb.get_item(table_name: "Testie#{count}_2", key: dynamo_item).item["hash"]["s"])
+    end
+  end
+end

data/spec/lib/moe_spec.rb

@@ -0,0 +1,19 @@
+describe Moe::ModuleFunctions do
+  let!(:setup) { Moe::Sequence.setup "name", 2, 5, 10 }
+
+  describe ".collection" do
+    it "is a convenience method returning a Moe::Sequence::Collection" do
+      collection = Moe.collection "name", "owner"
+
+      expect( collection ).to be_a(Moe::Sequence::Collection)
+    end
+  end
+
+  describe ".collector" do
+    it "is a convenience method returning a Moe::Sequence::Collector" do
+      collector = Moe.collector "name", "owner"
+
+      expect( collector ).to be_a(Moe::Sequence::Collector)
+    end
+  end
+end