monsoon-droplet 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7e6232ff9dd488d25b28e378d87905d90a8e80bf
-  data.tar.gz: 6628745f73d30cc0c87cb5d19c916fc9a4f38e3e
+  metadata.gz: 71c6d61b46942b8e4b68c0e52742786e9d6d5809
+  data.tar.gz: 220a3aafa400bd53007a7472008eaf4051d9b951
 SHA512:
-  metadata.gz: fb6209b101d0aa6437d9b90fba8e215f2f8d81439ec9754b5a1ecd8b35c93e19523e2c1dee06ecca4be7ce7073fc43aabf5e709e983b880edf5ad81045733e4f
-  data.tar.gz: 16ba8785319d604cf311db264f70cb00abc1482314b58f34d97f70831d2e7688827aaf9672c8e26131ba30fe39bd620b867342b1c9ce218e09073746449728d2
+  metadata.gz: c4d7e57af9b1e1f02e71ea69fe87003fe4bb90f53303709c18fbbc8be45ab59572c13cd272ad1f03e38e5b0188ad8922e56e4d05dcea867ac0302171a7cfb945
+  data.tar.gz: f61e8e456527c79551542091ccf7f94dbc75b68ca6abe2f098c6ff773b084c864e361fff65541ef22bdc9d045b0c2d70f0747ca27f3ab7f00743ad09c88177a7

data/README.md

@@ -32,17 +32,18 @@ require 'monsoon/streams/kinesis'
 require 'monsoon/streams/console'
 
 Monsoon.versions_schema = {
-  "analytics": {
-    "1.0.0": ["user_id", "event_type", "timestamp", "data"],
-    "1.0.1": ["user_id", "event_type", "timestamp", "data"],
-    "1.1.0": ["user_id", "event_type", "timestamp", "data", "resource_id"],
-    "2.0.0": ["event_type", "timestamp", "data", "resource_id"]
-  },
-  "transcodes": {
-    "1.0.0": ["filename", "status"],
-    "2.0.0": ["filename", "sources"]
+  "transcodes" => {
+    "download_complete" => {
+      "1.0" => ["filename", "media_id"],
+      "2.0" => ["url", "media_id"]
+    },
+    "transcode_complete" => {
+      "1.1" => ["filename", "media_id"],
+      "2.0" => ["media_id", "status"]
+    }
   }
 }
+Monsoon.default_stream = 'transcodes'
 
 # if using the Kinesis stream...
 ENV['AWS_ACCESS_KEY_ID'] = "YOUR AWS ACCESS KEY"
@@ -60,39 +61,43 @@ require 'monsoon/streams/kinesis'
 
 This will add the built-in AWS Kinesis adapter. `monsoon/streams/console` is also built-in for testing by writing your streams to STDOUT.
 
-You can write your own stream adapter by creating a class that implements `#put_records(stream_name, records_array)` and then add it to Monsoon with `Monsoon.streams << YourNewAdapter.new`. See `Monsoon::Streams::Console` for an example.
+You can write your own stream adapter by creating a class that implements `#put_records(stream_name, records_array, options_hsh = {})` and then add it to Monsoon with `Monsoon.streams << YourNewAdapter.new`. See `Monsoon::Streams::Console` for an example.
 
 Monsoon will send your messages to all added streams by default.
 
+Note on Kinesis Stream: Kinesis requires a partition key when streaming a message. Monsoon will look for a `partition_key` in your droplet's `options` hash. If no `partition_key` is provided, Monsoon will default to partitioning on the event name or droplet version. If neither of those are available, Monsoon will send everything to the `monsoon` partition.
+
 ### Versions Schema
 
-To auto-generate versioned copies of your message, configure Monsoon with your versions schema. This is just a hash of stream names and keys that tells Monsoon, given a stream name, here's a list of versions we support and the keys each version should include in its message. For example, if you have the following schema defined:
+To auto-generate versioned copies of your message, configure Monsoon with your versions schema. This is just a hash of stream names and keys that tells Monsoon, given a stream name and event name, here's a list of versions we support and the keys each version should include in its message. For example, if you have the following schema defined:
 
 ```ruby
 Monsoon.versions_schema = {
-  "analytics": {
-    "1.0.0": ["user_id", "event_type", "timestamp", "data"],
-    "1.0.1": ["user_id", "event_type", "timestamp", "data"],
-    "1.1.0": ["user_id", "event_type", "timestamp", "data", "resource_id"],
-    "2.0.0": ["event_type", "timestamp", "data", "resource_id"]
-  },
-  "transcodes": {
-    "1.0.0": ["filename", "status"],
-    "2.0.0": ["filename", "sources"]
+  "transcodes" => {
+    "download_complete" => {
+      "1.0" => ["filename", "media_id"],
+      "2.0" => ["url", "media_id"]
+    },
+    "transcode_complete" => {
+      "1.1" => ["filename", "media_id"],
+      "2.0" => ["media_id", "status"]
+    }
   }
 }
 ```
 
-and you want to send the message `{user_id: 1, event_type: 'play', timestamp: 23, data: 'some other data', resource_id: 10}` to your `analytics` stream, Monsoon will automatically generate 4 versions of your message for each defined version. The versioned message will only include the keys needed for that version, as well as 2 new keys: 'droplet_version' and 'droplet_deprecated'. `droplet_version` tells the receiving client which version they're reading, and `version_deprecated` is true if there is a newer version defined in the schema.
+and you want to send the message `{stream: 'transcodes', event: 'download_complete', filename: 'my_movie.mp4', media_id: 5, url: 'example.com/my_movie.mp4'}`, Monsoon will automatically generate 2 versions of your message for each defined version (1.0 and 2.0). The versioned message will only include the keys needed for that version, as well as 2 new keys: 'droplet_version' and 'droplet_deprecated'. `droplet_version` tells the receiving client which version they're reading, and `version_deprecated` is true if there is a newer version defined in the schema.
+
+If the original message does not include all of the keys needed for an older version (e.g., `filename` is left out in the above example), then Monsoon will only send the versions it has the data for (e.g., it will only send v2.0).
 
-If the original message does not include all of the keys needed for an older version (e.g., `user_id` is left out in the above example), then Abacus will only send the versions it has the data for (e.g., it will only send v2.0+ for the 'analytics' stream).
+Since many apps will only use 1 stream, you can also set `Monsoon.default_stream` to avoid passing the `stream` key with each message.
 
 ### Droplets
 
 To send a message after you've configured Monsoon, just create a new Droplet and stream it:
 
 ```ruby
-droplet = Monsoon::Droplet.new("analytics", {user_id: 1, event_type: 'play', timestamp: 23, data: 'some other data', resource_id: 10})
+droplet = Monsoon::Droplet.new({stream: 'transcodes', event: 'download_complete', filename: 'my_movie.mp4', media_id: 5, url: 'example.com/my_movie.mp4'})
 droplet.stream
 ```
 
@@ -106,7 +111,7 @@ Droplets can also take a hash with the following options:
 `#stream` can also take an optional stream adapter (any object that implements `#put_records`) to only stream to that adapter. For example:
 
 ```ruby
-droplet = Monsoon::Droplet.new("my_stream", {user_id: 1, event_type: 'play', timestamp: 23}, {versioning: :skip})
+droplet = Monsoon::Droplet.new({stream: 'transcodes', event: 'download_complete', media_id: 5, url: 'example.com/my_movie.mp4'}, {versioning: :skip})
 droplet.stream(YourNewAdapter.new)
 ```
 

data/lib/monsoon.rb

@@ -5,6 +5,7 @@ require 'monsoon/droplet'
 module Monsoon
   @@streams = []
   @@versions_schema = {}
+  @@default_stream = nil
 
   def self.streams
     @@streams
@@ -21,4 +22,12 @@ module Monsoon
   def self.versions_schema=(versions_schema)
     @@versions_schema = versions_schema
   end
+
+  def self.default_stream
+    @@default_stream
+  end
+
+  def self.default_stream=(stream_name)
+    @@default_stream = stream_name
+  end
 end

data/lib/monsoon/droplet.rb

@@ -6,21 +6,23 @@ module Monsoon
     attr_reader :data
 
     # new - creates a droplet
-    #   stream_name - name of the stream to write to (required)
     #   data - hash of data to write to stream (required)
     #   options - optional hash
     #     :versioning => can be :skip, :enforce, or nil
     #       :skip will ignore the versions schema and write the data exactly as passed
     #       :enforce will require using the schema and will write nothing if unable to version
     #       nil will try to write versioned droplets and fallback to raw data (default)
+    #     :partition_key => the partition key to use for kinesis
     #
     # @example
-    #   Monsoon::Droplet.new('analytics', {user_id: 3, event_type: 'play'}, {versioning: :enforce})
-    def initialize(stream_name, raw_data, options = {})
-      @stream_name = stream_name
+    #   Monsoon::Droplet.new({stream: 'transcodes', event: 'download_complete', filename: 'my_movie.mp4', media_id: 5, url: 'example.com/my_movie.mp4'}, {versioning: :enforce})
+    def initialize(raw_data, options = {})
+      @stream_name = raw_data.delete(:stream) || raw_data.delete('stream') || Monsoon.default_stream
+      raise ArgumentError, "stream not specified" unless @stream_name
+      @event = raw_data[:event] || raw_data['event']
       @options = options
       @raw_data = raw_data
-      @data = VersionsSchema.new(@stream_name).get_droplets(@raw_data) unless @options[:versioning] == :skip
+      @data = VersionsSchema.new(@stream_name, @event).get_droplets(@raw_data) unless @options[:versioning] == :skip
       @data = [@raw_data] if blank? && @options[:versioning] != :enforce
     end
 
@@ -28,11 +30,11 @@ module Monsoon
     #   streamer - (optional) stream adapter instance to limit which adapter is used (adapter must implement `#put_records`)
     #
     # @example
-    #   droplet = Monsoon::Droplet.new('analytics', {user_id: 3, event_type: 'play'})
+    #   droplet = Monsoon::Droplet.new({stream: 'transcodes', event: 'download_complete', filename: 'my_movie.mp4', media_id: 5, url: 'example.com/my_movie.mp4'})
     #   droplet.stream(Monsoon::Streams::Console.new)
     def stream(streamer = false)
       return if blank?
-      streamer ? streamer.put_records(@stream_name, @data) : Monsoon.streams.each {|s| stream(s) }
+      streamer ? streamer.put_records(@stream_name, @data, @options) : Monsoon.streams.each {|s| stream(s) }
     end
 
     def blank?

data/lib/monsoon/streams/console.rb

@@ -3,7 +3,7 @@ require 'json'
 module Monsoon
   module Streams
     class Console
-      def put_records(stream, records)
+      def put_records(stream, records, options = {})
         records.each do |r|
           puts "Streaming to #{stream}: #{JSON.pretty_generate(r)}"
         end

data/lib/monsoon/streams/kinesis.rb

@@ -7,11 +7,11 @@ module Monsoon
         @client = Aws::Kinesis::Client.new
       end
 
-      def put_records(stream, records)
+      def put_records(stream, records, options = {})
         data = records.map do |r|
           {
             data: JSON.generate(r),
-            partition_key: 'monsoon'
+            partition_key: options[:partition_key] || r['event'] || r[:event] || r['droplet_version'] || 'monsoon'
           }
         end
         @client.put_records(records: data, stream_name: stream)

data/lib/monsoon/version.rb

@@ -1,3 +1,3 @@
 module Monsoon
-  VERSION = '1.0.0'
+  VERSION = '1.1.0'
 end

data/lib/monsoon/versions_schema.rb

@@ -1,15 +1,17 @@
 module Monsoon
   class VersionsSchema
     attr_reader :stream
+    attr_reader :event
 
-    def initialize(stream)
+    def initialize(stream, event)
       @stream = stream
+      @event = event
     end
 
     def get_droplets(record)
       return if record.nil?
 
-      schema.map do |version, keys|
+      event_schema.map do |version, keys|
         symbolized_keys = keys.map(&:to_sym)
         # Only keep the record keys that are defined in this version's schema
         versioned_record = record.select{|k,v| symbolized_keys.include?(k.to_sym)}
@@ -18,7 +20,8 @@ module Monsoon
         # add versioning metadata
         versioned_record.merge({
           'droplet_version' => version.to_s,
-          'droplet_deprecated' => deprecated?(version)
+          'droplet_deprecated' => deprecated?(version),
+          'event' => @event
         }) if versioned_record.keys.length == keys.length # make sure we got all the keys for this version
       end.compact
     end
@@ -28,13 +31,19 @@ module Monsoon
     end
 
     def versions
-      schema.keys
+      event_schema.keys
     end
 
     private
 
-    def schema
-      @schema ||= Monsoon.versions_schema[@stream.to_sym] || Monsoon.versions_schema[@stream.to_s] || {}
+    def stream_schema
+      return {} unless @stream
+      @stream_schema ||= Monsoon.versions_schema[@stream.to_sym] || Monsoon.versions_schema[@stream.to_s] || {}
+    end
+
+    def event_schema
+      return {} unless @event
+      @event_schema ||= stream_schema[@event.to_sym] || stream_schema[@event.to_s] || {}
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: monsoon-droplet
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Neil Gupta
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-10 00:00:00.000000000 Z
+date: 2016-03-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk