eventosaurus 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 1a8ef4f20b8d48c0c0e035f514ba4088578e9068
+  data.tar.gz: 9038157130f2b42586d4fee746a3ebd394d3d0c2
+SHA512:
+  metadata.gz: 90643659fb8232e9f6d713d946adbd8f984d49038d00a9095bd30a79df287796603ffcb449104ea79829f8f4944dd0906d5a204742d72a66579739d8888116a9
+  data.tar.gz: 158959e5a2f144a853cf7fa300301dde2d08dc742143b98ab826984ee081d79981d9a40a73df7bce2b38d28602e10e94cfcfd076fc6000c504c1bc171b50cb02

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+/*.gem

data/.rspec

@@ -0,0 +1 @@
+--color

data/.rubocop.yml

@@ -0,0 +1,28 @@
+AllCops:
+  TargetRubyVersion: 2.3
+  Include:
+    - ./Rakefile
+  Exclude:
+    - bin/**/*
+    - vendor/bundle/**/*
+
+Lint/EndAlignment:
+  AlignWith: variable
+
+Metrics/LineLength:
+  Max: 120
+
+Style/AlignParameters:
+  Enabled: false
+
+Style/Documentation:
+  Enabled: false
+
+Style/DotPosition:
+  EnforcedStyle: leading
+
+Style/FrozenStringLiteralComment:
+  Enabled: false
+
+Style/StringLiterals:
+  Enabled: false

data/.ruby-version

@@ -0,0 +1 @@
+2.3.0

data/CHANGELOG.md

@@ -0,0 +1,46 @@
+# 1.0.0
+
+* changes name to 'eventosaurus'
+* adds a test_mode
+
+# 0.6.0
+
+* uses event_uuid as sort key rather than created_at
+
+# 0.5.5
+
+* fixes bug in chainable local secondary indices
+
+# 0.5.4
+
+* set default sidekiq retry count to 3
+* fix logger bug
+
+# 0.5.3
+
+* scope rake tasks to environment_prefix
+
+# 0.5.2
+
+* first release after code review
+
+# 0.5.1
+
+# 0.5.0
+
+* adds dynamic query methods based on table definition
+
+# 0.4.0
+
+* removes dynamoid gem in favor of native aws-sdk usage
+* adds rake tasks to build tables
+
+# 0.3.0
+
+* use Sidekiq for delayed jobs
+* utilize Configuration class
+* event jobs are idempotent
+
+# 0.2.0
+
+* uses DynamoID 1.1.0

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,49 @@
+# Contributor Code of Conduct
+
+As contributors and maintainers of this project, and in the interest of
+fostering an open and welcoming community, we pledge to respect all people who
+contribute through reporting issues, posting feature requests, updating
+documentation, submitting pull requests or patches, and other activities.
+
+We are committed to making participation in this project a harassment-free
+experience for everyone, regardless of level of experience, gender, gender
+identity and expression, sexual orientation, disability, personal appearance,
+body size, race, ethnicity, age, religion, or nationality.
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery
+* Personal attacks
+* Trolling or insulting/derogatory comments
+* Public or private harassment
+* Publishing other's private information, such as physical or electronic
+  addresses, without explicit permission
+* Other unethical or unprofessional conduct
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+By adopting this Code of Conduct, project maintainers commit themselves to
+fairly and consistently applying these principles to every aspect of managing
+this project. Project maintainers who do not follow or enforce the Code of
+Conduct may be permanently removed from the project team.
+
+This code of conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community.
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting a project maintainer at engineering@blueapron.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. Maintainers are
+obligated to maintain confidentiality with regard to the reporter of an
+incident.
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 1.3.0, available at
+[http://contributor-covenant.org/version/1/3/0/][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/3/0/

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in eventosaurus.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2016 Blue Apron, inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,307 @@
+# Eventosaurus
+
+[![CircleCI](https://circleci.com/gh/blueapron/eventosaurus.svg?style=shield)](https://circleci.com/gh/blueapron/eventosaurus)
+[![Code Climate](https://codeclimate.com/github/blueapron/eventosaurus/badges/gpa.svg)](https://codeclimate.com/github/blueapron/eventosaurus)
+[![Test Coverage](https://codeclimate.com/github/blueapron/eventosaurus/badges/coverage.svg)](https://codeclimate.com/github/blueapron/eventosaurus/coverage)
+[![Issue Count](https://codeclimate.com/github/blueapron/eventosaurus/badges/issue_count.svg)](https://codeclimate.com/github/blueapron/eventosaurus)
+[![Dependency Status](https://gemnasium.com/badges/github.com/blueapron/eventosaurus.svg)](https://gemnasium.com/github.com/blueapron/eventosaurus)
+
+Enables easy asynchronous event storing and querying on [DynamoDB][dynamodb]
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'eventosaurus'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install eventosaurus
+
+
+## Setting up a local DynamoDB server
+
+When developing, use the local DynamoDB server. Install and kick off your local instance:
+
+Set the following environment variables:
+
+```
+EVENT_ENVIRONMENT_PREFIX=localhost
+AWS_ENDPOINT=http://localhost:8000
+```
+
+Install and kick off DynamoDB:
+
+```bash
+$ brew install dynamodb-local
+$ ln -sfv /usr/local/opt/dynamodb-local/*.plist ~/Library/LaunchAgents
+$ launchctl load ~/Library/LaunchAgents/homebrew.mxcl.dynamodb-local.plist
+```
+
+## Configuration
+
+You need to add the following initializer (ex: `config/initializers/eventosaurus.rb`):
+
+```ruby
+Eventosaurus.configure do |config|
+  config.use_sidekiq
+
+  # ex: localhost, productions
+  config.environment_prefix = ENV['EVENT_ENVIRONMENT_PREFIX']
+
+  config.aws_access_key_id = ENV['AWS_ACCESS_KEY_ID']
+  config.aws_secret_access_key = ENV['AWS_SECRET_ACCESS_KEY']
+  config.aws_region = ENV['AWS_REGION']
+
+  # optional, used for local dynamodb
+  config.aws_endpoint = ENV['AWS_ENDPOINT']
+end
+```
+
+### Sidekiq Alternatives
+
+Eventosaurus ships with both synchronous and asynchronous options. By default Eventosaurus uses sidekiq to persist data to DynamoDB asynchronously.
+
+For synchronous persistance:
+```ruby
+ Eventosaurus.configure do |config|
+   # ...
+   config.use_synchronous
+   # ...
+end
+```
+
+To use your own persistence mechanism, reference the below two files:
+
+* [`lib/eventosaurus/persistors/sidekiq.rb`](https://github.com/blueapron/eventosaurus/blob/master/lib/eventosaurus/persistors/sidekiq.rb)
+* [`lib/eventosaurus/workers/sidekiq.rb`](https://github.com/blueapron/eventosaurus/blob/master/lib/eventosaurus/workers/sidekiq.rb)
+
+And include it in your configuration
+
+```ruby
+require 'custom_persistor'
+ Eventosaurus.configure do |config|
+   # ...
+   config.persistor = CustomPersistor
+   # ...
+end
+```
+
+## Event Representation
+
+Every event type is represented by a class that includes `Eventosaurus::Storable`. Each class must do two things:
+
+0. define the table using the `table_definition` macro
+1. define the `details` class method, which defines the event interface
+
+Here is an example of an event definition:
+
+```ruby
+module Events
+  class PhoneCall
+    include Eventosaurus::Storable
+
+    table_definition name: :phone_call, partition_key: { person_id: :n }
+
+    def self.details( person_id:, phone_number:, last_called:)
+        {
+          'person_id'    => person_id
+          'phone_number' => phone_number.to_s,
+          'last_called'  => last_called
+        }
+     end
+  end
+end
+```
+
+There are some built-in attributes for your events:
+
+0. the gem defines the range of your partition key to be `event_uuid.` When writing an event, this attribute is enforced to be unique, preventing duplicate writes. See Event Duplication Prevention below.
+0. the event also stores the timestamp, which represents the time when the gem client calls `.store`
+
+## Building the Tables
+
+DynamoDB must have the tables needed to run your events. Once you've written your event classes you must run a rake task to create the tables. The tables are namespaced by your environment, as defined in the environment_prefix variable mentioned above. So if you build locally, the table name might be `localhost_core_customer_audit`. This will allow us to quickly get up and running on new environments. For the time being, the rake task expects your event definitions to be in `app/models/events`. **Be sure to put them there!**. Rake tasks are scoped to only work with tables that begin with your environment_prefix. This means even if staging and production point to the same dynamodb account, the `drop_tables` task will only drop tables from the environment specified.
+
+```
+rake eventosaurus:create_tables
+```
+*(NOTE: in the short term, you will have to manually run this create tables task upon deploy, as well as staging environments and anyone who pulls code utilizing these tables. This is temporary and there is an [outstanding task](https://app.asana.com/0/35948616193167/101210257934375) to change this.)*
+
+
+You may then verify the tables were created:
+
+```
+rake eventosaurus:list_tables
+```
+
+See the JSON used to create your tables:
+
+```
+rake eventosaurus:describe_tables
+```
+
+If you decide to (╯°□°)╯︵ ┻━┻
+
+```
+rake eventosaurus:drop_tables
+```
+
+## Storing Data
+
+To store data use the `.store` class method on your event class. Use the same signature as your `details` method mentioned above:
+
+```ruby
+# Somewhere in your app:
+ def check_for_audit(row)
+    if audit_time?
+      Events::CoreCustomerAudit.store(
+          updated_row: row,
+          updated_attribute: 'first_name',
+          updated_by_user_id: 35,
+          audit_occurred_at: UTC.now
+      )
+    end
+ end
+```
+
+## Querying Data
+
+The gem gives you some dynamic methods to query your data based on your table definition. It's important to keep in mind that you are working with DynamoDB. It is not meant to be a data store that is accessed generically. It expects you to know the queries you want to run upfront. Good for us, we are storing each event type in its own table, so we can make good guesses about this. To this end, eventosaurus creates getters for the attributes you listed in your table definition:
+
+```ruby
+ Events::CoreCustomerAudit.by_customer_id(5)
+ Events::CoreCustomerAudit.by_customer_id(5).by_table_name('users').count
+
+ # event_uuid & created_at included for free :)
+ Events::CoreCustomerAudit.by_created_at('2015-01-04', 'GT')
+```
+
+The queries above return eventosaurus Query objects. To actually execute the query, use the `run` method:
+
+```ruby
+Events::CoreCustomerAudit.by_customer_id(5).count.run
+```
+
+Note:
+
+0. In the last example we query by created_at even though it was not listed in the table definition. This is because each table gets the created_at timestamp column as well as the event_uuid column added.
+1. The operator defaults to 'EQ' (equals) but there are many to choose from: EQ, NE, IN, LE, LT, GE, GT, CONTAINS, NOT_CONTAINS, BEGINS_WITH
+2. DynamoDB only allows a **single** secondary index to accompany the partition key. This means the following query will not work the way you think:
+
+```
+# too many secondary predicates. after one secondary index is used, the rest will be full scans on whatever comes back after the first local index.
+Events::CoreCustomerAudit.by_created_at('2015-01-04', 'GT').by_table_name('users')
+```
+
+**To sum it up: for speed, you are allowed 0||1 partition key condition and 0||1 secondary condition. No more than that.**
+
+## Test mode
+
+Test mode can be enabled by placing the following in rails_helper.rb (or equivalent):
+
+```ruby
+Eventosaurus.enable_test_mode
+```
+
+## On Choosing the proper table_definition for your event
+
+When considering the correct partition_key, there are a few considerations. The first is to consider the predicates you will filter by. The predicates you use the most should probably become your partition_key. The second is the number of different values you expect to see in your partition. The more you have, the better. This is a complicated subject, and understanding of how DynamoDB works (partion keys, local and global secondary indexes) should be understood before creating an event. [Here][dynamodbbestpractices] you can find more detail about best practices, and of course hopefully a co-worker Near You can help too.
+
+## Event Error Handling upon calling .store
+
+When you call `.store`, you will be utilizing your `.details` method. Sadly, sometimes you will make mistakes and the gem will raise. Happily, you can decide what to do about the errors. If your call to `.store` raises, the `on_error` class method is called with the error as an argument. Feel free to overwrite this class method in your Event class:
+
+```ruby
+module Events
+ class NeatEvent
+   include Eventosaurus::Storable
+
+   # ... your primary event code here...
+
+   def on_error(error)
+     HaikuNotifier.write_haiku_with_error(error)
+   end
+ end
+end
+```
+
+## Event Duplication Prevention
+
+This gem has two methods of preventing duplicate events from being written to DynamoDB, and each method addresses a different way duplication can occur.
+
+
+### Background on the event_uuid column
+
+Before getting into the two methods below, the foundational piece of information is that DynamoDB writes can be configured to fail if a duplicate value is found on an attribute. The Event Gem leverages this by using an 'event_uuid' as the table's sort key, and setting our writes to fail if the event_uuid already exists.
+
+### Duplication Cause 1: Double processing of asynchronous jobs.
+
+If the same job that sends an event to dynamodb gets run twice, we need to make sure we don't store two events. This is handled by the mechanism explained above: writes are instructed to fail if they see the event_uuid already exists.
+
+### Duplication Cause 2: Gem client erroneously calls the .store method multiple times
+
+If your (the gem client's) code has an error, and your code calls Events::NeatEvent.store more than intended, the Event Gem can be configured to help defend and ensure only 1 event is stored. To guard against this, you can create a composite primary key, based on the fields of your choosing. This composite key is then digested and used as the basis of the event_uuid. You may use the macro `compsite_primary_key` to achieve this duplication defense:
+
+
+```ruby
+module Events
+ class NeatEvent
+
+ composite_primary_key :location, :employee_name, :employee_action
+end
+```
+
+The above example will cause a string like the following to be generated and used as a digest for the event_uuid:
+
+```
+ location=gardens:employee_name=Markus:employee_action=ate grapes
+```
+
+Even if you call Events::NeatEvent.store(args) multiple times with the same args, only one event will be created.
+
+If you do not opt to use the composite_primary_key feature, the Event Gem will use `SecureRandom.uuid` to generate the uuid, which has a much less likely chance of collision than you winning the lottery (It follows RFC 4122)
+
+#### Do not add the created_at attr to your list of composite_primary_key attrs
+
+Using a timestamp representing the creation-time of the event (aka the created_at attr) in the composite_primary_key is not advisable, as accidental duplicate events might have slightly different timestamps, and thus slightly different UUIDs. Put simply: _do not add the created_at attr to your list of composite_primary_key attrs._
+
+**Note** that any attribute listed in the composite_primary_key macro promotes that attribute to a required attribute.
+
+
+## Using the AWS SDK (V2) Client Directly
+
+To access the [Aws SDK v2 client][awssdkv2] directly (for educational purposes only), access via `Eventosaurus.configuration.dynamodb_client`. For example:
+
+```ruby
+Eventosaurus.configuration.dynamodb_client.list_tables
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+## Releasing
+
+Version bumps should be done straight in master after appropriate PRs are merged.
+
+Gem release best practices [here](https://github.com/blueapron/guides/blob/master/best_practices/developing_ruby_gems.md)
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/blueapron/eventosaurus.
+
+## License
+
+The gem is Copyright 2015 Blue Apron, Inc.
+
+[dynamodbbestpractices]: http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/GuidelinesForTables.html#GuidelinesForTables.UniformWorkload
+[dynamodb]: https://aws.amazon.com/dynamodb/
+[localdynamo]: http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Tools.DynamoDBLocal.html
+[awssdkv2]: http://docs.aws.amazon.com/sdkforruby/api/Aws/DynamoDB/Client.html

data/Rakefile

@@ -0,0 +1,6 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/bin/console

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "eventosaurus"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+require "pry"
+Pry.start

data/bin/setup

@@ -0,0 +1,12 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here
+
+echo -e "\n************************************************************\n"
+echo -e "Install a local DynamoDB server: "
+echo -e "http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Tools.DynamoDBLocal.html"
+echo -e "\n************************************************************\n"

data/circle.yml

@@ -0,0 +1,6 @@
+dependencies:
+  pre:
+    - gem update bundler
+test:
+  pre:
+    - bundle exec rubocop -D -fp

data/eventosaurus.gemspec

@@ -0,0 +1,39 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'eventosaurus/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'eventosaurus'
+  spec.version       = Eventosaurus::VERSION
+  spec.authors       = ['Blue Apron Engineering']
+  spec.email         = ['engineering@blueapron.com']
+
+  spec.summary       = %q{Enables easy reporting of events to an event store.}
+  spec.description   = %q{Enables easy reporting of events to an event store.}
+  spec.homepage      = 'https://github.com/blueapron/eventosaurus'
+
+  spec.metadata['allowed_push_host'] = 'https://rubygems.org'
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = 'bin'
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'activesupport'
+  spec.add_dependency 'aws-sdk', '~> 2.0'
+
+  spec.add_development_dependency 'bundler', '~> 1.10'
+  spec.add_development_dependency 'pry'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'rubocop'
+  spec.add_development_dependency 'sidekiq'
+  spec.add_development_dependency 'simplecov'
+  spec.add_development_dependency 'timecop'
+
+  if ENV['CIRCLECI']
+    spec.add_development_dependency 'codeclimate-test-reporter'
+    spec.add_development_dependency 'rspec_junit_formatter'
+  end
+end

data/lib/eventosaurus/configuration.rb

@@ -0,0 +1,82 @@
+require 'aws-sdk'
+
+module Eventosaurus
+  class << self
+    attr_accessor :configuration
+
+    def configure
+      yield(configuration) if block_given?
+
+      configuration.configure_aws
+      configuration.configure_dynamodb
+    end
+
+    def configuration
+      @configuration ||= Configuration.new
+    end
+
+    def enable_test_mode
+      Storable::ClassMethods.redefine_method(:store) do
+        true
+      end
+
+      Models::Query.redefine_method(:run) do
+        true
+      end
+    end
+  end
+
+  class Configuration
+    attr_accessor :aws_access_key_id
+    attr_accessor :aws_endpoint
+    attr_accessor :aws_secret_access_key
+    attr_accessor :aws_region
+    attr_accessor :environment_prefix
+    attr_accessor :dynamodb_client
+    attr_accessor :dynamodb_table_namespace
+    attr_accessor :dynamodb_warn_on_scan
+    attr_accessor :logger
+    attr_accessor :persistor
+
+    def initialize
+      @logger = Logger.new(STDOUT)
+    end
+
+    def use_synchronous
+      require 'eventosaurus/persistors/synchronous'
+
+      @persistor = Eventosaurus::Persistors::Synchronous
+    end
+
+    def use_sidekiq
+      require 'eventosaurus/persistors/sidekiq'
+
+      @persistor = Eventosaurus::Persistors::Sidekiq
+    end
+
+    def configure_aws
+      Aws.config.update(region: aws_region, credentials: aws_credentials)
+    end
+
+    def aws_credentials
+      self.aws_region ||= 'us-east-2'
+      self.aws_access_key_id ||= 'unset'
+      self.aws_secret_access_key ||= 'unset'
+
+      @aws_credientials ||= Aws::Credentials.new(aws_access_key_id, aws_secret_access_key)
+    end
+
+    def configure_dynamodb
+      self.environment_prefix ||= 'unset'
+
+      args = {
+        region: aws_region,
+        credentials: aws_credentials
+      }
+
+      args[:endpoint] = aws_endpoint if aws_endpoint.present?
+
+      self.dynamodb_client = Aws::DynamoDB::Client.new(args)
+    end
+  end
+end