kinesis-stream-reader 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 6bb89a21a392fcfa49caca6f223d2a039961f130
+  data.tar.gz: f0a08dcbecf7208002b7524652211dbdfba2ff10
+SHA512:
+  metadata.gz: 9e679f5ec4da78c12a134fd6ca5eb4737df76e2a35c3e35277bfe8d27e3ab8fe022296873adea599e894af0098a6f7a2da4e2641f2b898f9c23be28550e68f66
+  data.tar.gz: 9e256e5f5f83308284fa4c4886d39d3fa62228052767c580c9bd6cf70ef5275ce044d7b19f8f0515f9a3380a3810d3610e53141506b03aed2fa6bc45a5c527b6

data/.gitignore

@@ -0,0 +1,5 @@
+/.bundle
+/Gemfile.lock
+pkg
+vendor
+/projectFilesBackup

data/.travis.yml

@@ -0,0 +1,7 @@
+language: ruby
+rvm:
+- 2.2.2
+notifications:
+  email: false
+  slack:
+    secure: GZdf4YAEI9uEly5FBAeYGl6grQi9YgtXImTCWBsrtSyqcVu19paorOe0zVIaXuQttv+8wNHcRx67U9Af/Mu3xWhLcNII4ukE9mKIvJm2TvQBbGpfv1slQ8K9QTTDrQZ4iH8X2ixt3FRseRywb/R4ANsqLbJn4hp/W81i8RG92kTREmR0oTT5Uya5eD2sb0iPOs9qy7gIxAdxIffUTmFiTlEo0R/BAiQFfpjGefbzYaMgi0fpTKvda8PN09itFuV2qNcFHClQHnFpuk3vpKO0WqQz0mfk453xPDihQXS0gneVvtIoqUJYuFy7RGep2fqlIccQV8U61HsVKSuEC3HIRXCdwJDpypOuxNXFuTmT0H6mfL05akUPGseG3S3Q4uTYeqYMjLOR+cDbNFJzsILDmJ0xCDvMnicoghFkB/eBpE9T5+VWiukI2KYo7comf+NTBKLAijxI8jrgZH5PjBcaotD4WQCqBZyEUs2wqWr8Xbup3hRsRzD3pG5PIlhy6LK/l7iXcJ/A2U2yI2ciKnU6Xqb/LjSAxOHwUimloKxwCvluRzVBbLHC21rKOIGT0wokXszpmhlRw2MDx1+9zssKMSS+REct1Ow+pFqCpVvPhV5mLdOjILcA3qeucFqH484SH8/GdfYTGYqSKmgLbTZezmEsnkDe9PmIxh3O7DKDIUY=

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Ello PBC
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1 @@
+Readme

data/Rakefile

@@ -0,0 +1,6 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/lib/stream_helper.rb

@@ -0,0 +1,6 @@
+require 'logger'
+require 'aws-sdk-core'
+require 'active_support/notifications'
+require './lib/stream_reader/sequence_number_tracker'
+require './lib/stream_reader/avro_parser'
+require './lib/stream_reader/version'

data/lib/stream_reader.rb

@@ -0,0 +1,76 @@
+require 'stream_helper'
+
+module StreamReader
+  class << self
+    def logger
+      @logger ||= Logger.new(STDOUT)
+    end
+  end
+
+  # Assume only one shard for now
+  BATCH_SIZE = 10
+
+  def initialize(stream_name:, prefix: '')
+    @stream_name = stream_name
+    @prefix      = prefix
+    @tracker     = SequenceNumberTracker.new(key_prefix: [ stream_name, prefix ].compact.join('-'))
+    @logger      = StreamReader.logger
+  end
+
+  def run!(&block)
+    # Locate a shard id to iterate through - we only have one for now
+    loop do
+      begin
+        iterator_opts = { stream_name: @stream_name, shard_id: shard_id }
+        if seq = @tracker.last_sequence_number
+          iterator_opts[:shard_iterator_type] = 'AFTER_SEQUENCE_NUMBER'
+          iterator_opts[:starting_sequence_number] = seq
+        else
+          iterator_opts[:shard_iterator_type] = 'TRIM_HORIZON'
+        end
+        @logger.debug "Getting shard iterator for #{@stream_name} / #{seq}"
+        resp = client.get_shard_iterator(iterator_opts)
+        shard_iterator = resp.shard_iterator
+
+        # Iterate!
+        loop do
+          sleep 1
+          @logger.debug "Getting records for #{shard_iterator}"
+          resp = client.get_records({
+            shard_iterator: shard_iterator,
+            limit: BATCH_SIZE,
+          })
+
+          resp.records.each do |record|
+            ActiveSupport::Notifications.instrument('stream_reader.process_record',
+                                                    stream_name: @stream_name,
+                                                    prefix: @prefix,
+                                                    shard_id: shard_id,
+                                                    ms_behind: resp.millis_behind_latest) do
+              AvroParser.new(record.data).each_with_schema_name(&block)
+              @tracker.last_sequence_number = record.sequence_number
+            end
+          end
+
+          shard_iterator = resp.next_shard_iterator
+        end
+
+      rescue Aws::Kinesis::Errors::ExpiredIteratorException
+        @logger.debug "Iterator expired! Fetching a new one."
+      end
+    end
+  end
+
+  private
+
+  def client
+    @client ||= Aws::Kinesis::Client.new
+  end
+
+  def shard_id
+    @shard_id ||= begin
+                    resp = client.describe_stream(stream_name: @stream_name, limit: 1)
+                    resp.stream_description.shards[0].shard_id
+                  end
+  end
+end

data/lib/stream_reader/avro_parser.rb

@@ -0,0 +1,17 @@
+require 'avro'
+
+class AvroParser
+  def initialize(data)
+    @data = data
+  end
+
+  def each_with_schema_name
+    buffer = StringIO.new(@data)
+    reader = Avro::DataFile::Reader.new(buffer, Avro::IO::DatumReader.new)
+    reader.each do |record|
+      schema_name = reader.datum_reader.readers_schema.name
+      yield record, schema_name
+    end
+  end
+end
+

data/lib/stream_reader/sequence_number_tracker.rb

@@ -0,0 +1,17 @@
+require 'redis'
+
+class SequenceNumberTracker
+  def initialize(key_prefix:, redis_url: ENV['REDIS_URL'], key: 'kinesis-last-seq')
+    uri    = URI.parse(redis_url || 'redis://localhost:6379')
+    @redis = Redis.new(url: uri)
+    @key   = "#{key_prefix}-#{key}"
+  end
+
+  def last_sequence_number
+    @redis.get(@key)
+  end
+
+  def last_sequence_number=(value)
+    @redis.set(@key, value)
+  end
+end

data/lib/stream_reader/version.rb

@@ -0,0 +1,3 @@
+module StreamReader
+  VERSION = '0.1.0'
+end

data/spec/spec_helper.rb

@@ -0,0 +1,9 @@
+ENV['RAILS_ENV'] ||= 'test'
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+require 'stream_reader'
+require 'pry'
+require 'fakeredis/rspec'
+
+RSpec.configure do |config|
+  config.color = true
+end

data/spec/stream_reader/avro_parser_spec.rb

@@ -0,0 +1,19 @@
+require 'spec_helper'
+
+describe AvroParser do
+  let(:data) { File.read(File.join(File.dirname(__FILE__), '..', 'support', 'user_was_created.avro')) }
+  let(:parser) { described_class.new(data) }
+
+  it 'parses an Avro message' do
+    parser.each_with_schema_name do |record, schema_name|
+      expect(schema_name).to eq('UserWasCreated')
+      expect(record).to eq({
+        "id"=>"1",
+        "username"=>"hello",
+        "created_at"=>"2015-08-17T13:57:09-06:00",
+        "email"=>"hello@example.com",
+        "analytics_id"=>"123123"
+      })
+    end
+  end
+end

data/spec/stream_reader/sequence_number_track_spec.rb

@@ -0,0 +1,26 @@
+require 'spec_helper'
+
+describe SequenceNumberTracker do
+  let(:tracker) { described_class.new(key_prefix: 'test-key')}
+  let(:tracker2) { described_class.new(key_prefix: 'test-key')}
+  let(:other_tracker) { described_class.new(key_prefix: 'other-test-key')}
+
+  it 'initially has a nil sequence number' do
+    expect(tracker.last_sequence_number).to be_nil
+  end
+
+  it 'stores the last saved value for the sequence number' do
+    tracker.last_sequence_number = '1000'
+    expect(tracker.last_sequence_number).to eq('1000')
+  end
+
+  it 'tracks the same value across individual tracker instances with the same key/prefix' do
+    tracker.last_sequence_number = '1000'
+    expect(tracker2.last_sequence_number).to eq('1000')
+  end
+
+  it 'does not track the same value across individual tracker instances with different keys/prefixes' do
+    tracker.last_sequence_number = '1000'
+    expect(other_tracker.last_sequence_number).to be_nil
+  end
+end

data/spec/stream_reader_spec.rb

@@ -0,0 +1,7 @@
+require 'spec_helper'
+
+describe StreamReader do
+  it 'has a logger' do
+    expect(described_class.logger).to be_a(Logger)
+  end
+end

data/stream_reader.gemspec

@@ -0,0 +1,32 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require './lib/stream_reader/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'kinesis-stream-reader'
+  spec.version       = StreamReader::VERSION
+  spec.authors       = ['Justin-Holmes']
+  spec.email         = ['justin.ryan.holmes@icloud.com']
+
+  spec.summary       = %q{Ruby interface for reading AWS Kinesis streams}
+  spec.homepage      = 'https://github.com/ello/kinesis-stream-reader'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'avro'
+  spec.add_dependency 'aws-sdk-core'
+  spec.add_dependency 'redis'
+  spec.add_dependency 'activesupport'
+
+  spec.add_development_dependency 'bundler', '~> 1.10'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'fakeredis'
+  spec.add_development_dependency 'pry'
+end
+

metadata

@@ -0,0 +1,192 @@
+--- !ruby/object:Gem::Specification
+name: kinesis-stream-reader
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Justin-Holmes
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-03-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: avro
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fakeredis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- justin.ryan.holmes@icloud.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/stream_helper.rb
+- lib/stream_reader.rb
+- lib/stream_reader/avro_parser.rb
+- lib/stream_reader/sequence_number_tracker.rb
+- lib/stream_reader/version.rb
+- spec/spec_helper.rb
+- spec/stream_reader/avro_parser_spec.rb
+- spec/stream_reader/sequence_number_track_spec.rb
+- spec/stream_reader_spec.rb
+- spec/support/user_was_created.avro
+- stream_reader.gemspec
+homepage: https://github.com/ello/kinesis-stream-reader
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5.1
+signing_key: 
+specification_version: 4
+summary: Ruby interface for reading AWS Kinesis streams
+test_files:
+- spec/spec_helper.rb
+- spec/stream_reader/avro_parser_spec.rb
+- spec/stream_reader/sequence_number_track_spec.rb
+- spec/stream_reader_spec.rb
+- spec/support/user_was_created.avro