kinesis-tools 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 72f013adab5a99e32e235b0aaef93c5ca50c824f
+  data.tar.gz: 055384b28ded2e8a4095e9fc7059c79aa94d6932
+SHA512:
+  metadata.gz: 89dd41eb9d524253c6e53f4a1ec243b531bf67c6f43cb8523cf245da2ea7189ee61663e55e2de442434f49d1f4d4f8a50fed22cf92f6540d75081a487e3eb3df
+  data.tar.gz: 962c232482bd398c16bd6d4606dd709e09266a656737e99f41666e6c022943d8ff666b92630620e6907036a6ab6a348c93c61f73d4e72f7f128bec5ed24f4aaf

data/.gitignore

@@ -0,0 +1,3 @@
+vendor/bundle/**
+.bundle/**
+pkg/**

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+gemspec
+

data/Gemfile.lock

@@ -0,0 +1,25 @@
+PATH
+  remote: .
+  specs:
+    kinesis-tools (0.1.0)
+      aws-sdk
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    aws-sdk (2.2.8)
+      aws-sdk-resources (= 2.2.8)
+    aws-sdk-core (2.2.8)
+      jmespath (~> 1.0)
+    aws-sdk-resources (2.2.8)
+      aws-sdk-core (= 2.2.8)
+    jmespath (1.1.3)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  kinesis-tools!
+
+BUNDLED WITH
+   1.11.2

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Johannes Müller, AutoScout24 GmbH
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,19 @@
+# kinesis-tools
+Tools for working with AWS Kinesis streams
+
+## kinesis-tail
+"tail -f" for AWS Kinesis streams
+
+### Usage
+```
+./kinesis-tail.rb streamName
+```
+
+## kinesis-resharding
+- Easy upscaling of AWS Kinesis streams
+- Doubles the amount of shards
+
+### Usage
+```
+./kinesis-resharding.rb streamName up
+```

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/bin/kinesis-tail

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+
+require 'aws-sdk'
+
+Aws.use_bundled_cert!
+kinesis = Aws::Kinesis::Client.new
+
+stream_name = ARGV.shift
+
+shards = kinesis.describe_stream(:stream_name => stream_name)[:stream_description][:shards]
+
+iterators = {}
+
+shards.each do |shard|
+  response = kinesis.get_shard_iterator(:stream_name => stream_name, :shard_id => shard[:shard_id], :shard_iterator_type => 'LATEST')
+  iterators[shard[:shard_id]] = response[:shard_iterator]
+end
+
+while true
+  shards.each do |shard|
+    response = kinesis.get_records(:shard_iterator => iterators[shard[:shard_id]], :limit => 100)
+
+    iterators[shard[:shard_id]] = response[:next_shard_iterator]
+    response[:records].each do |record|
+      puts record[:data]
+    end
+  end
+  sleep 10
+end

data/kinesis-resharding.rb

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+
+require 'aws-sdk'
+
+Aws.use_bundled_cert!
+def has_child_shard(shards,shard_id)
+  shards.select{ |s| s[:parent_shard_id] == shard_id}.length > 0
+end
+
+kinesis = Aws::Kinesis::Client.new
+
+stream_name = ARGV.shift
+updown = ARGV.shift
+
+stream = kinesis.describe_stream(:stream_name => stream_name)[:stream_description]
+cur_shards = stream[:shards]
+
+if updown == 'up'
+  puts "Scaling up, double shards"
+  cur_shards.select{ |s| !has_child_shard(cur_shards, s[:shard_id]) }.each do |shard|
+    puts "splitting shard #{shard[:shard_id]}"
+    new_starting_hash_key = ((shard[:hash_key_range][:starting_hash_key].to_i + shard[:hash_key_range][:ending_hash_key].to_i) / 2).to_s
+    response = kinesis.split_shard(:stream_name => stream_name, :shard_to_split => shard[:shard_id], :new_starting_hash_key => new_starting_hash_key)
+    while true
+      sleep 10
+      break if kinesis.describe_stream(:stream_name => stream_name)[:stream_description][:stream_status] == 'ACTIVE'
+    end
+  end
+end

data/kinesis-tools.gemspec

@@ -0,0 +1,20 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = "kinesis-tools"
+  s.version = "0.1.0"
+  s.authors = ["Marten Veldthuis"]
+  s.email = "marten@veldthuis.com"
+
+  s.summary = "Small CLI tool to tail an AWS Kinesis stream"
+  s.homepage = 'https://github.com/marten/kinesis-tools'
+  s.license = 'MIT'
+
+  s.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  s.bindir        = "bin"
+  s.executables   = ["kinesis-tail"]
+  s.require_paths = ["lib"]
+
+  s.add_dependency "aws-sdk"
+end
+

data/lib/kinesis-tools.rb

@@ -0,0 +1,3 @@
+module KinesisTools
+  VERSION = '0.1.0'
+end

metadata

@@ -0,0 +1,69 @@
+--- !ruby/object:Gem::Specification
+name: kinesis-tools
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Marten Veldthuis
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-02-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email: marten@veldthuis.com
+executables:
+- kinesis-tail
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- bin/kinesis-tail
+- kinesis-resharding.rb
+- kinesis-tools.gemspec
+- lib/kinesis-tools.rb
+homepage: https://github.com/marten/kinesis-tools
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Small CLI tool to tail an AWS Kinesis stream
+test_files: []
+has_rdoc: