fluent-plugin-dynamodb-streams 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 1fb8047ed140d8f219d2ef602911c1c74cb1f894
+  data.tar.gz: cb6f8f23baf265dee02865a9607e6bc159524ef1
+SHA512:
+  metadata.gz: 7944a9e15f65e021c653ac4c9c9d6029ba1b7a7309aa307c932896535d6a5956712e61942441863929647cc7864704c1274ceb6c60dfaf7c3f6e370a9686d07c
+  data.tar.gz: baa75db2e7d1bc2b130f79e81da2a359cb1b7072d5437026c75255fb38dcdba82d3771580aa3f7c313fbe95b89f6340c8a5968bfeb7e30c9307319541b826f63

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/.travis.yml

@@ -0,0 +1,12 @@
+language: ruby
+rvm:
+  - 2.2
+  - 2.1
+  - 2.0.0
+  - 1.9.3
+before_script:
+  - bundle exec dynamodb-local -inMemory &
+  - sleep 5
+script:
+  - bundle exec rake test
+  - bundle exec rake build

data/Gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+
+gemspec
+
+group :test do
+  gem "codeclimate-test-reporter", require: nil
+end

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 Takumi Sakamoto
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,99 @@
+# fluent-plugin-dynamodb-streams
+
+[![Build Status](https://travis-ci.org/takus/fluent-plugin-dynamodb-streams.svg?branch=master)](https://travis-ci.org/takus/fluent-plugin-dynamodb-streams)
+[![Code Climate](https://codeclimate.com/github/takus/fluent-plugin-dynamodb-streams/badges/gpa.svg)](https://codeclimate.com/github/takus/fluent-plugin-dynamodb-streams)
+[![Test Coverage](https://codeclimate.com/github/takus/fluent-plugin-dynamodb-streams/badges/coverage.svg)](https://codeclimate.com/github/takus/fluent-plugin-dynamodb-streams/coverage)
+
+Fluentd input plugin for [AWS DynamoDB Streams](http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Streams.html).
+
+## Preparation
+
+Create IAM user with a policy like the following.
+
+```
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "dynamodb:GetRecords",
+        "dynamodb:GetShardIterator",
+        "dynamodb:DescribeStream",
+        "dynamodb:ListStreams"
+      ],
+      "Resource": "*"
+    }
+  ]
+}
+```
+
+Or define `aws_key_id` and `aws_sec_key` in your config file.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'fluent-plugin-dynamodb-streams'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install fluent-plugin-dynamodb-streams
+
+## Configuration
+
+```apache
+<source>
+  type dynamodb_streams
+  #aws_key_id  AWS_ACCESS_KEY_ID
+  #aws_sec_key AWS_SECRET_ACCESS_KEY
+  #aws_region  AWS_DEFAULT_REGION
+  stream_arn arn:aws:dynamodb:ap-northeast-1:000000000000:table/table_name/stream/2015-01-01T00:00:00.000
+  pos_file /var/lib/fluent/dynamodb_streams_table_name
+  fetch_interval 1
+  fetch_size 1
+</source>
+```
+
+- `tag`: Fluentd tag.
+- `stream_arn`: DynamoDB Streams ARN.
+- `pos_file`: File to store last sequence number.
+- `fetch_interval`: The interval to fetch records in seconds. Default is 1 sec.
+- `fetch_size`: The maximum number of records fetches in each iteration. Default is 1.
+
+## Output
+
+```javascript
+{
+  "aws_region": "ap-northeast-1",
+  "event_source": "aws:dynamodb",
+  "event_version": "1.0",
+  "event_id": "dfbdf4fe-6f2b-4b34-9b17-4b8caae561fa",
+  "event_name": "INSERT",
+  "dynamodb": {
+    "stream_view_type": "NEW_AND_OLD_IMAGES",
+    "sequence_number": "000000000000000000001",
+    "size_bytes": 14,
+    "keys": {
+      "key": "value2"
+    },
+    "old_image": {
+      "key": "value1"
+    },
+    "new_image": {
+      "key": "value2"
+    }
+  }
+}
+```
+
+## TODO
+
+- store sequence number to DynamoDB
+- fetch records from each shards concurrently

data/Rakefile

@@ -0,0 +1,9 @@
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'test'
+  test.test_files = FileList['test/plugin/*.rb']
+end
+

data/example/fluentd.conf

@@ -0,0 +1,22 @@
+<source>
+  type dynamodb_streams
+  tag stream
+  aws_region ddblocal
+  stream_arn "#{ENV['STREAM_ARN']}"
+</source>
+
+# Only pass MODIFY event
+<filter stream>
+  type grep
+  regexp1 event_name MODIFY
+</filter>
+
+# Only keep new_image
+<filter stream>
+  type jq
+  jq '.dynamodb|{new_image:.new_image}'
+</filter>
+
+<match stream> 
+  type stdout
+</match>

data/fluent-plugin-dynamodb-streams.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-dynamodb-streams"
+  spec.version       = "0.0.1"
+  spec.authors       = ["Takumi Sakamoto"]
+  spec.email         = ["takumi.saka@gmail.com"]
+  spec.summary       = %q{Amazon DynamoDB Streams input plugin for Fluentd}
+  spec.description   = %q{Amazon DynamoDB Streams input plugin for Fluentd}
+  spec.homepage      = "https://github.com/takus/fluent-plugin-dynamodb-streams"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "fluentd", [">= 0.10.58", "< 2"]
+  spec.add_dependency "aws-sdk", '~> 2'
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "test-unit", ">= 3.1.0"
+  spec.add_development_dependency "dynamodb-local"
+end

data/lib/fluent/plugin/in_dynamodb_streams.rb

@@ -0,0 +1,192 @@
+module Fluent
+  class DynamoDBStreamsInput < Input
+    Fluent::Plugin.register_input('dynamodb_streams', self)
+
+    # Define `router` method of v0.12 to support v0.10 or earlier
+    unless method_defined?(:router)
+      define_method("router") { Fluent::Engine }
+    end
+
+    def initialize
+      super
+      require 'aws-sdk'
+      require 'bigdecimal'
+    end
+
+    config_param :tag, :string
+    config_param :aws_key_id, :string, :default => nil, :secret => true
+    config_param :aws_sec_key, :string, :default => nil, :secret => true
+    config_param :aws_region, :string, :default => "ap-northeast-1"
+    config_param :stream_arn, :string
+    config_param :fetch_interval, :time, :default => 1
+    config_param :fetch_size, :integer, :default => 1
+    config_param :pos_file, :string, :default => nil
+
+    def configure(conf)
+      super
+
+      if @aws_region == "ddblocal"
+        @aws_region = "ap-northeast-1" # dummy settings
+        @stream_endpoint = "http://localhost:8000"
+      else
+        @stream_endpoint = "https://streams.dynamodb.#{@aws_region}.amazonaws.com"
+      end
+
+      unless @pos_file
+        log.warn "dynamodb-streams: 'pos_file PATH' parameter is not set to a 'dynamodb-streams' source."
+        log.warn "dynamodb-streams: this parameter is highly recommended to save the position to resume."
+      end
+    end
+
+    def start
+      super
+
+      unless @pos_file
+        @pos_memory = {}
+      end
+        
+      options = {}
+      options[:region] = @aws_region if @aws_region
+      options[:credentials] = Aws::Credentials.new(@aws_key_id, @aws_sec_key) if @aws_key_id && @aws_sec_key
+      options[:endpoint] = @stream_endpoint
+      @client = Aws::DynamoDBStreams::Client.new(options)
+
+      @running = true
+      @thread = Thread.new(&method(:run))
+    end
+
+    def shutdown
+      @running = false
+      @thread.join
+    end
+
+    def run
+      while @running
+        sleep @fetch_interval
+
+        @client.describe_stream({
+          stream_arn: @stream_arn
+        }).stream_description.shards.each do |s|
+
+          if s.sequence_number_range.ending_sequence_number
+            remove_sequence(s.shard_id)
+            next
+          end
+
+          if load_sequence(s.shard_id)
+            iterator = @client.get_shard_iterator({
+              stream_arn: @stream_arn,
+              shard_id: s.shard_id,
+              shard_iterator_type: "AFTER_SEQUENCE_NUMBER",
+              sequence_number: load_sequence(s.shard_id),
+            }).shard_iterator
+          else
+            iterator = @client.get_shard_iterator({
+              stream_arn: @stream_arn,
+              shard_id: s.shard_id,
+              shard_iterator_type: "TRIM_HORIZON",
+            }).shard_iterator
+          end
+
+          resp = @client.get_records({
+            shard_iterator: iterator,
+            limit: @fetch_size,
+          })
+
+          resp.records.each do |r|
+            begin
+              emit(r)
+            rescue => e
+              log.error "dynamodb-streams: error has occoured.", error: e.message, error_class: e.class
+            end
+            save_sequence(s.shard_id, r.dynamodb.sequence_number)
+          end
+        end
+      end
+    end
+
+    def load_sequence(shard_id)
+      if @pos_file
+        return nil unless File.exist?("#{@pos_file}.#{shard_id}")
+        File.read("#{@pos_file}.#{shard_id}").chomp
+      else
+        return nil unless @pos_memory[shard_id]
+        @pos_memory[shard_id]
+      end
+    end
+
+    def save_sequence(shard_id, sequence)
+      if @pos_file
+        open("#{@pos_file}.#{shard_id}", 'w') do |f|
+          f.write sequence
+        end
+      else
+        @pos_memory[shard_id] = sequence
+      end
+      sequence
+    end
+
+    def remove_sequence(shard_id)
+      if @pos_file
+        return unless File.exist?("#{@pos_file}.#{shard_id}")
+        File.unlink("#{@pos_file}.#{shard_id}")
+      else
+        @pos_memory[shard_id] = nil
+      end
+    end
+
+    def emit(r)
+      record = {
+        "aws_region" => r.aws_region,
+        "event_source" => r.event_source,
+        "event_version" => r.event_version,
+        "event_id" => r.event_id,
+        "event_name" => r.event_name,
+        "dynamodb" => {
+          "stream_view_type" => r.dynamodb.stream_view_type,
+          "sequence_number" => r.dynamodb.sequence_number,
+          "size_bytes" => r.dynamodb.size_bytes,
+        }
+      }
+      record["dynamodb"]["keys"] = dynamodb_to_hash(r.dynamodb.keys) if r.dynamodb.keys
+      record["dynamodb"]["old_image"] = dynamodb_to_hash(r.dynamodb.old_image) if r.dynamodb.old_image
+      record["dynamodb"]["new_image"] = dynamodb_to_hash(r.dynamodb.new_image) if r.dynamodb.new_image
+      router.emit(@tag, Time.now.to_i, record)
+    end
+
+    def dynamodb_to_hash(hash)
+      hash.each do |k, v|
+        # delete binary attributes
+        if v.b || v.bs
+          hash.delete(k)
+        else
+          hash[k] = format_attribute_value(v)
+        end
+      end
+      return hash
+    end
+
+    def format_attribute_value(v)
+      if v.m
+        return dynamodb_to_hash(v.m)
+      elsif v.l
+        return v.l.map {|i| format_attribute_value(i) }
+      elsif v.ns
+        return v.ns.map {|i| BigDecimal.new(i).to_i }
+      elsif v.ss
+        return v.ss
+      elsif v.null
+        return null
+      elsif v.bool
+        return v.bool
+      elsif v.n
+        return BigDecimal.new(v.n).to_i
+      elsif v.s
+        return v.s
+      else
+        log.warn "dynamodb-streams: unknown attribute value."
+      end
+    end
+
+  end
+end

data/test/plugin/in_dynamodb_streams.rb

@@ -0,0 +1,88 @@
+require "test_helper"
+
+class DynamoDBStreamsInputTest < Test::Unit::TestCase
+  include DynamoDBStreamsTestHelper
+
+  def setup
+    Fluent::Test.setup
+  end
+
+  def test_configure
+    d = create_driver(
+      %[
+        tag             streams
+        aws_key_id      test_key_id
+        aws_sec_key     test_sec_key
+        aws_region      ap-northeast-1
+        stream_arn      arn:aws:dynamodb:ap-northeast-1:123456789012:table/fluent-plugin-dynamodb-streams/stream/2015-09-01T00:00:00.000
+        pos_file        /tmp/fluent-plugin-dynamodb-streams.pos
+        fetch_interval  5
+        fetch_size      100
+      ]
+    )
+
+    assert_equal 'test_key_id', d.instance.aws_key_id
+    assert_equal 'test_sec_key', d.instance.aws_sec_key
+    assert_equal 'ap-northeast-1', d.instance.aws_region
+    assert_equal 'arn:aws:dynamodb:ap-northeast-1:123456789012:table/fluent-plugin-dynamodb-streams/stream/2015-09-01T00:00:00.000', d.instance.stream_arn
+    assert_equal '/tmp/fluent-plugin-dynamodb-streams.pos', d.instance.pos_file
+    assert_equal 5.0, d.instance.fetch_interval
+    assert_equal 100, d.instance.fetch_size
+  end
+
+  def test_emit
+    delete_table
+    create_table
+
+    time_ms = (Time.now.to_f * 1000).floor
+
+    d = create_driver(
+      %[
+        tag         test
+        aws_key_id  dummy
+        aws_sec_key dummy
+        aws_region  ddblocal
+        stream_arn  #{@stream_arn}
+      ]
+    )
+
+    d.run do
+      sleep 1
+      put_records([
+        {key:"k1", timestamp:time_ms, bool:true, hash:{k:"v"}, l:[{k:"v"}], ns:[1,2,3], ss:["1","2","3"]},
+        {key:"k2", timestamp:time_ms},
+      ])
+      sleep 1
+    end
+
+    emits = d.emits
+
+    assert_equal(2, emits.size)
+
+    assert_equal("test", emits[0][0])
+    assert_equal(
+      {"key" => "k1", "timestamp" => time_ms, "bool" => true, "hash" => {"k" => "v"}, "l" => [{"k" => "v"}], "ns" => [1,2,3], "ss" => ["1","2","3"]},
+      emits[0][2]["dynamodb"]["new_image"],
+    )
+
+    assert_equal("test", emits[1][0])
+    assert_equal({"key" => "k2", "timestamp" => time_ms}, emits[1][2]["dynamodb"]["new_image"])
+
+    assert_equal("ddblocal", emits[0][2]["aws_region"])
+    assert_true(emits[0][2]["event_id"].size > 0)
+    assert_equal("aws:dynamodb", emits[0][2]["event_source"])
+    assert_true(emits[0][2]["event_version"].size > 0)
+    assert_equal("INSERT", emits[0][2]["event_name"])
+    assert_true(emits[0][2]["dynamodb"]["sequence_number"].size > 0)
+    assert_equal("NEW_AND_OLD_IMAGES", emits[0][2]["dynamodb"]["stream_view_type"])
+    assert_equal(78, emits[0][2]["dynamodb"]["size_bytes"])
+
+    delete_table
+  end
+
+  private
+  def create_driver(conf = CONFIG)
+    Fluent::Test::InputTestDriver.new(Fluent::DynamoDBStreamsInput).configure(conf)
+  end
+
+end

data/test/test_helper.rb

@@ -0,0 +1,66 @@
+require "codeclimate-test-reporter"
+CodeClimate::TestReporter.start
+
+require 'fluent/test'
+require 'fluent/plugin/in_dynamodb_streams'
+require 'aws-sdk'
+
+module DynamoDBStreamsTestHelper
+
+  TEST_TABLE_NAME = "in_dynamodb_streams"
+
+  private
+  def dynamodb
+    @ddb ||= Aws::DynamoDB::Client.new(
+      region: 'ap-northeast-1',
+      access_key_id: 'dummy',
+      secret_access_key: 'dummy',
+      endpoint: 'http://localhost:8000',
+    )
+  end
+
+  def create_table
+    @stream_arn = dynamodb.create_table({
+      table_name: TEST_TABLE_NAME,
+      key_schema: [
+        {
+          attribute_name: "key",
+          key_type: "HASH",
+        }
+      ],
+      attribute_definitions: [
+        {
+          attribute_name: "key",
+          attribute_type: "S",
+        }
+      ],
+      provisioned_throughput: {
+        read_capacity_units: 1,
+        write_capacity_units: 1,
+      },
+      stream_specification: {
+        stream_enabled: true,
+        stream_view_type: "NEW_AND_OLD_IMAGES",
+      }
+    }).table_description.latest_stream_arn
+  end
+  
+  def put_records(records)
+    records.each do |r|
+      dynamodb.put_item(
+        table_name: TEST_TABLE_NAME,
+        item: r,
+      )
+    end
+  end
+  
+  def delete_table
+    dynamodb.list_tables.table_names.each do |t|
+      if t == TEST_TABLE_NAME
+        dynamodb.delete_table({
+          table_name: TEST_TABLE_NAME,
+        })
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,147 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-dynamodb-streams
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Takumi Sakamoto
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-09-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.58
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.58
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+- !ruby/object:Gem::Dependency
+  name: dynamodb-local
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Amazon DynamoDB Streams input plugin for Fluentd
+email:
+- takumi.saka@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- example/fluentd.conf
+- fluent-plugin-dynamodb-streams.gemspec
+- lib/fluent/plugin/in_dynamodb_streams.rb
+- test/plugin/in_dynamodb_streams.rb
+- test/test_helper.rb
+homepage: https://github.com/takus/fluent-plugin-dynamodb-streams
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Amazon DynamoDB Streams input plugin for Fluentd
+test_files:
+- test/plugin/in_dynamodb_streams.rb
+- test/test_helper.rb