kcl-rb 1.0.0

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/demo/Gemfile

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+
+gem 'kcl-rb', path: '../'
+gem 'pry'
+gem 'rake', '~> 12.0'

data/demo/Gemfile.lock

@@ -0,0 +1,60 @@
+PATH
+  remote: ..
+  specs:
+    kcl-rb (1.0.0)
+      activesupport (>= 5.0)
+      aws-sdk-dynamodb (~> 1)
+      aws-sdk-kinesis (~> 1)
+      eventmachine (~> 1.2.7)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (6.0.3.1)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+      zeitwerk (~> 2.2, >= 2.2.2)
+    aws-eventstream (1.1.0)
+    aws-partitions (1.326.0)
+    aws-sdk-core (3.98.0)
+      aws-eventstream (~> 1, >= 1.0.2)
+      aws-partitions (~> 1, >= 1.239.0)
+      aws-sigv4 (~> 1.1)
+      jmespath (~> 1.0)
+    aws-sdk-dynamodb (1.48.0)
+      aws-sdk-core (~> 3, >= 3.71.0)
+      aws-sigv4 (~> 1.1)
+    aws-sdk-kinesis (1.23.0)
+      aws-sdk-core (~> 3, >= 3.71.0)
+      aws-sigv4 (~> 1.1)
+    aws-sigv4 (1.1.4)
+      aws-eventstream (~> 1.0, >= 1.0.2)
+    coderay (1.1.3)
+    concurrent-ruby (1.1.6)
+    eventmachine (1.2.7)
+    i18n (1.8.3)
+      concurrent-ruby (~> 1.0)
+    jmespath (1.4.0)
+    method_source (1.0.0)
+    minitest (5.14.1)
+    pry (0.13.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    rake (12.3.3)
+    thread_safe (0.3.6)
+    tzinfo (1.2.7)
+      thread_safe (~> 0.1)
+    zeitwerk (2.3.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  kcl-rb!
+  pry
+  rake (~> 12.0)
+
+BUNDLED WITH
+   2.1.4

data/demo/README.md

@@ -0,0 +1,38 @@
+# kcl-rb Demo App
+
+## Build and Run
+
+Run localstack container (mock for Kinesis and DynamoDB).
+
+```
+$ docker-compose up
+```
+
+Create resources on localstack using Terraform
+
+```
+$ cd terraform
+$ terraform init
+$ terraform plan
+$ terraform apply
+```
+
+Build dependencies
+
+```
+$ bundle install --path vendor/bundle
+```
+
+Run Demo KCL application
+
+```
+$ bundle exec rake run
+```
+
+Put records to Kinesis stream
+
+```
+$ RECORD_COUNT=10 bundle exec rake seed
+```
+
+You can see in console that the input data is distributed and processed by each consumer.

data/demo/Rakefile

@@ -0,0 +1,31 @@
+require 'bundler/setup'
+require 'pry'
+
+require 'kcl'
+require_relative './lib/kcl_demo'
+
+task :default => :run
+task :run do
+  KclDemo::App.initialize
+  KclDemo::App.run
+end
+
+task :seed do
+  KclDemo::App.initialize
+  record_count = Integer(ENV['RECORD_COUNT'] ||0)
+  if record_count.zero?
+    puts 'Set over 1 for RECORD_COUNT'
+    return
+  end
+  KclDemo::App.seed(record_count)
+end
+
+task :debug do
+  KclDemo::App.initialize
+
+  kinesis = Kcl::Proxies::KinesisProxy.new(KclDemo::App.config)
+  dynamodb = Kcl::Proxies::DynamoDbProxy.new(KclDemo::App.config)
+  # rubocop:disable Lint/Debugger
+  binding.pry
+  # rubocop:enable Lint/Debugger
+end

data/demo/aws/config

@@ -0,0 +1,3 @@
+[default]
+region = ap-northeast-1
+output = json

data/demo/aws/credentials

@@ -0,0 +1,3 @@
+[default]
+aws_access_key_id     = dummy
+aws_secret_access_key = dummy

data/demo/docker-compose.yml

@@ -0,0 +1,23 @@
+version: '3'
+
+volumes:
+  localstack-data:
+    driver: local
+
+services:
+  localstack:
+    image: localstack/localstack:0.11.0
+    container_name: localstack-for-kcl-demo
+    ports:
+      - "8080:8080"
+      - "4566:4566"
+    environment:
+      - DATA_DIR=/tmp/localstack/data
+      - DEBUG=${LOCALSTACK_DEBUG:-true}
+      - DEFAULT_REGION=ap-northeast-1
+      - SERVICES=dynamodb,kinesis
+      - USE_SSL=true
+    volumes:
+      - "${PWD}/aws:/root/.aws"
+      - "/var/run/docker.sock:/var/run/docker.sock"
+      - "localstack-data:/tmp/localstack"

data/demo/lib/kcl_demo.rb

@@ -0,0 +1,49 @@
+require 'json'
+require 'securerandom'
+
+require_relative './kcl_demo/demo_record_processor'
+require_relative './kcl_demo/demo_record_processor_factory'
+
+module KclDemo
+  class App
+    def self.initialize
+      Kcl.configure do |config|
+        config.aws_region = 'ap-northeast-1'
+        config.aws_access_key_id = 'dummy'
+        config.aws_secret_access_key = 'dummy'
+        config.dynamodb_endpoint = 'https://localhost:4566'
+        config.dynamodb_table_name = 'kcl-rb-demo'
+        config.kinesis_endpoint = 'https://localhost:4566'
+        config.kinesis_stream_name = 'kcl-rb-demo'
+        config.use_ssl = false
+      end
+    end
+
+    def self.config
+      Kcl.config
+    end
+
+    def self.run
+      factory = KclDemo::DemoRecordProcessorFactory.new
+      Kcl::Worker.run('kcl-demo', factory)
+    end
+
+    def self.seed(record_count = 1000)
+      proxy = Kcl::Proxies::KinesisProxy.new(config)
+
+      # puts records
+      record_count.times do |i|
+        str = SecureRandom.alphanumeric
+        hash = JSON.generate({ id: i, name: str })
+        resp = proxy.put_record(
+          {
+            stream_name: config.kinesis_stream_name,
+            data: Base64.strict_encode64(hash),
+            partition_key: str
+          }
+        )
+        puts resp
+      end
+    end
+  end
+end

data/demo/lib/kcl_demo/demo_record_processor.rb

@@ -0,0 +1,43 @@
+require 'pry'
+
+module KclDemo
+  class DemoRecordProcessor < Kcl::RecordProcessor
+    # @implement
+    def after_initialize(initialization_input)
+      Kcl.logger.info("Initialization at #{initialization_input}")
+    end
+
+    # @implement
+    def process_records(records_input)
+      Kcl.logger.info('Processing records...')
+
+      # レコードのリストを取得
+      return if records_input.records.empty?
+
+      # rubocop:disable Lint/Debugger
+      binding.pry if ENV['DEBUG'] == '1'
+      # rubocop:enable Lint/Debugger
+
+      records_input.records.each do |record|
+        Kcl.logger.info("Record = #{record}")
+      end
+
+      # チェックポイントを記録
+      last_sequence_number = records_input.records[-1].sequence_number
+      Kcl.logger.info(
+        "Checkpoint progress at: #{last_sequence_number}" \
+        ", MillisBehindLatest = #{records_input.millis_behind_latest}"
+      )
+      records_input.record_checkpointer.update_checkpoint(last_sequence_number)
+    end
+
+    # @implement
+    def shutdown(shutdown_input)
+      Kcl.logger.info("Shutdown reason: #{shutdown_input.shutdown_reason}")
+
+      if shutdown_input.shutdown_reason == Kcl::Workers::ShutdownReason::TERMINATE
+        shutdown_input.record_checkpointer.update_checkpoint(nil)
+      end
+    end
+  end
+end

data/demo/lib/kcl_demo/demo_record_processor_factory.rb

@@ -0,0 +1,7 @@
+module KclDemo
+  class DemoRecordProcessorFactory < Kcl::RecordProcessorFactory
+    def create_processor
+      KclDemo::DemoRecordProcessor.new
+    end
+  end
+end

data/demo/terraform/main.tf

@@ -0,0 +1,35 @@
+#-------------------------------------------------------------------------------
+# 開発環境 (LocalStack)
+#-------------------------------------------------------------------------------
+
+provider "aws" {
+  version                     = "~> 2.60"
+  access_key                  = "dummy"
+  secret_key                  = "dummy"
+  region                      = "ap-northeast-1"
+  insecure                    = true
+  skip_credentials_validation = true
+  skip_metadata_api_check     = true
+  skip_requesting_account_id  = true
+  
+  endpoints {
+    dynamodb = "https://localhost:4566"
+    kinesis  = "https://localhost:4566"
+  }
+}
+
+
+#-------------------------------------------------------------------------------
+# Kinesis stream
+#-------------------------------------------------------------------------------
+
+resource "aws_kinesis_stream" "kcl-rb-demo_stream" {
+  name             = "kcl-rb-demo"
+  shard_count      = 5
+  retention_period = 24
+
+  tags = {
+    Environment = "test"
+  }
+}
+

data/docker-compose.yml

@@ -0,0 +1,22 @@
+version: '3'
+
+volumes:
+  localstack-data:
+    driver: local
+
+services:
+  localstack:
+    image: localstack/localstack:0.11.0
+    container_name: localstack-for-kcl
+    ports:
+      - "8080:8080"
+      - "4566:4566"
+    environment:
+      - DATA_DIR=/tmp/localstack/data
+      - DEBUG=${LOCALSTACK_DEBUG:-true}
+      - DEFAULT_REGION=ap-northeast-1
+      - SERVICES=dynamodb,kinesis
+      - USE_SSL=true
+    volumes:
+      - "${PWD}/aws:/root/.aws"
+      - "localstack-data:/tmp/localstack"

data/kcl-rb.gemspec

@@ -0,0 +1,36 @@
+require_relative 'lib/kcl/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'kcl-rb'
+  spec.version       = Kcl::VERSION
+  spec.authors       = ['yo_waka']
+  spec.email         = ['y.wakahara@gmail.com']
+
+  spec.summary       = 'Amazon.Kinesis Client Library for Ruby.'
+  spec.description   = 'A pure ruby interface for Amazon Kinesis Client.'
+  spec.homepage      = 'https://github.com/waka/kcl-rb'
+  spec.license       = 'MIT'
+  spec.required_ruby_version = Gem::Requirement.new('>= 2.3.0')
+
+  spec.metadata['homepage_uri'] = spec.homepage
+  spec.metadata['source_code_uri'] = 'https://github.com/waka/kcl-rb'
+  spec.metadata['changelog_uri'] = 'https://github.com/waka/kcl-rb/CHANGELOG'
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'activesupport', '>= 5.0'
+  spec.add_dependency 'aws-sdk-dynamodb', '~> 1'
+  spec.add_dependency 'aws-sdk-kinesis', '~> 1'
+  spec.add_dependency 'eventmachine', '~> 1.2.7'
+
+  spec.add_development_dependency 'rake', '~> 12.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'rubocop', '~> 0.86.0'
+end

data/lib/kcl.rb

@@ -0,0 +1,32 @@
+require 'kcl/checkpointer'
+require 'kcl/checkpoints/sentinel'
+require 'kcl/config'
+require 'kcl/errors'
+require 'kcl/logger'
+require 'kcl/proxies/dynamo_db_proxy'
+require 'kcl/proxies/kinesis_proxy'
+require 'kcl/record_processor'
+require 'kcl/record_processor_factory'
+require 'kcl/types/extended_sequence_number'
+require 'kcl/types/initialization_input'
+require 'kcl/types/records_input'
+require 'kcl/types/shutdown_input'
+require 'kcl/worker'
+require 'kcl/workers/consumer'
+require 'kcl/workers/record_checkpointer'
+require 'kcl/workers/shard_info'
+require 'kcl/workers/shutdown_reason'
+
+module Kcl
+  def self.configure
+    yield config
+  end
+
+  def self.config
+    @_config ||= Kcl::Config.new
+  end
+
+  def self.logger
+    @_logger ||= (config.logger || Kcl::Logger.new($stdout))
+  end
+end

data/lib/kcl/checkpointer.rb

@@ -0,0 +1,179 @@
+require 'time'
+
+class Kcl::Checkpointer
+  DYNAMO_DB_LEASE_PRIMARY_KEY = 'shard_id'.freeze
+  DYNAMO_DB_LEASE_OWNER_KEY   = 'assigned_to'.freeze
+  DYNAMO_DB_LEASE_TIMEOUT_KEY = 'lease_timeout'.freeze
+  DYNAMO_DB_CHECKPOINT_SEQUENCE_NUMBER_KEY = 'checkpoint'.freeze
+  DYNAMO_DB_PARENT_SHARD_KEY  = 'parent_shard_id'.freeze
+
+  attr_reader :dynamodb
+
+  # @param [Kcl::Config] config
+  def initialize(config)
+    @dynamodb = Kcl::Proxies::DynamoDbProxy.new(config)
+    @table_name = config.dynamodb_table_name
+
+    return if @dynamodb.exists?(@table_name)
+    @dynamodb.create_table(
+      @table_name,
+      [{
+        attribute_name: DYNAMO_DB_LEASE_PRIMARY_KEY,
+        attribute_type: 'S'
+      }],
+      [{
+        attribute_name: DYNAMO_DB_LEASE_PRIMARY_KEY,
+        key_type: 'HASH'
+      }],
+      {
+        read_capacity_units: config.dynamodb_read_capacity,
+        write_capacity_units: config.dynamodb_write_capacity
+      }
+    )
+    Kcl.logger.info("Created DynamoDB table: #{@table_name}")
+  end
+
+  # Retrieves the checkpoint for the given shard
+  # @params [Kcl::Workers::ShardInfo] shard
+  # @return [Kcl::Workers::ShardInfo]
+  def fetch_checkpoint(shard)
+    checkpoint = @dynamodb.get_item(
+      @table_name,
+      { "#{DYNAMO_DB_LEASE_PRIMARY_KEY}" => shard.shard_id }
+    )
+    return shard if checkpoint.nil?
+
+    if checkpoint[DYNAMO_DB_CHECKPOINT_SEQUENCE_NUMBER_KEY]
+      shard.checkpoint = checkpoint[DYNAMO_DB_CHECKPOINT_SEQUENCE_NUMBER_KEY]
+    end
+    if checkpoint[DYNAMO_DB_LEASE_OWNER_KEY]
+      shard.assigned_to = checkpoint[DYNAMO_DB_LEASE_OWNER_KEY]
+    end
+    Kcl.logger.info("Retrieves checkpoint of shard at #{shard.to_h}")
+
+    shard
+  end
+
+  # Write the checkpoint for the given shard
+  # @params [Kcl::Workers::ShardInfo] shard
+  # @return [Kcl::Workers::ShardInfo]
+  def update_checkpoint(shard)
+    item = {
+      "#{DYNAMO_DB_LEASE_PRIMARY_KEY}" => shard.shard_id,
+      "#{DYNAMO_DB_CHECKPOINT_SEQUENCE_NUMBER_KEY}" => shard.checkpoint,
+      "#{DYNAMO_DB_LEASE_OWNER_KEY}" => shard.assigned_to,
+      "#{DYNAMO_DB_LEASE_TIMEOUT_KEY}" => shard.lease_timeout.to_s
+    }
+    if shard.parent_shard_id > 0
+      item[DYNAMO_DB_PARENT_SHARD_KEY] = shard.parent_shard_id
+    end
+
+    result = @dynamodb.put_item(@table_name, item)
+    if result
+      Kcl.logger.info("Write checkpoint of shard at #{shard.to_h}")
+    else
+      Kcl.logger.info("Failed to write checkpoint for shard at #{shard.to_h}")
+    end
+
+    shard
+  end
+
+  # Attempt to gain a lock on the given shard
+  # @params [Kcl::Workers::ShardInfo] shard
+  # @params [String] next_assigned_to
+  # @return [Kcl::Workers::ShardInfo]
+  def lease(shard, next_assigned_to)
+    now = Time.now.utc
+    next_lease_timeout = now + Kcl.config.dynamodb_failover_seconds
+
+    checkpoint = @dynamodb.get_item(
+      @table_name,
+      { "#{DYNAMO_DB_LEASE_PRIMARY_KEY}" => shard.shard_id }
+    )
+    assigned_to   = checkpoint && checkpoint[DYNAMO_DB_LEASE_OWNER_KEY]
+    lease_timeout = checkpoint && checkpoint[DYNAMO_DB_LEASE_TIMEOUT_KEY]
+
+    if assigned_to && lease_timeout
+      if now > Time.parse(lease_timeout) && assigned_to != next_assigned_to
+        raise Kcl::Errors::LeaseNotAquiredError
+      end
+      condition_expression = 'shard_id = :shard_id AND assigned_to = :assigned_to AND lease_timeout = :lease_timeout'
+      expression_attributes = {
+        ':shard_id' => shard.shard_id,
+        ':assigned_to' => assigned_to,
+        ':lease_timeout' => lease_timeout
+      }
+      Kcl.logger.info("Attempting to get a lock for shard: #{shard.to_h}")
+    else
+      condition_expression = 'attribute_not_exists(assigned_to)'
+      expression_attributes = nil
+    end
+
+    item = {
+      "#{DYNAMO_DB_LEASE_PRIMARY_KEY}" => shard.shard_id,
+      "#{DYNAMO_DB_LEASE_OWNER_KEY}"   => next_assigned_to,
+      "#{DYNAMO_DB_LEASE_TIMEOUT_KEY}" => next_lease_timeout.to_s
+    }
+    if shard.checkpoint != ''
+      item[DYNAMO_DB_CHECKPOINT_SEQUENCE_NUMBER_KEY] = shard.checkpoint
+    end
+    if shard.parent_shard_id > 0
+      item[DYNAMO_DB_PARENT_SHARD_KEY] = shard.parent_shard_id
+    end
+
+    result = @dynamodb.conditional_update_item(
+      @table_name,
+      item,
+      condition_expression,
+      expression_attributes
+    )
+    if result
+      shard.assigned_to   = next_assigned_to
+      shard.lease_timeout = next_lease_timeout
+      Kcl.logger.info("Get lease for shard at #{shard.to_h}")
+    else
+      Kcl.logger.info("Failed to get lease for shard at #{shard.to_h}")
+    end
+
+    shard
+  end
+
+  # Remove the shard entry
+  # @params [Kcl::Workers::ShardInfo] shard
+  # @return [Kcl::Workers::ShardInfo]
+  def remove_lease(shard)
+    result = @dynamodb.remove_item(
+      @table_name,
+      { "#{DYNAMO_DB_LEASE_PRIMARY_KEY}" => shard.shard_id }
+    )
+    if result
+      shard.assigned_to   = nil
+      shard.checkpoint    = nil
+      shard.lease_timeout = nil
+      Kcl.logger.info("Remove lease for shard at #{shard.to_h}")
+    else
+      Kcl.logger.info("Failed to remove lease for shard at #{shard.to_h}")
+    end
+
+    shard
+  end
+
+  # Remove lease owner for the shard entry
+  # @params [Kcl::Workers::ShardInfo] shard
+  # @return [Kcl::Workers::ShardInfo]
+  def remove_lease_owner(shard)
+    result = @dynamodb.update_item(
+      @table_name,
+      { "#{DYNAMO_DB_LEASE_PRIMARY_KEY}" => shard.shard_id },
+      "remove #{DYNAMO_DB_LEASE_OWNER_KEY}"
+    )
+    if result
+      shard.assigned_to = nil
+      Kcl.logger.info("Remove lease owner for shard at #{shard.to_h}")
+    else
+      Kcl.logger.info("Failed to remove lease owner for shard at #{shard.to_h}")
+    end
+
+    shard
+  end
+end