kcl-rb 1.0.0

data/lib/kcl/workers/consumer.rb

@@ -0,0 +1,80 @@
+module Kcl::Workers
+  # Shard : Consumer = 1 : 1
+  # - get records from stream
+  # - send to record processor
+  # - create record checkpoint
+  class Consumer
+    def initialize(shard, record_processor, kinesis_proxy, checkpointer)
+      @shard = shard
+      @record_processor = record_processor
+      @kinesis = kinesis_proxy
+      @checkpointer = checkpointer
+    end
+
+    def consume!
+      initialize_input = create_initialize_input
+      @record_processor.after_initialize(initialize_input)
+
+      record_checkpointer = Kcl::Workers::RecordCheckpointer.new(@shard, @checkpointer)
+      shard_iterator = start_shard_iterator
+
+      loop do
+        result = @kinesis.get_records(shard_iterator)
+
+        records_input = create_records_input(
+          result[:records],
+          result[:millis_behind_latest],
+          record_checkpointer
+        )
+        @record_processor.process_records(records_input)
+
+        shard_iterator = result[:next_shard_iterator]
+        break if result[:records].empty? || shard_iterator.nil?
+      end
+
+      shutdown_reason = shard_iterator.nil? ?
+        Kcl::Workers::ShutdownReason::TERMINATE :
+        Kcl::Workers::ShutdownReason::REQUESTED
+      shutdown_input = create_shutdown_input(shutdown_reason, record_checkpointer)
+      @record_processor.shutdown(shutdown_input)
+    end
+
+    def start_shard_iterator
+      shard = @checkpointer.fetch_checkpoint(@shard)
+      if shard.checkpoint.nil?
+        return @kinesis.get_shard_iterator(
+          @shard.shard_id,
+          Kcl::Checkpoints::Sentinel::TRIM_HORIZON
+        )
+      end
+
+      @kinesis.get_shard_iterator(
+        @shard.shard_id,
+        Kcl::Checkpoints::Sentinel::AFTER_SEQUENCE_NUMBER,
+        @shard.checkpoint
+      )
+    end
+
+    def create_initialize_input
+      Kcl::Types::InitializationInput.new(
+        @shard.shard_id,
+        Kcl::Types::ExtendedSequenceNumber.new(@shard.checkpoint)
+      )
+    end
+
+    def create_records_input(records, millis_behind_latest, record_checkpointer)
+      Kcl::Types::RecordsInput.new(
+        records,
+        millis_behind_latest,
+        record_checkpointer
+      )
+    end
+
+    def create_shutdown_input(shutdown_reason, record_checkpointer)
+      Kcl::Types::ShutdownInput.new(
+        shutdown_reason,
+        record_checkpointer
+      )
+    end
+  end
+end

data/lib/kcl/workers/record_checkpointer.rb

@@ -0,0 +1,14 @@
+module Kcl::Workers
+  class RecordCheckpointer
+    def initialize(shard, checkpointer)
+      @shard = shard
+      @checkpointer = checkpointer
+    end
+
+    def update_checkpoint(sequence_number)
+      # checkpoint the last sequence of a closed shard
+      @shard.checkpoint = sequence_number || Kcl::Checkpoints::Sentinel::SHARD_END
+      @checkpointer.update_checkpoint(@shard)
+    end
+  end
+end

data/lib/kcl/workers/shard_info.rb

@@ -0,0 +1,47 @@
+module Kcl::Workers
+  class ShardInfo
+    attr_reader :shard_id,
+      :parent_shard_id,
+      :starting_sequence_number,
+      :ending_sequence_number
+    attr_accessor :assigned_to, :checkpoint, :lease_timeout
+
+    # @param [String] shard_id
+    # @param [String] parent_shard_id
+    # @param [Hash] sequence_number_range
+    def initialize(shard_id, parent_shard_id, sequence_number_range)
+      @shard_id        = shard_id
+      @parent_shard_id = parent_shard_id || 0
+      @starting_sequence_number = sequence_number_range[:starting_sequence_number]
+      @ending_sequence_number   = sequence_number_range[:ending_sequence_number]
+      @assigned_to     = nil
+      @checkpoint      = nil
+      @lease_timeout   = nil
+    end
+
+    def lease_owner
+      @assigned_to
+    end
+
+    def lease_owner=(assigned_to)
+      @assigned_to = assigned_to
+    end
+
+    def completed?
+      @checkpoint == Kcl::Checkpoints::Sentinel::SHARD_END
+    end
+
+    # For debug
+    def to_h
+      {
+        shard_id: shard_id,
+        parent_shard_id: parent_shard_id,
+        starting_sequence_number: starting_sequence_number,
+        ending_sequence_number: ending_sequence_number,
+        assigned_to: assigned_to,
+        checkpoint: checkpoint,
+        lease_timeout: lease_timeout
+      }
+    end
+  end
+end

data/lib/kcl/workers/shutdown_reason.rb

@@ -0,0 +1,6 @@
+module Kcl::Workers
+  module ShutdownReason
+    TERMINATE = 'TERMINATE'.freeze
+    REQUESTED = 'REQUESTED'.freeze
+  end
+end

data/terraform/main.tf

@@ -0,0 +1,35 @@
+#-------------------------------------------------------------------------------
+# 開発環境 (LocalStack)
+#-------------------------------------------------------------------------------
+
+provider "aws" {
+  version                     = "~> 2.60"
+  access_key                  = "dummy"
+  secret_key                  = "dummy"
+  region                      = "ap-northeast-1"
+  insecure                    = true
+  skip_credentials_validation = true
+  skip_metadata_api_check     = true
+  skip_requesting_account_id  = true
+  
+  endpoints {
+    dynamodb = "https://localhost:4566"
+    kinesis  = "https://localhost:4566"
+  }
+}
+
+
+#-------------------------------------------------------------------------------
+# Kinesis stream
+#-------------------------------------------------------------------------------
+
+resource "aws_kinesis_stream" "kcl-rb-test_stream" {
+  name             = "kcl-rb-test"
+  shard_count      = 5
+  retention_period = 24
+
+  tags = {
+    Environment = "test"
+  }
+}
+

metadata

@@ -0,0 +1,191 @@
+--- !ruby/object:Gem::Specification
+name: kcl-rb
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- yo_waka
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2020-07-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-dynamodb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-kinesis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: eventmachine
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.2.7
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.2.7
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.86.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.86.0
+description: A pure ruby interface for Amazon Kinesis Client.
+email:
+- y.wakahara@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".github/workflows/main.yml"
+- ".gitignore"
+- ".rubocop.yml"
+- ".ruby-version"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- aws/config
+- aws/credentials
+- bin/console
+- bin/setup
+- demo/Gemfile
+- demo/Gemfile.lock
+- demo/README.md
+- demo/Rakefile
+- demo/aws/config
+- demo/aws/credentials
+- demo/docker-compose.yml
+- demo/lib/kcl_demo.rb
+- demo/lib/kcl_demo/demo_record_processor.rb
+- demo/lib/kcl_demo/demo_record_processor_factory.rb
+- demo/terraform/main.tf
+- docker-compose.yml
+- kcl-rb.gemspec
+- lib/kcl.rb
+- lib/kcl/checkpointer.rb
+- lib/kcl/checkpoints/sentinel.rb
+- lib/kcl/config.rb
+- lib/kcl/errors.rb
+- lib/kcl/logger.rb
+- lib/kcl/proxies/dynamo_db_proxy.rb
+- lib/kcl/proxies/kinesis_proxy.rb
+- lib/kcl/record_processor.rb
+- lib/kcl/record_processor_factory.rb
+- lib/kcl/types/extended_sequence_number.rb
+- lib/kcl/types/initialization_input.rb
+- lib/kcl/types/records_input.rb
+- lib/kcl/types/shutdown_input.rb
+- lib/kcl/version.rb
+- lib/kcl/worker.rb
+- lib/kcl/workers/consumer.rb
+- lib/kcl/workers/record_checkpointer.rb
+- lib/kcl/workers/shard_info.rb
+- lib/kcl/workers/shutdown_reason.rb
+- terraform/main.tf
+homepage: https://github.com/waka/kcl-rb
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/waka/kcl-rb
+  source_code_uri: https://github.com/waka/kcl-rb
+  changelog_uri: https://github.com/waka/kcl-rb/CHANGELOG
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.3.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.2
+signing_key: 
+specification_version: 4
+summary: Amazon.Kinesis Client Library for Ruby.
+test_files: []