kcl-rb 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 647ea8327b5ed00e957b90c4072b05fa7898ebf2ba3b64c6b8a7e2e13e327445
+  data.tar.gz: f5664d982cd02f065bd8d995e574e781864d6d3aa3808e96a13994a82de814af
+SHA512:
+  metadata.gz: 4f6e8f989caa626203dfaa6a681ff766299e9337a054e70ad98422ebb57eae11ba108a9651216b818ff010bf46c0d722a76c7b4bcd06b7bdb13b9451f50ff8ff
+  data.tar.gz: e4534afe312c6bf4915d263116163f53dcab44d7bb92928d27a5f2e9f2b6e68b2db987b2a9038b6c89e5e75a101ace4369740b9e939abfcd39b85ed1b5c8b796

data/.github/workflows/main.yml

@@ -0,0 +1,58 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - master
+    paths-ignore:
+      - '**.md'
+      - '**.txt'
+  pull_request:
+    paths-ignore:
+      - '**.md'
+      - '**.txt'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+
+    - name: Up docker-compose
+      run: docker-compose -f docker-compose.yml up -d
+
+    - name: Sleep for 30 seconds
+      uses: jakejarvis/wait-action@master
+      with:
+        time: '30s'
+
+    - name: Set up Terraform
+      uses: hashicorp/setup-terraform@v1
+      with:
+        terraform_version: 0.12.24
+    - id: init
+      run: terraform init
+      working-directory: ${{ github.workspace }}/terraform
+    - id: apply
+      run: terraform apply -auto-approve -no-color
+      working-directory: ${{ github.workspace }}/terraform
+
+    - name: Set up Ruby 2.7
+      uses: actions/setup-ruby@v1
+      with:
+        ruby-version: 2.7
+
+    - name: Bundle install
+      run: |
+        gem install bundler
+        bundle install --jobs 4 --retry 3
+
+    - name: Rubocop
+      run: |
+        bundle exec rubocop
+
+    - name: RSpec
+      run: |
+        bundle exec rspec

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/terraform/terraform.tfstate*
+/terraform/.terraform
+/vendor
+
+/demo/.bundle/
+/demo/terraform/terraform.tfstate*
+/demo/terraform/.terraform
+/demo/vendor
+
+/pkg

data/.rubocop.yml

@@ -0,0 +1,93 @@
+AllCops:
+  Exclude:
+    - "vendor/**/*"
+    - "demo/vendor/**/*"
+  DisplayCopNames: true
+
+Layout/ArgumentAlignment:
+  Enabled: false
+
+Layout/ExtraSpacing:
+  Enabled: false
+
+Layout/HashAlignment:
+  Enabled: false
+
+Lint/NonDeterministicRequireOrder:
+  Enabled: false
+
+Layout/SpaceAroundOperators:
+  Enabled: false
+
+Metrics/AbcSize:
+  Enabled: false
+
+Metrics/BlockLength:
+  Enabled: false
+
+Metrics/ClassLength:
+  Max: 300
+
+Metrics/MethodLength:
+  Max: 50
+
+Naming/MemoizedInstanceVariableName:
+  Enabled: false
+
+Metrics/CyclomaticComplexity:
+  Max: 15
+
+Metrics/PerceivedComplexity:
+  Max: 15
+
+# 日本語のコメントを許可する
+Style/AsciiComments:
+  Enabled: false
+
+Style/ClassAndModuleChildren:
+  Enabled: false
+
+Style/Documentation:
+  Enabled: false
+
+Style/EmptyLineAfterGuardClause:
+  Enabled: false
+
+Style/EmptyMethod:
+  Enabled: false
+
+Style/FrozenStringLiteralComment:
+  Enabled: false
+
+Style/GuardClause:
+  MinBodyLength: 5
+
+Style/HashSyntax:
+  Enabled: false
+ 
+Style/IfUnlessModifier:
+  Enabled: false
+
+Style/MultilineTernaryOperator:
+  Enabled: false
+
+Style/NumericPredicate:
+  Enabled: false
+
+Style/RaiseArgs:
+  Enabled: false
+
+Style/RedundantBegin:
+  Enabled: false
+
+Style/RedundantInterpolation:
+  Enabled: false
+
+Style/RedundantSelf:
+  Enabled: false
+
+Style/RescueStandardError:
+  Enabled: false
+
+Style/SymbolArray:
+  Enabled: false

data/.ruby-version

@@ -0,0 +1 @@
+2.7.1

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in kcl-rb.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,90 @@
+PATH
+  remote: .
+  specs:
+    kcl-rb (1.0.0)
+      activesupport (>= 5.0)
+      aws-sdk-dynamodb (~> 1)
+      aws-sdk-kinesis (~> 1)
+      eventmachine (~> 1.2.7)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (6.0.3.1)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+      zeitwerk (~> 2.2, >= 2.2.2)
+    ast (2.4.1)
+    aws-eventstream (1.1.0)
+    aws-partitions (1.326.0)
+    aws-sdk-core (3.98.0)
+      aws-eventstream (~> 1, >= 1.0.2)
+      aws-partitions (~> 1, >= 1.239.0)
+      aws-sigv4 (~> 1.1)
+      jmespath (~> 1.0)
+    aws-sdk-dynamodb (1.48.0)
+      aws-sdk-core (~> 3, >= 3.71.0)
+      aws-sigv4 (~> 1.1)
+    aws-sdk-kinesis (1.23.0)
+      aws-sdk-core (~> 3, >= 3.71.0)
+      aws-sigv4 (~> 1.1)
+    aws-sigv4 (1.1.4)
+      aws-eventstream (~> 1.0, >= 1.0.2)
+    concurrent-ruby (1.1.6)
+    diff-lcs (1.4.2)
+    eventmachine (1.2.7)
+    i18n (1.8.3)
+      concurrent-ruby (~> 1.0)
+    jmespath (1.4.0)
+    minitest (5.14.1)
+    parallel (1.19.2)
+    parser (2.7.1.4)
+      ast (~> 2.4.1)
+    rainbow (3.0.0)
+    rake (12.3.3)
+    regexp_parser (1.7.1)
+    rexml (3.2.4)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.2)
+      rspec-support (~> 3.9.3)
+    rspec-expectations (3.9.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.3)
+    rubocop (0.86.0)
+      parallel (~> 1.10)
+      parser (>= 2.7.0.1)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.7)
+      rexml
+      rubocop-ast (>= 0.0.3, < 1.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 2.0)
+    rubocop-ast (0.1.0)
+      parser (>= 2.7.0.1)
+    ruby-progressbar (1.10.1)
+    thread_safe (0.3.6)
+    tzinfo (1.2.7)
+      thread_safe (~> 0.1)
+    unicode-display_width (1.7.0)
+    zeitwerk (2.3.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  kcl-rb!
+  rake (~> 12.0)
+  rspec (~> 3.0)
+  rubocop (~> 0.86.0)
+
+BUNDLED WITH
+   2.1.4

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2020 yo_waka
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,130 @@
+# kcl-rb
+
+## Overview
+
+The Amazon Kinesis Client Library for Pure Ruby (Amazon KCL) enables Ruby developers to easily consume and process data from [Amazon Kinesis](http://aws.amazon.com/kinesis).
+
+Already [KCL for Ruby](https://github.com/awslabs/amazon-kinesis-client-ruby) is provided by AWS, but Java is required for the operating environment because MultiLangDaemon is used.
+**kcl-rb** is built on Pure Ruby, not depend on Java.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'kcl-rb'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install kcl-rb
+
+## Usage
+
+It's okay if you develop it according to [the KCL specifications](https://docs.aws.amazon.com/streams/latest/dev/kinesis-record-processor-implementation-app-java.html).
+
+### Implement the RecordProcessor
+
+```rb
+class RecordProcessor < Kcl::RecordProcessor
+  def after_initialize(initialization_input)
+    puts "SHARD_ID: #{initialization_input.shard_id}"
+  end
+
+  def process_records(records_input)
+    puts "Current behind: #{records_input.millis_behind_latest}"
+    records_input.records.each do |record|
+      puts "Record: #{record}"
+    end
+  end
+
+  def shutdown(shutdown_input)
+    puts "Shutdown reason: #{shutdown_input.shutdown_reason}"
+
+    if shutdown_input.shutdown_reason == Kcl::Workers::ShutdownReason::TERMINATE
+      shutdown_input.record_checkpointer.update_checkpoint(nil)
+    end
+  end
+end
+```
+
+### Implement a Class Factory for the RecordProcessor
+
+```rb
+class RecordProcessorFactory < Kcl::RecordProcessorFactory
+  def create_processor
+    RecordProcessor.new
+  end
+end
+```
+
+### Initialize KCL configurations
+
+```rb
+Kcl.configure do |config|
+  config.aws_region = 'ap-northeast-1'
+  config.aws_access_key_id = 'dummy'
+  config.aws_secret_access_key = 'dummy'
+  config.dynamodb_endpoint = 'https://localhost:4566'
+  config.dynamodb_table_name = 'kcl-rb'
+  config.kinesis_endpoint = 'https://localhost:4566'
+  config.kinesis_stream_name = 'kcl-rb'
+  config.use_ssl = false
+end
+```
+
+If you want to see all the setting items, please see [config class file](https://github.com/waka/kcl-rb/blob/master/lib/kcl/config.rb).
+
+### Run a Worker
+
+```rb
+worker_id = 'kcl-worker'
+factory = RecordProcessorFactory.new
+Kcl::Worker.run(worker_id, factory)
+```
+
+If you want more concrete example, look under [the demo directory](https://github.com/waka/kcl-rb/tree/master/demo).
+
+## Development
+
+### Prerequisites
+
+- Install Ruby 2.7.1
+- Install docker
+- Install Terraform
+
+### Build & Run for RSpec
+
+Create Kinesis resources on localstack using Terraform
+
+```sh
+$ docker-compose up -d
+$ cd terraform
+$ terraform init
+$ terraform apply
+```
+
+Build dependencies.
+
+```
+$ bundle install --path vendor/bundle
+```
+
+And run RSpec.
+
+```sh
+$ bundle exec rspec
+```
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/waka/kcl-rb.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,2 @@
+require 'bundler/gem_tasks'
+task :default => :spec

data/aws/config

@@ -0,0 +1,3 @@
+[default]
+region = ap-northeast-1
+output = json

data/aws/credentials

@@ -0,0 +1,3 @@
+[default]
+aws_access_key_id     = dummy
+aws_secret_access_key = dummy

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'kcl'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+require 'pry'
+Pry.start
+
+# require 'irb'
+# IRB.start(__FILE__)