cyclone_lariat 0.3.5 → 0.3.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1a2cb273b81db1d5340fb4e0082e2d672365ebf75dcdfa25681f18d1d3c1fcb6
-  data.tar.gz: c5a458e595a79ed1c11b8560fe92eed4c72d6f763bc0912fe6caa4b83c7011ac
+  metadata.gz: 9cc046e8076d845378a3e32c67b9afe728f59e98d3728246b7835480464cb6f0
+  data.tar.gz: 7e521071b62ec40f378cee86d06b42d5a7e1857ca4ad72ddf9a92d97d7e2a2fa
 SHA512:
-  metadata.gz: 7220818e4617f98673116c0f952ae011bb278a7b0bde70f39f7b1d603db5665f1c8667c4242dd20b56be8ed55439554bcbd8fee52de4618a82864820774e0bfe
-  data.tar.gz: bddaa075da7cafba0037e69e2bdd53520dc6afc257deb2dedf7726ef2beb622dc46b9225c8fdd74e11441f9b03defe0949db4b48044de807f4c8b4c806a56732
+  metadata.gz: 93830e825f2035d114ebb96ab8692d27670a644429675ddd8f21b582109adb4f7c8a0285bdd0f3c4b55d63af2da1b8f25155de1681250a2aef1f6d053366c012
+  data.tar.gz: ab100001f182c78762f0278ced7e6b404c25ef02380540946137e1022bda883359300029dce9e44457bc4fce0dd649a0d72ff6f6b4e2f0ffa66fb09c8d1ac834

data/.rubocop.yml

@@ -7,7 +7,7 @@ inherit_mode:
     - Exclude
 
 AllCops:
-  TargetRubyVersion: 2.6
+  TargetRubyVersion: 2.4
   Exclude:
     - Rakefile
   UseCache: true

data/.ruby-version

@@ -1 +1 @@
-ruby-2.6.0
+ruby-2.4.6

data/CHANGELOG.md

@@ -4,6 +4,19 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.3.8] - 2022-09-05
+Changed
+- Added pagination to sns topics list for receiving all topics
+- Added list topics store for reduce unnecessary requests to aws
+
+## [0.3.7] - 2022-02-10
+Changed
+- Add exception when received broken JSON
+
+## [0.3.6] - 2022-02-09
+Changed
+- Downgrade minimum ruby version to 2.4.0
+
 ## [0.3.3] - 2021-07-14
 Changed
 - Bugfix of message equality check
@@ -32,8 +45,8 @@ Added
 
 ## [0.2.2] - 2021-06-08
 Changed
-- Fix save to database 
-- Rename error to client error 
+- Fix save to database
+- Rename error to client error
 
 ## [0.2.1] - 2021-06-02
 - Fix can load from database if error_details is nil

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    cyclone_lariat (0.3.5)
+    cyclone_lariat (0.3.8)
       aws-sdk-sns
       aws-sdk-sqs
       luna_park (~> 0.11)
@@ -9,75 +9,40 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.7.0)
+    addressable (2.8.0)
       public_suffix (>= 2.0.2, < 5.0)
     ast (2.4.2)
     aws-eventstream (1.2.0)
-    aws-partitions (1.541.0)
-    aws-sdk-core (3.124.0)
+    aws-partitions (1.626.0)
+    aws-sdk-core (3.140.0)
       aws-eventstream (~> 1, >= 1.0.2)
       aws-partitions (~> 1, >= 1.525.0)
       aws-sigv4 (~> 1.1)
-      jmespath (~> 1.0)
-    aws-sdk-sns (1.49.0)
-      aws-sdk-core (~> 3, >= 3.122.0)
+      jmespath (~> 1, >= 1.6.1)
+    aws-sdk-sns (1.54.0)
+      aws-sdk-core (~> 3, >= 3.127.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-sqs (1.48.0)
-      aws-sdk-core (~> 3, >= 3.122.0)
+    aws-sdk-sqs (1.51.1)
+      aws-sdk-core (~> 3, >= 3.127.0)
       aws-sigv4 (~> 1.1)
-    aws-sigv4 (1.4.0)
+    aws-sigv4 (1.5.1)
       aws-eventstream (~> 1, >= 1.0.2)
     byebug (11.1.3)
     coderay (1.1.3)
-    concurrent-ruby (1.1.8)
     crack (0.4.5)
       rexml
     database_cleaner-core (2.0.1)
     database_cleaner-sequel (2.0.0)
       database_cleaner-core (~> 2.0.0)
       sequel
-    diff-lcs (1.4.4)
-    docile (1.4.0)
-    dry-configurable (0.12.1)
-      concurrent-ruby (~> 1.0)
-      dry-core (~> 0.5, >= 0.5.0)
-    dry-container (0.7.2)
-      concurrent-ruby (~> 1.0)
-      dry-configurable (~> 0.1, >= 0.1.3)
-    dry-core (0.5.0)
-      concurrent-ruby (~> 1.0)
-    dry-equalizer (0.3.0)
-    dry-inflector (0.2.0)
-    dry-initializer (3.0.4)
-    dry-logic (1.2.0)
-      concurrent-ruby (~> 1.0)
-      dry-core (~> 0.5, >= 0.5)
-    dry-schema (1.6.2)
-      concurrent-ruby (~> 1.0)
-      dry-configurable (~> 0.8, >= 0.8.3)
-      dry-core (~> 0.5, >= 0.5)
-      dry-initializer (~> 3.0)
-      dry-logic (~> 1.0)
-      dry-types (~> 1.5)
-    dry-types (1.5.1)
-      concurrent-ruby (~> 1.0)
-      dry-container (~> 0.3)
-      dry-core (~> 0.5, >= 0.5)
-      dry-inflector (~> 0.1, >= 0.1.2)
-      dry-logic (~> 1.0, >= 1.0.2)
-    dry-validation (1.6.0)
-      concurrent-ruby (~> 1.0)
-      dry-container (~> 0.7, >= 0.7.1)
-      dry-core (~> 0.4)
-      dry-equalizer (~> 0.2)
-      dry-initializer (~> 3.0)
-      dry-schema (~> 1.5, >= 1.5.2)
+    diff-lcs (1.5.0)
+    docile (1.3.5)
     hashdiff (1.0.1)
-    jmespath (1.4.0)
-    luna_park (0.11.3)
+    jmespath (1.6.1)
+    luna_park (0.11.1)
     method_source (1.0.0)
     parallel (1.20.1)
-    parser (3.0.1.1)
+    parser (3.1.0.0)
       ast (~> 2.4.1)
     pg (1.2.3)
     pry (0.13.1)
@@ -87,23 +52,23 @@ GEM
       byebug (~> 11.0)
       pry (~> 0.13.0)
     public_suffix (4.0.6)
-    rainbow (3.0.0)
-    rake (13.0.3)
-    regexp_parser (2.1.1)
+    rainbow (3.1.1)
+    rake (13.0.6)
+    regexp_parser (2.2.0)
     rexml (3.2.5)
-    rspec (3.10.0)
-      rspec-core (~> 3.10.0)
-      rspec-expectations (~> 3.10.0)
-      rspec-mocks (~> 3.10.0)
-    rspec-core (3.10.1)
-      rspec-support (~> 3.10.0)
-    rspec-expectations (3.10.1)
+    rspec (3.11.0)
+      rspec-core (~> 3.11.0)
+      rspec-expectations (~> 3.11.0)
+      rspec-mocks (~> 3.11.0)
+    rspec-core (3.11.0)
+      rspec-support (~> 3.11.0)
+    rspec-expectations (3.11.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-mocks (3.10.2)
+      rspec-support (~> 3.11.0)
+    rspec-mocks (3.11.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-support (3.10.2)
+      rspec-support (~> 3.11.0)
+    rspec-support (3.11.0)
     rubocop (0.93.1)
       parallel (~> 1.10)
       parser (>= 2.7.1.5)
@@ -113,34 +78,32 @@ GEM
       rubocop-ast (>= 0.6.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 2.0)
-    rubocop-ast (1.5.0)
-      parser (>= 3.0.1.1)
+    rubocop-ast (1.4.1)
+      parser (>= 2.7.1.5)
     ruby-progressbar (1.11.0)
-    sequel (5.44.0)
-    simplecov (0.21.2)
+    sequel (5.53.0)
+    simplecov (0.18.5)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
-      simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
-    simplecov_json_formatter (0.1.3)
     timecop (0.9.4)
-    unicode-display_width (1.7.0)
+    unicode-display_width (1.8.0)
     webmock (3.7.6)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
-    yard (0.9.26)
+    webrick (1.7.0)
+    yard (0.9.27)
+      webrick (~> 1.7.0)
 
 PLATFORMS
   ruby
-  x86_64-linux
 
 DEPENDENCIES
   bundler (~> 1.17)
   byebug (~> 11.1)
   cyclone_lariat!
   database_cleaner-sequel (~> 2.0)
-  dry-validation (~> 1.1)
   pg (~> 1.2)
   pry (~> 0.13)
   pry-byebug (~> 3.9)

data/README.md

@@ -147,6 +147,7 @@ If you use middleware:
 ```ruby
 require 'cyclone_lariat/middleware' # If require: false in Gemfile
 
+
 class Receiver
   include Shoryuken::Worker
   

data/cyclone_lariat.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |spec|
   spec.summary       = 'Shoryuken middleware for LunaPark based application.'
   spec.homepage      = 'https://am-team.github.io/cyclone_lariat/#/'
   spec.license       = 'MIT'
-  spec.required_ruby_version = '>= 2.6.0'
+  spec.required_ruby_version = '>= 2.4.0'
 
   # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
   # to allow pushing to a single host or delete this section to allow pushing to any host.
@@ -39,7 +39,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'bundler', '~> 1.17'
   spec.add_development_dependency 'byebug', '~> 11.1'
   spec.add_development_dependency 'database_cleaner-sequel', '~> 2.0'
-  spec.add_development_dependency 'dry-validation', '~> 1.1'
   spec.add_development_dependency 'pg', '~> 1.2'
   spec.add_development_dependency 'pry', '~> 0.13'
   spec.add_development_dependency 'pry-byebug', '~> 3.9'

data/lib/cyclone_lariat/list_topics_store.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require 'singleton'
+
+module CycloneLariat
+  class ListTopicsStore
+    include Singleton
+
+    def topic_arn(topic_name)
+      @topics[topic_name.to_sym]
+    end
+
+    def list
+      @topics.keys
+    end
+
+    def add_topics(aws_client)
+      @topics ||= {}
+      @aws_client = aws_client
+      fetch
+    end
+
+    def clear_store!
+      @topics = {}
+    end
+
+    private
+
+    def fetch
+      return unless @topics.empty?
+
+      @next_token = ''
+      topics_from_aws until @next_token.nil?
+    end
+
+    def topics_from_aws
+      result = @aws_client.list_topics(next_token: @next_token)
+      @next_token = result.next_token
+      result.topics.each do |topic|
+        topic_name = topic.topic_arn.split(':').last
+        @topics[topic_name.to_sym] = topic.topic_arn
+      end
+    end
+  end
+end

data/lib/cyclone_lariat/messages_mapper.rb

@@ -12,18 +12,17 @@ module CycloneLariat
       end
 
       def to_row(input)
-        row = {
-          uuid:                   input.uuid,
-          kind:                   input.kind,
-          type:                   input.type,
-          publisher:              input.publisher,
-          data:                   JSON.generate(input.data),
-          client_error_message:   input.client_error&.message,
-          client_error_details:   JSON.generate(input.client_error&.details),
-          version:                input.version,
-          sent_at:                input.sent_at
+        {
+          uuid: input.uuid,
+          kind: input.kind,
+          type: input.type,
+          publisher: input.publisher,
+          data: JSON.generate(input.data),
+          client_error_message: input.client_error&.message,
+          client_error_details: JSON.generate(input.client_error&.details),
+          version: input.version,
+          sent_at: input.sent_at
         }
-        row
       end
 
       private
@@ -45,5 +44,3 @@ module CycloneLariat
     end
   end
 end
-
-

data/lib/cyclone_lariat/middleware.rb

@@ -13,9 +13,10 @@ module CycloneLariat
     end
 
     def call(_worker_instance, queue, _sqs_msg, body, &block)
-      msg = receive_message(body)
+      msg = receive_message!(body)
 
       message_notifier&.info 'Receive message', message: msg, queue: queue
+      return if msg.is_a? String
 
       catch_standard_error(queue, msg) do
         event = Event.wrap(msg)
@@ -30,8 +31,11 @@ module CycloneLariat
 
     attr_reader :errors_notifier, :message_notifier, :events_repo
 
-    def receive_message(body)
-      body[:Message] ? JSON.parse(body[:Message], symbolize_names: true ) : body
+    def receive_message!(body)
+      body[:Message] ? JSON.parse(body[:Message], symbolize_names: true) : body
+    rescue JSON::ParserError => e
+      errors_notifier&.error(e, message: body[:Message])
+      body[:Message]
     end
 
     def store_in_dataset(event)

data/lib/cyclone_lariat/sns_client.rb

@@ -2,6 +2,7 @@
 
 require 'aws-sdk-sns'
 require_relative 'abstract/client'
+require_relative 'list_topics_store'
 
 module CycloneLariat
   class SnsClient < Abstract::Client
@@ -31,11 +32,21 @@ module CycloneLariat
     private
 
     def topic_arn(topic_name)
-      list  = aws_client.list_topics.topics
-      topic = list.find { |t| t.topic_arn.match?(topic_name) }
-      raise Errors::TopicNotFound.new(expected_topic: topic_name, existed_topics: list.map(&:topic_arn)) if topic.nil?
+      topics_store.add_topics(aws_client)
+      topic_arn = topics_store.topic_arn(topic_name)
 
-      topic.topic_arn
+      if topic_arn.nil?
+        raise Errors::TopicNotFound.new(
+          expected_topic: topic_name,
+          existed_topics: topics_store.list
+        )
+      end
+
+      topic_arn
+    end
+
+    def topics_store
+      ListTopicsStore.instance
     end
   end
 end

data/lib/cyclone_lariat/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module CycloneLariat
-  VERSION = '0.3.5'
+  VERSION = '0.3.8'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cyclone_lariat
 version: !ruby/object:Gem::Version
-  version: 0.3.5
+  version: 0.3.8
 platform: ruby
 authors:
 - Alexander Kudrin
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-12-16 00:00:00.000000000 Z
+date: 2022-09-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-sns
@@ -95,20 +95,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
-- !ruby/object:Gem::Dependency
-  name: dry-validation
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.1'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.1'
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
@@ -285,6 +271,7 @@ files:
 - lib/cyclone_lariat/command.rb
 - lib/cyclone_lariat/errors.rb
 - lib/cyclone_lariat/event.rb
+- lib/cyclone_lariat/list_topics_store.rb
 - lib/cyclone_lariat/messages_mapper.rb
 - lib/cyclone_lariat/messages_repo.rb
 - lib/cyclone_lariat/middleware.rb
@@ -305,7 +292,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.6.0
+      version: 2.4.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="