cyclone_lariat 0.3.1 → 0.3.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5458f171a6a306641b03d57341f62228b73a49c2aa222c49e764dd2194c1ab61
-  data.tar.gz: 3fd96a4dcd07b7a52203161b75023b480312b0a6a4bf3eb76ecbaecb8122d8a9
+  metadata.gz: 1a2cb273b81db1d5340fb4e0082e2d672365ebf75dcdfa25681f18d1d3c1fcb6
+  data.tar.gz: c5a458e595a79ed1c11b8560fe92eed4c72d6f763bc0912fe6caa4b83c7011ac
 SHA512:
-  metadata.gz: 22b7f9b1c2e438368a329d118f6fcfc0b659d12d42590cea25db4961ed17ac0f36904f67a0ea18d7c6498dae5ab8b66033155bec6a883a9c0fceb1024c8b8a1a
-  data.tar.gz: ee83d106d2cf8fcee7831aece4b52ebf1083f6ec1ceb37adaea106fda27bd747907301b5baaa6efd6c420c970a6a45495a976508bf0683be71fb17b09a6634d4
+  metadata.gz: 7220818e4617f98673116c0f952ae011bb278a7b0bde70f39f7b1d603db5665f1c8667c4242dd20b56be8ed55439554bcbd8fee52de4618a82864820774e0bfe
+  data.tar.gz: bddaa075da7cafba0037e69e2bdd53520dc6afc257deb2dedf7726ef2beb622dc46b9225c8fdd74e11441f9b03defe0949db4b48044de807f4c8b4c806a56732

data/.gitignore

@@ -16,3 +16,7 @@ config/db.rb
 
 # gem builds
 *.gem
+
+# docker-compose
+/local/
+/privatekey

data/CHANGELOG.md

@@ -4,7 +4,15 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [0.3.0] - 2021-06-11
+## [0.3.3] - 2021-07-14
+Changed
+- Bugfix of message equality check
+
+## [0.3.2] - 2021-06-11
+Changed
+- Bugfix
+
+## [0.3.1] - 2021-06-11
 Changed
 - Command
 - SqsClient

data/Dockerfile

@@ -0,0 +1,10 @@
+FROM ruby:2.6.5-buster
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+        postgresql-client
+WORKDIR /gem
+COPY lib/cyclone_lariat/version.rb ./lib/cyclone_lariat/version.rb
+COPY cyclone_lariat.gemspec ./
+COPY Gemfile* ./
+RUN bundle install --jobs 20 --retry 5
+CMD ["rake"]

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    cyclone_lariat (0.3.1)
+    cyclone_lariat (0.3.5)
       aws-sdk-sns
       aws-sdk-sqs
       luna_park (~> 0.11)
@@ -12,20 +12,20 @@ GEM
     addressable (2.7.0)
       public_suffix (>= 2.0.2, < 5.0)
     ast (2.4.2)
-    aws-eventstream (1.1.1)
-    aws-partitions (1.462.0)
-    aws-sdk-core (3.114.0)
+    aws-eventstream (1.2.0)
+    aws-partitions (1.541.0)
+    aws-sdk-core (3.124.0)
       aws-eventstream (~> 1, >= 1.0.2)
-      aws-partitions (~> 1, >= 1.239.0)
+      aws-partitions (~> 1, >= 1.525.0)
       aws-sigv4 (~> 1.1)
       jmespath (~> 1.0)
-    aws-sdk-sns (1.40.0)
-      aws-sdk-core (~> 3, >= 3.112.0)
+    aws-sdk-sns (1.49.0)
+      aws-sdk-core (~> 3, >= 3.122.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-sqs (1.39.0)
-      aws-sdk-core (~> 3, >= 3.112.0)
+    aws-sdk-sqs (1.48.0)
+      aws-sdk-core (~> 3, >= 3.122.0)
       aws-sigv4 (~> 1.1)
-    aws-sigv4 (1.2.3)
+    aws-sigv4 (1.4.0)
       aws-eventstream (~> 1, >= 1.0.2)
     byebug (11.1.3)
     coderay (1.1.3)
@@ -74,7 +74,7 @@ GEM
       dry-schema (~> 1.5, >= 1.5.2)
     hashdiff (1.0.1)
     jmespath (1.4.0)
-    luna_park (0.11.1)
+    luna_park (0.11.3)
     method_source (1.0.0)
     parallel (1.20.1)
     parser (3.0.1.1)
@@ -132,10 +132,11 @@ GEM
     yard (0.9.26)
 
 PLATFORMS
+  ruby
   x86_64-linux
 
 DEPENDENCIES
-  bundler (~> 2.1)
+  bundler (~> 1.17)
   byebug (~> 11.1)
   cyclone_lariat!
   database_cleaner-sequel (~> 2.0)
@@ -152,4 +153,4 @@ DEPENDENCIES
   yard (~> 0.9)
 
 BUNDLED WITH
-   2.2.19
+   1.17.3

data/README.md

@@ -159,7 +159,6 @@ class Receiver
                     queue: 'your_sqs_queue_name'
 
   server_middleware do |chain|
-    
     # Options dataset, errors_notifier and message_notifier is optionals.
     # If you dont define notifiers - middleware does not notify
     # If you dont define dataset - middleware does store events in db

data/Rakefile

@@ -1,11 +1,15 @@
 # frozen_string_literal: true
 
 require 'rake'
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
 
 # tasks from lib directory
 Dir[File.expand_path('lib/tasks/**/*.rake', __dir__)].each do |entity|
-  puts entity
-  load entity
+  print "#{entity} : "
+  puts load entity
 end
 
 task default: %i[spec] # rubocop]

data/bin/docker/diff

@@ -0,0 +1,3 @@
+# Show differences between local Dockerfile and production Dockerfile
+
+diff Dockerfile local.Dockerfile -aBw

data/bin/docker/init

@@ -0,0 +1,38 @@
+# Initialize project to use Docker and prepare app in Docker
+#
+# WARNING
+#   if your default ssh key `id_rsa` is protected by password,
+#   you need to create a new key without password only for Github (and authorize this key in github)
+#   @example
+#     `ssh-keygen -b 4096 -f ~/.ssh/id_rsa_git -N ""`
+#
+#   And define a name of new key when calling this script.
+#   @example:
+#     `bin/docker-init id_rsa_git`
+
+bin/docker-privatekey $@
+
+echo "> Preparing volume directories"
+mkdir local
+mkdir local/postgres
+mkdir local/postgres/pgdata
+grep -q "# docker-compose" .gitignore || echo "\n# docker-compose\n/local/\n/privatekey" >> .gitignore
+
+echo "> Build"
+docker compose down
+docker compose build
+docker compose up -d
+
+# Restart because of some problems with PG first startup
+echo "> Restart"
+docker compose down
+docker compose up -d
+
+echo "> Wait for pg startup"
+sleep 10
+
+echo "> Create databases"
+docker compose exec app rake db:create
+docker compose exec app rake db:create INSTANCE=test
+docker compose exec app rake db:migrate
+docker compose exec app rake db:migrate INSTANCE=test

data/bin/docker/privatekey

@@ -0,0 +1,17 @@
+# Note:
+#   Docker needs for ssh privatekey to fetch data from github
+#   To do that, this script creates hardlink to privatekey
+#
+# WARNING
+#   privatekey MUST NOT be protected by password
+#   If your `id_rsa` is protected by password, you need to create new key without password (for example, `id_rsa_git`)
+#   And use that key when running this script:
+#     `bin/docker-init id_rsa_git`
+
+if test "$@"
+then privatekey_name="$@"
+else privatekey_name="id_rsa"
+fi
+
+echo "> Trying to create link 'privatekey'"
+ln ~/.ssh/$privatekey_name privatekey && echo "> Created link to '~/.ssh/$privatekey_name' named 'privatekey'"

data/bin/docker/rm

@@ -0,0 +1,5 @@
+# Remove Doсker local files created by docker-init
+
+docker compose down
+rm -rf local
+rm privatekey

data/config/db.example.rb

@@ -2,8 +2,8 @@
 
 DB_CONF = {
   adapter: 'postgresql',
-  host: 'host',
-  username: 'postgres',
-  password: 'password',
-  database: 'cyclone-lariat-test'
+  host: ENV.fetch('DB_HOST', 'host'),
+  username: ENV.fetch('DB_USER', 'postgres'),
+  password: ENV.fetch('DB_PASSWORD', 'password'),
+  database: ENV.fetch('DB_NAME', 'cyclone-lariat-test')
 }.freeze

data/cyclone_lariat.gemspec

@@ -36,7 +36,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'aws-sdk-sqs'
   spec.add_dependency 'luna_park', '~> 0.11'
 
-  spec.add_development_dependency 'bundler', '~> 2.1'
+  spec.add_development_dependency 'bundler', '~> 1.17'
   spec.add_development_dependency 'byebug', '~> 11.1'
   spec.add_development_dependency 'database_cleaner-sequel', '~> 2.0'
   spec.add_development_dependency 'dry-validation', '~> 1.1'

data/docker-compose.yml

@@ -0,0 +1,24 @@
+version: "3.9"
+
+services:
+  app:
+    build: .
+    volumes:
+        - .:/gem
+    command: /bin/bash -c "while true; do sleep 1; done"
+    links:
+      - pg
+    environment:
+      - DB_HOST=pg
+      - DB_USER=ruby
+      - DB_PASSWORD=ruby
+    restart: always
+
+  pg:
+    image: postgres
+    volumes:
+      - ./local/postgres/pgdata:/var/lib/postgresql/data/pgdata
+    environment:
+      POSTGRES_USER: ruby
+      POSTGRES_PASSWORD: ruby
+      PGDATA: /var/lib/postgresql/data/pgdata

data/lib/cyclone_lariat/abstract/message.rb

@@ -9,13 +9,8 @@ module CycloneLariat
       attr :uuid,      String, :new
       attr :publisher, String, :new
       attr :type,      String, :new
-      attr :client_error
-      attr :version
-      attr :data
-
-      attr_reader :sent_at,
-                  :processed_at,
-                  :received_at
+      attrs :client_error, :version, :data,
+            :sent_at, :processed_at, :received_at
 
       def kind
         raise LunaPark::Errors::AbstractMethod
@@ -37,6 +32,10 @@ module CycloneLariat
         @processed_at = wrap_time(value)
       end
 
+      def processed?
+        !@processed_at.nil?
+      end
+
       def client_error_message=(txt)
         return unless txt
 
@@ -57,11 +56,10 @@ module CycloneLariat
           publisher == other.publisher &&
           type == other.type &&
           client_error&.message == other.client_error&.message &&
-          client_error&.details == other.client_error&.details &&
           version == other.version &&
           sent_at.to_i == other.sent_at.to_i &&
-          received_at.to_i == other.received_at.to_i
-        processed_at.to_i == other.processed_at.to_i
+          received_at.to_i == other.received_at.to_i &&
+          processed_at.to_i == other.processed_at.to_i
       end
 
       def to_json(*args)
@@ -77,7 +75,7 @@ module CycloneLariat
         when String   then Time.parse(value)
         when Time     then value
         when NilClass then nil
-        else raise Argumentevent.rbError, "Unknown type `#{value.class}`"
+        else raise ArgumentError, "Unknown type `#{value.class}`"
         end
       end
     end

data/lib/cyclone_lariat/messages_mapper.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module CycloneLariat
+  class MessagesMapper
+    class << self
+      def from_row(row)
+        return if row.nil?
+
+        row[:data] = hash_from_json_column(row[:data])
+        row[:client_error_details] = hash_from_json_column(row[:client_error_details]) if row[:client_error_details]
+        row
+      end
+
+      def to_row(input)
+        row = {
+          uuid:                   input.uuid,
+          kind:                   input.kind,
+          type:                   input.type,
+          publisher:              input.publisher,
+          data:                   JSON.generate(input.data),
+          client_error_message:   input.client_error&.message,
+          client_error_details:   JSON.generate(input.client_error&.details),
+          version:                input.version,
+          sent_at:                input.sent_at
+        }
+        row
+      end
+
+      private
+
+      def hash_from_json_column(data)
+        return JSON.parse(data) if data.is_a?(String)
+
+        if pg_json_extension_enabled?
+          return data.to_h             if data.is_a?(Sequel::Postgres::JSONHash)
+          return JSON.parse(data.to_s) if data.is_a?(Sequel::Postgres::JSONString)
+        end
+
+        raise ArgumentError, "Unknown type of `#{data}`"
+      end
+
+      def pg_json_extension_enabled?
+        Object.const_defined?('Sequel::Postgres::JSONHash')
+      end
+    end
+  end
+end
+
+

data/lib/cyclone_lariat/messages_repo.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative 'event'
+require_relative 'messages_mapper'
 
 module CycloneLariat
   class MessagesRepo
@@ -11,17 +12,7 @@ module CycloneLariat
     end
 
     def create(msg)
-      dataset.insert(
-        uuid: msg.uuid,
-        kind: msg.kind,
-        type: msg.type,
-        publisher: msg.publisher,
-        data: JSON.generate(msg.data),
-        client_error_message: msg.client_error&.message,
-        client_error_details: JSON.generate(msg.client_error&.details),
-        version: msg.version,
-        sent_at: msg.sent_at
-      )
+      dataset.insert MessagesMapper.to_row(msg)
     end
 
     def exists?(uuid:)
@@ -36,32 +27,22 @@ module CycloneLariat
     end
 
     def find(uuid:)
-      raw = dataset.where(uuid: uuid).first
-      raw[:data] = JSON.parse(raw[:data], symbolize_names: true)
-      if raw[:client_error_details]
-        raw[:client_error_details] = JSON.parse(raw[:client_error_details], symbolize_names: true)
-      end
-      build raw
+      row = dataset.where(uuid: uuid).first
+      return if row.nil?
+
+      build MessagesMapper.from_row(row)
     end
 
     def each_unprocessed
-      dataset.where(processed_at: nil).each do |raw|
-        raw[:data]                 = JSON.parse(raw[:data], symbolize_names: true)
-        if raw[:client_error_details]
-          raw[:client_error_details] = JSON.parse(raw[:client_error_details], symbolize_names: true)
-        end
-        msg = build raw
+      dataset.where(processed_at: nil).each do |row|
+        msg = build MessagesMapper.from_row(row)
         yield(msg)
       end
     end
 
     def each_with_client_errors
-      dataset.where { (processed_at !~ nil) & (client_error_message !~ nil) }.each do |raw|
-        raw[:data] = JSON.parse(raw[:data], symbolize_names: true)
-        if raw[:client_error_details]
-          raw[:client_error_details] = JSON.parse(raw[:client_error_details], symbolize_names: true)
-        end
-        msg = build raw
+      dataset.where { (processed_at !~ nil) & (client_error_message !~ nil) }.each do |row|
+        msg = build MessagesMapper.from_row(row)
         yield(msg)
       end
     end

data/lib/cyclone_lariat/middleware.rb

@@ -20,8 +20,8 @@ module CycloneLariat
       catch_standard_error(queue, msg) do
         event = Event.wrap(msg)
 
-        catch_business_error(event) do
-          store_in_dataset(event, &block)
+        store_in_dataset(event) do
+          catch_business_error(event, &block)
         end
       end
     end
@@ -36,9 +36,11 @@ module CycloneLariat
 
     def store_in_dataset(event)
       return yield if events_repo.nil?
-      return true  if events_repo.exists?(uuid: event.uuid)
 
-      events_repo.create(event)
+      existed = events_repo.find(uuid: event.uuid)
+      return true if existed&.processed?
+
+      events_repo.create(event) unless existed
       yield
       events_repo.processed! uuid: event.uuid, error: event.client_error
     end
@@ -52,7 +54,7 @@ module CycloneLariat
 
     def catch_standard_error(queue, msg)
       yield
-    rescue StandardError => e
+    rescue Exception => e
       errors_notifier&.error(e, queue: queue, message: msg)
       raise e
     end

data/lib/cyclone_lariat/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module CycloneLariat
-  VERSION = '0.3.1'
+  VERSION = '0.3.5'
 end

data/lib/cyclone_lariat.rb

@@ -3,6 +3,7 @@
 require_relative 'cyclone_lariat/sns_client'
 require_relative 'cyclone_lariat/errors'
 require_relative 'cyclone_lariat/event'
+require_relative 'cyclone_lariat/messages_mapper'
 require_relative 'cyclone_lariat/messages_repo'
 require_relative 'cyclone_lariat/middleware'
 require_relative 'cyclone_lariat/version'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cyclone_lariat
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.5
 platform: ruby
 authors:
 - Alexander Kudrin
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-06-11 00:00:00.000000000 Z
+date: 2021-12-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-sns
@@ -59,14 +59,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '1.17'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '1.17'
 - !ruby/object:Gem::Dependency
   name: byebug
   requirement: !ruby/object:Gem::Requirement
@@ -263,14 +263,20 @@ files:
 - ".ruby-gemset"
 - ".ruby-version"
 - CHANGELOG.md
+- Dockerfile
 - Gemfile
 - Gemfile.lock
 - README.md
 - Rakefile
+- bin/docker/diff
+- bin/docker/init
+- bin/docker/privatekey
+- bin/docker/rm
 - config/db.example.rb
 - cyclone_lariat.gemspec
 - db/migrate/01_add_uuid_extensions.rb
 - db/migrate/02_add_events.rb
+- docker-compose.yml
 - docs/_imgs/diagram.png
 - docs/_imgs/lariat.jpg
 - lib/cyclone_lariat.rb
@@ -279,6 +285,7 @@ files:
 - lib/cyclone_lariat/command.rb
 - lib/cyclone_lariat/errors.rb
 - lib/cyclone_lariat/event.rb
+- lib/cyclone_lariat/messages_mapper.rb
 - lib/cyclone_lariat/messages_repo.rb
 - lib/cyclone_lariat/middleware.rb
 - lib/cyclone_lariat/sns_client.rb