rimless 0.1.4 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cb723bcb062c5d9a9266ef38998fe17ab11975cbf943333235cc5fd878201399
-  data.tar.gz: a827fa49f7d2ab7e486ec7d26bdeb312fa9c6027a156292508d212e5e8c30d89
+  metadata.gz: a7bb2e99346e0f71cdd800b802a9d405e3188303a7017b375442c50a66872cfb
+  data.tar.gz: 73f37b16200f0ac00cec1756ffd085a8fcd1a3265fc8af8b652aca130161bf95
 SHA512:
-  metadata.gz: 22366b62a462bbe2ffbce01c3382a53e7773d125442f6e4f516aab2a225abe5fbb3920c3c22323df43094daaf07bcdc5307e7ed9b80e2f6d428c4485d3779781
-  data.tar.gz: 4f53842c9b8f721a42ba275abbcad699d0349d2db11aa8f9492b62d8405621ccd3d1a2e2d7b168a55a758e46fdf8dce5b0a692c7943de5aa954d2526f8c13693
+  metadata.gz: 42541dacf4cd9c0ce8c7dc837e0f0d289d4be93c0b940f3b2fd724ec08ed70f4d4a69e92b0726a81951e509c83c2c4e69a48b92884862404225754fbce4ef109
+  data.tar.gz: 84986a34d1f651dcb1f215d25762f1e1b6c9f18aa8068ed34090fa15f0185284be3ab1b77e8b29c2dddb9b74b3a6f655728b42d196af9650ed7b7253710ad4c9

data/CHANGELOG.md

@@ -1,3 +1,22 @@
+### 0.2.0
+
+* Added the `Rimless.encode` (`.avro_encode`) and `Rimless.decode`
+  (`.avro_decode`) helpers/shortcuts to simplify the interaction with the
+  Apache Avro library
+  * The `.encode` method automatically performs input data sanitation and
+    supports deep relative (to the local namespace) schema resolution. This
+    allows you to access deeply located schemes relative by just providing a
+    leading period (eg. `.deep.a.b.c` becomes
+    `development.your_app.deep.a.b.c`)
+* The `.message`, '.sync_message', '.async_message' helpers now make use of the
+  new `.encode` functionality which adds transparent data sanitation and schema
+  name features
+* At the `test` environment the compiled Avro schemas output path is not
+  deleted anymore, instead the compiled schemas are overwritten. This may keep
+  dead schemas, but it allows parallel test execution without flaws. The removal
+  of the compiled schema directory caused previously file read errors when a
+  parallel process started a recompilation.
+
 ### 0.1.4
 
 * Reconfigure (reset) the AvroTurf instance on tests to avoid caching issues

data/Dockerfile

@@ -11,7 +11,8 @@ RUN apt-get update -yqqq && \
     ca-certificates \
     bash-completion inotify-tools && \
   echo 'en_US.UTF-8 UTF-8' >> /etc/locale.gen && /usr/sbin/locale-gen && \
-  gem install bundler --no-document --no-prerelease
+  gem install bundler -v '~> 2.0' --no-document --no-prerelease && \
+  gem install bundler -v '~> 1.0' --no-document --no-prerelease
 
 # Add new web user
 RUN mkdir /app && \

data/Envfile

@@ -0,0 +1,6 @@
+LANG=en_US.UTF-8
+LANGUAGE=en_US.UTF-8
+LC_ALL=en_US.UTF-8
+
+# Just fix this, so Appraisals with Rails 4 are working fine
+BUNDLER_VERSION=1.17.3

data/Makefile

@@ -116,7 +116,7 @@ endif
 clean-images:
 	# Clean build images
 ifeq ($(MAKE_ENV),docker)
-	@-$(DOCKER) images | $(GREP) hausgold-sdk \
+	@-$(DOCKER) images | $(GREP) rimless \
 		| $(AWK) '{ print $$3 }' \
 		| $(XARGS) -rn1 $(DOCKER) rmi -f
 endif

data/README.md

@@ -23,6 +23,7 @@ opinionated framework which sets up solid conventions for producing messages.
     - [Confluent Schema Registry Subject](#confluent-schema-registry-subject)
   - [Organize and write schema definitions](#organize-and-write-schema-definitions)
   - [Producing messages](#producing-messages)
+  - [Encoding/Decoding messages](#encodingdecoding-messages)
     - [Handling of schemaless deep blobs](#handling-of-schemaless-deep-blobs)
   - [Writing tests for your messages](#writing-tests-for-your-messages)
 - [Development](#development)
@@ -254,8 +255,11 @@ Rimless.message(data: user, schema: :user_v1, topic: :users)
 Rimless.async_message(data: user, schema: :user_v1, topic: :users)
 
 # In cases you just want the encoded Apache Avro binary blob, you can encode it
-# directly via the AvroTurf gem
-encoded = Rimless.avro.encode(user, schema_name: 'user_v1')
+# directly with our simple helper like this:
+encoded = Rimless.encode(user, schema: 'user_v1')
+# Next to this wrapped shortcut (see Encoding/Decoding messages section for
+# details), we provide access to our configured AvroTurf gem instance via
+# +Rimless.avro+, so you can also use +Rimless.avro.encode(user, ..)+
 
 # You can also send raw messages with the rimless gem, so encoding of your
 # message must be done before
@@ -275,6 +279,30 @@ Rimless.message(data: user, schema: :user_v1,
 Rimless.async_raw_message(data: encoded, topic: :users)
 ```
 
+### Encoding/Decoding messages
+
+By convention we focus on the [Apache Avro](https://avro.apache.org/) data
+format. This is provided by the [AvroTurf](https://rubygems.org/gems/avro_turf)
+gem and the rimless gem adds some neat helpers on top of it. Here are a few
+examples to show how rimless can be used to encode/decode Apache Avro data:
+
+```ruby
+# Encode a data structure (no matter of symbolized, or stringified keys, or
+# non-simple types) to Apache Avro format
+encoded = Rimless.encode(user, schema: 'user_v1')
+
+# Works the same for symbolized schema names
+encoded = Rimless.encode(user, schema: :user_v1)
+
+# Also supports the resolution of deep relative schemes
+# (+.user.address+ becomes +<ENV>.<APP>.user.address+)
+encoded = Rimless.encode(user.address, schema: '.user.address')
+
+# Decoding Apache Avro data is even more simple. The resulting data structure
+# is deeply key-symbolized.
+decoded = Rimless.decode('your-avro-binary-data-here')
+```
+
 #### Handling of schemaless deep blobs
 
 Apache Avro is by design a strict, type casted format which does not allow

data/docker-compose.yml

@@ -2,6 +2,7 @@ version: "3"
 services:
   test:
     build: .
+    env_file: Envfile
     network_mode: bridge
     working_dir: /app
     volumes:

data/lib/rimless/avro_helpers.rb

@@ -8,6 +8,41 @@ module Rimless
     class_methods do
       # A top-level avro instance
       mattr_accessor :avro
+      # A shared AvroUtils instance
+      mattr_accessor :avro_utils
+
+      # A shortcut to encode data using the specified schema to the Apache Avro
+      # format. This also applies data sanitation to avoid issues with the low
+      # level Apache Avro library (symbolized keys, etc) and it allows
+      # deep-relative schema names. When you pass +.deep.deep+ for example
+      # (leading period) it will prefix the schema name with the local
+      # namespace (so it becomes absolute).
+      #
+      # @param data [Mixed] the data structure to encode
+      # @param schema [String, Symbol] name of the schema that should be used
+      # @param opts [Hash{Symbol => Mixed}] additional options
+      # @return [String] the Apache Avro blob
+      def avro_encode(data, schema:, **opts)
+        data = avro_sanitize(data)
+
+        # When the deep-relative form (+.deep.deep[..]+) is present, we add our
+        # local namespace, so Avro can resolve it
+        schema = avro_utils.namespace + schema.to_s \
+          if schema.to_s.start_with? '.'
+
+        avro.encode(data, schema_name: schema.to_s, **opts)
+      end
+      alias_method :encode, :avro_encode
+
+      # A shortcut to parse a blob of Apache Avro data.
+      #
+      # @param data [String] the Apache Avro blob
+      # @param opts [Hash{Symbol => Mixed}] additional options
+      # @return [Mixed] the decoded data structure
+      def avro_decode(data, **opts)
+        avro.decode(data, **opts).deep_symbolize_keys!
+      end
+      alias_method :decode, :avro_decode
 
       # The Apache Avro Ruby gem requires simple typed hashes for encoding.
       # This forces us to convert eg. Grape entity representations into simple

data/lib/rimless/avro_utils.rb

@@ -61,7 +61,12 @@ module Rimless
 
     # Clear previous compiled Avro schema files to provide a clean rebuild.
     def clear
-      FileUtils.rm_rf(output_path)
+      # In a test environment, especially with parallel test execution the
+      # recompiling of Avro schemas is error prone due to the deletion of the
+      # configuration (output) directory. This leads to random failures due to
+      # file read calls to temporary not existing files. So we just keep the
+      # files and just overwrite them in place while testing.
+      FileUtils.rm_rf(output_path) unless Rimless.env.test?
       FileUtils.mkdir_p(output_path)
     end
 

data/lib/rimless/dependencies.rb

@@ -2,6 +2,8 @@
 
 module Rimless
   # The top-level dependencies helpers.
+  #
+  # rubocop:disable Metrics/BlockLength because its an Active Support concern
   module Dependencies
     extend ActiveSupport::Concern
 
@@ -53,17 +55,18 @@ module Rimless
 
         # Setup a global available Apache Avro decoder/encoder with support for
         # the Confluent Schema Registry, but first create a helper instance
-        avro_utils = Rimless::AvroUtils.new
+        Rimless.avro_utils = Rimless::AvroUtils.new
         # Compile our Avro schema templates to ready-to-consume Avro schemas
-        avro_utils.recompile_schemas
+        Rimless.avro_utils.recompile_schemas
         # Register a global Avro messaging instance
         Rimless.avro = AvroTurf::Messaging.new(
           logger: Rimless.logger,
-          namespace: avro_utils.namespace,
-          schemas_path: avro_utils.output_path,
+          namespace: Rimless.avro_utils.namespace,
+          schemas_path: Rimless.avro_utils.output_path,
           registry_url: Rimless.configuration.schema_registry_url
         )
       end
     end
   end
+  # rubocop:enable Metrics/BlockLength
 end

data/lib/rimless/kafka_helpers.rb

@@ -50,7 +50,7 @@ module Rimless
       # @param topic [String, Symbol, Hash{Symbol => Mixed}] the destination
       #   Apache Kafka topic
       def sync_message(data:, schema:, topic:, **args)
-        encoded = Rimless.avro.encode(data, schema_name: schema.to_s)
+        encoded = Rimless.encode(data, schema: schema)
         sync_raw_message(data: encoded, topic: topic, **args)
       end
       alias_method :message, :sync_message
@@ -66,7 +66,7 @@ module Rimless
       # @param topic [String, Symbol, Hash{Symbol => Mixed}] the destination
       #   Apache Kafka topic
       def async_message(data:, schema:, topic:, **args)
-        encoded = Rimless.avro.encode(data, schema_name: schema.to_s)
+        encoded = Rimless.encode(data, schema: schema)
         async_raw_message(data: encoded, topic: topic, **args)
       end
 

data/lib/rimless/rspec/helpers.rb

@@ -8,9 +8,10 @@ module Rimless
       # A simple helper to parse a blob of Apache Avro data.
       #
       # @param data [String] the Apache Avro blob
+      # @param opts [Hash{Symbol => Mixed}] additional options
       # @return [Hash{String => Mixed}] the parsed payload
-      def avro_parse(data)
-        Rimless.avro.decode(data)
+      def avro_parse(data, **opts)
+        Rimless.avro_decode(data, **opts)
       end
     end
   end

data/lib/rimless/version.rb

@@ -2,5 +2,5 @@
 
 module Rimless
   # The version of the +rimless+ gem
-  VERSION = '0.1.4'
+  VERSION = '0.2.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rimless
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.2.0
 platform: ruby
 authors:
 - Hermann Mayer
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-04-17 00:00:00.000000000 Z
+date: 2019-06-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -327,6 +327,7 @@ files:
 - Appraisals
 - CHANGELOG.md
 - Dockerfile
+- Envfile
 - Gemfile
 - LICENSE
 - Makefile