rimless 1.2.0 → 1.3.0

data/doc/kafka-playground/Makefile

@@ -0,0 +1,209 @@
+MAKEFLAGS += --warn-undefined-variables -j1
+SHELL := bash
+.SHELLFLAGS := -eu -o pipefail -c
+.DEFAULT_GOAL := all
+.DELETE_ON_ERROR:
+.SUFFIXES:
+.PHONY:
+
+# Environment switches
+MAKE_ENV ?= docker
+MAKE_SUB_ENV ?= false
+DOCKER_MOUNT_MODE ?= rw
+IMAGE_VENDOR ?= hausgold
+PREBUILD_IMAGE_SEARCH ?= kafka-playground
+PROJECT_NAME ?= kafka-playground
+START ?= foreground
+START_CONTAINERS ?= kafka schema-registry schema-registry-ui
+COMPOSE_RUN_SHELL_FLAGS ?= --rm
+BASH_RUN_SHELL_FLAGS ?=
+BUNDLE_FLAGS ?=
+
+# Directories
+LOG_DIR ?= log
+
+# Host binaries
+AWK ?= awk
+BASH ?= bash
+CHMOD ?= chmod
+COMPOSE ?= docker-compose
+CP ?= cp
+CUT ?= cut
+DOCKER ?= docker
+ECHO ?= echo
+FIND ?= find
+GREP ?= grep
+HEAD ?= head
+LS ?= ls
+MKDIR ?= mkdir
+MV ?= mv
+NODE ?= node
+NPM ?= npm
+NPROC ?= nproc
+PRINTF ?= printf
+RM ?= rm
+TAIL ?= tail
+TEE ?= tee
+TEST ?= test
+WC ?= wc
+XARGS ?= xargs
+
+# Container binaries
+BUNDLE ?= bundle
+
+# Check all binaries which needs to be available
+CHECK_BINS ?= AWK BASH CHMOD ECHO HEAD FIND GREP LS MKDIR \
+	MV NODE NPM NPROC PRINTF TAIL TEE TEST WC XARGS
+
+ifeq ($(MAKE_ENV),docker)
+# Check also the docker binaries
+CHECK_BINS += COMPOSE DOCKER
+else ifeq ($(MAKE_ENV),baremetal)
+# Nothing to do here - just a env check
+else
+$(error MAKE_ENV got an invalid value. Use `docker` or `baremetal`)
+endif
+
+all:
+	# Apache Kafka Playground
+	#
+	# install                 Install the dependencies
+	# start                   Start the containers
+	# stop                    Stop all running containers
+	# logs                    Monitor the started containers
+	# update-images           Pull the latest Docker images and rebuild ours
+	#
+	# shell                   Start an interactive shell session
+	#
+	# clean                   Clean all temporary application files
+	# clean-containers        Clean the Docker containers (also database data)
+	# distclean               Same as clean and cleans Docker images
+
+# Check a binary
+# $1 - The binary
+define check-binary
+	$(shell if [ -n "`which g$($(1)) 2>/dev/null`" ]; then \
+		echo 'g$($(1))'; \
+		elif [ -n "`which $($(1)) 2>/dev/null`" ]; then \
+		echo '$($(1))'; \
+		else \
+		echo '$$(error Neither "$($(1))" nor "g$($(1))" is available ($(1)))'; \
+		fi)
+endef
+
+# Define a generic shell run wrapper
+# $1 - The command to run
+ifeq ($(MAKE_ENV),docker)
+define run-shell
+	$(PRINTF) '# (Docker mount mode: $(DOCKER_MOUNT_MODE))\n'; \
+	$(COMPOSE) run $(COMPOSE_RUN_SHELL_FLAGS) \
+		-e LANG=en_US.UTF-8 -e LANGUAGE=en_US.UTF-8 -e LC_ALL=en_US.UTF-8 \
+		-u app app bash $(BASH_RUN_SHELL_FLAGS) -c 'sleep 0.1; echo; $(1)'
+endef
+else ifeq ($(MAKE_ENV),baremetal)
+define run-shell
+	$(1)
+endef
+endif
+
+# Define a retry helper
+# $1 - The command to run
+define retry
+	if eval "$(call run-shell,$(1))"; then exit 0; fi; \
+	for i in 1; do sleep 10s; echo "Retrying $$i..."; \
+		if eval "$(call run-shell,$(1))"; then exit 0; fi; \
+	done; \
+	exit 1
+endef
+
+# Check all binaries
+_ := $(foreach BIN,$(CHECK_BINS),$(eval $(BIN) := $(call check-binary,$(BIN))))
+
+COMPOSE := $(COMPOSE) -p $(PROJECT_NAME)
+PREBUILT_IMAGE ?= $(PROJECT_NAME)_app:latest
+
+.interactive:
+	@$(eval BASH_RUN_SHELL_FLAGS = --login)
+
+.not-implemented:
+	# Not yet implemented.
+
+install:
+	# Install the dependencies
+ifeq ($(MAKE_ENV),docker)
+	@$(eval INSTALL_NAME = $(PROJECT_NAME)_install)
+	@$(eval COMPOSE_RUN_SHELL_FLAGS = --no-deps --name $(INSTALL_NAME))
+	@$(DOCKER) rm -f $(INSTALL_NAME) 2>/dev/null || true
+endif
+	@$(call retry,$(BUNDLE) check || \
+		$(BUNDLE) install --jobs $(shell $(NPROC)) \
+			--retry 3 $(BUNDLE_FLAGS))
+ifeq ($(MAKE_ENV),docker)
+	@$(DOCKER) commit $(INSTALL_NAME) $(PREBUILT_IMAGE)
+	@$(DOCKER) rm -f $(INSTALL_NAME) 2>/dev/null || true
+endif
+
+start: stop
+	# Start the application
+ifeq ($(START),foreground)
+	@$(COMPOSE) up $(START_CONTAINERS)
+else
+	$(error START got an invalid value. Use `foreground`.)
+endif
+
+restart:
+	# Restart the application
+	@$(MAKE) stop start
+
+logs:
+	# Monitor the started application
+	@$(COMPOSE) logs -f --tail='all'
+
+stop: clean-containers
+stop-containers:
+	# Stop all running containers
+	@$(COMPOSE) stop -t 5 || true
+	@$(DOCKER) ps -a | $(GREP) $(PROJECT_NAME)_ | $(CUT) -d ' ' -f1 \
+		| $(XARGS) -rn10 $(DOCKER) stop -t 5 || true
+
+shell:
+	# Start an interactive shell session
+	@$(call run-shell,$(BASH) -i)
+
+update-images: clean-containers clean-images
+	# Pull latest Docker images
+	@$(GREP) -Pih 'from|image:' docker-compose.yml Dockerfile \
+		| $(GREP) -Po '$(IMAGE_VENDOR).*' \
+		| $(XARGS) -rn1 $(DOCKER) pull
+	@$(MAKE) install
+
+clean-logs:
+	# Clean logs
+	@$(RM) -rf $(LOG_DIR)
+	@$(MKDIR) -p $(LOG_DIR)
+
+clean-containers: stop-containers
+	# Stop and kill all containers
+	@$(COMPOSE) rm -vf || true
+	@$(DOCKER) ps -a | $(GREP) $(PROJECT_NAME)_ | $(CUT) -d ' ' -f1 \
+		| $(XARGS) -rn10 $(DOCKER) rm -vf || true
+
+clean-images: clean-containers
+	# Remove all docker images
+	$(eval APP_NAME = $(shell $(CUT) -d: -f2 <<< $(PREBUILD_IMAGE_SEARCH)))
+	$(eval EMPTY = ) $(eval CLEAN_IMAGES = $(PROJECT_NAME)_ $(PREBUILT_IMAGE))
+	$(eval CLEAN_IMAGES += $(PREBUILD_IMAGE_SEARCH) \s+$(APP_NAME): <none>)
+	@$(DOCKER) images -a --format '{{.ID}} {{.Repository}}:{{.Tag}}' \
+		| $(GREP) -P "$(subst $(EMPTY) $(EMPTY),|,$(CLEAN_IMAGES))" \
+		| $(AWK) '{print $$0}' \
+		| $(XARGS) -rn1 $(DOCKER) rmi -f 2>&1 \
+		| $(GREP) -vP 'cannot be forced|invalid reference' || true
+
+clean: clean-logs clean-containers
+distclean: clean clean-images
+
+usage: .not-implemented
+docs: .not-implemented
+stats: .not-implemented
+test: .not-implemented
+watch: .not-implemented

data/doc/kafka-playground/README.md

@@ -0,0 +1,185 @@
+![Apache Kafka Playground](doc/assets/project.svg)
+
+This sub-project is dedicated to allow a simple local bootstrap of the Apache
+Kafka ecosystem with the help of containers/Docker. **Heads up!** This
+configuration is not designed to be used in production.
+
+- [Requirements](#requirements)
+- [Getting started](#getting-started)
+- [What's in the box](#whats-in-the-box)
+- [Examples](#examples)
+  - [Simple Message Producing/Consuming](#simple-message-producingconsuming)
+  - [Message Producing/Consuming with Rimless (Apache Avro)](#message-producingconsuming-with-rimless-apache-avro)
+
+## Requirements
+
+* [GNU Make](https://www.gnu.org/software/make/) (>=4.2.1)
+* [Docker](https://www.docker.com/get-docker) (>=17.06.0-ce)
+* [Docker Compose](https://docs.docker.com/compose/install/) (>=1.15.0)
+* [Host enabled mDNS stack](#mdns-host-configuration)
+
+## Getting started
+
+First you need to clone this repository from Github:
+
+```bash
+# Clone the repository
+$ git clone git@github.com:hausgold/rimless.git
+# Go in the repository directory
+$ cd rimless/doc/kafka-playground
+```
+
+We assume you have prepared the requirements in advance. The only thing
+which is left, is to install and start the application:
+
+```shell
+$ make install
+$ make start
+```
+
+## mDNS host configuration
+
+If you running Ubuntu/Debian, all required packages should be in place out of
+the box. On older versions (Ubuntu < 18.10, Debian < 10) the configuration is
+also fine out of the box. When you however find yourself unable to resolve the
+domains or if you are a lucky user of newer Ubuntu/Debian versions, read on.
+
+**Heads up:** This is the Arch Linux way. (package and service names may
+differ, config is the same) Install the `nss-mdns` and `avahi` packages, enable
+and start the `avahi-daemon.service`. Then, edit the file `/etc/nsswitch.conf`
+and change the hosts line like this:
+
+```bash
+hosts: ... mdns4 [NOTFOUND=return] resolve [!UNAVAIL=return] dns ...
+```
+
+Afterwards create (or overwrite) the `/etc/mdns.allow` file when not yet
+present with the following content:
+
+```bash
+.local.
+.local
+```
+
+This is the regular way for nss-mdns > 0.10 package versions (the
+default now). If you use a system with 0.10 or lower take care of using
+`mdns4_minimal` instead of `mdns4` on the `/etc/nsswitch.conf` file and skip
+the creation of the `/etc/mdns.allow` file.
+
+**Further readings**
+* Archlinux howto: https://wiki.archlinux.org/index.php/avahi
+* Ubuntu/Debian howto: https://wiki.ubuntuusers.de/Avahi/
+* Further detail on nss-mdns: https://github.com/lathiat/nss-mdns
+
+## What's in the box
+
+After the installation and bootup processes are finished you should have a
+working Apache Kafka setup which includes the following:
+
+* A single node [Apache Kafka](https://kafka.apache.org/) broker via [Zookeeper](https://zookeeper.apache.org/)
+* [Confluent Schema Registry](https://docs.confluent.io/platform/current/schema-registry/index.html), used for [Apache Avro](https://avro.apache.org/docs/current/) schemas
+* [Lenses.io Schema Registry UI](https://github.com/lensesio/schema-registry-ui), you can access it via mDNS at http://schema-registry-ui.playground.local
+* A Ruby 2.5 enabled playground container with configured Rimless support
+
+## Examples
+
+### Simple Message Producing/Consuming
+
+Start a playground container with `$ make start` and run the following:
+
+```shell
+$ create-topic -v test
+```
+
+```shell
+$ list-topics
+
+Metadata for all topics (from broker 1001: kafka.playground.local:9092/1001):
+ 1 brokers:
+  broker 1001 at kafka.playground.local:9092 (controller)
+ 2 topics:
+  topic "_schemas" with 1 partitions:
+    partition 0, leader 1001, replicas: 1001, isrs: 1001
+  topic "test" with 1 partitions:
+```
+
+Now start a second teminal playground container with `$ make shell` and run:
+
+```shell
+# Terminal B
+
+$ consume-topic test
+
+% Waiting for group rebalance
+% Group kcat rebalanced (memberid kcat-1ec7324b-463c-4c1e-ab47-b58aa886a98d): assigned: test [0]
+% Reached end of topic test [0] at offset 0
+```
+
+At the first container session run:
+
+```shell
+# Terminal A
+
+$ echo '{"test":true}' | produce-event test -
+
+Processing lines of '/dev/stdin' ..
+{"test":true}
+```
+
+And see that the consumer at the second terminal output changed to:
+
+```shell
+# Terminal B
+
+$ consume-topic test
+
+% Waiting for group rebalance
+% Group kcat rebalanced (memberid kcat-1ec7324b-463c-4c1e-ab47-b58aa886a98d): assigned: test [0]
+% Reached end of topic test [0] at offset 0
+{"test":true}
+
+% Reached end of topic test [0] at offset 1
+```
+
+### Message Producing/Consuming with Rimless (Apache Avro)
+
+Setup two terminal playground session containers with `$ make shell` and run
+the following snippets to produce an Apache Avro message and consume it with
+[kcat](https://github.com/edenhill/kcat):
+
+```shell
+# Terminal A
+
+$ create-topic production.playground-app.payments
+$ consume-topic -s value=avro production.playground-app.payments
+```
+
+And at the otherside run:
+
+```shell
+# Terminal B
+
+$ examples/rimless-produce
+
+{"event"=>"payment_authorized",
+ "payment"=>
+  {"gid"=>"gid://playground-app/Payment/19da4f09-56c8-47d6-8a01-dc7ec2f9daff",
+   "currency"=>"eur",
+   "net_amount_sum"=>500,
+   "items"=>
+    [{"gid"=>
+       "gid://playground-app/PaymentItem/9f2d9746-52a8-4b8a-a614-4f8f8b4ef4a5",
+      "net_amount"=>499,
+      "tax_rate"=>19,
+      "created_at"=>"2021-10-27T15:09:06.990+00:00",
+      "updated_at"=>nil},
+     {"gid"=>
+       "gid://playground-app/PaymentItem/c8f9f718-03fd-442a-9677-1a4e64349c2c",
+      "net_amount"=>1,
+      "tax_rate"=>19,
+      "created_at"=>"2021-10-27T15:09:06.990+00:00",
+      "updated_at"=>nil}],
+   "state"=>"authorized",
+   "created_at"=>"2021-10-27T15:09:06.990+00:00",
+   "updated_at"=>"2021-10-27T15:09:06.990+00:00"}}
+```

data/doc/kafka-playground/bin/consume-topic

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+if [ -z "${GROUP}" ]; then
+  GROUP="kcat"
+fi
+
+kcat -G "${GROUP}" ${@}

data/doc/kafka-playground/bin/create-topic

@@ -0,0 +1,42 @@
+#!/usr/bin/env ruby
+
+require_relative '../config/environment'
+
+class CreateTopic < Thor
+  default_command :create
+
+  desc 'NAME [CONFIGS...]', 'create a new Apache Kafka topic'
+  option :partitions, aliases: '-p', type: :numeric, default: 1,
+         desc: 'The number of partitions'
+  option :replicas, aliases: '-r', type: :numeric, default: 1,
+         desc: 'The number of replications'
+  option :verbose, aliases: '-v', type: :boolean,
+         desc: 'Enable verbose outputs'
+  def create(name, *configs)
+    debug! options
+
+    opts = {
+      num_partitions: options[:partitions].to_i,
+      replication_factor: options[:replicas].to_i,
+    }
+    config = configs.map { |conf| conf.split('=').map(&:strip) }.to_h
+
+    if topic?(name)
+      STDERR.puts "The topic '#{name}' already exists."
+      puts JSON.pretty_generate(@topic_conf)
+      exit
+    end
+
+    # Create the topic
+    KafkaClient.create_topic(name, **opts, config: config)
+
+    # Fetch the topic config
+    puts JSON.pretty_generate(KafkaClient.describe_topic(name))
+  rescue Kafka::InvalidConfig
+    STDOUT.puts "Could not create the topic '#{name}'."
+    STDOUT.puts "The given configuration is invalid:\n\n"
+    puts JSON.pretty_generate(config)
+    exit 1
+  end
+end
+CreateTopic.start(args!)

data/doc/kafka-playground/bin/delete-topic

@@ -0,0 +1,22 @@
+#!/usr/bin/env ruby
+
+require_relative '../config/environment'
+
+class DeleteTopic < Thor
+  default_command :delete
+
+  desc 'NAME', 'delete an existing Apache Kafka topic'
+  option :verbose, aliases: '-v', type: :boolean,
+         desc: 'Enable verbose outputs'
+  def delete(name)
+    debug! options
+
+    unless topic?(name)
+      STDERR.puts "The topic '#{name}' does not exists."
+      exit 1
+    end
+
+    KafkaClient.delete_topic name
+  end
+end
+DeleteTopic.start(args!)

data/doc/kafka-playground/bin/list-topics

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+kcat -L ${@}

data/doc/kafka-playground/bin/produce-event

@@ -0,0 +1,64 @@
+#!/usr/bin/env ruby
+
+require_relative '../config/environment'
+
+class ProduceTopic < Thor
+  default_command :produce
+
+  desc 'TOPIC FILE...', 'produce a new event at a given Apache Kafka topic'
+  option :partition, aliases: '-p', type: :numeric,
+         desc: 'The topic partitions to write to'
+  option :partition_key, aliases: '-k', type: :string,
+         desc: 'The partition key to use to select the partition'
+  option :verbose, aliases: '-v', type: :boolean,
+         desc: 'Enable verbose outputs'
+  def produce(topic, *files)
+    debug! options
+
+    opts = {
+      topic: topic,
+      partition: options[:partition]&.to_i,
+      partition_key: options[:partition_key]
+    }.compact
+
+    if options.key?(:partition) && options.key?(:partition_key)
+      STDERR.puts 'Either use the fixed partition or a partition key.'
+      STDERR.puts 'But not both together.'
+      exit 1
+    end
+
+    files = files.map do |file|
+      next '/dev/stdin' if file == '-'
+
+      unless File.file? file
+        STDERR.puts "File '#{file}' does not exist."
+        next
+      end
+
+      file
+    end.compact.uniq
+
+    if files.empty?
+      STDERR.puts 'No files given or exists.'
+      STDERR.puts 'You have to specify file(s) or use `-\' for stdin.'
+      exit 1
+    end
+
+    producer = KafkaClient.producer
+
+    files.each do |file|
+      puts "Processing lines of '#{file}' .."
+      File.open(file, 'r') do |f|
+        f.each_line.lazy.each do |line|
+          puts line
+          producer.produce(line, **opts)
+          puts
+        end
+      end
+      producer.deliver_messages
+    end
+  rescue Interrupt
+    producer.deliver_messages
+  end
+end
+ProduceTopic.start(args!)

data/doc/kafka-playground/config/avro_schemas/.gitignore

@@ -0,0 +1 @@
+compiled/

data/doc/kafka-playground/config/avro_schemas/playground_app/item_v1.avsc.erb

@@ -0,0 +1,36 @@
+{
+  "name": "item_v1",
+  "doc": "Playground App - item attached to a payment",
+  "namespace": "<%= namespace %>",
+  "type": "record",
+  "fields": [
+    {
+      "name": "gid",
+      "type": "string",
+      "doc": "The global ID of the item that is being sold"
+    },
+    {
+      "name": "net_amount",
+      "type": "int",
+      "doc": "The price of the item in cents (without tax)"
+    },
+    {
+      "name": "tax_rate",
+      "type": "float",
+      "doc": "The tax rate for the item"
+    },
+    {
+      "name": "created_at",
+      "type": "string",
+      "doc": "When the user was created (ISO 8601)"
+    },
+    {
+      "name": "updated_at",
+      "type": [
+        "null",
+        "string"
+      ],
+      "doc": "When the item was last updated (ISO 8601)"
+    }
+  ]
+}

data/doc/kafka-playground/config/avro_schemas/playground_app/payment_v1.avsc.erb

@@ -0,0 +1,59 @@
+{
+  "name": "payment_v1",
+  "doc": "Playground App - the payment",
+  "namespace": "<%= namespace %>",
+  "type": "record",
+  "fields": [
+    {
+      "name": "gid",
+      "type": "string",
+      "doc": "Global ID of the payment (UUID)"
+    },
+    {
+      "name": "currency",
+      "type": "string",
+      "doc": "The currency used for the payment"
+    },
+    {
+      "name": "net_amount_sum",
+      "type": "int",
+      "doc": "The price sum with tax included in cents"
+    },
+    {
+      "name": "items",
+      "type": {
+        "items": "<%= namespace %>.item_v1",
+        "type": "array"
+      },
+      "doc": "All connected payment items"
+    },
+    {
+      "name": "state",
+      "type": {
+        "name": "state",
+        "type": "enum",
+        "symbols": [
+          "pending",
+          "authorized",
+          "settled",
+          "cancelled",
+          "failed"
+        ]
+      },
+      "doc": "The current state of the payment"
+    },
+    {
+      "name": "created_at",
+      "type": "string",
+      "doc": "When the payment was created (ISO 8601)"
+    },
+    {
+      "name": "updated_at",
+      "type": [
+        "null",
+        "string"
+      ],
+      "doc": "When the payment was last updated (ISO 8601)"
+    }
+  ]
+}

data/doc/kafka-playground/config/avro_schemas/playground_app/payment_v1_event.avsc.erb

@@ -0,0 +1,18 @@
+{
+  "name": "payment_v1_event",
+  "namespace": "<%= namespace %>",
+  "type": "record",
+  "doc": "Playground App - payment event representation",
+  "fields": [
+    {
+      "doc": "The event name/type",
+      "name": "event",
+      "type": "string"
+    },
+    {
+      "doc": "The corresponding payment of the event",
+      "name": "payment",
+      "type": "<%= namespace %>.payment_v1"
+    }
+  ]
+}

data/doc/kafka-playground/config/docker/shell/.bash_profile

@@ -0,0 +1,3 @@
+if [ -f ~/.bashrc ]; then
+  . ~/.bashrc
+fi