message-db 0.0.0 → 1.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 56d836080e68d18d26462b409d0262ec20762315f1d06cd2e574497b7e3e8efd
-  data.tar.gz: de652f57e7272a2fb4b61ff98730259a5a906670e08cb933c98a10681330cd04
+  metadata.gz: e483e428c6311bd88f5c9eb6b10567a26657fc7bb324b807ac29493ad4e1a5cf
+  data.tar.gz: 69fbdccd72ae4e34f9155b235356c2f0daa2b18c5135499cf133f427b541a607
 SHA512:
-  metadata.gz: 72f2836385d3c7333b2b7cccf608cc00c2836dece71448f7443c6c89f8342632ce4dfdbff648483c6e8d8b5dc1c7aa760aa2803c0643e2b831239625c0e0f827
-  data.tar.gz: 781ef096eaed6a556d1ebd24486420f7697470da926bd49b77b27fa962e7d0d9668452b7c9b5ddc82b9be44b6cc77c07ec81c0f95188dcb21f0a6b13e4b393bf
+  metadata.gz: da65afe3dedb6426ae5d7fd2db99e368d76450c450ddbe7f38a430f64353c4a956d77f0dba8fc5c59c6d459ec6669f0ecfc3a968941ce05138a9aa75d8092954
+  data.tar.gz: 73322f41132a625db09c30d780dfc808315344ab804ef143cbe3048ef03a670f194dc94505801c33b10c776332e80625cc0df72f31c6e3a34d60aa2559349bf4

data/database/VERSION.txt

@@ -0,0 +1 @@
+1.1.3

data/database/benchmark.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+
+set -u
+
+uuid=$(echo $(uuidgen) | tr '[:upper:]' '[:lower:]')
+
+stream_name="testStream-$uuid"
+if [ ! -z ${STREAM_NAME+x} ]; then
+  stream_name=$STREAM_NAME
+fi
+
+cycles=1000
+if [ ! -z ${CYCLES+x} ]; then
+  cycles=$CYCLES
+fi
+
+function script_dir {
+  val="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+  echo "$val"
+}
+
+base=$(script_dir)
+
+echo
+echo "Benchmark $cycles cycles (Stream Name: $stream_name)"
+echo "= = ="
+echo
+
+if [ -z ${DATABASE_NAME+x} ]; then
+  echo "(DATABASE_NAME is not set)"
+  database=message_store
+else
+  database=$DATABASE_NAME
+fi
+echo "Database name is: $database"
+echo
+
+echo "Installing benchmark scripts"
+echo
+
+psql $database -q -f $base/benchmark_write.sql
+psql $database -q -f $base/benchmark_get.sql
+
+echo
+echo "Benchmarking write"
+echo "- - -"
+echo
+
+psql $database -U message_store -c "EXPLAIN ANALYZE SELECT benchmark_write('$stream_name'::varchar, $cycles::int);"
+
+echo
+
+echo
+echo "Benchmarking get"
+echo "- - -"
+echo
+
+psql $database -U message_store -c "EXPLAIN ANALYZE SELECT benchmark_get('$stream_name'::varchar, $cycles::int);"
+
+echo "= = ="
+echo "Done"
+echo

data/database/benchmark_get.sql

@@ -0,0 +1,24 @@
+CREATE OR REPLACE FUNCTION benchmark_get(
+  stream_name varchar,
+  cycles int DEFAULT 1000
+)
+RETURNS void
+AS $$
+BEGIN
+  RAISE NOTICE '» benchmark_get';
+  RAISE NOTICE 'stream_name: %', benchmark_get.stream_name;
+  RAISE NOTICE 'cycles: %', benchmark_get.cycles;
+
+  FOR i IN 1..cycles LOOP
+    IF current_setting('message_store.debug_benchmark', true) = 'on' OR current_setting('message_store.debug', true) = 'on' THEN
+      RAISE NOTICE '%', i;
+    END IF;
+
+    PERFORM get_stream_messages(stream_name, "position" => i - 1, batch_size => 1);
+  END LOOP;
+END;
+$$ LANGUAGE plpgsql
+VOLATILE;
+
+
+

data/database/benchmark_write.sql

@@ -0,0 +1,24 @@
+CREATE OR REPLACE FUNCTION benchmark_write(
+  stream_name varchar,
+  cycles int DEFAULT 1000
+)
+RETURNS void
+AS $$
+BEGIN
+  RAISE NOTICE '» benchmark_write';
+  RAISE NOTICE 'stream_name: %', benchmark_write.stream_name;
+  RAISE NOTICE 'cycles: %', benchmark_write.cycles;
+
+  FOR i IN 1..cycles LOOP
+    IF current_setting('message_store.debug_benchmark', true) = 'on' OR current_setting('message_store.debug', true) = 'on' THEN
+      RAISE NOTICE '%', i;
+    END IF;
+
+    PERFORM write_message(gen_random_uuid()::varchar, stream_name::varchar, 'SomeType'::varchar, '{"attribute": "some value"}'::jsonb, '{"metaAttribute": "some meta value", "correlationStreamName": "someCorrelation-123"}'::jsonb);
+  END LOOP;
+END;
+$$ LANGUAGE plpgsql
+VOLATILE;
+
+
+

data/database/clear-messages.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+set -e
+
+echo
+echo "Clearing Messages Table"
+echo "= = ="
+echo
+
+default_name=message_store
+
+if [ -z ${DATABASE_NAME+x} ]; then
+  echo "(DATABASE_NAME is not set)"
+  database=$default_name
+else
+  database=$DATABASE_NAME
+fi
+echo "Database name is: $database"
+
+echo
+
+psql $database -q -c "TRUNCATE message_store.messages RESTART IDENTITY;"
+
+echo "= = ="
+echo "Done Clearing Messages Table"
+echo

data/database/extensions/pgcrypto.sql

@@ -0,0 +1 @@
+CREATE EXTENSION IF NOT EXISTS pgcrypto;

data/database/functions/acquire-lock.sql

@@ -0,0 +1,24 @@
+CREATE OR REPLACE FUNCTION message_store.acquire_lock(
+  stream_name varchar
+)
+RETURNS bigint
+AS $$
+DECLARE
+  _category varchar;
+  _category_name_hash bigint;
+BEGIN
+  _category := category(acquire_lock.stream_name);
+  _category_name_hash := hash_64(_category);
+  PERFORM pg_advisory_xact_lock(_category_name_hash);
+
+  IF current_setting('message_store.debug_write', true) = 'on' OR current_setting('message_store.debug', true) = 'on' THEN
+    RAISE NOTICE '» acquire_lock';
+    RAISE NOTICE 'stream_name: %', acquire_lock.stream_name;
+    RAISE NOTICE '_category: %', _category;
+    RAISE NOTICE '_category_name_hash: %', _category_name_hash;
+  END IF;
+
+  RETURN _category_name_hash;
+END;
+$$ LANGUAGE plpgsql
+VOLATILE;

data/database/functions/cardinal-id.sql

@@ -0,0 +1,18 @@
+CREATE OR REPLACE FUNCTION message_store.cardinal_id(
+  stream_name varchar
+)
+RETURNS varchar
+AS $$
+DECLARE
+  _id varchar;
+BEGIN
+  _id := id(cardinal_id.stream_name);
+
+  IF _id IS NULL THEN
+    RETURN NULL;
+  END IF;
+
+  RETURN SPLIT_PART(_id, '+', 1);
+END;
+$$ LANGUAGE plpgsql
+IMMUTABLE;

data/database/functions/category.sql

@@ -0,0 +1,10 @@
+CREATE OR REPLACE FUNCTION message_store.category(
+  stream_name varchar
+)
+RETURNS varchar
+AS $$
+BEGIN
+  RETURN SPLIT_PART(category.stream_name, '-', 1);
+END;
+$$ LANGUAGE plpgsql
+IMMUTABLE;

data/database/functions/get-category-messages.sql

@@ -0,0 +1,129 @@
+CREATE OR REPLACE FUNCTION message_store.get_category_messages(
+  category varchar,
+  "position" bigint DEFAULT 1,
+  batch_size bigint DEFAULT 1000,
+  correlation varchar DEFAULT NULL,
+  consumer_group_member bigint DEFAULT NULL,
+  consumer_group_size bigint DEFAULT NULL,
+  condition varchar DEFAULT NULL
+)
+RETURNS SETOF message_store.message
+AS $$
+DECLARE
+  _command text;
+BEGIN
+  IF NOT is_category(get_category_messages.category) THEN
+    RAISE EXCEPTION
+      'Must be a category: %',
+      get_category_messages.category;
+  END IF;
+
+  position := COALESCE(position, 1);
+  batch_size := COALESCE(batch_size, 1000);
+
+  _command := '
+    SELECT
+      id::varchar,
+      stream_name::varchar,
+      type::varchar,
+      position::bigint,
+      global_position::bigint,
+      data::varchar,
+      metadata::varchar,
+      time::timestamp
+    FROM
+      messages
+    WHERE
+      category(stream_name) = $1 AND
+      global_position >= $2';
+
+  IF get_category_messages.correlation IS NOT NULL THEN
+    IF position('-' IN get_category_messages.correlation) > 0 THEN
+      RAISE EXCEPTION
+        'Correlation must be a category (Correlation: %)',
+        get_category_messages.correlation;
+    END IF;
+
+    _command := _command || ' AND
+      category(metadata->>''correlationStreamName'') = $4';
+  END IF;
+
+  IF (get_category_messages.consumer_group_member IS NOT NULL AND
+      get_category_messages.consumer_group_size IS NULL) OR
+      (get_category_messages.consumer_group_member IS NULL AND
+      get_category_messages.consumer_group_size IS NOT NULL) THEN
+
+    RAISE EXCEPTION
+      'Consumer group member and size must be specified (Consumer Group Member: %, Consumer Group Size: %)',
+      get_category_messages.consumer_group_member,
+      get_category_messages.consumer_group_size;
+  END IF;
+
+  IF get_category_messages.consumer_group_member IS NOT NULL AND
+      get_category_messages.consumer_group_size IS NOT NULL THEN
+
+    IF get_category_messages.consumer_group_size < 1 THEN
+      RAISE EXCEPTION
+        'Consumer group size must not be less than 1 (Consumer Group Member: %, Consumer Group Size: %)',
+        get_category_messages.consumer_group_member,
+        get_category_messages.consumer_group_size;
+    END IF;
+
+    IF get_category_messages.consumer_group_member < 0 THEN
+      RAISE EXCEPTION
+        'Consumer group member must not be less than 0 (Consumer Group Member: %, Consumer Group Size: %)',
+        get_category_messages.consumer_group_member,
+        get_category_messages.consumer_group_size;
+    END IF;
+
+    IF get_category_messages.consumer_group_member >= get_category_messages.consumer_group_size THEN
+      RAISE EXCEPTION
+        'Consumer group member must be less than the group size (Consumer Group Member: %, Consumer Group Size: %)',
+        get_category_messages.consumer_group_member,
+        get_category_messages.consumer_group_size;
+    END IF;
+
+    _command := _command || ' AND
+      MOD(@hash_64(cardinal_id(stream_name)), $6) = $5';
+  END IF;
+
+  IF get_category_messages.condition IS NOT NULL THEN
+    IF current_setting('message_store.sql_condition', true) IS NULL OR
+        current_setting('message_store.sql_condition', true) = 'off' THEN
+      RAISE EXCEPTION
+        'Retrieval with SQL condition is not activated';
+    END IF;
+
+    _command := _command || ' AND
+      (%s)';
+    _command := format(_command, get_category_messages.condition);
+  END IF;
+
+  _command := _command || '
+    ORDER BY
+      global_position ASC
+    LIMIT
+      $3';
+
+  IF current_setting('message_store.debug_get', true) = 'on' OR current_setting('message_store.debug', true) = 'on' THEN
+    RAISE NOTICE '» get_category_messages';
+    RAISE NOTICE 'category ($1): %', get_category_messages.category;
+    RAISE NOTICE 'position ($2): %', get_category_messages.position;
+    RAISE NOTICE 'batch_size ($3): %', get_category_messages.batch_size;
+    RAISE NOTICE 'correlation ($4): %', get_category_messages.correlation;
+    RAISE NOTICE 'consumer_group_member ($5): %', get_category_messages.consumer_group_member;
+    RAISE NOTICE 'consumer_group_size ($6): %', get_category_messages.consumer_group_size;
+    RAISE NOTICE 'condition: %', get_category_messages.condition;
+    RAISE NOTICE 'Generated Command: %', _command;
+  END IF;
+
+  RETURN QUERY EXECUTE _command USING
+    get_category_messages.category,
+    get_category_messages.position,
+    get_category_messages.batch_size,
+    get_category_messages.correlation,
+    get_category_messages.consumer_group_member,
+    get_category_messages.consumer_group_size::smallint;
+END;
+$$ LANGUAGE plpgsql
+VOLATILE;

data/database/functions/get-last-stream-message.sql

@@ -0,0 +1,37 @@
+CREATE OR REPLACE FUNCTION message_store.get_last_stream_message(
+  stream_name varchar
+)
+RETURNS SETOF message_store.message
+AS $$
+DECLARE
+  _command text;
+BEGIN
+  _command := '
+    SELECT
+      id::varchar,
+      stream_name::varchar,
+      type::varchar,
+      position::bigint,
+      global_position::bigint,
+      data::varchar,
+      metadata::varchar,
+      time::timestamp
+    FROM
+      messages
+    WHERE
+      stream_name = $1
+    ORDER BY
+      position DESC
+    LIMIT
+      1';
+
+  IF current_setting('message_store.debug_get', true) = 'on' OR current_setting('message_store.debug', true) = 'on' THEN
+    RAISE NOTICE '» get_last_message';
+    RAISE NOTICE 'stream_name ($1): %', get_last_stream_message.stream_name;
+    RAISE NOTICE 'Generated Command: %', _command;
+  END IF;
+
+  RETURN QUERY EXECUTE _command USING get_last_stream_message.stream_name;
+END;
+$$ LANGUAGE plpgsql
+VOLATILE;

data/database/functions/get-stream-messages.sql

@@ -0,0 +1,71 @@
+CREATE OR REPLACE FUNCTION message_store.get_stream_messages(
+  stream_name varchar,
+  "position" bigint DEFAULT 0,
+  batch_size bigint DEFAULT 1000,
+  condition varchar DEFAULT NULL
+)
+RETURNS SETOF message_store.message
+AS $$
+DECLARE
+  _command text;
+  _setting text;
+BEGIN
+  IF is_category(get_stream_messages.stream_name) THEN
+    RAISE EXCEPTION
+      'Must be a stream name: %',
+      get_stream_messages.stream_name;
+  END IF;
+
+  position := COALESCE(position, 0);
+  batch_size := COALESCE(batch_size, 1000);
+
+  _command := '
+    SELECT
+      id::varchar,
+      stream_name::varchar,
+      type::varchar,
+      position::bigint,
+      global_position::bigint,
+      data::varchar,
+      metadata::varchar,
+      time::timestamp
+    FROM
+      messages
+    WHERE
+      stream_name = $1 AND
+      position >= $2';
+
+  IF get_stream_messages.condition IS NOT NULL THEN
+    IF current_setting('message_store.sql_condition', true) IS NULL OR
+        current_setting('message_store.sql_condition', true) = 'off' THEN
+      RAISE EXCEPTION
+        'Retrieval with SQL condition is not activated';
+    END IF;
+
+    _command := _command || ' AND
+      (%s)';
+    _command := format(_command, get_stream_messages.condition);
+  END IF;
+
+  _command := _command || '
+    ORDER BY
+      position ASC
+    LIMIT
+      $3';
+
+  IF current_setting('message_store.debug_get', true) = 'on' OR current_setting('message_store.debug', true) = 'on' THEN
+    RAISE NOTICE '» get_stream_messages';
+    RAISE NOTICE 'stream_name ($1): %', get_stream_messages.stream_name;
+    RAISE NOTICE 'position ($2): %', get_stream_messages.position;
+    RAISE NOTICE 'batch_size ($3): %', get_stream_messages.batch_size;
+    RAISE NOTICE 'condition ($4): %', get_stream_messages.condition;
+    RAISE NOTICE 'Generated Command: %', _command;
+  end if;
+
+  RETURN QUERY EXECUTE _command USING
+    get_stream_messages.stream_name,
+    get_stream_messages.position,
+    get_stream_messages.batch_size;
+END;
+$$ LANGUAGE plpgsql
+VOLATILE;