evt-message_store-postgres-database 0.1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: a0f4bb6cb696f88af5f25636de90de8a77dd47c1998c7b8a92a4a053debd2328
+  data.tar.gz: b5f19af7297e09cee6cc1a15d708756618ebf499be46435d2c0bf468540a810e
+SHA512:
+  metadata.gz: 59b9a5d456bfbb21649322559cbf9335478c1afc78012dac917985c5687f43dfaf32fb0a370771a1e05ca6c934258b43f51919ba9d8c045f5c4170aabb6e86f8
+  data.tar.gz: e026fca219bf6bfa0abc15d70f683f9a2c32c113d25b81e46fadbdc74674f72441c8aa56722a681939c0088957c61e3b7709a558c7abf869e9c7a4596b6d60da

data/database/clear-events-table.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+set -e
+
+echo
+echo "Clearing Events Table"
+echo "= = ="
+echo
+
+default_name=message_store
+
+if [ -z ${DATABASE_USER+x} ]; then
+  echo "(DATABASE_USER is not set)"
+  user=$default_name
+else
+  user=$DATABASE_USER
+fi
+echo "Database user is: $user"
+
+if [ -z ${DATABASE_NAME+x} ]; then
+  echo "(DATABASE_NAME is not set)"
+  database=$default_name
+else
+  database=$DATABASE_NAME
+fi
+echo "Database name is: $database"
+
+default_table_name=messages
+
+if [ -z ${TABLE_NAME+x} ]; then
+  echo "(TABLE_NAME is not set)"
+  table=$default_table_name
+else
+  table=$TABLE_NAME
+fi
+echo "Table name is: $table"
+
+echo
+
+psql $database -c "TRUNCATE $table RESTART IDENTITY;"

data/database/extensions.sql

@@ -0,0 +1 @@
+CREATE EXTENSION IF NOT EXISTS "pgcrypto";

data/database/functions/category.sql

@@ -0,0 +1,10 @@
+CREATE OR REPLACE FUNCTION category(
+  _stream_name varchar
+)
+RETURNS varchar
+AS $$
+BEGIN
+  return split_part(_stream_name, '-', 1);
+END;
+$$ LANGUAGE plpgsql
+IMMUTABLE;

data/database/functions/get-category-messages.sql

@@ -0,0 +1,45 @@
+CREATE OR REPLACE FUNCTION get_category_messages(
+  _stream_name varchar,
+  _position bigint DEFAULT 0,
+  _batch_size bigint DEFAULT 1000,
+  _condition varchar DEFAULT NULL
+)
+RETURNS SETOF message
+AS $$
+DECLARE
+  command text;
+BEGIN
+  command := '
+    SELECT
+      id::varchar,
+      stream_name::varchar,
+      type::varchar,
+      position::bigint,
+      global_position::bigint,
+      data::varchar,
+      metadata::varchar,
+      time::timestamp
+    FROM
+      messages
+    WHERE
+      category(stream_name) = $1 AND
+      global_position >= $2';
+
+  if _condition is not null then
+    command := command || ' AND
+      %s';
+    command := format(command, _condition);
+  end if;
+
+  command := command || '
+    ORDER BY
+      global_position ASC
+    LIMIT
+      $3';
+
+  -- RAISE NOTICE '%', command;
+
+  RETURN QUERY EXECUTE command USING _stream_name, _position, _batch_size;
+END;
+$$ LANGUAGE plpgsql
+VOLATILE;

data/database/functions/get-last-message.sql

@@ -0,0 +1,33 @@
+CREATE OR REPLACE FUNCTION get_last_message(
+  _stream_name varchar
+)
+RETURNS SETOF message
+AS $$
+DECLARE
+  command text;
+BEGIN
+  command := '
+    SELECT
+      id::varchar,
+      stream_name::varchar,
+      type::varchar,
+      position::bigint,
+      global_position::bigint,
+      data::varchar,
+      metadata::varchar,
+      time::timestamp
+    FROM
+      messages
+    WHERE
+      stream_name = $1
+    ORDER BY
+      position DESC
+    LIMIT
+      1';
+
+  -- RAISE NOTICE '%', command;
+
+  RETURN QUERY EXECUTE command USING _stream_name;
+END;
+$$ LANGUAGE plpgsql
+VOLATILE;

data/database/functions/get-stream-messages.sql

@@ -0,0 +1,45 @@
+CREATE OR REPLACE FUNCTION get_stream_messages(
+  _stream_name varchar,
+  _position bigint DEFAULT 0,
+  _batch_size bigint DEFAULT 1000,
+  _condition varchar DEFAULT NULL
+)
+RETURNS SETOF message
+AS $$
+DECLARE
+  command text;
+BEGIN
+  command := '
+    SELECT
+      id::varchar,
+      stream_name::varchar,
+      type::varchar,
+      position::bigint,
+      global_position::bigint,
+      data::varchar,
+      metadata::varchar,
+      time::timestamp
+    FROM
+      messages
+    WHERE
+      stream_name = $1 AND
+      position >= $2';
+
+  if _condition is not null then
+    command := command || ' AND
+      %s';
+    command := format(command, _condition);
+  end if;
+
+  command := command || '
+    ORDER BY
+      position ASC
+    LIMIT
+      $3';
+
+  -- RAISE NOTICE '%', command;
+
+  RETURN QUERY EXECUTE command USING _stream_name, _position, _batch_size;
+END;
+$$ LANGUAGE plpgsql
+VOLATILE;

data/database/functions/hash-64.sql

@@ -0,0 +1,13 @@
+CREATE OR REPLACE FUNCTION hash_64(
+  _stream_name varchar
+)
+RETURNS bigint
+AS $$
+DECLARE
+  hash bigint;
+BEGIN
+  select left('x' || md5(_stream_name), 16)::bit(64)::bigint into hash;
+  return hash;
+END;
+$$ LANGUAGE plpgsql
+IMMUTABLE;

data/database/functions/scratch.sql

@@ -0,0 +1,68 @@
+:stream_name, na(:position), na(:batch_size), :condition
+
+
+module Defaults
+  def self.position
+    0
+  end
+
+  def self.batch_size
+    1000
+  end
+end
+
+
+- - -
+
+command := 'SELECT * FROM messages WHERE $1';
+RETURN QUERY EXECUTE command USING where_clause;
+
+- - -
+select * from messages where metadata @> '{"correlationStreamName":"someCorrelation"}';
+- - -
+select * from get_messages($$metadata @> '{"correlationStreamName":"someCorrelation"}'$$);
+- - -
+
+
+_sql := format('
+SELECT *
+FROM  (
+   SELECT p.person_id, p.name, p.team, %1$s(s.score)::int AS score
+         ,rank() OVER (PARTITION BY p.team
+                       ORDER BY %1$s(s.score) DESC)::int AS rnk
+    FROM  person p
+    %2$s  score  s USING (person_id)
+    %3$s
+    GROUP BY 1
+   ) sub
+WHERE  rnk < 3
+ORDER  BY team, rnk'
+   , _agg
+   , CASE WHEN _left_join THEN 'LEFT JOIN' ELSE 'JOIN' END
+   , CASE WHEN _where_name <> '' THEN 'WHERE p.name LIKE $1' ELSE '' END
+);
+
+-- debug   -- quote when tested ok
+-- RAISE NOTICE '%', _sql;
+
+-- execute -- unquote when tested ok
+RETURN QUERY EXECUTE _sql
+
+
+- - -
+
+CREATE OR REPLACE FUNCTION get_messages()
+RETURNS SETOF messages
+AS $$
+DECLARE
+  _sql text;
+BEGIN
+
+  _sql := 'RETURN QUERY SELECT * FROM messages'
+
+  -- RETURN QUERY EXECUTE _sql
+
+  RETURN QUERY SELECT * FROM messages;
+END;
+$$ LANGUAGE plpgsql
+VOLATILE;

data/database/functions/stream-version.sql

@@ -0,0 +1,14 @@
+CREATE OR REPLACE FUNCTION stream_version(
+  _stream_name varchar
+)
+RETURNS bigint
+AS $$
+DECLARE
+  stream_version bigint;
+BEGIN
+  select max(position) into stream_version from messages where stream_name = _stream_name;
+
+  return stream_version;
+END;
+$$ LANGUAGE plpgsql
+VOLATILE;

data/database/functions/write-message.sql

@@ -0,0 +1,60 @@
+CREATE OR REPLACE FUNCTION write_message(
+  _id varchar,
+  _stream_name varchar,
+  _type varchar,
+  _data jsonb,
+  _metadata jsonb DEFAULT NULL,
+  _expected_version bigint DEFAULT NULL
+)
+RETURNS bigint
+AS $$
+DECLARE
+  message_id uuid;
+  stream_version bigint;
+  position bigint;
+  category varchar;
+  stream_name_hash bigint;
+BEGIN
+  message_id = uuid(_id);
+
+  stream_name_hash = hash_64(_stream_name);
+  PERFORM pg_advisory_xact_lock(stream_name_hash);
+
+  stream_version := stream_version(_stream_name);
+
+  if stream_version is null then
+    stream_version := -1;
+  end if;
+
+  if _expected_version is not null then
+    if _expected_version != stream_version then
+      raise exception 'Wrong expected version: % (Stream: %, Stream Version: %)', _expected_version, _stream_name, stream_version;
+    end if;
+  end if;
+
+  position := stream_version + 1;
+
+  insert into "messages"
+    (
+      "id",
+      "stream_name",
+      "position",
+      "type",
+      "data",
+      "metadata"
+    )
+  values
+    (
+      message_id,
+      _stream_name,
+      position,
+      _type,
+      _data,
+      _metadata
+    )
+  ;
+
+  return position;
+END;
+$$ LANGUAGE plpgsql
+VOLATILE;

data/database/indexes/messages-category-global-position.sql

@@ -0,0 +1 @@
+CREATE INDEX CONCURRENTLY "messages_category_global_position_idx" ON "public"."messages" USING btree(category(stream_name) COLLATE "default" "pg_catalog"."text_ops" ASC NULLS LAST, "global_position" "pg_catalog"."int8_ops" ASC NULLS LAST);

data/database/indexes/messages-id.sql

@@ -0,0 +1 @@
+CREATE INDEX CONCURRENTLY  "messages_id_idx" ON "public"."messages" USING btree(id ASC NULLS LAST);

data/database/indexes/messages-stream-name-position-uniq.sql

@@ -0,0 +1 @@
+CREATE UNIQUE INDEX CONCURRENTLY "messages_stream_name_position_uniq_idx" ON "public"."messages" USING btree(stream_name COLLATE "default" "pg_catalog"."text_ops" ASC NULLS LAST, "position" "pg_catalog"."int8_ops" ASC NULLS LAST);

data/database/install.sh

@@ -0,0 +1,119 @@
+#!/usr/bin/env bash
+
+set -e
+
+echo
+echo "Installing Database"
+echo "= = ="
+echo
+
+default_name=message_store
+
+if [ -z ${DATABASE_USER+x} ]; then
+  echo "(DATABASE_USER is not set. Default will be used.)"
+  user=$default_name
+else
+  user=$DATABASE_USER
+fi
+
+if [ -z ${DATABASE_NAME+x} ]; then
+  echo "(DATABASE_NAME is not set. Default will be used.)"
+  database=$default_name
+else
+  database=$DATABASE_NAME
+fi
+
+echo
+
+function create-user {
+  echo "Database user is: $user"
+  createuser -s $user
+  echo
+}
+
+function create-database {
+  echo "Database name is: $database"
+  echo "Creating database \"$database\"..."
+  createdb $database
+  echo
+}
+
+function script_dir {
+  val="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+  echo "$val"
+}
+
+function create-extensions {
+  echo "Creating extensions..."
+  base=$(script_dir)
+  psql $database -f $base/extensions.sql
+  echo
+}
+
+function create-table {
+  echo "Creating messages table..."
+  psql $database -f $base/table/messages-table.sql
+  echo
+}
+
+function create-types {
+  base=$(script_dir)
+  echo "Creating types..."
+
+  echo "message type"
+  psql $database -f $base/types/message.sql
+
+  echo
+}
+
+function create-functions {
+  echo "Creating functions..."
+  base=$(script_dir)
+
+  echo "hash_64 function"
+  psql $database -f $base/functions/hash-64.sql
+
+  echo "category function"
+  psql $database -f $base/functions/category.sql
+
+  echo "stream_version function"
+  psql $database -f $base/functions/stream-version.sql
+
+  echo "write_message function"
+  psql $database -f $base/functions/write-message.sql
+
+  echo "get_stream_messages function"
+  psql $database -f $base/functions/get-stream-messages.sql
+
+  echo "get_category_messages function"
+  psql $database -f $base/functions/get-category-messages.sql
+
+  echo "get_last_message function"
+  psql $database -f $base/functions/get-last-message.sql
+
+  echo
+}
+
+function create-indexes {
+  echo "Creating indexes..."
+  base=$(script_dir)
+
+  echo "messages_id_idx"
+  psql $database -f $base/indexes/messages-id.sql
+
+  echo "messages_category_global_position_idx"
+  psql $database -f $base/indexes/messages-category-global-position.sql
+
+  echo "messages_stream_name_position_uniq_idx"
+  psql $database -f $base/indexes/messages-stream-name-position-uniq.sql
+
+  echo
+}
+
+create-user
+create-database
+create-extensions
+create-table
+create-types
+create-functions
+create-indexes

data/database/install_functions.sh

@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+
+set -e
+
+echo
+echo "Installing Functions"
+echo "= = ="
+echo
+
+default_name=message_store
+
+if [ -z ${DATABASE_NAME+x} ]; then
+  echo "(DATABASE_NAME is not set. Default will be used.)"
+  database=$default_name
+else
+  database=$DATABASE_NAME
+fi
+
+echo
+
+function script_dir {
+  val="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+  echo "$val"
+}
+
+function create-types {
+  base=$(script_dir)
+  echo "Creating types..."
+
+  echo "message type"
+  psql $database -f $base/types/message.sql
+
+  echo
+}
+
+function create-functions {
+  echo "Creating functions..."
+  base=$(script_dir)
+
+  echo "hash_64 function"
+  psql $database -f $base/functions/hash-64.sql
+
+  echo "category function"
+  psql $database -f $base/functions/category.sql
+
+  echo "stream_version function"
+  psql $database -f $base/functions/stream-version.sql
+
+  echo "write_message function"
+  psql $database -f $base/functions/write-message.sql
+
+  echo "get_stream_messages function"
+  psql $database -f $base/functions/get-stream-messages.sql
+
+  echo "get_category_messages function"
+  psql $database -f $base/functions/get-category-messages.sql
+
+  echo "get_last_message function"
+  psql $database -f $base/functions/get-last-message.sql
+
+  echo
+}
+
+create-types
+create-functions

data/database/print-messages.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+
+set -e
+
+echo
+echo "Printing Messages"
+echo "= = ="
+echo
+
+default_name=message_store
+
+if [ -z ${DATABASE_USER+x} ]; then
+  echo "(DATABASE_USER is not set)"
+  user=$default_name
+else
+  user=$DATABASE_USER
+fi
+echo "Database user is: $user"
+
+if [ -z ${DATABASE_NAME+x} ]; then
+  echo "(DATABASE_NAME is not set)"
+  database=$default_name
+else
+  database=$DATABASE_NAME
+fi
+echo "Database name is: $database"
+
+default_table_name=messages
+
+if [ -z ${TABLE_NAME+x} ]; then
+  echo "(TABLE_NAME is not set)"
+  table=$default_table_name
+else
+  table=$TABLE_NAME
+fi
+echo "Table name is: $table"
+
+if [ -z ${STREAM_NAME+x} ]; then
+  echo "(STREAM_NAME is not set)"
+  stream_name=''
+else
+  stream_name=$STREAM_NAME
+  echo "Stream name is: $STREAM_NAME"
+fi
+
+echo
+
+if [ -z $stream_name ]; then
+  psql $database -x -c "SELECT * FROM $table"
+else
+  psql $database -x -c "SELECT * FROM $table WHERE stream_name = '$stream_name'"
+fi

data/database/table/messages-table.sql

@@ -0,0 +1,19 @@
+-- ----------------------------
+--  Table structure for messages
+-- ----------------------------
+CREATE TABLE "public"."messages" (
+  "id" UUID NOT NULL DEFAULT gen_random_uuid(),
+  "stream_name" varchar(255) NOT NULL COLLATE "default",
+  "type" varchar(255) NOT NULL COLLATE "default",
+  "position" bigint NOT NULL,
+  "global_position" bigserial NOT NULL ,
+  "data" jsonb,
+  "metadata" jsonb,
+  "time" TIMESTAMP WITHOUT TIME ZONE DEFAULT (now() AT TIME ZONE 'utc') NOT NULL
+)
+WITH (OIDS=FALSE);
+
+-- ----------------------------
+--  Primary key structure for table messages
+-- ----------------------------
+ALTER TABLE "public"."messages" ADD PRIMARY KEY ("global_position") NOT DEFERRABLE INITIALLY IMMEDIATE;

data/database/types/message.sql

@@ -0,0 +1,15 @@
+DO $$
+BEGIN
+  IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'message') THEN
+    CREATE TYPE message AS (
+      id varchar,
+      stream_name varchar,
+      type varchar,
+      position bigint,
+      global_position bigint,
+      data varchar,
+      metadata varchar,
+      time timestamp
+    );
+  END IF;
+END$$;

data/database/uninstall.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+
+set -e
+
+echo
+echo "Uninstalling Database"
+echo "= = ="
+echo
+
+default_name=message_store
+
+if [ -z ${DATABASE_USER+x} ]; then
+  echo "(DATABASE_USER is not set. Default will be used.)"
+  user=$default_name
+else
+  user=$DATABASE_USER
+fi
+
+if [ -z ${DATABASE_NAME+x} ]; then
+  echo "(DATABASE_NAME is not set. Default will be used.)"
+  database=$default_name
+else
+  database=$DATABASE_NAME
+fi
+
+echo
+
+function delete-user {
+  echo "Database user is: $user"
+
+  user_exists=`psql postgres -qtAXc "SELECT 1 FROM pg_roles WHERE rolname='$user'"`
+
+  if [ "$user_exists" = "1" ]; then
+    echo "Deleting database user \"$user\"..."
+    dropuser $user
+  else
+    echo "Database user \"$user\" does not exist. Not deleting."
+  fi
+
+  echo
+}
+
+function delete-database {
+  echo "Database name is: $database"
+
+  database_exists=`psql postgres -qtAXc "SELECT 1 FROM pg_database WHERE datname='$database'"`
+
+  if [ "$database_exists" = "1" ]; then
+    echo "Deleting database \"$database\"..."
+    dropdb $database
+  else
+    echo "Database \"$database\" does not exist. Not deleting."
+  fi
+
+  echo
+}
+
+
+delete-database
+delete-user

data/scripts/evt-pg-create-db

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+root = File.expand_path '../database', __dir__
+script_filename = 'install.sh'
+script_filepath = File.join root, script_filename
+
+system script_filepath

data/scripts/evt-pg-delete-db

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+root = File.expand_path '../database', __dir__
+script_filename = 'uninstall.sh'
+script_filepath = File.join root, script_filename
+
+system script_filepath

data/scripts/evt-pg-print-messages

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+root = File.expand_path '../database', __dir__
+script_filename = 'print-messages.sh'
+script_filepath = File.join root, script_filename
+
+system script_filepath

data/scripts/evt-pg-recreate-db

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+root = File.expand_path '../database', __dir__
+
+script_filename = 'uninstall.sh'
+script_filepath = File.join root, script_filename
+system script_filepath
+
+script_filename = 'install.sh'
+script_filepath = File.join root, script_filename
+system script_filepath

metadata

@@ -0,0 +1,70 @@
+--- !ruby/object:Gem::Specification
+name: evt-message_store-postgres-database
+version: !ruby/object:Gem::Version
+  version: 0.1.0.0
+platform: ruby
+authors:
+- The Eventide Project
+autorequire: 
+bindir: scripts
+cert_chain: []
+date: 2018-06-08 00:00:00.000000000 Z
+dependencies: []
+description: " "
+email: opensource@eventide-project.org
+executables:
+- evt-pg-print-messages
+- evt-pg-delete-db
+- evt-pg-create-db
+- evt-pg-recreate-db
+extensions: []
+extra_rdoc_files: []
+files:
+- database/clear-events-table.sh
+- database/extensions.sql
+- database/functions/category.sql
+- database/functions/get-category-messages.sql
+- database/functions/get-last-message.sql
+- database/functions/get-stream-messages.sql
+- database/functions/hash-64.sql
+- database/functions/scratch.sql
+- database/functions/stream-version.sql
+- database/functions/write-message.sql
+- database/indexes/messages-category-global-position.sql
+- database/indexes/messages-id.sql
+- database/indexes/messages-stream-name-position-uniq.sql
+- database/install.sh
+- database/install_functions.sh
+- database/print-messages.sh
+- database/table/messages-table.sql
+- database/types/message.sql
+- database/uninstall.sh
+- scripts/evt-pg-create-db
+- scripts/evt-pg-delete-db
+- scripts/evt-pg-print-messages
+- scripts/evt-pg-recreate-db
+homepage: https://github.com/eventide-project/message-store-postgres-database
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.3
+signing_key: 
+specification_version: 4
+summary: Message store PostgreSQL database definition
+test_files: []