message-db 0.0.0 → 1.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (73) hide show
  1. checksums.yaml +4 -4
  2. data/database/VERSION.txt +1 -0
  3. data/database/benchmark.sh +62 -0
  4. data/database/benchmark_get.sql +24 -0
  5. data/database/benchmark_write.sql +24 -0
  6. data/database/clear-messages.sh +26 -0
  7. data/database/extensions/pgcrypto.sql +1 -0
  8. data/database/functions/acquire-lock.sql +24 -0
  9. data/database/functions/cardinal-id.sql +18 -0
  10. data/database/functions/category.sql +10 -0
  11. data/database/functions/get-category-messages.sql +129 -0
  12. data/database/functions/get-last-stream-message.sql +37 -0
  13. data/database/functions/get-stream-messages.sql +71 -0
  14. data/database/functions/hash-64.sql +13 -0
  15. data/database/functions/id.sql +18 -0
  16. data/database/functions/is-category.sql +14 -0
  17. data/database/functions/message-store-version.sql +8 -0
  18. data/database/functions/stream-version.sql +19 -0
  19. data/database/functions/write-message.sql +77 -0
  20. data/database/indexes/messages-category.sql +7 -0
  21. data/database/indexes/messages-id.sql +5 -0
  22. data/database/indexes/messages-stream.sql +6 -0
  23. data/database/install-functions.sh +79 -0
  24. data/database/install-indexes.sh +33 -0
  25. data/database/install-privileges.sh +36 -0
  26. data/database/install-views.sh +42 -0
  27. data/database/install.sh +102 -0
  28. data/database/print-category-type-summary.sh +44 -0
  29. data/database/print-message-store-version.sh +32 -0
  30. data/database/print-messages.sh +44 -0
  31. data/database/print-stream-summary.sh +44 -0
  32. data/database/print-stream-type-summary.sh +44 -0
  33. data/database/print-type-category-summary.sh +44 -0
  34. data/database/print-type-stream-summary.sh +44 -0
  35. data/database/print-type-summary.sh +44 -0
  36. data/database/privileges/functions.sql +14 -0
  37. data/database/privileges/sequence.sql +1 -0
  38. data/database/privileges/table.sql +1 -0
  39. data/database/privileges/views.sql +6 -0
  40. data/database/roles/message-store.sql +7 -0
  41. data/database/schema/message-store.sql +1 -0
  42. data/database/tables/messages.sql +12 -0
  43. data/database/types/message.sql +15 -0
  44. data/database/uninstall.sh +54 -0
  45. data/database/update.sh +171 -0
  46. data/database/views/category-type-summary.sql +32 -0
  47. data/database/views/stream-summary.sql +28 -0
  48. data/database/views/stream-type-summary.sql +32 -0
  49. data/database/views/type-category-summary.sql +32 -0
  50. data/database/views/type-stream-summary.sql +32 -0
  51. data/database/views/type-summary.sql +28 -0
  52. data/database/write-test-message.sh +63 -0
  53. data/scripts/mdb-clear-messages +7 -0
  54. data/scripts/mdb-create-db +7 -0
  55. data/scripts/mdb-delete-db +7 -0
  56. data/scripts/mdb-install-functions +7 -0
  57. data/scripts/mdb-install-indexes +7 -0
  58. data/scripts/mdb-install-privileges +7 -0
  59. data/scripts/mdb-install-views +7 -0
  60. data/scripts/mdb-open-database-scripts-dir +5 -0
  61. data/scripts/mdb-print-category-type-summary +7 -0
  62. data/scripts/mdb-print-message-store-version +7 -0
  63. data/scripts/mdb-print-messages +7 -0
  64. data/scripts/mdb-print-stream-summary +7 -0
  65. data/scripts/mdb-print-stream-type-summary +7 -0
  66. data/scripts/mdb-print-type-category-summary +7 -0
  67. data/scripts/mdb-print-type-stream-summary +7 -0
  68. data/scripts/mdb-print-type-summary +7 -0
  69. data/scripts/mdb-recreate-db +11 -0
  70. data/scripts/mdb-update-db +7 -0
  71. data/scripts/mdb-write-test-message +7 -0
  72. metadata +94 -6
  73. data/MIT-License.txt +0 -20
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 56d836080e68d18d26462b409d0262ec20762315f1d06cd2e574497b7e3e8efd
4
- data.tar.gz: de652f57e7272a2fb4b61ff98730259a5a906670e08cb933c98a10681330cd04
3
+ metadata.gz: 933944df84aeb9ae8e8a0d265b7ef3b95a82df0d58eceed0ca31507f96f04299
4
+ data.tar.gz: 307379e48df23685b9ad494315568f152b57a3481b8f17a1dcf128fabefded9c
5
5
  SHA512:
6
- metadata.gz: 72f2836385d3c7333b2b7cccf608cc00c2836dece71448f7443c6c89f8342632ce4dfdbff648483c6e8d8b5dc1c7aa760aa2803c0643e2b831239625c0e0f827
7
- data.tar.gz: 781ef096eaed6a556d1ebd24486420f7697470da926bd49b77b27fa962e7d0d9668452b7c9b5ddc82b9be44b6cc77c07ec81c0f95188dcb21f0a6b13e4b393bf
6
+ metadata.gz: e730605cb9dbad8cf9072df8ec1fde409edaa6765e852893117a0a99e60bd387f94a0dbd6ffc895ca69ab3180d7b4217a2858ac640b6e272dc94038510ebf2be
7
+ data.tar.gz: 664e88ea262afa6f2771563b0926fc406e3585c2a112534ba80270de13a55de5816b437217ed21265308a517fc45989994dfa2b54945b1472c204aeb47b5e326
@@ -0,0 +1 @@
1
+ 1.0.0
@@ -0,0 +1,62 @@
1
+ #!/usr/bin/env bash
2
+
3
+ set -u
4
+
5
+ uuid=$(echo $(uuidgen) | tr '[:upper:]' '[:lower:]')
6
+
7
+ stream_name="testStream-$uuid"
8
+ if [ ! -z ${STREAM_NAME+x} ]; then
9
+ stream_name=$STREAM_NAME
10
+ fi
11
+
12
+ cycles=1000
13
+ if [ ! -z ${CYCLES+x} ]; then
14
+ cycles=$CYCLES
15
+ fi
16
+
17
+ function script_dir {
18
+ val="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
19
+ echo "$val"
20
+ }
21
+
22
+ base=$(script_dir)
23
+
24
+ echo
25
+ echo "Benchmark $cycles cycles (Stream Name: $stream_name)"
26
+ echo "= = ="
27
+ echo
28
+
29
+ if [ -z ${DATABASE_NAME+x} ]; then
30
+ echo "(DATABASE_NAME is not set)"
31
+ database=message_store
32
+ else
33
+ database=$DATABASE_NAME
34
+ fi
35
+ echo "Database name is: $database"
36
+ echo
37
+
38
+ echo "Installing benchmark scripts"
39
+ echo
40
+
41
+ psql $database -q -f $base/benchmark_write.sql
42
+ psql $database -q -f $base/benchmark_get.sql
43
+
44
+ echo
45
+ echo "Benchmarking write"
46
+ echo "- - -"
47
+ echo
48
+
49
+ psql $database -U message_store -c "EXPLAIN ANALYZE SELECT benchmark_write('$stream_name'::varchar, $cycles::int);"
50
+
51
+ echo
52
+
53
+ echo
54
+ echo "Benchmarking get"
55
+ echo "- - -"
56
+ echo
57
+
58
+ psql $database -U message_store -c "EXPLAIN ANALYZE SELECT benchmark_get('$stream_name'::varchar, $cycles::int);"
59
+
60
+ echo "= = ="
61
+ echo "Done"
62
+ echo
@@ -0,0 +1,24 @@
1
+ CREATE OR REPLACE FUNCTION benchmark_get(
2
+ stream_name varchar,
3
+ cycles int DEFAULT 1000
4
+ )
5
+ RETURNS void
6
+ AS $$
7
+ BEGIN
8
+ RAISE NOTICE '» benchmark_get';
9
+ RAISE NOTICE 'stream_name: %', benchmark_get.stream_name;
10
+ RAISE NOTICE 'cycles: %', benchmark_get.cycles;
11
+
12
+ FOR i IN 1..cycles LOOP
13
+ IF current_setting('message_store.debug_benchmark', true) = 'on' OR current_setting('message_store.debug', true) = 'on' THEN
14
+ RAISE NOTICE '%', i;
15
+ END IF;
16
+
17
+ PERFORM get_stream_messages(stream_name, "position" => i - 1, batch_size => 1);
18
+ END LOOP;
19
+ END;
20
+ $$ LANGUAGE plpgsql
21
+ VOLATILE;
22
+
23
+
24
+
@@ -0,0 +1,24 @@
1
+ CREATE OR REPLACE FUNCTION benchmark_write(
2
+ stream_name varchar,
3
+ cycles int DEFAULT 1000
4
+ )
5
+ RETURNS void
6
+ AS $$
7
+ BEGIN
8
+ RAISE NOTICE '» benchmark_write';
9
+ RAISE NOTICE 'stream_name: %', benchmark_write.stream_name;
10
+ RAISE NOTICE 'cycles: %', benchmark_write.cycles;
11
+
12
+ FOR i IN 1..cycles LOOP
13
+ IF current_setting('message_store.debug_benchmark', true) = 'on' OR current_setting('message_store.debug', true) = 'on' THEN
14
+ RAISE NOTICE '%', i;
15
+ END IF;
16
+
17
+ PERFORM write_message(gen_random_uuid()::varchar, stream_name::varchar, 'SomeType'::varchar, '{"attribute": "some value"}'::jsonb, '{"metaAttribute": "some meta value", "correlationStreamName": "someCorrelation-123"}'::jsonb);
18
+ END LOOP;
19
+ END;
20
+ $$ LANGUAGE plpgsql
21
+ VOLATILE;
22
+
23
+
24
+
@@ -0,0 +1,26 @@
1
+ #!/usr/bin/env bash
2
+
3
+ set -e
4
+
5
+ echo
6
+ echo "Clearing Messages Table"
7
+ echo "= = ="
8
+ echo
9
+
10
+ default_name=message_store
11
+
12
+ if [ -z ${DATABASE_NAME+x} ]; then
13
+ echo "(DATABASE_NAME is not set)"
14
+ database=$default_name
15
+ else
16
+ database=$DATABASE_NAME
17
+ fi
18
+ echo "Database name is: $database"
19
+
20
+ echo
21
+
22
+ psql $database -q -c "TRUNCATE message_store.messages RESTART IDENTITY;"
23
+
24
+ echo "= = ="
25
+ echo "Done Clearing Messages Table"
26
+ echo
@@ -0,0 +1 @@
1
+ CREATE EXTENSION IF NOT EXISTS pgcrypto SCHEMA message_store;
@@ -0,0 +1,24 @@
1
+ CREATE OR REPLACE FUNCTION message_store.acquire_lock(
2
+ stream_name varchar
3
+ )
4
+ RETURNS bigint
5
+ AS $$
6
+ DECLARE
7
+ _category varchar;
8
+ _category_name_hash bigint;
9
+ BEGIN
10
+ _category := category(acquire_lock.stream_name);
11
+ _category_name_hash := hash_64(_category);
12
+ PERFORM pg_advisory_xact_lock(_category_name_hash);
13
+
14
+ IF current_setting('message_store.debug_write', true) = 'on' OR current_setting('message_store.debug', true) = 'on' THEN
15
+ RAISE NOTICE '» acquire_lock';
16
+ RAISE NOTICE 'stream_name: %', acquire_lock.stream_name;
17
+ RAISE NOTICE '_category: %', _category;
18
+ RAISE NOTICE '_category_name_hash: %', _category_name_hash;
19
+ END IF;
20
+
21
+ RETURN _category_name_hash;
22
+ END;
23
+ $$ LANGUAGE plpgsql
24
+ VOLATILE;
@@ -0,0 +1,18 @@
1
+ CREATE OR REPLACE FUNCTION message_store.cardinal_id(
2
+ stream_name varchar
3
+ )
4
+ RETURNS varchar
5
+ AS $$
6
+ DECLARE
7
+ _id varchar;
8
+ BEGIN
9
+ _id := id(cardinal_id.stream_name);
10
+
11
+ IF _id IS NULL THEN
12
+ RETURN NULL;
13
+ END IF;
14
+
15
+ RETURN SPLIT_PART(_id, '+', 1);
16
+ END;
17
+ $$ LANGUAGE plpgsql
18
+ IMMUTABLE;
@@ -0,0 +1,10 @@
1
+ CREATE OR REPLACE FUNCTION message_store.category(
2
+ stream_name varchar
3
+ )
4
+ RETURNS varchar
5
+ AS $$
6
+ BEGIN
7
+ RETURN SPLIT_PART(category.stream_name, '-', 1);
8
+ END;
9
+ $$ LANGUAGE plpgsql
10
+ IMMUTABLE;
@@ -0,0 +1,129 @@
1
+ CREATE OR REPLACE FUNCTION message_store.get_category_messages(
2
+ category varchar,
3
+ "position" bigint DEFAULT 1,
4
+ batch_size bigint DEFAULT 1000,
5
+ correlation varchar DEFAULT NULL,
6
+ consumer_group_member bigint DEFAULT NULL,
7
+ consumer_group_size bigint DEFAULT NULL,
8
+ condition varchar DEFAULT NULL
9
+ )
10
+ RETURNS SETOF message_store.message
11
+ AS $$
12
+ DECLARE
13
+ _command text;
14
+ BEGIN
15
+ IF NOT is_category(get_category_messages.category) THEN
16
+ RAISE EXCEPTION
17
+ 'Must be a category: %',
18
+ get_category_messages.category;
19
+ END IF;
20
+
21
+ position := COALESCE(position, 1);
22
+ batch_size := COALESCE(batch_size, 1000);
23
+
24
+ _command := '
25
+ SELECT
26
+ id::varchar,
27
+ stream_name::varchar,
28
+ type::varchar,
29
+ position::bigint,
30
+ global_position::bigint,
31
+ data::varchar,
32
+ metadata::varchar,
33
+ time::timestamp
34
+ FROM
35
+ messages
36
+ WHERE
37
+ category(stream_name) = $1 AND
38
+ global_position >= $2';
39
+
40
+ IF get_category_messages.correlation IS NOT NULL THEN
41
+ IF position('-' IN get_category_messages.correlation) > 0 THEN
42
+ RAISE EXCEPTION
43
+ 'Correlation must be a category (Correlation: %)',
44
+ get_category_messages.correlation;
45
+ END IF;
46
+
47
+ _command := _command || ' AND
48
+ category(metadata->>''correlationStreamName'') = $4';
49
+ END IF;
50
+
51
+ IF (get_category_messages.consumer_group_member IS NOT NULL AND
52
+ get_category_messages.consumer_group_size IS NULL) OR
53
+ (get_category_messages.consumer_group_member IS NULL AND
54
+ get_category_messages.consumer_group_size IS NOT NULL) THEN
55
+
56
+ RAISE EXCEPTION
57
+ 'Consumer group member and size must be specified (Consumer Group Member: %, Consumer Group Size: %)',
58
+ get_category_messages.consumer_group_member,
59
+ get_category_messages.consumer_group_size;
60
+ END IF;
61
+
62
+ IF get_category_messages.consumer_group_member IS NOT NULL AND
63
+ get_category_messages.consumer_group_size IS NOT NULL THEN
64
+
65
+ IF get_category_messages.consumer_group_size < 1 THEN
66
+ RAISE EXCEPTION
67
+ 'Consumer group size must not be less than 1 (Consumer Group Member: %, Consumer Group Size: %)',
68
+ get_category_messages.consumer_group_member,
69
+ get_category_messages.consumer_group_size;
70
+ END IF;
71
+
72
+ IF get_category_messages.consumer_group_member < 0 THEN
73
+ RAISE EXCEPTION
74
+ 'Consumer group member must not be less than 0 (Consumer Group Member: %, Consumer Group Size: %)',
75
+ get_category_messages.consumer_group_member,
76
+ get_category_messages.consumer_group_size;
77
+ END IF;
78
+
79
+ IF get_category_messages.consumer_group_member >= get_category_messages.consumer_group_size THEN
80
+ RAISE EXCEPTION
81
+ 'Consumer group member must be less than the group size (Consumer Group Member: %, Consumer Group Size: %)',
82
+ get_category_messages.consumer_group_member,
83
+ get_category_messages.consumer_group_size;
84
+ END IF;
85
+
86
+ _command := _command || ' AND
87
+ MOD(@hash_64(cardinal_id(stream_name)), $6) = $5';
88
+ END IF;
89
+
90
+ IF get_category_messages.condition IS NOT NULL THEN
91
+ IF current_setting('message_store.sql_condition', true) IS NULL OR
92
+ current_setting('message_store.sql_condition', true) = 'off' THEN
93
+ RAISE EXCEPTION
94
+ 'Retrieval with SQL condition is not activated';
95
+ END IF;
96
+
97
+ _command := _command || ' AND
98
+ (%s)';
99
+ _command := format(_command, get_category_messages.condition);
100
+ END IF;
101
+
102
+ _command := _command || '
103
+ ORDER BY
104
+ global_position ASC
105
+ LIMIT
106
+ $3';
107
+
108
+ IF current_setting('message_store.debug_get', true) = 'on' OR current_setting('message_store.debug', true) = 'on' THEN
109
+ RAISE NOTICE '» get_category_messages';
110
+ RAISE NOTICE 'category ($1): %', get_category_messages.category;
111
+ RAISE NOTICE 'position ($2): %', get_category_messages.position;
112
+ RAISE NOTICE 'batch_size ($3): %', get_category_messages.batch_size;
113
+ RAISE NOTICE 'correlation ($4): %', get_category_messages.correlation;
114
+ RAISE NOTICE 'consumer_group_member ($5): %', get_category_messages.consumer_group_member;
115
+ RAISE NOTICE 'consumer_group_size ($6): %', get_category_messages.consumer_group_size;
116
+ RAISE NOTICE 'condition: %', get_category_messages.condition;
117
+ RAISE NOTICE 'Generated Command: %', _command;
118
+ END IF;
119
+
120
+ RETURN QUERY EXECUTE _command USING
121
+ get_category_messages.category,
122
+ get_category_messages.position,
123
+ get_category_messages.batch_size,
124
+ get_category_messages.correlation,
125
+ get_category_messages.consumer_group_member,
126
+ get_category_messages.consumer_group_size::smallint;
127
+ END;
128
+ $$ LANGUAGE plpgsql
129
+ VOLATILE;
@@ -0,0 +1,37 @@
1
+ CREATE OR REPLACE FUNCTION message_store.get_last_stream_message(
2
+ stream_name varchar
3
+ )
4
+ RETURNS SETOF message_store.message
5
+ AS $$
6
+ DECLARE
7
+ _command text;
8
+ BEGIN
9
+ _command := '
10
+ SELECT
11
+ id::varchar,
12
+ stream_name::varchar,
13
+ type::varchar,
14
+ position::bigint,
15
+ global_position::bigint,
16
+ data::varchar,
17
+ metadata::varchar,
18
+ time::timestamp
19
+ FROM
20
+ messages
21
+ WHERE
22
+ stream_name = $1
23
+ ORDER BY
24
+ position DESC
25
+ LIMIT
26
+ 1';
27
+
28
+ IF current_setting('message_store.debug_get', true) = 'on' OR current_setting('message_store.debug', true) = 'on' THEN
29
+ RAISE NOTICE '» get_last_message';
30
+ RAISE NOTICE 'stream_name ($1): %', get_last_stream_message.stream_name;
31
+ RAISE NOTICE 'Generated Command: %', _command;
32
+ END IF;
33
+
34
+ RETURN QUERY EXECUTE _command USING get_last_stream_message.stream_name;
35
+ END;
36
+ $$ LANGUAGE plpgsql
37
+ VOLATILE;
@@ -0,0 +1,71 @@
1
+ CREATE OR REPLACE FUNCTION message_store.get_stream_messages(
2
+ stream_name varchar,
3
+ "position" bigint DEFAULT 0,
4
+ batch_size bigint DEFAULT 1000,
5
+ condition varchar DEFAULT NULL
6
+ )
7
+ RETURNS SETOF message_store.message
8
+ AS $$
9
+ DECLARE
10
+ _command text;
11
+ _setting text;
12
+ BEGIN
13
+ IF is_category(get_stream_messages.stream_name) THEN
14
+ RAISE EXCEPTION
15
+ 'Must be a stream name: %',
16
+ get_stream_messages.stream_name;
17
+ END IF;
18
+
19
+ position := COALESCE(position, 0);
20
+ batch_size := COALESCE(batch_size, 1000);
21
+
22
+ _command := '
23
+ SELECT
24
+ id::varchar,
25
+ stream_name::varchar,
26
+ type::varchar,
27
+ position::bigint,
28
+ global_position::bigint,
29
+ data::varchar,
30
+ metadata::varchar,
31
+ time::timestamp
32
+ FROM
33
+ messages
34
+ WHERE
35
+ stream_name = $1 AND
36
+ position >= $2';
37
+
38
+ IF get_stream_messages.condition IS NOT NULL THEN
39
+ IF current_setting('message_store.sql_condition', true) IS NULL OR
40
+ current_setting('message_store.sql_condition', true) = 'off' THEN
41
+ RAISE EXCEPTION
42
+ 'Retrieval with SQL condition is not activated';
43
+ END IF;
44
+
45
+ _command := _command || ' AND
46
+ (%s)';
47
+ _command := format(_command, get_stream_messages.condition);
48
+ END IF;
49
+
50
+ _command := _command || '
51
+ ORDER BY
52
+ position ASC
53
+ LIMIT
54
+ $3';
55
+
56
+ IF current_setting('message_store.debug_get', true) = 'on' OR current_setting('message_store.debug', true) = 'on' THEN
57
+ RAISE NOTICE '» get_stream_messages';
58
+ RAISE NOTICE 'stream_name ($1): %', get_stream_messages.stream_name;
59
+ RAISE NOTICE 'position ($2): %', get_stream_messages.position;
60
+ RAISE NOTICE 'batch_size ($3): %', get_stream_messages.batch_size;
61
+ RAISE NOTICE 'condition ($4): %', get_stream_messages.condition;
62
+ RAISE NOTICE 'Generated Command: %', _command;
63
+ end if;
64
+
65
+ RETURN QUERY EXECUTE _command USING
66
+ get_stream_messages.stream_name,
67
+ get_stream_messages.position,
68
+ get_stream_messages.batch_size;
69
+ END;
70
+ $$ LANGUAGE plpgsql
71
+ VOLATILE;