message-db 0.0.0 → 1.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. checksums.yaml +4 -4
  2. data/database/VERSION.txt +1 -0
  3. data/database/benchmark.sh +62 -0
  4. data/database/benchmark_get.sql +24 -0
  5. data/database/benchmark_write.sql +24 -0
  6. data/database/clear-messages.sh +26 -0
  7. data/database/extensions/pgcrypto.sql +1 -0
  8. data/database/functions/acquire-lock.sql +24 -0
  9. data/database/functions/cardinal-id.sql +18 -0
  10. data/database/functions/category.sql +10 -0
  11. data/database/functions/get-category-messages.sql +129 -0
  12. data/database/functions/get-last-stream-message.sql +37 -0
  13. data/database/functions/get-stream-messages.sql +71 -0
  14. data/database/functions/hash-64.sql +13 -0
  15. data/database/functions/id.sql +18 -0
  16. data/database/functions/is-category.sql +14 -0
  17. data/database/functions/message-store-version.sql +8 -0
  18. data/database/functions/stream-version.sql +19 -0
  19. data/database/functions/write-message.sql +73 -0
  20. data/database/indexes/messages-category.sql +7 -0
  21. data/database/indexes/messages-id.sql +5 -0
  22. data/database/indexes/messages-stream.sql +6 -0
  23. data/database/install-functions.sh +79 -0
  24. data/database/install-indexes.sh +33 -0
  25. data/database/install-privileges.sh +39 -0
  26. data/database/install-views.sh +42 -0
  27. data/database/install.sh +102 -0
  28. data/database/print-category-type-summary.sh +44 -0
  29. data/database/print-message-store-version.sh +32 -0
  30. data/database/print-messages.sh +44 -0
  31. data/database/print-stream-summary.sh +44 -0
  32. data/database/print-stream-type-summary.sh +44 -0
  33. data/database/print-type-category-summary.sh +44 -0
  34. data/database/print-type-stream-summary.sh +44 -0
  35. data/database/print-type-summary.sh +44 -0
  36. data/database/privileges/functions.sql +15 -0
  37. data/database/privileges/schema.sql +1 -0
  38. data/database/privileges/sequence.sql +1 -0
  39. data/database/privileges/table.sql +1 -0
  40. data/database/privileges/views.sql +6 -0
  41. data/database/roles/message-store.sql +7 -0
  42. data/database/schema/message-store.sql +1 -0
  43. data/database/tables/messages.sql +12 -0
  44. data/database/types/message.sql +15 -0
  45. data/database/uninstall.sh +54 -0
  46. data/database/update.sh +171 -0
  47. data/database/views/category-type-summary.sql +32 -0
  48. data/database/views/stream-summary.sql +28 -0
  49. data/database/views/stream-type-summary.sql +32 -0
  50. data/database/views/type-category-summary.sql +32 -0
  51. data/database/views/type-stream-summary.sql +32 -0
  52. data/database/views/type-summary.sql +28 -0
  53. data/database/write-test-message.sh +63 -0
  54. data/scripts/mdb-clear-messages +7 -0
  55. data/scripts/mdb-create-db +7 -0
  56. data/scripts/mdb-delete-db +7 -0
  57. data/scripts/mdb-install-functions +7 -0
  58. data/scripts/mdb-install-indexes +7 -0
  59. data/scripts/mdb-install-privileges +7 -0
  60. data/scripts/mdb-install-views +7 -0
  61. data/scripts/mdb-open-database-scripts-dir +5 -0
  62. data/scripts/mdb-print-category-type-summary +7 -0
  63. data/scripts/mdb-print-message-store-version +7 -0
  64. data/scripts/mdb-print-messages +7 -0
  65. data/scripts/mdb-print-stream-summary +7 -0
  66. data/scripts/mdb-print-stream-type-summary +7 -0
  67. data/scripts/mdb-print-type-category-summary +7 -0
  68. data/scripts/mdb-print-type-stream-summary +7 -0
  69. data/scripts/mdb-print-type-summary +7 -0
  70. data/scripts/mdb-recreate-db +11 -0
  71. data/scripts/mdb-update-db +7 -0
  72. data/scripts/mdb-write-test-message +7 -0
  73. metadata +95 -6
  74. data/MIT-License.txt +0 -20
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 56d836080e68d18d26462b409d0262ec20762315f1d06cd2e574497b7e3e8efd
4
- data.tar.gz: de652f57e7272a2fb4b61ff98730259a5a906670e08cb933c98a10681330cd04
3
+ metadata.gz: e483e428c6311bd88f5c9eb6b10567a26657fc7bb324b807ac29493ad4e1a5cf
4
+ data.tar.gz: 69fbdccd72ae4e34f9155b235356c2f0daa2b18c5135499cf133f427b541a607
5
5
  SHA512:
6
- metadata.gz: 72f2836385d3c7333b2b7cccf608cc00c2836dece71448f7443c6c89f8342632ce4dfdbff648483c6e8d8b5dc1c7aa760aa2803c0643e2b831239625c0e0f827
7
- data.tar.gz: 781ef096eaed6a556d1ebd24486420f7697470da926bd49b77b27fa962e7d0d9668452b7c9b5ddc82b9be44b6cc77c07ec81c0f95188dcb21f0a6b13e4b393bf
6
+ metadata.gz: da65afe3dedb6426ae5d7fd2db99e368d76450c450ddbe7f38a430f64353c4a956d77f0dba8fc5c59c6d459ec6669f0ecfc3a968941ce05138a9aa75d8092954
7
+ data.tar.gz: 73322f41132a625db09c30d780dfc808315344ab804ef143cbe3048ef03a670f194dc94505801c33b10c776332e80625cc0df72f31c6e3a34d60aa2559349bf4
@@ -0,0 +1 @@
1
+ 1.1.3
@@ -0,0 +1,62 @@
1
+ #!/usr/bin/env bash
2
+
3
+ set -u
4
+
5
+ uuid=$(echo $(uuidgen) | tr '[:upper:]' '[:lower:]')
6
+
7
+ stream_name="testStream-$uuid"
8
+ if [ ! -z ${STREAM_NAME+x} ]; then
9
+ stream_name=$STREAM_NAME
10
+ fi
11
+
12
+ cycles=1000
13
+ if [ ! -z ${CYCLES+x} ]; then
14
+ cycles=$CYCLES
15
+ fi
16
+
17
+ function script_dir {
18
+ val="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
19
+ echo "$val"
20
+ }
21
+
22
+ base=$(script_dir)
23
+
24
+ echo
25
+ echo "Benchmark $cycles cycles (Stream Name: $stream_name)"
26
+ echo "= = ="
27
+ echo
28
+
29
+ if [ -z ${DATABASE_NAME+x} ]; then
30
+ echo "(DATABASE_NAME is not set)"
31
+ database=message_store
32
+ else
33
+ database=$DATABASE_NAME
34
+ fi
35
+ echo "Database name is: $database"
36
+ echo
37
+
38
+ echo "Installing benchmark scripts"
39
+ echo
40
+
41
+ psql $database -q -f $base/benchmark_write.sql
42
+ psql $database -q -f $base/benchmark_get.sql
43
+
44
+ echo
45
+ echo "Benchmarking write"
46
+ echo "- - -"
47
+ echo
48
+
49
+ psql $database -U message_store -c "EXPLAIN ANALYZE SELECT benchmark_write('$stream_name'::varchar, $cycles::int);"
50
+
51
+ echo
52
+
53
+ echo
54
+ echo "Benchmarking get"
55
+ echo "- - -"
56
+ echo
57
+
58
+ psql $database -U message_store -c "EXPLAIN ANALYZE SELECT benchmark_get('$stream_name'::varchar, $cycles::int);"
59
+
60
+ echo "= = ="
61
+ echo "Done"
62
+ echo
@@ -0,0 +1,24 @@
1
+ CREATE OR REPLACE FUNCTION benchmark_get(
2
+ stream_name varchar,
3
+ cycles int DEFAULT 1000
4
+ )
5
+ RETURNS void
6
+ AS $$
7
+ BEGIN
8
+ RAISE NOTICE '» benchmark_get';
9
+ RAISE NOTICE 'stream_name: %', benchmark_get.stream_name;
10
+ RAISE NOTICE 'cycles: %', benchmark_get.cycles;
11
+
12
+ FOR i IN 1..cycles LOOP
13
+ IF current_setting('message_store.debug_benchmark', true) = 'on' OR current_setting('message_store.debug', true) = 'on' THEN
14
+ RAISE NOTICE '%', i;
15
+ END IF;
16
+
17
+ PERFORM get_stream_messages(stream_name, "position" => i - 1, batch_size => 1);
18
+ END LOOP;
19
+ END;
20
+ $$ LANGUAGE plpgsql
21
+ VOLATILE;
22
+
23
+
24
+
@@ -0,0 +1,24 @@
1
+ CREATE OR REPLACE FUNCTION benchmark_write(
2
+ stream_name varchar,
3
+ cycles int DEFAULT 1000
4
+ )
5
+ RETURNS void
6
+ AS $$
7
+ BEGIN
8
+ RAISE NOTICE '» benchmark_write';
9
+ RAISE NOTICE 'stream_name: %', benchmark_write.stream_name;
10
+ RAISE NOTICE 'cycles: %', benchmark_write.cycles;
11
+
12
+ FOR i IN 1..cycles LOOP
13
+ IF current_setting('message_store.debug_benchmark', true) = 'on' OR current_setting('message_store.debug', true) = 'on' THEN
14
+ RAISE NOTICE '%', i;
15
+ END IF;
16
+
17
+ PERFORM write_message(gen_random_uuid()::varchar, stream_name::varchar, 'SomeType'::varchar, '{"attribute": "some value"}'::jsonb, '{"metaAttribute": "some meta value", "correlationStreamName": "someCorrelation-123"}'::jsonb);
18
+ END LOOP;
19
+ END;
20
+ $$ LANGUAGE plpgsql
21
+ VOLATILE;
22
+
23
+
24
+
@@ -0,0 +1,26 @@
1
+ #!/usr/bin/env bash
2
+
3
+ set -e
4
+
5
+ echo
6
+ echo "Clearing Messages Table"
7
+ echo "= = ="
8
+ echo
9
+
10
+ default_name=message_store
11
+
12
+ if [ -z ${DATABASE_NAME+x} ]; then
13
+ echo "(DATABASE_NAME is not set)"
14
+ database=$default_name
15
+ else
16
+ database=$DATABASE_NAME
17
+ fi
18
+ echo "Database name is: $database"
19
+
20
+ echo
21
+
22
+ psql $database -q -c "TRUNCATE message_store.messages RESTART IDENTITY;"
23
+
24
+ echo "= = ="
25
+ echo "Done Clearing Messages Table"
26
+ echo
@@ -0,0 +1 @@
1
+ CREATE EXTENSION IF NOT EXISTS pgcrypto;
@@ -0,0 +1,24 @@
1
+ CREATE OR REPLACE FUNCTION message_store.acquire_lock(
2
+ stream_name varchar
3
+ )
4
+ RETURNS bigint
5
+ AS $$
6
+ DECLARE
7
+ _category varchar;
8
+ _category_name_hash bigint;
9
+ BEGIN
10
+ _category := category(acquire_lock.stream_name);
11
+ _category_name_hash := hash_64(_category);
12
+ PERFORM pg_advisory_xact_lock(_category_name_hash);
13
+
14
+ IF current_setting('message_store.debug_write', true) = 'on' OR current_setting('message_store.debug', true) = 'on' THEN
15
+ RAISE NOTICE '» acquire_lock';
16
+ RAISE NOTICE 'stream_name: %', acquire_lock.stream_name;
17
+ RAISE NOTICE '_category: %', _category;
18
+ RAISE NOTICE '_category_name_hash: %', _category_name_hash;
19
+ END IF;
20
+
21
+ RETURN _category_name_hash;
22
+ END;
23
+ $$ LANGUAGE plpgsql
24
+ VOLATILE;
@@ -0,0 +1,18 @@
1
+ CREATE OR REPLACE FUNCTION message_store.cardinal_id(
2
+ stream_name varchar
3
+ )
4
+ RETURNS varchar
5
+ AS $$
6
+ DECLARE
7
+ _id varchar;
8
+ BEGIN
9
+ _id := id(cardinal_id.stream_name);
10
+
11
+ IF _id IS NULL THEN
12
+ RETURN NULL;
13
+ END IF;
14
+
15
+ RETURN SPLIT_PART(_id, '+', 1);
16
+ END;
17
+ $$ LANGUAGE plpgsql
18
+ IMMUTABLE;
@@ -0,0 +1,10 @@
1
+ CREATE OR REPLACE FUNCTION message_store.category(
2
+ stream_name varchar
3
+ )
4
+ RETURNS varchar
5
+ AS $$
6
+ BEGIN
7
+ RETURN SPLIT_PART(category.stream_name, '-', 1);
8
+ END;
9
+ $$ LANGUAGE plpgsql
10
+ IMMUTABLE;
@@ -0,0 +1,129 @@
1
+ CREATE OR REPLACE FUNCTION message_store.get_category_messages(
2
+ category varchar,
3
+ "position" bigint DEFAULT 1,
4
+ batch_size bigint DEFAULT 1000,
5
+ correlation varchar DEFAULT NULL,
6
+ consumer_group_member bigint DEFAULT NULL,
7
+ consumer_group_size bigint DEFAULT NULL,
8
+ condition varchar DEFAULT NULL
9
+ )
10
+ RETURNS SETOF message_store.message
11
+ AS $$
12
+ DECLARE
13
+ _command text;
14
+ BEGIN
15
+ IF NOT is_category(get_category_messages.category) THEN
16
+ RAISE EXCEPTION
17
+ 'Must be a category: %',
18
+ get_category_messages.category;
19
+ END IF;
20
+
21
+ position := COALESCE(position, 1);
22
+ batch_size := COALESCE(batch_size, 1000);
23
+
24
+ _command := '
25
+ SELECT
26
+ id::varchar,
27
+ stream_name::varchar,
28
+ type::varchar,
29
+ position::bigint,
30
+ global_position::bigint,
31
+ data::varchar,
32
+ metadata::varchar,
33
+ time::timestamp
34
+ FROM
35
+ messages
36
+ WHERE
37
+ category(stream_name) = $1 AND
38
+ global_position >= $2';
39
+
40
+ IF get_category_messages.correlation IS NOT NULL THEN
41
+ IF position('-' IN get_category_messages.correlation) > 0 THEN
42
+ RAISE EXCEPTION
43
+ 'Correlation must be a category (Correlation: %)',
44
+ get_category_messages.correlation;
45
+ END IF;
46
+
47
+ _command := _command || ' AND
48
+ category(metadata->>''correlationStreamName'') = $4';
49
+ END IF;
50
+
51
+ IF (get_category_messages.consumer_group_member IS NOT NULL AND
52
+ get_category_messages.consumer_group_size IS NULL) OR
53
+ (get_category_messages.consumer_group_member IS NULL AND
54
+ get_category_messages.consumer_group_size IS NOT NULL) THEN
55
+
56
+ RAISE EXCEPTION
57
+ 'Consumer group member and size must be specified (Consumer Group Member: %, Consumer Group Size: %)',
58
+ get_category_messages.consumer_group_member,
59
+ get_category_messages.consumer_group_size;
60
+ END IF;
61
+
62
+ IF get_category_messages.consumer_group_member IS NOT NULL AND
63
+ get_category_messages.consumer_group_size IS NOT NULL THEN
64
+
65
+ IF get_category_messages.consumer_group_size < 1 THEN
66
+ RAISE EXCEPTION
67
+ 'Consumer group size must not be less than 1 (Consumer Group Member: %, Consumer Group Size: %)',
68
+ get_category_messages.consumer_group_member,
69
+ get_category_messages.consumer_group_size;
70
+ END IF;
71
+
72
+ IF get_category_messages.consumer_group_member < 0 THEN
73
+ RAISE EXCEPTION
74
+ 'Consumer group member must not be less than 0 (Consumer Group Member: %, Consumer Group Size: %)',
75
+ get_category_messages.consumer_group_member,
76
+ get_category_messages.consumer_group_size;
77
+ END IF;
78
+
79
+ IF get_category_messages.consumer_group_member >= get_category_messages.consumer_group_size THEN
80
+ RAISE EXCEPTION
81
+ 'Consumer group member must be less than the group size (Consumer Group Member: %, Consumer Group Size: %)',
82
+ get_category_messages.consumer_group_member,
83
+ get_category_messages.consumer_group_size;
84
+ END IF;
85
+
86
+ _command := _command || ' AND
87
+ MOD(@hash_64(cardinal_id(stream_name)), $6) = $5';
88
+ END IF;
89
+
90
+ IF get_category_messages.condition IS NOT NULL THEN
91
+ IF current_setting('message_store.sql_condition', true) IS NULL OR
92
+ current_setting('message_store.sql_condition', true) = 'off' THEN
93
+ RAISE EXCEPTION
94
+ 'Retrieval with SQL condition is not activated';
95
+ END IF;
96
+
97
+ _command := _command || ' AND
98
+ (%s)';
99
+ _command := format(_command, get_category_messages.condition);
100
+ END IF;
101
+
102
+ _command := _command || '
103
+ ORDER BY
104
+ global_position ASC
105
+ LIMIT
106
+ $3';
107
+
108
+ IF current_setting('message_store.debug_get', true) = 'on' OR current_setting('message_store.debug', true) = 'on' THEN
109
+ RAISE NOTICE '» get_category_messages';
110
+ RAISE NOTICE 'category ($1): %', get_category_messages.category;
111
+ RAISE NOTICE 'position ($2): %', get_category_messages.position;
112
+ RAISE NOTICE 'batch_size ($3): %', get_category_messages.batch_size;
113
+ RAISE NOTICE 'correlation ($4): %', get_category_messages.correlation;
114
+ RAISE NOTICE 'consumer_group_member ($5): %', get_category_messages.consumer_group_member;
115
+ RAISE NOTICE 'consumer_group_size ($6): %', get_category_messages.consumer_group_size;
116
+ RAISE NOTICE 'condition: %', get_category_messages.condition;
117
+ RAISE NOTICE 'Generated Command: %', _command;
118
+ END IF;
119
+
120
+ RETURN QUERY EXECUTE _command USING
121
+ get_category_messages.category,
122
+ get_category_messages.position,
123
+ get_category_messages.batch_size,
124
+ get_category_messages.correlation,
125
+ get_category_messages.consumer_group_member,
126
+ get_category_messages.consumer_group_size::smallint;
127
+ END;
128
+ $$ LANGUAGE plpgsql
129
+ VOLATILE;
@@ -0,0 +1,37 @@
1
+ CREATE OR REPLACE FUNCTION message_store.get_last_stream_message(
2
+ stream_name varchar
3
+ )
4
+ RETURNS SETOF message_store.message
5
+ AS $$
6
+ DECLARE
7
+ _command text;
8
+ BEGIN
9
+ _command := '
10
+ SELECT
11
+ id::varchar,
12
+ stream_name::varchar,
13
+ type::varchar,
14
+ position::bigint,
15
+ global_position::bigint,
16
+ data::varchar,
17
+ metadata::varchar,
18
+ time::timestamp
19
+ FROM
20
+ messages
21
+ WHERE
22
+ stream_name = $1
23
+ ORDER BY
24
+ position DESC
25
+ LIMIT
26
+ 1';
27
+
28
+ IF current_setting('message_store.debug_get', true) = 'on' OR current_setting('message_store.debug', true) = 'on' THEN
29
+ RAISE NOTICE '» get_last_message';
30
+ RAISE NOTICE 'stream_name ($1): %', get_last_stream_message.stream_name;
31
+ RAISE NOTICE 'Generated Command: %', _command;
32
+ END IF;
33
+
34
+ RETURN QUERY EXECUTE _command USING get_last_stream_message.stream_name;
35
+ END;
36
+ $$ LANGUAGE plpgsql
37
+ VOLATILE;
@@ -0,0 +1,71 @@
1
+ CREATE OR REPLACE FUNCTION message_store.get_stream_messages(
2
+ stream_name varchar,
3
+ "position" bigint DEFAULT 0,
4
+ batch_size bigint DEFAULT 1000,
5
+ condition varchar DEFAULT NULL
6
+ )
7
+ RETURNS SETOF message_store.message
8
+ AS $$
9
+ DECLARE
10
+ _command text;
11
+ _setting text;
12
+ BEGIN
13
+ IF is_category(get_stream_messages.stream_name) THEN
14
+ RAISE EXCEPTION
15
+ 'Must be a stream name: %',
16
+ get_stream_messages.stream_name;
17
+ END IF;
18
+
19
+ position := COALESCE(position, 0);
20
+ batch_size := COALESCE(batch_size, 1000);
21
+
22
+ _command := '
23
+ SELECT
24
+ id::varchar,
25
+ stream_name::varchar,
26
+ type::varchar,
27
+ position::bigint,
28
+ global_position::bigint,
29
+ data::varchar,
30
+ metadata::varchar,
31
+ time::timestamp
32
+ FROM
33
+ messages
34
+ WHERE
35
+ stream_name = $1 AND
36
+ position >= $2';
37
+
38
+ IF get_stream_messages.condition IS NOT NULL THEN
39
+ IF current_setting('message_store.sql_condition', true) IS NULL OR
40
+ current_setting('message_store.sql_condition', true) = 'off' THEN
41
+ RAISE EXCEPTION
42
+ 'Retrieval with SQL condition is not activated';
43
+ END IF;
44
+
45
+ _command := _command || ' AND
46
+ (%s)';
47
+ _command := format(_command, get_stream_messages.condition);
48
+ END IF;
49
+
50
+ _command := _command || '
51
+ ORDER BY
52
+ position ASC
53
+ LIMIT
54
+ $3';
55
+
56
+ IF current_setting('message_store.debug_get', true) = 'on' OR current_setting('message_store.debug', true) = 'on' THEN
57
+ RAISE NOTICE '» get_stream_messages';
58
+ RAISE NOTICE 'stream_name ($1): %', get_stream_messages.stream_name;
59
+ RAISE NOTICE 'position ($2): %', get_stream_messages.position;
60
+ RAISE NOTICE 'batch_size ($3): %', get_stream_messages.batch_size;
61
+ RAISE NOTICE 'condition ($4): %', get_stream_messages.condition;
62
+ RAISE NOTICE 'Generated Command: %', _command;
63
+ end if;
64
+
65
+ RETURN QUERY EXECUTE _command USING
66
+ get_stream_messages.stream_name,
67
+ get_stream_messages.position,
68
+ get_stream_messages.batch_size;
69
+ END;
70
+ $$ LANGUAGE plpgsql
71
+ VOLATILE;