pg_eventstore 0.2.3 → 0.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fd2524ab67ce1fa009de14a032cdc3d922051420bc2c02e933cb63ee245db02d
-  data.tar.gz: f49867dab08825770aae44e3e1bec38a8144504e3bea2e6618ce47a0d6e8cb54
+  metadata.gz: bc158c2f99fee36514e1902199691dfb93e882ecdebe4738bbcf787fadc8f514
+  data.tar.gz: a70d00d4bdaef48223e35234f01d938a713f9306aa1bfb45bde99733b3654c39
 SHA512:
-  metadata.gz: e9925413067ca47159cbd5902f2716bbd9e5fe1e0c63aa7397f077c95878bacc2da57cd65f32160a0e1d70d855e101d16fb6d5aa5b537c9630c4290449db8c16
-  data.tar.gz: 65add6ce6e11ebe0634cec3949709ad3c8d42b105ba9325c5d6d45c45bf2339c0e47f9286dbb71edc8e1c99636aa449adc34926f4a6d57c695c50655609d5451
+  metadata.gz: bf38d241b001b4244d9fa915d2dfcabec7ac9456aa83832b8d850fe3fd34e48b0ef8b9ef5f0cca91b776124ec3333dea658220f3bd427309b3b1ae09a12714ae
+  data.tar.gz: af5b5e4394ee696790145852ac6a7761728e5cc7e7aca127e6cde112b857e5516b121ac2bb7bf2280e7e66c93cf0823e33bcbe1f258cadbff31d50f83d9972a8

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## [0.2.4] - 2023-12-20
+
+Due to performance issues under certain circumstances, searching by event type was giving bad performance. I decided to extract `type` column from `events` table into separated table. **No breaking changes in public API though.**
+
+**Warning** The migrations this version has, requires you to shut down applications that use `pg_eventstore` and only then run `rake pg_eventstore:migrate`.
+
 ## [0.2.3] - 2023-12-18
 
 - Fix performance when searching by event type only(under certain circumstances PosetgreSQL was picking wrong index).

data/db/migrations/3_extract_type_into_separate_table.sql

@@ -0,0 +1,17 @@
+CREATE TABLE public.event_types
+(
+    id              bigserial         NOT NULL,
+    type            character varying NOT NULL
+);
+
+ALTER TABLE ONLY public.events ADD COLUMN event_type_id bigint;
+
+ALTER TABLE ONLY public.event_types
+    ADD CONSTRAINT event_types_pkey PRIMARY KEY (id);
+
+ALTER TABLE ONLY public.events
+    ADD CONSTRAINT events_event_type_fk FOREIGN KEY (event_type_id)
+        REFERENCES public.event_types (id);
+
+CREATE UNIQUE INDEX idx_event_types_type ON public.event_types USING btree (type);
+CREATE INDEX idx_events_event_type_id ON public.events USING btree (event_type_id);

data/db/migrations/4_populate_event_types.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+PgEventstore.connection.with do |conn|
+  types = conn.exec('select type from events group by type').to_a.map { |attrs| attrs['type'] }
+  types.each.with_index(1) do |type, index|
+    id = conn.exec_params('SELECT id FROM event_types WHERE type = $1', [type]).to_a.first['id']
+    id ||= conn.exec_params('INSERT INTO event_types (type) VALUES ($1) RETURNING *', [type]).to_a.first['id']
+    conn.exec_params('UPDATE events SET event_type_id = $1 WHERE type = $2 AND event_type_id IS NULL', [id, type])
+    puts "Processed #{index} types of #{types.size}"
+  end
+end

data/db/migrations/5_adjust_indexes.sql

@@ -0,0 +1,6 @@
+CREATE INDEX idx_events_global_position ON public.events USING btree (global_position);
+
+DROP INDEX idx_events_stream_id_and_type_and_revision;
+DROP INDEX idx_events_type_and_stream_id_and_position;
+DROP INDEX idx_events_global_position_including_type;
+DROP INDEX idx_events_type_and_position;

data/db/migrations/6_change_events_event_type_id_null_constraint.sql

@@ -0,0 +1 @@
+ALTER TABLE public.events ALTER COLUMN event_type_id SET NOT NULL;

data/db/migrations/7_change_events_type_constraint.sql

@@ -0,0 +1 @@
+ALTER TABLE public.events ALTER COLUMN type DROP NOT NULL;

data/lib/pg_eventstore/queries/event_queries.rb

@@ -22,6 +22,7 @@ module PgEventstore
     # @param options [Hash]
     # @return [Array<PgEventstore::Event>]
     def stream_events(stream, options)
+      options = include_event_types_ids(options)
       exec_params = events_filtering(stream, options).to_exec_params
       pg_result = connection.with do |conn|
         conn.exec_params(*exec_params)
@@ -35,17 +36,18 @@ module PgEventstore
     def insert(stream, event)
       serializer.serialize(event)
 
-      attributes = event.options_hash.slice(:id, :type, :data, :metadata, :stream_revision, :link_id).compact
+      attributes = event.options_hash.slice(:id, :data, :metadata, :stream_revision, :link_id).compact
       attributes[:stream_id] = stream.id
+      attributes[:event_type_id] = event_type_queries.find_or_create_type(event.type)
 
       sql = <<~SQL
         INSERT INTO events (#{attributes.keys.join(', ')}) 
-          VALUES (#{(1..attributes.values.size).map { |n| "$#{n}" }.join(', ')}) 
-          RETURNING *
+          VALUES (#{positional_vars(attributes.values)}) 
+          RETURNING *, $#{attributes.values.size + 1} as type
       SQL
 
       pg_result = connection.with do |conn|
-        conn.exec_params(sql, attributes.values)
+        conn.exec_params(sql, [*attributes.values, event.type])
       end
       deserializer.without_middlewares.deserialize_one(pg_result).tap do |persisted_event|
         persisted_event.stream = stream
@@ -63,5 +65,29 @@ module PgEventstore
 
       QueryBuilders::EventsFiltering.specific_stream_filtering(stream, options, offset: offset)
     end
+
+    # Replaces filter by event type strings with filter by event type ids
+    # @param options [Hash]
+    # @return [Hash]
+    def include_event_types_ids(options)
+      options in { filter: { event_types: Array => event_types } }
+      return options unless event_types
+
+      filter = options[:filter].dup
+      filter[:event_type_ids] = event_type_queries.find_event_types(event_types).uniq
+      filter.delete(:event_types)
+      options.merge(filter: filter)
+    end
+
+    # @param array [Array]
+    # @return [String] positional variables, based on array size. Example: "$1, $2, $3"
+    def positional_vars(array)
+      array.size.times.map { |t| "$#{t + 1}" }.join(', ')
+    end
+
+    # @return [PgEventstore::EventTypeQueries]
+    def event_type_queries
+      EventTypeQueries.new(connection)
+    end
   end
 end

data/lib/pg_eventstore/queries/event_type_queries.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  # @!visibility private
+  class EventTypeQueries
+    attr_reader :connection
+    private :connection
+
+    # @param connection [PgEventstore::Connection]
+    def initialize(connection)
+      @connection = connection
+    end
+
+    # @param type [String]
+    # @return [Integer] event type's id
+    def find_or_create_type(type)
+      find_type(type) || create_type(type)
+    end
+
+    # @param type [String]
+    # @return [Integer, nil] event type's id
+    def find_type(type)
+      connection.with do |conn|
+        conn.exec_params('SELECT id FROM event_types WHERE type = $1', [type])
+      end.to_a.dig(0, 'id')
+    end
+
+    # @param type [String]
+    # @return [Integer] event type's id
+    def create_type(type)
+      connection.with do |conn|
+        conn.exec_params('INSERT INTO event_types (type) VALUES ($1) RETURNING id', [type])
+      end.to_a.dig(0, 'id')
+    end
+
+    # @param types [Array<String>]
+    # @return [Array<Integer, nil>]
+    def find_event_types(types)
+      connection.with do |conn|
+        conn.exec_params(<<~SQL, [types])
+          SELECT event_types.id, types.type
+            FROM event_types 
+            RIGHT JOIN (
+              SELECT unnest($1::varchar[]) type
+            ) types ON types.type = event_types.type
+        SQL
+      end.to_a.map { |attrs| attrs['id'] }
+    end
+  end
+end

data/lib/pg_eventstore/queries/transaction_queries.rb

@@ -19,14 +19,23 @@ module PgEventstore
           next yield
         end
 
-        conn.transaction do
-          conn.exec("SET TRANSACTION ISOLATION LEVEL SERIALIZABLE")
+        pg_transaction(conn) do
           yield
         end
       end
-    rescue PG::TRSerializationFailure, PG::TRDeadlockDetected => e
-      retry if [PG::PQTRANS_IDLE, PG::PQTRANS_UNKNOWN].include?(e.connection.transaction_status)
-      raise
+    end
+
+    private
+
+    # @param pg_connection [PG::Connection]
+    # @return [void]
+    def pg_transaction(pg_connection)
+      pg_connection.transaction do
+        pg_connection.exec("SET TRANSACTION ISOLATION LEVEL SERIALIZABLE")
+        yield
+      end
+    rescue PG::TRSerializationFailure, PG::TRDeadlockDetected
+      retry
     end
   end
 end

data/lib/pg_eventstore/queries.rb

@@ -3,6 +3,7 @@
 require_relative 'queries/transaction_queries'
 require_relative 'queries/event_queries'
 require_relative 'queries/stream_queries'
+require_relative 'queries/event_type_queries'
 
 module PgEventstore
   # @!visibility private

data/lib/pg_eventstore/query_builders/events_filtering_query.rb

@@ -23,8 +23,8 @@ module PgEventstore
         # @return [PgEventstore::QueryBuilders::EventsFiltering]
         def all_stream_filtering(options, offset: 0)
           event_filter = new
-          options in { filter: { event_types: Array => event_types } }
-          event_filter.add_event_types(event_types)
+          options in { filter: { event_type_ids: Array => event_type_ids } }
+          event_filter.add_event_types(event_type_ids)
           event_filter.add_limit(options[:max_count])
           event_filter.add_offset(offset)
           event_filter.resolve_links(options[:resolve_link_tos])
@@ -41,8 +41,8 @@ module PgEventstore
         # @return [PgEventstore::QueryBuilders::EventsFiltering]
         def specific_stream_filtering(stream, options, offset: 0)
           event_filter = new
-          options in { filter: { event_types: Array => event_types } }
-          event_filter.add_event_types(event_types)
+          options in { filter: { event_type_ids: Array => event_type_ids } }
+          event_filter.add_event_types(event_type_ids)
           event_filter.add_limit(options[:max_count])
           event_filter.add_offset(offset)
           event_filter.resolve_links(options[:resolve_link_tos])
@@ -58,8 +58,10 @@ module PgEventstore
           SQLBuilder.new.
             select('events.*').
             select('row_to_json(streams.*) as stream').
+            select('event_types.type as type').
             from('events').
             join('JOIN streams ON streams.id = events.stream_id').
+            join('JOIN event_types ON event_types.id = events.event_type_id').
             limit(DEFAULT_LIMIT).
             offset(DEFAULT_OFFSET)
       end
@@ -85,16 +87,16 @@ module PgEventstore
         @sql_builder.where("streams.id = ?", stream.id)
       end
 
-      # @param event_types [Array, nil]
+      # @param event_type_ids [Array<Integer>, nil]
       # @return [void]
-      def add_event_types(event_types)
-        return if event_types.nil?
-        return if event_types.empty?
-
-        sql = event_types.size.times.map do
-          "events.type = ?"
-        end.join(" OR ")
-        @sql_builder.where(sql, *event_types)
+      def add_event_types(event_type_ids)
+        return if event_type_ids.nil?
+        return if event_type_ids.empty?
+
+        sql = event_type_ids.size.times.map do
+          "?"
+        end.join(", ")
+        @sql_builder.where("event_types.id IN (#{sql})", *event_type_ids)
       end
 
       # @param revision [Integer, nil]

data/lib/pg_eventstore/tasks/setup.rake

@@ -31,12 +31,16 @@ namespace :pg_eventstore do
       latest_migration =
         conn.exec('SELECT number FROM migrations ORDER BY number DESC LIMIT 1').to_a.dig(0, 'number') || -1
 
-      Dir["#{migration_files_root}/*.sql"].each do |f_name|
-        number = File.basename(f_name).split('_')[0].to_i
-        next if latest_migration >= number
+      Dir.chdir migration_files_root do
+        Dir["*.{sql,rb}"].each do |f_name|
+          number = File.basename(f_name).split('_')[0].to_i
+          next if latest_migration >= number
 
-        conn.transaction do
-          conn.exec(File.read(f_name))
+          if File.extname(f_name) == '.rb'
+            load f_name
+          else
+            conn.exec(File.read(f_name))
+          end
           conn.exec_params('INSERT INTO migrations (number) VALUES ($1)', [number])
         end
       end
@@ -53,6 +57,7 @@ namespace :pg_eventstore do
       conn.exec <<~SQL
         DROP TABLE IF EXISTS public.events;
         DROP TABLE IF EXISTS public.streams;
+        DROP TABLE IF EXISTS public.event_types;
         DROP TABLE IF EXISTS public.migrations;
         DROP EXTENSION IF EXISTS "uuid-ossp";
         DROP EXTENSION IF EXISTS pgcrypto;

data/lib/pg_eventstore/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PgEventstore
-  VERSION = "0.2.3"
+  VERSION = "0.2.4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pg_eventstore
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.2.4
 platform: ruby
 authors:
 - Ivan Dzyzenko
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2023-12-18 00:00:00.000000000 Z
+date: 2023-12-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -56,6 +56,11 @@ files:
 - db/migrations/0_improve_all_stream_indexes.sql
 - db/migrations/1_improve_specific_stream_indexes.sql
 - db/migrations/2_adjust_global_position_index.sql
+- db/migrations/3_extract_type_into_separate_table.sql
+- db/migrations/4_populate_event_types.rb
+- db/migrations/5_adjust_indexes.sql
+- db/migrations/6_change_events_event_type_id_null_constraint.sql
+- db/migrations/7_change_events_type_constraint.sql
 - docs/appending_events.md
 - docs/configuration.md
 - docs/events_and_streams.md
@@ -80,6 +85,7 @@ files:
 - lib/pg_eventstore/pg_result_deserializer.rb
 - lib/pg_eventstore/queries.rb
 - lib/pg_eventstore/queries/event_queries.rb
+- lib/pg_eventstore/queries/event_type_queries.rb
 - lib/pg_eventstore/queries/stream_queries.rb
 - lib/pg_eventstore/queries/transaction_queries.rb
 - lib/pg_eventstore/query_builders/events_filtering_query.rb