evt-event_source-postgres 0.10.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: cbb3a9b61a7f4a379cd86bc1fca7462a767dce24
+  data.tar.gz: d47872d6c1d73a06db41da3bc77cbf06ac652e16
+SHA512:
+  metadata.gz: 96ab2094cec2f21a4b38967f76ca90c7b97cf11050b496d3aee119878d4d7b5550921126b6693e1cee9367580ed82709767f379a17cc1ea3a41e7403b51e39e5
+  data.tar.gz: 579b568775c5a3ed2b9542d5c10f0208c07f3896996fb2b2aacb70e172434b6d9b531c17bbe0470a0c6bddcfe8167746300a003441f35deaf31f0caca2c490bf

data/lib/event_source/loader.rb

@@ -0,0 +1 @@
+postgres.rb

data/lib/event_source/postgres.rb

@@ -0,0 +1,20 @@
+require 'pg'
+
+require 'event_source'
+require 'cycle'
+
+require 'log'
+require 'telemetry'
+require 'settings'; Settings.activate
+
+require 'event_source/postgres/log'
+
+require 'event_source/postgres/settings'
+require 'event_source/postgres/session'
+
+require 'event_source/postgres/put'
+require 'event_source/postgres/write'
+
+require 'event_source/postgres/get/select_statement'
+require 'event_source/postgres/get'
+require 'event_source/postgres/read'

data/lib/event_source/postgres/controls.rb

@@ -0,0 +1,7 @@
+require 'event_source/controls'
+
+require 'event_source/postgres/controls/category'
+require 'event_source/postgres/controls/stream_name'
+require 'event_source/postgres/controls/stream'
+require 'event_source/postgres/controls/event_data'
+require 'event_source/postgres/controls/put'

data/lib/event_source/postgres/controls/category.rb

@@ -0,0 +1,7 @@
+module EventSource
+  module Postgres
+    module Controls
+      Category = EventSource::Controls::Category
+    end
+  end
+end

data/lib/event_source/postgres/controls/event_data.rb

@@ -0,0 +1,7 @@
+module EventSource
+  module Postgres
+    module Controls
+      EventData = EventSource::Controls::EventData
+    end
+  end
+end

data/lib/event_source/postgres/controls/put.rb

@@ -0,0 +1,19 @@
+module EventSource
+  module Postgres
+    module Controls
+      module Put
+        def self.call(instances: nil, stream_name: nil, event: nil, category: nil)
+          instances ||= 1
+          stream_name ||= StreamName.example(category: category)
+          event ||= EventData::Write.example
+
+          instances.times do
+            EventSource::Postgres::Put.(event, stream_name)
+          end
+
+          stream_name
+        end
+      end
+    end
+  end
+end

data/lib/event_source/postgres/controls/stream.rb

@@ -0,0 +1,7 @@
+module EventSource
+  module Postgres
+    module Controls
+      Stream = EventSource::Controls::Stream
+    end
+  end
+end

data/lib/event_source/postgres/controls/stream_name.rb

@@ -0,0 +1,7 @@
+module EventSource
+  module Postgres
+    module Controls
+      StreamName = EventSource::Controls::StreamName
+    end
+  end
+end

data/lib/event_source/postgres/get.rb

@@ -0,0 +1,91 @@
+module EventSource
+  module Postgres
+    class Get
+      include Log::Dependency
+
+      initializer :batch_size, :precedence
+
+      dependency :session, Session
+
+      def self.build(batch_size: nil, precedence: nil, session: nil)
+        new(batch_size, precedence).tap do |instance|
+          instance.configure(session: session)
+        end
+      end
+
+      def self.configure(receiver, attr_name: nil, position: nil, batch_size: nil, precedence: nil, session: nil)
+        attr_name ||= :get
+        instance = build(batch_size: batch_size, precedence: precedence, session: session)
+        receiver.public_send "#{attr_name}=", instance
+      end
+
+      def configure(session: nil)
+        Session.configure self, session: session
+      end
+
+      def self.call(stream_name, position: nil, batch_size: nil, precedence: nil, session: nil)
+        instance = build(batch_size: batch_size, precedence: precedence, session: session)
+        instance.(stream_name, position: position)
+      end
+
+      def call(stream_name, position: nil)
+        logger.trace { "Getting event data (Position: #{position.inspect}, Stream Name: #{stream_name}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" }
+
+        records = get_records(stream_name, position)
+
+        events = convert(records)
+
+        logger.info { "Finished getting event data (Count: #{events.length}, Position: #{position.inspect}, Stream Name: #{stream_name}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" }
+        logger.info(tags: [:data, :event_data]) { events.pretty_inspect }
+
+        events
+      end
+
+      def get_records(stream_name, position)
+        logger.trace { "Getting records (Stream: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" }
+
+        select_statement = SelectStatement.build(stream_name, offset: position, batch_size: batch_size, precedence: precedence)
+
+        records = session.execute(select_statement.sql)
+
+        logger.debug { "Finished getting records (Count: #{records.ntuples}, Stream: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Precedence: #{precedence.inspect})" }
+
+        records
+      end
+
+      def convert(records)
+        logger.trace { "Converting records to event data (Records Count: #{records.ntuples})" }
+
+        events = records.map do |record|
+          record['data'] = Deserialize.data(record['data'])
+          record['metadata'] = Deserialize.metadata(record['metadata'])
+          record['time'] = Time.utc_coerced(record['time'])
+
+          EventData::Read.build record
+        end
+
+        logger.debug { "Converted records to event data (Event Data Count: #{events.length})" }
+
+        events
+      end
+
+      module Deserialize
+        def self.data(serialized_data)
+          return nil if serialized_data.nil?
+          Transform::Read.(serialized_data, EventData::Hash, :json)
+        end
+
+        def self.metadata(serialized_metadata)
+          return nil if serialized_metadata.nil?
+          Transform::Read.(serialized_metadata, EventData::Hash, :json)
+        end
+      end
+
+      module Time
+        def self.utc_coerced(local_time)
+          Clock::UTC.coerce(local_time)
+        end
+      end
+    end
+  end
+end

data/lib/event_source/postgres/get/select_statement.rb

@@ -0,0 +1,89 @@
+module EventSource
+  module Postgres
+    class Get
+      class SelectStatement
+        include Log::Dependency
+
+        initializer :stream, w(:offset), w(:batch_size), w(:precedence)
+
+        def offset
+          @offset ||= Defaults.offset
+        end
+
+        def batch_size
+          @batch_size ||= Defaults.batch_size
+        end
+
+        def precedence
+          @precedence ||= Defaults.precedence
+        end
+
+        def stream_name
+          stream.name
+        end
+
+        def stream_type
+          stream.type
+        end
+
+        def self.build(stream_name, offset: nil, batch_size: nil, precedence: nil)
+          stream = Stream.new(stream_name)
+          new(stream, offset, batch_size, precedence)
+        end
+
+        def sql
+          logger.trace(tag: :sql) { "Composing select statement (Stream: #{stream_name}, Category: #{stream.category?}, Type: #{stream_type}, Position: #{offset}, Batch Size: #{batch_size}, Precedence: #{precedence})" }
+
+          statement = <<-SQL
+            SELECT
+              stream_name::varchar,
+              position::int,
+              type::varchar,
+              global_position::bigint,
+              data::varchar,
+              metadata::varchar,
+              time::timestamp
+            FROM
+              events
+            WHERE
+              #{where_clause_field} = '#{stream_name}'
+            ORDER BY
+              global_position #{precedence.to_s.upcase}
+            LIMIT
+              #{batch_size}
+            OFFSET
+              #{offset}
+            ;
+          SQL
+
+          logger.debug(tag: :sql) { "Composed select statement (Stream: #{stream_name}, Category: #{stream.category?}, Type: #{stream_type}, Position: #{offset}, Batch Size: #{batch_size}, Precedence: #{precedence})" }
+          logger.debug(tags: [:data, :sql]) { "Statement: #{statement}" }
+
+          statement
+        end
+
+        def where_clause_field
+          unless stream.category?
+            'stream_name'
+          else
+            'category(stream_name)'
+          end
+        end
+
+        module Defaults
+          def self.offset
+            0
+          end
+
+          def self.batch_size
+            1000
+          end
+
+          def self.precedence
+            :asc
+          end
+        end
+      end
+    end
+  end
+end

data/lib/event_source/postgres/log.rb

@@ -0,0 +1,11 @@
+module EventSource
+  module Postgres
+    class Log < ::Log
+      def tag!(tags)
+        tags << :event_source_postgres
+        tags << :library
+        tags << :verbose
+      end
+    end
+  end
+end

data/lib/event_source/postgres/put.rb

@@ -0,0 +1,137 @@
+module EventSource
+  module Postgres
+    class Put
+      include Log::Dependency
+
+      dependency :session, Session
+
+      def self.build(session: nil)
+        new.tap do |instance|
+          instance.configure(session: session)
+        end
+      end
+
+      def configure(session: nil)
+        Session.configure(self, session: session)
+      end
+
+      def self.configure(receiver, session: nil, attr_name: nil)
+        attr_name ||= :put
+        instance = build(session: session)
+        receiver.public_send "#{attr_name}=", instance
+      end
+
+      def self.call(write_event, stream_name, expected_version: nil, session: nil)
+        instance = build(session: session)
+        instance.(write_event, stream_name, expected_version: expected_version)
+      end
+
+      def call(write_event, stream_name, expected_version: nil)
+        logger.trace { "Putting event data (Stream Name: #{stream_name}, Type: #{write_event.type}, Expected Version: #{expected_version.inspect})" }
+        logger.trace(tags: [:data, :event_data]) { write_event.pretty_inspect }
+
+        type, data, metadata = destructure_event(write_event)
+        expected_version = ExpectedVersion.canonize(expected_version)
+
+        insert_event(stream_name, type, data, metadata, expected_version).tap do |position|
+          logger.info { "Put event data (Position: #{position}, Stream Name: #{stream_name}, Type: #{write_event.type}, Expected Version: #{expected_version.inspect})" }
+          logger.info(tags: [:data, :event_data]) { write_event.pretty_inspect }
+        end
+      end
+
+      def destructure_event(write_event)
+        type = write_event.type
+        data = write_event.data
+        metadata = write_event.metadata
+
+        logger.debug(tags: [:data, :event_data]) { "Data: #{data.pretty_inspect}" }
+        logger.debug(tags: [:data, :event_data]) { "Metadata: #{metadata.pretty_inspect}" }
+
+        return type, data, metadata
+      end
+
+      def insert_event(stream_name, type, data, metadata, expected_version)
+        serialized_data = serialized_data(data)
+        serialized_metadata = serialized_metadata(metadata)
+        records = execute_query(stream_name, type, serialized_data, serialized_metadata, expected_version)
+        position(records)
+      end
+
+      def execute_query(stream_name, type, serialized_data, serialized_metadata, expected_version)
+        logger.trace { "Executing insert (Stream Name: #{stream_name}, Type: #{type}, Expected Version: #{expected_version.inspect})" }
+
+        params = [
+          stream_name,
+          type,
+          serialized_data,
+          serialized_metadata,
+          expected_version
+        ]
+
+        begin
+          records = session.execute(self.class.statement, params)
+        rescue PG::RaiseException => e
+          raise_error e
+        end
+
+        logger.debug { "Executed insert (Stream Name: #{stream_name}, Type: #{type}, Expected Version: #{expected_version.inspect})" }
+
+        records
+      end
+
+      def self.statement
+        @statement ||= "SELECT write_event($1::varchar, $2::varchar, $3::jsonb, $4::jsonb, $5::int);"
+      end
+
+      def serialized_data(data)
+        serialized_data = nil
+
+        if data.is_a?(Hash) && data.empty?
+          data = nil
+        end
+
+        unless data.nil?
+          serializable_data = EventData::Hash[data]
+          serialized_data = Transform::Write.(serializable_data, :json)
+        end
+
+        logger.debug(tags: [:data, :serialize]) { "Serialized Data: #{serialized_data.inspect}" }
+        serialized_data
+      end
+
+      def serialized_metadata(metadata)
+        serialized_metadata = nil
+
+        if metadata.is_a?(Hash) && metadata.empty?
+          metadata = nil
+        end
+
+        unless metadata.nil?
+          serializable_metadata = EventData::Hash[metadata]
+          serialized_metadata = Transform::Write.(serializable_metadata, :json)
+        end
+
+        logger.debug(tags: [:data, :serialize]) { "Serialized Metadata: #{serialized_metadata.inspect}" }
+        serialized_metadata
+      end
+
+      def position(records)
+        position = nil
+        unless records[0].nil?
+          position = records[0].values[0]
+        end
+        position
+      end
+
+      def raise_error(pg_error)
+        error_message = pg_error.message
+        if error_message.include? 'Wrong expected version'
+          error_message.gsub!('ERROR:', '').strip!
+          logger.error { error_message }
+          raise ExpectedVersion::Error, error_message
+        end
+        raise pg_error
+      end
+    end
+  end
+end

data/lib/event_source/postgres/read.rb

@@ -0,0 +1,11 @@
+module EventSource
+  module Postgres
+    class Read
+      include EventSource::Read
+
+      def configure(batch_size: nil, precedence: nil, session: nil)
+        Get.configure(self, batch_size: batch_size, precedence: precedence, session: session)
+      end
+    end
+  end
+end

data/lib/event_source/postgres/session.rb

@@ -0,0 +1,120 @@
+module EventSource
+  module Postgres
+    class Session
+      include Log::Dependency
+
+      def self.settings
+        Settings.names
+      end
+
+      settings.each do |s|
+        setting s
+      end
+
+      attr_accessor :connection
+
+      def self.build(settings: nil)
+        new.tap do |instance|
+          settings ||= Settings.instance
+          settings.set(instance)
+        end
+      end
+
+      def self.configure(receiver, session: nil, attr_name: nil)
+        attr_name ||= :session
+
+        instance = session || build
+        receiver.public_send "#{attr_name}=", instance
+        instance
+      end
+
+      def connect
+        logger.trace "Connecting to database"
+
+        if connected?
+          logger.debug { "Already connected. A new connection will not be built." }
+          return
+        end
+
+        logger.debug { "Not connected. A new connection will be built." }
+        connection = self.class.build_connection(self)
+        self.connection = connection
+
+        logger.debug { "Connected to database" }
+
+        connection
+      end
+
+      def self.build_connection(instance)
+        settings = instance.settings
+        logger.trace { "Building new connection to database (Settings: #{LogText.settings(settings).inspect})" }
+
+        connection = PG::Connection.open(settings)
+        connection.type_map_for_results = PG::BasicTypeMapForResults.new(connection)
+
+        logger.trace { "Built new connection to database (Settings: #{LogText.settings(settings).inspect})" }
+
+        connection
+      end
+
+      def connected?
+        !connection.nil? && connection.status == PG::CONNECTION_OK
+      end
+      alias :open? :connected?
+
+      def close
+        connection.close
+        connection = nil
+      end
+
+      def reset
+        connection.reset
+      end
+
+      def settings
+        settings = {}
+        self.class.settings.each do |s|
+          val = public_send(s)
+          settings[s] = val unless val.nil?
+        end
+        settings
+      end
+
+      def execute(statement, params=nil)
+        unless connected?
+          connect
+        end
+
+        if params.nil?
+          connection.exec(statement)
+        else
+          connection.exec_params(statement, params)
+        end
+      end
+
+      def transaction(&blk)
+        unless connected?
+          connect
+        end
+
+        connection.transaction(&blk)
+      end
+
+      def self.logger
+        @logger ||= Log.get self
+      end
+
+      module LogText
+        def self.settings(settings)
+          s = settings.dup
+
+          if s.has_key?(:password)
+            s[:password] = '(hidden)'
+          end
+
+          s
+        end
+      end
+    end
+  end
+end

data/lib/event_source/postgres/settings.rb

@@ -0,0 +1,30 @@
+module EventSource
+  module Postgres
+    class Settings < ::Settings
+      def self.instance
+        @instance ||= build
+      end
+
+      def self.data_source
+        'settings/event_source_postgres.json'
+      end
+
+      def self.names
+        [
+          :dbname,
+          :host,
+          :hostaddr,
+          :port,
+          :user,
+          :password,
+          :connect_timeout,
+          :options,
+          :sslmode,
+          :krbsrvname,
+          :gsslib,
+          :service
+        ]
+      end
+    end
+  end
+end

data/lib/event_source/postgres/write.rb

@@ -0,0 +1,11 @@
+module EventSource
+  module Postgres
+    class Write
+      include EventSource::Write
+
+      def configure(session: nil)
+        Put.configure(self, session: session)
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,143 @@
+--- !ruby/object:Gem::Specification
+name: evt-event_source-postgres
+version: !ruby/object:Gem::Version
+  version: 0.10.0.2
+platform: ruby
+authors:
+- The Eventide Project
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-12-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: evt-event_source
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: evt-log
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: evt-cycle
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: evt-settings
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: test_bench
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: " "
+email: opensource@eventide-project.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/event_source/loader.rb
+- lib/event_source/postgres.rb
+- lib/event_source/postgres/controls.rb
+- lib/event_source/postgres/controls/category.rb
+- lib/event_source/postgres/controls/event_data.rb
+- lib/event_source/postgres/controls/put.rb
+- lib/event_source/postgres/controls/stream.rb
+- lib/event_source/postgres/controls/stream_name.rb
+- lib/event_source/postgres/get.rb
+- lib/event_source/postgres/get/select_statement.rb
+- lib/event_source/postgres/log.rb
+- lib/event_source/postgres/put.rb
+- lib/event_source/postgres/read.rb
+- lib/event_source/postgres/session.rb
+- lib/event_source/postgres/settings.rb
+- lib/event_source/postgres/write.rb
+homepage: https://github.com/eventide-project/event-source-postgres
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.3.3
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2
+signing_key: 
+specification_version: 4
+summary: Event source client for PostgreSQL
+test_files: []