evt-view_data-pg 0.1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 3a446a1a6d3124fa58e589403e99c2c2ea670283
+  data.tar.gz: 26ef15ffe45116c6002f983091c0c2538f4aba9f
+SHA512:
+  metadata.gz: 33d0921b2fcf365b97f321bc94acdb2cb9c5a7e4dd87c5fdddef6922628a74b9891b1833009a1f64b81c658c6c10f3899b3e40405047c1dd5c76ae01406b2c8e
+  data.tar.gz: aee4c805bb139e93d6ef9a758f1f3eeda1905863a4dc58d401e9c9e1e03f31f9fed07b72f782733bb06632243b65575338ab35dbe46f88d05c9cb0348c788419

data/lib/view_data/pg.rb

@@ -0,0 +1,10 @@
+require 'consumer/postgres'
+require 'view_data/commands'
+
+require 'view_data/pg/log'
+require 'view_data/pg/session'
+require 'view_data/pg/settings'
+
+require 'view_data/pg/primary_key/get_columns'
+
+require 'view_data/pg/handler'

data/lib/view_data/pg/controls.rb

@@ -0,0 +1,18 @@
+require 'view_data/commands/controls'
+require 'clock/controls'
+
+require 'view_data/pg/controls/id'
+require 'view_data/pg/controls/commands'
+require 'view_data/pg/controls/consumer'
+require 'view_data/pg/controls/data'
+require 'view_data/pg/controls/date'
+require 'view_data/pg/controls/enum'
+require 'view_data/pg/controls/json'
+require 'view_data/pg/controls/primary_key'
+require 'view_data/pg/controls/row/get'
+require 'view_data/pg/controls/row/put'
+require 'view_data/pg/controls/table'
+require 'view_data/pg/controls/table/composite_primary_key'
+require 'view_data/pg/controls/table/all_data_types'
+require 'view_data/pg/controls/time'
+require 'view_data/pg/controls/timestamp'

data/lib/view_data/pg/controls/commands.rb

@@ -0,0 +1,47 @@
+module ViewData
+  module PG
+    module Controls
+      module Commands
+        module Create
+          def self.example(primary_key: nil, column_value: nil, table: nil)
+            primary_key ||= PrimaryKey::UUID.example
+            table ||= Table.name
+            data = Data.example(column_value: column_value)
+
+            ViewData::Commands::Controls::Create.example(
+              name: table,
+              identifier: primary_key,
+              data: data
+            )
+          end
+        end
+
+        module Update
+          def self.example(primary_key: nil, column_value: nil, table: nil)
+            primary_key ||= PrimaryKey::UUID.example
+            table ||= Table.name
+            data = Data.example(column_value: column_value)
+
+            ViewData::Commands::Controls::Update.example(
+              name: table,
+              identifier: primary_key,
+              data: data
+            )
+          end
+        end
+
+        module Delete
+          def self.example(primary_key: nil, table: nil)
+            primary_key ||= PrimaryKey::UUID.example
+            table ||= Table.name
+
+            ViewData::Commands::Controls::Delete.example(
+              name: table,
+              identifier: primary_key
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/view_data/pg/controls/consumer.rb

@@ -0,0 +1,17 @@
+module ViewData
+  module PG
+    module Controls
+      module Consumer
+        def self.example
+          Example.build
+        end
+
+        class Example
+          include ::Consumer::Postgres
+
+          handler ViewData::PG::Handler
+        end
+      end
+    end
+  end
+end

data/lib/view_data/pg/controls/data.rb

@@ -0,0 +1,19 @@
+module ViewData
+  module PG
+    module Controls
+      module Data
+        def self.example(column_value: nil)
+          column_value ||= self.column_value
+
+          {
+            :some_column => column_value
+          }
+        end
+
+        def self.column_value
+          'some-column-value'
+        end
+      end
+    end
+  end
+end

data/lib/view_data/pg/controls/date.rb

@@ -0,0 +1,19 @@
+module ViewData
+  module PG
+    module Controls
+      module Date
+        def self.example(date=nil)
+          date ||= Raw.example
+
+          date.iso8601
+        end
+
+        module Raw
+          def self.example
+            ::Date.new(2000, 1, 1)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/view_data/pg/controls/enum.rb

@@ -0,0 +1,19 @@
+module ViewData
+  module PG
+    module Controls
+      module Enum
+        def self.example
+          'some-enum-value'
+        end
+
+        def self.alternate
+          'other-enum-value'
+        end
+
+        def self.unknown
+          'unknown-enum-value'
+        end
+      end
+    end
+  end
+end

data/lib/view_data/pg/controls/id.rb

@@ -0,0 +1,10 @@
+module ViewData
+  module PG
+    module Controls
+      module ID
+        Incrementing = Identifier::UUID::Controls::Incrementing
+        Random = Identifier::UUID::Controls::Random
+      end
+    end
+  end
+end

data/lib/view_data/pg/controls/json.rb

@@ -0,0 +1,22 @@
+module ViewData
+  module PG
+    module Controls
+      module JSON
+        def self.data
+          {
+            'someString' => 'some-string',
+            'someNumber' => 1,
+            'someList' => ['a', 'b', 'c'],
+            'someNestedObject' => {
+              'someNestedAttribute' => 'some-nested-value'
+            }
+          }
+        end
+
+        def self.text
+          ::JSON.generate(data)
+        end
+      end
+    end
+  end
+end

data/lib/view_data/pg/controls/primary_key.rb

@@ -0,0 +1,27 @@
+module ViewData
+  module PG
+    module Controls
+      module PrimaryKey
+        module UUID
+          def self.example
+            ID::Random.example
+          end
+
+          def self.column_name
+            'id'
+          end
+        end
+
+        module Composite
+          def self.example
+            [UUID.example, UUID.example]
+          end
+
+          def self.column_names
+            ['id_1', 'id_2']
+          end
+        end
+      end
+    end
+  end
+end

data/lib/view_data/pg/controls/row/get.rb

@@ -0,0 +1,41 @@
+module ViewData
+  module PG
+    module Controls
+      module Row
+        module Get
+          def self.call(primary_key, table: nil)
+            table ||= Table.name
+
+            session = Session.build
+
+            result = session.execute(<<~SQL, [primary_key])
+              SELECT * FROM #{table} WHERE id = $1 LIMIT 1
+            SQL
+
+            session.close
+
+            result.to_a[0]
+          end
+
+          module CompositePrimaryKey
+            def self.call(ids)
+              id_1, id_2 = ids
+
+              table = Table::CompositePrimaryKey.name
+
+              session = Session.build
+
+              result = session.execute(<<~SQL, [id_1, id_2])
+                SELECT * FROM #{table} WHERE id_1 = $1 AND id_2 = $2 LIMIT 1
+              SQL
+
+              session.close
+
+              result.to_a[0]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/view_data/pg/controls/row/put.rb

@@ -0,0 +1,43 @@
+module ViewData
+  module PG
+    module Controls
+      module Row
+        module Put
+          def self.call
+            primary_key = PrimaryKey::UUID.example
+            column_value = Controls::Data.column_value
+            table_name = Table.name
+
+            session = Session.build
+
+            session.execute(<<~SQL, [primary_key, column_value])
+              INSERT INTO #{table_name} (id, some_column) VALUES ($1, $2)
+            SQL
+
+            session.close
+
+            primary_key
+          end
+
+          module CompositePrimaryKey
+            def self.call
+              id_1, id_2 = PrimaryKey::Composite.example
+              column_value = Controls::Data.column_value
+              table_name = Table::CompositePrimaryKey.name
+
+              session = Session.build
+
+              session.execute(<<~SQL, [id_1, id_2, column_value])
+                INSERT INTO #{table_name} (id_1, id_2, some_column) VALUES ($1, $2, $3)
+              SQL
+
+              session.close
+
+              return id_1, id_2
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/view_data/pg/controls/table.rb

@@ -0,0 +1,28 @@
+module ViewData
+  module PG
+    module Controls
+      module Table
+        def self.create(drop: nil)
+          session = Session.build
+
+          if drop
+            session.execute("DROP TABLE IF EXISTS test_table")
+          end
+
+          session.execute(<<~SQL)
+            CREATE TABLE #{name} (
+              id uuid PRIMARY KEY,
+              some_column text
+            )
+          SQL
+
+          session.close
+        end
+
+        def self.name
+          'test_table'
+        end
+      end
+    end
+  end
+end

data/lib/view_data/pg/controls/table/all_data_types.rb

@@ -0,0 +1,85 @@
+module ViewData
+  module PG
+    module Controls
+      module Table
+        module AllDataTypes
+          def self.create(drop: nil)
+            session = Session.build
+
+            if drop
+              session.execute("DROP TABLE IF EXISTS #{name}")
+              session.execute("DROP TYPE some_enum_type")
+            end
+
+            begin
+              session.execute(<<~SQL)
+                CREATE TYPE some_enum_type AS ENUM (
+                  '#{Enum.example}',
+                  '#{Enum.alternate}'
+                  )
+              SQL
+            rescue ::PG::DuplicateObject
+            end
+
+            session.execute(<<~SQL)
+              CREATE TABLE #{name} (
+                id uuid PRIMARY KEY,
+
+                -- Numbers
+                some_smallint smallint,
+                some_integer integer,
+                some_bigint bigint,
+                some_decimal_3_1 decimal(4, 1),
+                some_numeric_3_1 numeric(4, 1),
+                some_real real,
+                some_double_precision double precision,
+                some_smallserial smallserial,
+                some_serial serial,
+                some_bigserial bigserial,
+
+                -- Money
+                some_money money,
+
+                -- Character
+                some_varchar_3 character varying(3),
+                some_char_3 character(3),
+                some_text text,
+
+                -- Binary
+                some_bytea bytea,
+
+                -- Date/Time
+                some_timestamp_3 timestamp(3) without time zone,
+                some_timestamp_3_with_tz timestamp(3) with time zone,
+                some_date date,
+                some_time_3 time(3) without time zone,
+                some_time_3_with_tz time(3) with time zone,
+                some_interval_year interval YEAR,
+                some_interval_second_3 interval SECOND(3),
+
+                -- Boolean
+                some_boolean bool,
+
+                -- Enum
+                some_enum some_enum_type,
+
+                -- UUID
+                some_uuid uuid,
+
+                -- JSON
+                some_json_text json,
+                some_json_binary jsonb
+              )
+            SQL
+
+            session.close
+          end
+
+          def self.name
+            'all_data_types'
+          end
+        end
+      end
+    end
+  end
+end

data/lib/view_data/pg/controls/table/composite_primary_key.rb

@@ -0,0 +1,34 @@
+module ViewData
+  module PG
+    module Controls
+      module Table
+        module CompositePrimaryKey
+          def self.create(drop: nil)
+            session = Session.build
+
+            if drop
+              session.execute("DROP TABLE IF EXISTS #{name}")
+            end
+
+            session.execute(<<~SQL)
+              CREATE TABLE #{name} (
+                id_1 uuid NOT NULL,
+                id_2 uuid NOT NULL,
+
+                some_column text,
+
+                PRIMARY KEY(id_1, id_2)
+              )
+            SQL
+
+            session.close
+          end
+
+          def self.name
+            'test_composite_primary_key'
+          end
+        end
+      end
+    end
+  end
+end

data/lib/view_data/pg/controls/time.rb

@@ -0,0 +1,47 @@
+module ViewData
+  module PG
+    module Controls
+      Time = Clock::Controls::Time
+
+      module Time
+        module Local
+          def self.example(time=nil, precision: nil)
+            time = Raw.example(time)
+
+            Time.example(time, precision: precision)
+          end
+
+          module Offset
+            def self.name
+              'CET'
+            end
+
+            def self.hours
+              1
+            end
+
+            def self.seconds
+              hours * 60 * 60
+            end
+          end
+
+          module Raw
+            def self.example(time=nil)
+              time ||= Time::Raw.example
+
+              ::Time.new(
+                time.year,
+                time.month,
+                time.day,
+                time.hour,
+                time.min,
+                time.sec,
+                Offset.seconds
+              )
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/view_data/pg/controls/timestamp.rb

@@ -0,0 +1,7 @@
+module ViewData
+  module PG
+    module Controls
+      Timestamp = Time
+    end
+  end
+end

data/lib/view_data/pg/handler.rb

@@ -0,0 +1,129 @@
+module ViewData
+  module PG
+    class Handler
+      include Messaging::Handle
+      include Log::Dependency
+
+      include ViewData::Commands
+
+      dependency :session, MessageStore::Postgres::Session
+      dependency :get_primary_key_columns, PrimaryKey::GetColumns
+
+      def configure
+        Session.configure(self)
+        PrimaryKey::GetColumns.configure(self, session: session)
+      end
+
+      handle Create do |create|
+        table_name = create.name
+
+        logger.trace { "Inserting row (Table: #{table_name}, Identifier: #{create.identifier.inspect})" }
+
+        pkey_columns = get_primary_key_columns.(table_name)
+        pkey_values = Array(create.identifier)
+
+        data_columns = create.data.keys
+        data_values = create.data.values
+
+        columns = pkey_columns + data_columns
+
+        values = pkey_values + data_values
+
+        values_clause = values.count.times.map do |i|
+          "$#{i + 1}"
+        end
+
+        statement = <<~SQL.chomp
+          INSERT INTO #{table_name} (#{columns * ', '})
+          VALUES (#{values_clause * ', '})
+        SQL
+
+        begin
+          logger.trace { "Inserting row (Table: #{table_name}, Identifier: #{create.identifier.inspect})" }
+          logger.trace(tag: :data) { "SQL: #{statement}" }
+          logger.trace(tag: :data) { values.pretty_inspect }
+
+          session.execute(statement, values)
+
+          logger.info { "Inserted row (Table: #{table_name}, Identifier: #{create.identifier.inspect})" }
+          logger.info(tag: :data) { "SQL: #{statement}" }
+          logger.info(tag: :data) { values.pretty_inspect }
+
+        rescue ::PG::UniqueViolation
+        end
+      end
+
+      handle Update do |update|
+        table_name = update.name
+
+        logger.trace { "Updating row (Table: #{table_name}, Identifier: #{update.identifier.inspect})" }
+
+        pkey_columns = get_primary_key_columns.(table_name)
+        pkey_values = Array(update.identifier)
+
+        data_columns = update.data.keys
+        data_values = update.data.values
+
+        set_clause = data_columns.map.with_index do |column_name, index|
+          reference = index + 1
+
+          "#{column_name} = $#{reference}"
+        end
+
+        pkey_clause = pkey_columns.map.with_index do |column_name, index|
+          reference = index + data_columns.count + 1
+
+          "#{column_name} = $#{reference}"
+        end
+
+        values = data_values + pkey_values
+
+        statement = <<~SQL.chomp
+          UPDATE #{table_name}
+          SET #{set_clause * ', '}
+          WHERE #{pkey_clause * ' AND '}
+        SQL
+
+        logger.trace { "Updating row (Table: #{table_name}, Identifier: #{update.identifier.inspect})" }
+        logger.trace(tag: :data) { "SQL: #{statement}" }
+        logger.trace(tag: :data) { values.pretty_inspect }
+
+        session.execute(statement, values)
+
+        logger.info { "Updated row (Table: #{table_name}, Identifier: #{update.identifier.inspect})" }
+        logger.info(tag: :data) { "SQL: #{statement}" }
+        logger.info(tag: :data) { values.pretty_inspect }
+      end
+
+      handle Delete do |delete|
+        table_name = delete.name
+
+        logger.trace { "Deleting row (Table: #{table_name}, Identifier: #{delete.identifier.inspect})" }
+
+        pkey_columns = get_primary_key_columns.(table_name)
+        pkey_values = Array(delete.identifier)
+
+        pkey_clause = pkey_columns.map.with_index do |column_name, index|
+          reference = index + 1
+
+          "#{column_name} = $#{reference}"
+        end
+
+        statement = <<~SQL.chomp
+          DELETE FROM #{table_name}
+          WHERE #{pkey_clause * ' AND '}
+        SQL
+
+        logger.trace { "Deleting row (Table: #{table_name}, Identifier: #{delete.identifier.inspect})" }
+        logger.trace(tag: :data) { "SQL: #{statement}" }
+        logger.trace(tag: :data) { pkey_values.pretty_inspect }
+
+        session.execute(statement, pkey_values)
+
+        logger.info { "Deleted row (Table: #{table_name}, Identifier: #{delete.identifier.inspect})" }
+        logger.info(tag: :data) { "SQL: #{statement}" }
+        logger.info(tag: :data) { pkey_values.pretty_inspect }
+      end
+    end
+  end
+end

data/lib/view_data/pg/log.rb

@@ -0,0 +1,11 @@
+module ViewData
+  module PG
+    class Log < ::Log
+      def tag!(tags)
+        tags << :view_data_pg
+        tags << :view_data
+        tags << :library
+      end
+    end
+  end
+end

data/lib/view_data/pg/primary_key/get_columns.rb

@@ -0,0 +1,71 @@
+module ViewData
+  module PG
+    module PrimaryKey
+      class GetColumns
+        configure :get_primary_key_columns
+
+        dependency :session, Session
+        dependency :telemetry, ::Telemetry
+
+        def self.build(session: nil)
+          instance = new
+          Session.configure(instance, session: session)
+          ::Telemetry.configure(instance)
+          instance
+        end
+
+        def self.call(table)
+          instance = build()
+          instance.(table)
+        end
+
+        def call(table)
+          cache[table]
+        end
+
+        def query(table)
+          result = session.execute(<<~SQL, [table])
+            SELECT
+              TEXT(attr.attname) AS column_name
+            FROM pg_index AS index
+            JOIN pg_attribute AS attr
+              ON attr.attrelid = index.indrelid
+              AND attr.attnum = ANY(index.indkey)
+            WHERE index.indrelid = $1::regclass
+            AND index.indisprimary
+          SQL
+
+          columns = result.to_a.map do |hash|
+            hash['column_name']
+          end
+
+          telemetry.record(:columns_queried, Telemetry::Data.new(table, columns))
+
+          columns
+        end
+
+        def cache
+          @cache ||= Hash.new do |hash, table|
+            hash[table] = query(table)
+          end
+        end
+
+        def self.register_telemetry_sink(instance)
+          sink = Telemetry::Sink.new
+          instance.telemetry.register(sink)
+          sink
+        end
+
+        module Telemetry
+          class Sink
+            include ::Telemetry::Sink
+
+            record :columns_queried
+          end
+
+          Data = Struct.new(:table, :columns)
+        end
+      end
+    end
+  end
+end

data/lib/view_data/pg/session.rb

@@ -0,0 +1,46 @@
+module ViewData
+  module PG
+    class Session < MessageStore::Postgres::Session
+      settings.each do |setting_name|
+        setting setting_name
+      end
+
+      def self.build(settings: nil)
+        settings ||= Settings.instance
+
+        super(settings: settings)
+      end
+
+      def self.build_connection(*)
+        connection = super
+
+        type_map_for_results = connection.type_map_for_results
+
+        name_decoder = ::PG::TextDecoder::String.new({
+          :oid => 19,
+          :name => 'name'
+        })
+
+        regproc_decoder = ::PG::TextDecoder::String.new({
+          :oid => 24,
+          :name => 'regproc'
+        })
+
+        numeric_decoder = ::PG::TextDecoder::Float.new({
+          :oid => 1700,
+          :name => 'numeric'
+        })
+
+        type_map_for_results.add_coder(name_decoder)
+        type_map_for_results.add_coder(regproc_decoder)
+        type_map_for_results.add_coder(numeric_decoder)
+
+        type_map_for_queries = ::PG::BasicTypeMapForQueries.new(connection)
+
+        connection.type_map_for_queries = type_map_for_queries
+
+        connection
+      end
+    end
+  end
+end

data/lib/view_data/pg/settings.rb

@@ -0,0 +1,9 @@
+module ViewData
+  module PG
+    class Settings < MessageStore::Postgres::Settings
+      def self.data_source
+        'settings/view_data_pg.json'
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,107 @@
+--- !ruby/object:Gem::Specification
+name: evt-view_data-pg
+version: !ruby/object:Gem::Version
+  version: 0.1.0.0
+platform: ruby
+authors:
+- The Eventide Project
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-08-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: evt-consumer-postgres
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: evt-view_data-commands
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: test_bench
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: " "
+email: opensource@eventide-project.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/view_data/pg.rb
+- lib/view_data/pg/controls.rb
+- lib/view_data/pg/controls/commands.rb
+- lib/view_data/pg/controls/consumer.rb
+- lib/view_data/pg/controls/data.rb
+- lib/view_data/pg/controls/date.rb
+- lib/view_data/pg/controls/enum.rb
+- lib/view_data/pg/controls/id.rb
+- lib/view_data/pg/controls/json.rb
+- lib/view_data/pg/controls/primary_key.rb
+- lib/view_data/pg/controls/row/get.rb
+- lib/view_data/pg/controls/row/put.rb
+- lib/view_data/pg/controls/table.rb
+- lib/view_data/pg/controls/table/all_data_types.rb
+- lib/view_data/pg/controls/table/composite_primary_key.rb
+- lib/view_data/pg/controls/time.rb
+- lib/view_data/pg/controls/timestamp.rb
+- lib/view_data/pg/handler.rb
+- lib/view_data/pg/log.rb
+- lib/view_data/pg/primary_key/get_columns.rb
+- lib/view_data/pg/session.rb
+- lib/view_data/pg/settings.rb
+homepage: https://github.com/eventide-contrib/view-data-pg
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.4.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.11
+signing_key: 
+specification_version: 4
+summary: Create and populate PostgreSQL view databases from event streams
+test_files: []