db_schema-reader-postgres 0.1.1 → 0.2.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9f12b3dcea5dc1a301418825e2b8da015e078c520195e42517da991830d3bc22
-  data.tar.gz: 52778450688b800dccf4934525fc567f0d99a9b426095298f198728cb827a9e6
+  metadata.gz: 8b2b3ebe11fc219d52a9576207a384fa6dd8f9b6178a5a65efaadbb215c3dcd9
+  data.tar.gz: 0fbc9926bcf4429875278c1d9b9157cf91618f6dad67c245c109fd85dbfc04e8
 SHA512:
-  metadata.gz: 7c222884746207d2c46ab112ed40b7d78b4de10feef39dbeee650fa0b40c58bd00307c121ce1fa550391379aa61e80a4d78275e7c2782394f4e91153be986997
-  data.tar.gz: 7ec7a253c3ca0c576b6935df764a24456994b8a26a045d5614445164399dff84568c424d11955ab4560542eef9ec8d340c215ee82145c7b585b27c38b5cf003a
+  metadata.gz: 3f55b980f9ceb11448e1699da20f071dcc75cbd79c3dfee4869081f2fb1450b04d0027be2df92310082a5c54144c5ce8df27efa21cc1d9d19365a0127280512e
+  data.tar.gz: 809c304408c6f03014db7ad5169f36f6616d3765d6e31a96f6ee6e566d3e3c963788a6df796eac373e59851cc42c0b69ae21a335ceb2f2a7399adde32d1ac1db

data/.travis.yml

@@ -1,10 +1,10 @@
 sudo: false
 language: ruby
 rvm:
-  - 2.2.8
-  - 2.3.5
-  - 2.4.2
-before_install: gem install bundler -v 1.16.0
+  - 2.3.8
+  - 2.4.5
+  - 2.5.3
+  - 2.6.0
 services:
   - postgresql
 addons:

data/Gemfile

@@ -1,3 +1,5 @@
 source 'https://rubygems.org'
 
+gem 'db_schema-definitions', github: 'db-schema/definitions', branch: 'primary_keys'
+
 gemspec

data/db_schema-reader-postgres.gemspec

@@ -20,7 +20,7 @@ Gem::Specification.new do |spec|
 
   spec.add_runtime_dependency 'sequel'
   spec.add_runtime_dependency 'pg'
-  spec.add_runtime_dependency 'db_schema-definitions', '~> 0.1.0'
+  spec.add_runtime_dependency 'db_schema-definitions', '= 0.2.rc1'
 
   spec.add_development_dependency 'bundler', '~> 1.16'
   spec.add_development_dependency 'rake', '~> 10.0'

data/lib/db_schema/reader/postgres.rb

@@ -5,6 +5,84 @@ require_relative 'postgres/version'
 module DbSchema
   module Reader
     class Postgres
+      COLUMNS_QUERY = <<-SQL.freeze
+   SELECT c.table_name,
+          c.column_name AS name,
+          c.ordinal_position AS pos,
+          c.column_default AS default,
+          c.is_nullable AS null,
+          c.data_type AS type,
+          c.is_identity,
+          c.udt_name AS custom_type_name,
+          c.character_maximum_length AS char_length,
+          c.numeric_precision AS num_precision,
+          c.numeric_scale AS num_scale,
+          c.datetime_precision AS dt_precision,
+          c.interval_type,
+          e.data_type AS element_type,
+          e.udt_name AS element_custom_type_name
+     FROM information_schema.columns AS c
+LEFT JOIN information_schema.element_types AS e
+       ON e.object_catalog = c.table_catalog
+      AND e.object_schema = c.table_schema
+      AND e.object_name = c.table_name
+      AND e.object_type = 'TABLE'
+      AND e.collection_type_identifier = c.dtd_identifier
+    WHERE c.table_schema = 'public'
+      SQL
+
+      INDEXES_QUERY = <<-SQL.freeze
+   SELECT table_rel.relname AS table_name,
+          pg_class.relname AS name,
+          indkey AS column_positions,
+          indisprimary AS primary,
+          indisunique AS unique,
+          indoption AS index_options,
+          pg_get_expr(indpred, indrelid, true) AS condition,
+          amname AS index_type,
+          indexrelid AS index_oid
+     FROM pg_class, pg_index
+LEFT JOIN pg_opclass
+       ON pg_opclass.oid = ANY(pg_index.indclass::int[])
+LEFT JOIN pg_am
+       ON pg_am.oid = pg_opclass.opcmethod
+     JOIN pg_class AS table_rel
+       ON table_rel.oid = pg_index.indrelid
+     JOIN pg_namespace
+       ON pg_namespace.oid = table_rel.relnamespace
+    WHERE pg_class.oid = pg_index.indexrelid
+      AND pg_namespace.nspname = 'public'
+ GROUP BY table_name, name, column_positions, indisprimary, indisunique, index_options, condition, index_type, index_oid
+      SQL
+
+      EXPRESSION_INDEXES_QUERY = <<-SQL.freeze
+    WITH index_ids AS (SELECT unnest(?) AS index_id),
+         elements AS (SELECT unnest(?) AS element)
+  SELECT index_id,
+         array_agg(pg_get_indexdef(index_id, element, 't')) AS definitions
+    FROM index_ids, elements
+GROUP BY index_id;
+      SQL
+
+      CONSTRAINTS_QUERY = <<-SQL.freeze
+   SELECT owner_table.relname AS table_name,
+          constr.conname AS name,
+          pg_get_expr(conbin, conrelid, true) AS condition,
+          referenced_table.relname AS referenced,
+          conkey,
+          confkey,
+          confupdtype AS on_update,
+          confdeltype AS on_delete,
+          condeferrable AS deferrable,
+          constr.contype AS type
+     FROM pg_constraint AS constr
+     JOIN pg_class AS owner_table
+       ON owner_table.oid = constr.conrelid
+LEFT JOIN pg_class AS referenced_table
+       ON referenced_table.oid = constr.confrelid
+    WHERE contype IN ('c', 'f');
+      SQL
+
       ENUMS_QUERY = <<-SQL.freeze
   SELECT t.typname AS name,
          array_agg(e.enumlabel ORDER BY e.enumsortorder) AS values
@@ -35,13 +113,17 @@ SELECT extname
       end
 
       def read_tables
-        connection.tables.map do |table_name|
-          read_table(table_name)
-        end
-      end
+        checks_data, foreign_keys_data = constraints_data
 
-      def read_table(table_name)
-        Table.new(connection, table_name).read
+        columns_data.keys.map do |table_name|
+          Table.new(
+            table_name.to_sym,
+            columns_data[table_name],
+            indexes_data[table_name],
+            checks_data[table_name],
+            foreign_keys_data[table_name]
+          ).definition
+        end
       end
 
       def read_enums
@@ -55,6 +137,120 @@ SELECT extname
           Definitions::Extension.new(extension_data[:extname].to_sym)
         end
       end
+
+    private
+      def columns_data
+        @columns_data ||= connection[COLUMNS_QUERY].to_a.group_by do |column|
+          column[:table_name]
+        end
+      end
+
+      def indexes_data
+        @indexes_data ||= begin
+          raw_data = connection[INDEXES_QUERY].map do |index_data|
+            index_data.merge(
+              column_positions: index_data[:column_positions].split(' ').map(&:to_i),
+              index_options:    index_data[:index_options].split(' ').map(&:to_i)
+            )
+          end
+
+          expressions_data = index_expressions_data(raw_data)
+
+          raw_data.map do |index_data|
+            columns = index_data[:column_positions].map do |position|
+              if position.zero?
+                expressions_data.fetch(index_data[:index_oid]).shift
+              else
+                get_field_name(index_data[:table_name], position)
+              end
+            end
+
+            index_data.delete(:index_oid)
+            index_data.delete(:column_positions)
+            index_data.merge(columns: columns)
+          end.group_by { |index| index[:table_name] }.tap { |h| h.default = [] }
+        end
+      end
+
+      def index_expressions_data(indexes_data)
+        all_positions, max_position = {}, 0
+
+        indexes_data.each do |index_data|
+          positions = index_data[:column_positions]
+          expression_positions = positions.each_index.select do |i|
+            positions[i].zero?
+          end
+
+          if expression_positions.any?
+            all_positions[index_data[:index_oid]] = expression_positions
+            max_position = [max_position, expression_positions.max].max
+          end
+        end
+
+        if all_positions.any?
+          connection[
+            EXPRESSION_INDEXES_QUERY,
+            Sequel.pg_array(all_positions.keys),
+            Sequel.pg_array((1..max_position.succ).to_a)
+          ].each_with_object({}) do |index_data, indexes_data|
+            index_id = index_data[:index_id]
+            expressions = all_positions[index_id].map { |pos| index_data[:definitions][pos] }
+
+            indexes_data[index_id] = expressions
+          end
+        else
+          {}
+        end
+      end
+
+      def constraints_data
+        checks       = Hash.new { |h, k| h[k] = [] }
+        foreign_keys = Hash.new { |h, k| h[k] = [] }
+
+        connection[CONSTRAINTS_QUERY].each do |constraint|
+          case constraint[:type]
+          when 'c'
+            checks[constraint[:table_name]] << {
+              name:      constraint[:name],
+              condition: constraint[:condition]
+            }
+          when 'f'
+            fields = constraint[:conkey].map do |position|
+              get_field_name(constraint[:table_name], position)
+            end
+
+            keys = constraint[:confkey].map do |position|
+              get_field_name(constraint[:referenced], position)
+            end
+
+            pkey_columns = indexes_data.fetch(constraint[:referenced]).find do |index|
+              index[:primary]
+            end.fetch(:columns)
+
+            keys = [] if keys == pkey_columns # this foreign key references a primary key
+
+            foreign_keys[constraint[:table_name]] << {
+              name:       constraint[:name],
+              referenced: constraint[:referenced],
+              fields:     fields,
+              keys:       keys,
+              on_update:  constraint[:on_update],
+              on_delete:  constraint[:on_delete],
+              deferrable: constraint[:deferrable]
+            }
+          else
+            raise "Unknown constraint type #{constraint[:type].inspect}"
+          end
+        end
+
+        [checks, foreign_keys]
+      end
+
+      def get_field_name(table, position)
+        columns_data.fetch(table).find do |column|
+          column[:pos] == position
+        end.fetch(:name).to_sym
+      end
     end
   end
 end

data/lib/db_schema/reader/postgres/table.rb

@@ -2,6 +2,12 @@ module DbSchema
   module Reader
     class Postgres
       class Table
+        SERIAL_TYPES = {
+          smallint: :smallserial,
+          integer:  :serial,
+          bigint:   :bigserial
+        }.freeze
+
         DEFAULT_VALUE = /\A(
           ('(?<date>\d{4}-\d{2}-\d{2})'::date)
             |
@@ -16,102 +22,25 @@ module DbSchema
           (?<boolean>true|false)
         )/x
 
-        COLUMN_NAMES_QUERY = <<-SQL.freeze
-   SELECT c.column_name AS name,
-          c.ordinal_position AS pos,
-          c.column_default AS default,
-          c.is_nullable AS null,
-          c.data_type AS type,
-          c.udt_name AS custom_type_name,
-          c.character_maximum_length AS char_length,
-          c.numeric_precision AS num_precision,
-          c.numeric_scale AS num_scale,
-          c.datetime_precision AS dt_precision,
-          c.interval_type,
-          e.data_type AS element_type,
-          e.udt_name AS element_custom_type_name
-     FROM information_schema.columns AS c
-LEFT JOIN information_schema.element_types AS e
-       ON e.object_catalog = c.table_catalog
-      AND e.object_schema = c.table_schema
-      AND e.object_name = c.table_name
-      AND e.object_type = 'TABLE'
-      AND e.collection_type_identifier = c.dtd_identifier
-    WHERE c.table_schema = 'public'
-      AND c.table_name = ?
-        SQL
-
-        CONSTRAINTS_QUERY = <<-SQL.freeze
-SELECT conname AS name,
-       pg_get_expr(conbin, conrelid, true) AS condition
-  FROM pg_constraint, pg_class
- WHERE conrelid = pg_class.oid
-   AND relname = ?
-   AND contype = 'c'
-        SQL
-
-        INDEXES_QUERY = <<-SQL.freeze
-   SELECT relname AS name,
-          indkey AS column_positions,
-          indisunique AS unique,
-          indoption AS index_options,
-          pg_get_expr(indpred, indrelid, true) AS condition,
-          amname AS index_type,
-          indexrelid AS index_oid
-     FROM pg_class, pg_index
-LEFT JOIN pg_opclass
-       ON pg_opclass.oid = ANY(pg_index.indclass::int[])
-LEFT JOIN pg_am
-       ON pg_am.oid = pg_opclass.opcmethod
-    WHERE pg_class.oid = pg_index.indexrelid
-      AND pg_class.oid IN (
-     SELECT indexrelid
-       FROM pg_index, pg_class
-      WHERE pg_class.relname = ?
-        AND pg_class.oid = pg_index.indrelid
-        AND indisprimary != 't'
-)
-  GROUP BY name, column_positions, indisunique, index_options, condition, index_type, index_oid
-        SQL
-
-        EXPRESSION_INDEXES_QUERY = <<-SQL.freeze
-    WITH index_ids AS (SELECT unnest(?) AS index_id),
-         elements AS (SELECT unnest(?) AS element)
-  SELECT index_id,
-         array_agg(pg_get_indexdef(index_id, element, 't')) AS definitions
-    FROM index_ids, elements
-GROUP BY index_id;
-        SQL
-
-        attr_reader :connection, :table_name
-
-        def initialize(connection, table_name)
-          @connection = connection
-          @table_name = table_name
+        FKEY_ACTIONS = {
+          a: :no_action,
+          r: :restrict,
+          c: :cascade,
+          n: :set_null,
+          d: :set_default
+        }.freeze
+
+        attr_reader :table_name, :fields_data, :indexes_data, :checks_data, :fkeys_data
+
+        def initialize(table_name, fields_data, indexes_data, checks_data, fkeys_data)
+          @table_name   = table_name
+          @fields_data  = fields_data
+          @indexes_data = indexes_data
+          @checks_data  = checks_data
+          @fkeys_data   = fkeys_data
         end
 
-        def read
-          primary_key_name = connection.primary_key(table_name)
-
-          fields = columns_data.map do |column_data|
-            build_field(column_data, primary_key: column_data[:name] == primary_key_name)
-          end
-
-          indexes = indexes_data.map do |index_data|
-            Definitions::Index.new(index_data)
-          end.sort_by(&:name)
-
-          foreign_keys = connection.foreign_key_list(table_name).map do |foreign_key_data|
-            build_foreign_key(foreign_key_data)
-          end
-
-          checks = connection[CONSTRAINTS_QUERY, table_name.to_s].map do |check_data|
-            Definitions::CheckConstraint.new(
-              name:      check_data[:name].to_sym,
-              condition: check_data[:condition]
-            )
-          end
-
+        def definition
           Definitions::Table.new(
             table_name,
             fields:       fields,
@@ -121,107 +50,71 @@ GROUP BY index_id;
           )
         end
 
-      private
-        def columns_data
-          @columns_data ||= connection[COLUMN_NAMES_QUERY, table_name.to_s]
-        end
-
-        def indexes_data
-          column_names = columns_data.reduce({}) do |names, column|
-            names.merge(column[:pos] => column[:name].to_sym)
-          end
-
-          indexes_data     = connection[INDEXES_QUERY, table_name.to_s].to_a
-          expressions_data = index_expressions_data(indexes_data)
-
-          indexes_data.map do |index|
-            positions = index[:column_positions].split(' ').map(&:to_i)
-            options   = index[:index_options].split(' ').map(&:to_i)
-
-            columns = positions.zip(options).map do |column_position, column_order_options|
-              options = case column_order_options
-              when 0
-                {}
-              when 3
-                { order: :desc }
-              when 2
-                { nulls: :first }
-              when 1
-                { order: :desc, nulls: :last }
-              end
-
-              if column_position.zero?
-                expression = expressions_data.fetch(index[:index_oid]).shift
-                DbSchema::Definitions::Index::Expression.new(expression, **options)
-              else
-                DbSchema::Definitions::Index::TableField.new(column_names.fetch(column_position), **options)
-              end
-            end
-
-            {
-              name:      index[:name].to_sym,
-              columns:   columns,
-              unique:    index[:unique],
-              type:      index[:index_type].to_sym,
-              condition: index[:condition]
-            }
+        def fields
+          fields_data.map do |field_data|
+            build_field(field_data)
           end
         end
 
-        def index_expressions_data(indexes_data)
-          all_positions, max_position = {}, 0
-
-          indexes_data.each do |index_data|
-            positions = index_data[:column_positions].split(' ').map(&:to_i)
-            expression_positions = positions.each_index.select { |i| positions[i].zero? }
+        def indexes
+          indexes_data.map do |index_data|
+            build_index(index_data)
+          end.sort_by(&:name)
+        end
 
-            if expression_positions.any?
-              all_positions[index_data[:index_oid]] = expression_positions
-              max_position = [max_position, expression_positions.max].max
-            end
+        def checks
+          checks_data.map do |check_data|
+            Definitions::CheckConstraint.new(
+              name:      check_data[:name].to_sym,
+              condition: check_data[:condition]
+            )
           end
+        end
 
-          if all_positions.any?
-            connection[
-              EXPRESSION_INDEXES_QUERY,
-              Sequel.pg_array(all_positions.keys),
-              Sequel.pg_array((1..max_position.succ).to_a)
-            ].each_with_object({}) do |index_data, indexes_data|
-              index_id = index_data[:index_id]
-              expressions = all_positions[index_id].map { |pos| index_data[:definitions][pos] }
-
-              indexes_data[index_id] = expressions
-            end
-          else
-            {}
+        def foreign_keys
+          fkeys_data.map do |foreign_key_data|
+            build_foreign_key(foreign_key_data)
           end
         end
 
-        def build_field(data, primary_key: false)
+      private
+        def build_field(data)
           type = data[:type].to_sym.downcase
           if type == :'user-defined'
             type = data[:custom_type_name].to_sym
           end
 
+          serial_type = SERIAL_TYPES[type]
+
           nullable = (data[:null] != 'NO')
 
-          unless primary_key || data[:default].nil?
-            default = if match = DEFAULT_VALUE.match(data[:default])
-              if match[:date]
-                Date.parse(match[:date])
-              elsif match[:time]
-                Time.parse(match[:time])
-              elsif match[:string]
-                match[:string]
-              elsif match[:integer]
-                match[:integer].to_i
-              elsif match[:float]
-                match[:float].to_f
-              elsif match[:boolean]
-                match[:boolean] == 'true'
-              end
+          if data[:is_identity] == 'YES'
+            type    = serial_type
+            default = nil
+          elsif !data[:default].nil?
+            serial_field_default = "nextval('#{table_name}_#{data[:name]}_seq'::regclass)"
+
+            if !serial_type.nil? && !nullable && data[:default] == serial_field_default
+              type    = serial_type
+              default = nil
             else
-              data[:default].to_sym
+              default = if match = DEFAULT_VALUE.match(data[:default])
+                if match[:date]
+                  Date.parse(match[:date])
+                elsif match[:time]
+                  Time.parse(match[:time])
+                elsif match[:string]
+                  match[:string]
+                elsif match[:integer]
+                  match[:integer].to_i
+                elsif match[:float]
+                  match[:float].to_f
+                elsif match[:boolean]
+                  match[:boolean] == 'true'
+                end
+              else
+                data[:default].to_sym
+              end
             end
           end
 
@@ -263,35 +156,52 @@ GROUP BY index_id;
           Definitions::Field.build(
             data[:name].to_sym,
             type,
-            primary_key: primary_key,
-            null:        nullable,
-            default:     default,
+            null:    nullable,
+            default: default,
             **options
           )
         end
 
-        def build_foreign_key(data)
-          keys = if data[:key] == [primary_key_for(data[:table])]
-            [] # this foreign key references a primary key
-          else
-            data[:key]
+        def build_index(data)
+          columns = data[:columns].zip(data[:index_options]).map do |column, order_options|
+            options = case order_options
+            when 0
+              {}
+            when 3
+              { order: :desc }
+            when 2
+              { nulls: :first }
+            when 1
+              { order: :desc, nulls: :last }
+            end
+
+            if column.is_a?(String)
+              DbSchema::Definitions::Index::Expression.new(column, **options)
+            else
+              DbSchema::Definitions::Index::TableField.new(column, **options)
+            end
           end
 
-          Definitions::ForeignKey.new(
-            name:       data[:name],
-            fields:     data[:columns],
-            table:      data[:table],
-            keys:       keys,
-            on_delete:  data[:on_delete],
-            on_update:  data[:on_update],
-            deferrable: data[:deferrable]
+          Definitions::Index.new(
+            name:      data[:name].to_sym,
+            columns:   columns,
+            unique:    data[:unique],
+            primary:   data[:primary],
+            type:      data[:index_type].to_sym,
+            condition: data[:condition]
           )
         end
 
-        def primary_key_for(table_name)
-          if pkey = connection.primary_key(table_name)
-            pkey.to_sym
-          end
+        def build_foreign_key(data)
+          Definitions::ForeignKey.new(
+            name:       data[:name].to_sym,
+            fields:     data[:fields],
+            table:      data[:referenced].to_sym,
+            keys:       data[:keys],
+            on_update:  FKEY_ACTIONS.fetch(data[:on_update].to_sym),
+            on_delete:  FKEY_ACTIONS.fetch(data[:on_delete].to_sym),
+            deferrable: data[:deferrable]
+          )
         end
 
         # TODO: replace following methods with Transproc

data/lib/db_schema/reader/postgres/version.rb

@@ -1,7 +1,7 @@
 module DbSchema
   module Reader
     class Postgres
-      VERSION = '0.1.1'
+      VERSION = '0.2.rc1'
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: db_schema-reader-postgres
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.rc1
 platform: ruby
 authors:
 - Vsevolod Romashov
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-10-31 00:00:00.000000000 Z
+date: 2019-02-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel
@@ -42,16 +42,16 @@ dependencies:
   name: db_schema-definitions
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: 0.1.0
+        version: 0.2.rc1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: 0.1.0
+        version: 0.2.rc1
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -202,12 +202,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.0.1
 signing_key: 
 specification_version: 4
 summary: Database schema reader for PostgreSQL