pg_meta 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 9a7711ea44e1ae9ff9fd3ed45016e7ba2a00a1554af08e5b311e036cb29c6f01
+  data.tar.gz: 5045d4b33fe8a0b2aeacac9958d7a2f80e8ed2ae3b27c1b69a457acc4d343875
+SHA512:
+  metadata.gz: 2e342aef3595dd33c6728ad702efa4a7e4e80640cecccf6c345ad5839f5d3c913b30215b8be7f858fa90faa84eef5c497083978d423c271e008dfb13737c8ba7
+  data.tar.gz: 38b665922c5005182d764db3126948defc7e87aeab3355782f695f3d89a3976e648aa405e191b871863da1ca42377be3b6ff1fcaeb9ea4657c5c272ddc40c3f4

data/.gitignore

@@ -0,0 +1,19 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status
+
+# Ignore auto-generated main file
+/main
+
+# Ignore Gemfile.lock. See https://stackoverflow.com/questions/4151495/should-gemfile-lock-be-included-in-gitignore
+/Gemfile.lock
+
+# Put your personal ignore files in /home/clr/.config/git/ignore

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.ruby-version

@@ -0,0 +1 @@
+ruby-2.7.1

data/.travis.yml

@@ -0,0 +1,7 @@
+---
+sudo: false
+language: ruby
+cache: bundler
+rvm:
+  - 2.6.6
+before_install: gem install bundler -v 1.17.3

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in pg_meta.gemspec
+gemspec

data/README.md

@@ -0,0 +1,35 @@
+# PgMeta
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/pg_meta`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'pg_meta'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install pg_meta
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/pg_meta.

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/TODO

@@ -0,0 +1,3 @@
+
+o Needs tests of functions and triggers (works on mikras)
+o ::load_yaml doesn't create depending/defining tables/relations on Table and View

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "pg_meta"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/pg_meta

@@ -0,0 +1,28 @@
+#!/usr/bin/env ruby
+
+require 'shellopts'
+require 'pg_meta'
+
+include ShellOpts
+
+SPEC = %(
+  -f,format=FORMAT:yaml,marshal,dump
+    Control format of output. FORMAT can be one of 'yaml', 'marshal' or the
+    human-readable 'dump' format (the default)
+
+  -- DATABASE
+)
+
+opts, args = ShellOpts.process(SPEC, ARGV, version: PgMeta::VERSION)
+
+format = opts.format
+database = args.expect(1)
+conn = PgConn.new(database)
+meta = PgMeta.new(conn)
+
+case format
+  when "yaml"; puts meta.to_yaml
+  when "marshal"; puts meta.to_marshal
+  else meta.dump
+end
+

data/lib/ext/hash.rb

@@ -0,0 +1,9 @@
+
+module Ext
+  module Hash
+    refine ::Hash do
+      def project(*keys) keys.map { |key| self[key] } end
+    end
+  end
+end
+

data/lib/pg_meta/dump.rb

@@ -0,0 +1,37 @@
+
+require 'indented_io'
+
+module PgMeta
+  class Node
+    def dump
+      dump_value(to_h)
+    end
+
+    def dump_value(v)
+      case v
+        when Hash
+          puts "{"
+          indent {
+            v.each { |k,v| 
+              print "#{k}: "
+              dump_value(v)
+            }
+          }
+          puts "}"
+        when []
+          puts "[]"
+        when Array
+          puts "["
+          indent {
+            v.each { |v| 
+              dump_value(v)
+            }
+          }
+          puts "]"
+      else
+        puts (v.nil? ? "nil" : v)
+      end
+    end
+  end
+end
+

data/lib/pg_meta/load_conn.rb

@@ -0,0 +1,356 @@
+
+module PgMeta
+  class Database
+    using Ext::Hash
+
+    # Load the database from the given PgConn::Connection object. This method is a
+    # transaction wrapper around #do_load_conn
+    def self.load_conn(pg_conn)
+      pg_conn.pg_connection.transaction { |conn| 
+        conn.exec "set transaction isolation level serializable read only deferrable"
+        do_load_from_connection(conn) 
+      }
+    end
+
+  private
+    # Load the given database into self. Note that +conn+ is a PG::Connection object
+    def self.do_load_from_connection(conn)
+      # Get database owner
+      r = conn.exec %(
+        select  rolname::varchar
+        from    pg_database d
+        join    pg_roles r on r.oid = d.datdba
+        where   d.datname = '#{conn.db}'
+      )
+      owner = r.values.first.first
+
+      # Create database object
+      db = Database.new(conn.db, owner)
+
+      # Build schemas
+      conn.exec(%(
+        select  schema_name::text as name,
+                schema_owner::text as owner
+        from    information_schema.schemata
+        where   schema_name !~ '^pg_' and schema_name <> 'information_schema' 
+      )).each { |row|
+        Schema.new(db, *row.project(:name, :owner))
+      }
+
+      # Build tables and views
+      conn.exec(%(
+        select  n.nspname::text as "schema",
+                c.relname::text as "name",
+                c.relkind,
+                coalesce(i.is_insertable_into, 'NO') = 'YES' as "insertable?",
+                coalesce(i.is_typed, 'NO') = 'YES' as "typed?"
+        from    pg_class c
+        join    pg_namespace n on n.oid = c.relnamespace
+        left join information_schema.tables i on
+                    i.table_schema = n.nspname
+                and i.table_name = c.relname
+        where   n.nspname !~ '^pg_' and n.nspname <> 'information_schema'
+        and     c.relkind in ('r', 'v', 'm')
+      )).each { |row|
+        schema = db.schemas[row[:schema]]
+        klass = 
+            case row[:relkind]
+              when 'r'; Table
+              when 'v'; View
+              when 'm'; MaterializedView
+            end
+        klass.new(schema, *row.project(:name, :insertable?, :typed?))
+      }
+
+      # Find transitive closure of dependencies between tables
+      conn.exec(%(
+        with recursive chain as (
+          select distinct
+                  c.conrelid as this,
+                  c.confrelid as that
+          from    pg_depend d
+          join    pg_constraint c on c.oid = d.objid
+          join    pg_class t1 on t1.oid = c.conrelid
+          join    pg_namespace n1 on n1.oid = t1.relnamespace
+          join    pg_class t2 on t2.oid = c.confrelid
+          join    pg_namespace n2 on n2.oid = t2.relnamespace
+          where   n1.nspname = 'auth'
+          and     deptype = 'n'
+        union
+          select  ch.this as this,
+                  c.confrelid as that
+          from    pg_depend d
+          join    pg_constraint c on c.oid = d.objid
+          join    chain ch on ch.that = c.conrelid
+        )
+        select  n1.nspname as this_schema, 
+                t1.relname as this_table, 
+                n2.nspname as that_schema, 
+                t2.relname as that_table
+        from chain c
+        join pg_class t1 on t1.oid = c.this
+        join pg_namespace n1 on n1.oid = t1.relnamespace
+        join pg_class t2 on t2.oid = c.that
+        join pg_namespace n2 on n2.oid = t2.relnamespace
+      )).each { |row|
+        this_table = db.schemas[row[:this_schema]].tables[row[:this_table]]
+        that_table = db.schemas[row[:that_schema]].tables[row[:that_table]]
+        that_table.send(:add_depending_table, this_table)
+      }
+
+      # Build defining-relation dependencies
+      conn.exec(%(
+        select distinct
+                n2.nspname::text as "dep_schema",
+                c2.relname::text as "dep_view",
+                n.nspname::text as "def_schema",
+                c.relname::text as "def_relation"
+        from    pg_class c
+        join    pg_namespace n on n.oid = c.relnamespace
+        join    pg_depend d on d.refobjid = c.oid
+        join    pg_rewrite r on r.oid = d.objid
+        join    pg_class c2 on c2.oid = r.ev_class
+        join    pg_namespace n2 on n2.oid = c2.relnamespace
+        where   n.nspname !~ '^pg_' and n.nspname <> 'information_schema'
+        and     c.relkind in ('r', 'v', 'm')
+        and     d.classid = 'pg_rewrite'::regclass
+        and     d.refclassid = 'pg_class'::regclass
+        and     d.deptype = 'n'
+        and     c2.relkind in ('r̈́', 'v', 'm')
+        and     c.oid <> c2.oid
+      )).each { |row|
+        depending_view = db.schemas[row[:dep_schema]].tables[row[:dep_view]]
+        defining_relation = db.schemas[row[:def_schema]].tables[row[:def_relation]]
+
+        defining_relation.send(:add_depending_view, depending_view)
+        depending_view.send(:add_defining_relation, defining_relation)
+      }
+
+      # Build columns
+      conn.exec(%(
+        select  n.nspname::text as "schema",
+                c.relname::text as "table",
+                a.attname::text as "name",
+                a.attnum as "ordinal",
+                t.typname::text as "type",
+                t2.typname::text as "element_type",
+                a.attndims::integer as "dimensions",
+                i.column_default::text as "default",
+                coalesce(i.is_identity, 'NO') = 'YES' as "identity?",
+                coalesce(i.is_generated, 'NO') = 'ALWAYS' as "generated?",
+                coalesce(i.is_nullable, 'NO') = 'YES' as "nullable?",
+                coalesce(i.is_updatable, 'NO') = 'YES' as "updatable?"
+        from    pg_namespace n 
+        join    pg_class c on c.relnamespace = n.oid
+        join    pg_attribute a on a.attrelid = c.oid
+        join    pg_type t on t.oid = a.atttypid
+        left join pg_type t2 on t2.oid = t.typelem
+        left join information_schema.columns i on 
+                    i.table_schema = n.nspname
+                and i.table_name = c.relname
+                and i.column_name = a.attname
+        where   n.nspname !~ '^pg_' and n.nspname <> 'information_schema'
+        and     c.relkind in ('r', 'v', 'm')
+        and     a.attnum > 0
+      )).each { |row|
+        table = db.schemas[row[:schema]].tables[row[:table]]
+        Column.new(table, *row.project(
+            :name, :ordinal, :type, :element_type, :dimensions, :default, :identity?, :generated?,
+            :nullable?, :updatable?))
+      }
+
+      # Build simple constraints
+      conn.exec(%(
+        select  c.table_schema::text as schema,
+                c.table_name::text as table,
+                c.constraint_type::text,
+                c.constraint_name::text as name,
+                cc.check_clause as expression,
+                (
+                  select  array_agg(column_name::text)
+                  from    information_schema.constraint_column_usage ccu
+                  where   ccu.table_schema = c.table_schema
+                  and     ccu.table_name = c.table_name
+                  and     ccu.constraint_schema = c.constraint_schema
+                  and     ccu.constraint_name = c.constraint_name
+                ) as columns
+        from    information_schema.table_constraints c
+        left join information_schema.check_constraints cc
+                  on  cc.constraint_schema = c.table_schema and
+                      cc.constraint_name = c.constraint_name
+        where   c.table_schema !~ '^pg_' and c.table_schema <> 'information_schema'
+        and     c.constraint_type in ('PRIMARY KEY', 'UNIQUE', 'CHECK')
+      )).each { |row|
+        table = db.schemas[row[:schema]].tables[row[:table]]
+        name = row[:name]
+        columns = lookup_columns(table, row[:columns] || [])
+        expression = row[:expression]
+        case row[:constraint_type]
+          when "PRIMARY KEY"; PrimaryKeyConstraint.new(table, name, columns)
+          when "UNIQUE"; UniqueConstraint.new(table, name, columns)
+          when "CHECK"
+            constraint = CheckConstraint.new(table, name, expression)
+            # patch-up column if constraint is a not null constraint
+            if constraint.not_null?
+              constraint.column.instance_variable_set(:@is_nullable, false)
+            end
+          else
+            raise ArgumentError
+        end
+      }
+
+      # Build implicit unique constraints from indexes. Only indexes not used
+      # in constraints are considered
+      conn.exec(%(
+        select  n.nspname as "schema",
+                c.relname as "table",
+                array_agg(a.attname::text) as "fields",
+                i.relname as "index",
+                x.indisunique as "unique"
+        from    pg_class c
+        join    pg_namespace n on n.oid = c.relnamespace
+        join    pg_index x on x.indrelid = c.oid
+        join    pg_class i on i.oid = x.indexrelid
+        join    ( select attrelid, attname from pg_attribute order by attnum ) a on a.attrelid = i.oid
+        where   n.nspname !~ '^pg_' and n.nspname <> 'information_schema'
+        and     i.oid not in (select conindid from pg_constraint)
+        and     x.indisunique
+        group by
+                "schema",
+                "table",
+                "index",
+                "unique"
+        order by 
+                "schema",
+                "table",
+                "index"
+      )).each { |row|
+        table = db.schemas[row[:schema]].tables[row[:table]]
+        name = "#{row[:schema]}.#{row[:table]}.#{row[:index]}_constr"
+        columns = lookup_columns(table, row[:fields] || [])
+        UniqueConstraint.new(table, name, columns)
+      }
+
+      # Build referential constraints
+      #
+      # Referential constraints has to be initialized after unique constraints
+      #
+      # The GROUP BY is necessary because we re-assign constraints from schema to
+      # table. This requires joining key_column_usage again to get the name of
+      # the referenced table and that yields a row for each column in the unique
+      # key (TODO: Can this be omitted?)
+      #
+      conn.exec(%(
+        select  rc.constraint_schema::text as schema,
+                rc.constraint_name::text as name,
+                cu_refing.table_schema::text as "referencing_schema",
+                cu_refing.table_name::text as "referencing_table",
+                (
+                  select  array_agg(column_name::text order by ordinal_position)
+                  from    information_schema.key_column_usage kcu
+                  where   kcu.constraint_schema = rc.constraint_schema
+                  and     kcu.constraint_name = rc.constraint_name
+                ) as "referencing_columns",
+                cu_refed.table_schema::text as referenced_schema,
+                cu_refed.table_name::text as referenced_table,
+                cu_refed.constraint_name::text as referenced_constraint
+        from    information_schema.referential_constraints rc
+        join    information_schema.key_column_usage cu_refing 
+                on  cu_refing.constraint_schema = rc.constraint_schema
+                    and cu_refing.constraint_name = rc.constraint_name
+        join    information_schema.key_column_usage cu_refed 
+                on  cu_refed.constraint_schema = rc.unique_constraint_schema
+                    and cu_refed.constraint_name = rc.unique_constraint_name
+        where   cu_refing.table_schema !~ '^pg_' and cu_refing.table_schema <> 'information_schema'
+        group by
+                rc.constraint_schema,
+                rc.constraint_name,
+                cu_refing.table_schema,
+                cu_refing.table_name,
+                cu_refed.table_schema,
+                cu_refed.table_name,
+                cu_refed.constraint_name
+      )).each { |row|
+        schema = db.schemas[row[:schema]]
+        name = row[:name]
+        referencing_table = schema.tables[row[:referencing_table]]
+        referencing_columns = lookup_columns(referencing_table, row[:referencing_columns])
+        referenced_constraint = 
+            db.schemas[row[:referenced_schema]] \
+            .tables[row[:referenced_table]] \
+            .constraints[row[:referenced_constraint]]
+        ref = ReferentialConstraint.new(referencing_table, name, referencing_columns, referenced_constraint)
+        ref.referenced_table.send(:add_depending_table, referencing_table)
+      }
+
+      # Build functions and procedures
+      conn.exec(%(
+        select  s.nspname::text as "schema",
+                pg_get_userbyid(p.proowner)::text as "owner",
+                format('%I(%s)', p.proname, oidvectortypes(p.proargtypes)) 
+                  || ': ' 
+                  || format_type(p.prorettype, null) as "name",
+                case format_type(p.prorettype, null) 
+                  when 'void' then 'procedure' 
+                  else 'function' 
+                end as "kind",
+                case 
+                  when prosecdef then 'definer' 
+                  else 'invoker' 
+                end as "security"
+        from    pg_proc p
+        join    pg_namespace s on (p.pronamespace = s.oid)
+        where   s.nspname !~ '^pg_' and s.nspname <> 'information_schema'
+      )).each { |row|
+        schema = db.schemas[row[:schema]]
+        klass = (row[:kind] == 'function' ? Function : Procedure)
+        klass.new(schema, *row.project(:name, :owner, :security))
+      }
+
+      # Build user-defined triggers
+      conn.exec(%(
+        select  n.nspname::text as "schema", 
+                c.relname::text as "table", 
+                t.tgname::text as "name",
+                fn.nspname::text as "function_schema",
+                format('%I(%s)', p.proname::text, oidvectortypes(p.proargtypes)) 
+                  || ': ' 
+                  || format_type(p.prorettype, null) as "function_name",
+                case when (tgtype::int::bit(7) & b'0000001')::int = 0 then 'stmt' else 'row' end as "level",
+                coalesce(
+                  case when (tgtype::int::bit(7) & b'0000010')::int = 0 then null else 'before' end,
+                  case when (tgtype::int::bit(7) & b'0000010')::int = 0 then 'after' else null end,
+                  case when (tgtype::int::bit(7) & b'1000000')::int = 0 then null else 'instead' end,
+                  ''
+                )::text as "timing", 
+                  (case when (tgtype::int::bit(7) & b'0000100')::int = 0 then '' else ' insert' end) ||
+                  (case when (tgtype::int::bit(7) & b'0001000')::int = 0 then '' else ' delete' end) ||
+                  (case when (tgtype::int::bit(7) & b'0010000')::int = 0 then '' else ' update' end) ||
+                  (case when (tgtype::int::bit(7) & b'0100000')::int = 0 then '' else ' truncate' end)
+                as "events"
+        from    pg_trigger t
+        join    pg_proc p on t.tgfoid = p.oid
+        join    pg_class c on c.oid = t.tgrelid
+        join    pg_namespace n on n.oid = c.relnamespace
+        join    pg_namespace fn on fn.oid = p.pronamespace
+        where   not t.tgisinternal
+      )).each { |row|
+        schema = db.schemas[row[:schema]]
+        table = schema.tables[row[:table]]
+        name, level, timing = *row.project(:name, :level, :timing)
+        function_schema = db.schemas[row[:function_schema]]
+        function = function_schema.functions[row[:function_name]]
+        events = row[:events].split.map(&:to_sym)
+        Trigger.new(table, name, function, level, timing, events)
+      }
+
+      # Return database object
+      db
+    end
+
+    def self.lookup_columns(table, column_names)
+      column_names.map { |n| table.columns[n] }
+    end
+  end
+end
+

data/lib/pg_meta/load_yaml.rb

@@ -0,0 +1,76 @@
+
+module PgMeta
+  class Database
+    using Ext::Hash
+
+    # Create a Database object from the given YAML representation and return
+    # it. The YAML representation can be generated by Database#to_h
+    def self.load_yaml(database_yaml)
+      database = Database.new *database_yaml.project(:name, :owner)
+
+      # Accumulators for second pass
+      views = []
+      referential_constraints = []
+
+      # First pass: Build everything except referential constraints
+      for schema_yaml in database_yaml[:schemas]
+        schema = Schema.new database, *schema_yaml.project(:name, :owner)
+        for table_yaml in schema_yaml[:tables]
+          klass = table_yaml[:table?] ? Table : (table_yaml[:materialized?] ? MaterializedView : View)
+          table = klass.new schema, *table_yaml.project(:name, :insertable?, :typed?)
+          views << [table, table_yaml[:defining_relations]] if klass <= View || klass <= MaterializedView
+          for column_yaml in table_yaml[:columns]
+            column = Column.new(table, 
+                *column_yaml.project(
+                    :name, :ordinal, :type, :element_type, :dimensions, :default, :identity?, :generated?,
+                    :nullable?, :updatable?))
+          end
+          for constraint_yaml in table_yaml[:constraints]
+            name = constraint_yaml[:name]
+            case kind = constraint_yaml[:kind]
+              when :primary_key
+                columns = constraint_yaml[:columns]&.map { |name| table.columns[name] }
+                PrimaryKeyConstraint.new(table, name, columns)
+              when :unique
+                columns = constraint_yaml[:columns]&.map { |name| table.columns[name] }
+                UniqueConstraint.new(table, name, columns)
+              when :check
+                CheckConstraint.new(table, name, constraint_yaml[:expression])
+              when :referential
+                # Postpone to second pass
+                columns = constraint_yaml[:referencing_columns]&.map { |name| table.columns[name] }
+                referential_constraints << [table, name, columns, constraint_yaml[:referenced_constraint]]
+            end
+          end
+          for trigger_yaml in table_yaml[:triggers]
+            Trigger.new(table, *trigger_yaml.project(:name, :function, :level, :timing, :events))
+          end
+        end
+        for function_yaml in schema_yaml[:functions]
+          klass = function_yaml[:function?] ? Function : Procedure
+          klass.new(schema, *function_yaml.project(:name, :owner, :security))
+        end
+      end
+
+      # Second pass: Add referential constraints
+      referential_constraints.each { |table, name, columns, referenced_constraint_uid|
+        referenced_constraint = database[referenced_constraint_uid] or 
+            raise Error, "Can't find UID '#{referenced_constraint_uid}'"
+        ReferentialConstraint.new(table, name, columns, referenced_constraint)
+        referenced_constraint.table.send(:add_depending_table, table)
+      }
+
+      # Second pass: Build defining tables
+      views.each { |view, defining_relation_uids|
+        defining_relation_uids.each { |uid|
+          relation = database[uid]
+          relation.send(:add_depending_view, view)
+          view.send(:add_defining_relation, relation)
+        }
+      }
+
+      database
+    end
+  end
+end
+