pg_graph 0.1.0

data/lib/data/render.rb

@@ -0,0 +1,237 @@
+
+module PgGraph::Data
+
+  class SqlRender
+    attr_reader :database
+
+    attr_reader :format
+    def format=(format)
+      constrain format, lambda { |v| [:sql, :exec, :psql] }, "Illegal value"
+      @format = format
+    end
+
+    # Which data to delete:
+    #   none - don't delete any data
+    #   touched - delete data for tables in the fox file
+    #   recursive - delete data for table in the fox file including recursively depending tables
+    #   all - delete data from the whole database
+    attr_reader :delete
+      
+    # +ids+ is a map from table UID to ID. Records with larger IDs will
+    # be emitted as insert statements, records with IDs less or equal to the
+    # given ID is emitted as update statements
+    #
+    # +delete+ control which tables are deleted. It can be :none, :touched,
+    # :recursive, :all Only records with an ID greater than the corresponding
+    # ID from +ids+ will be deleted
+    #
+    # +files+ is a list of source file names to be included in the psql SQL
+    # header as documentation. It can be set explicitly when #to_a or #to_h is
+    # called (FIXME: is this used?)
+    def initialize(database, format, ids: {}, delete: :all, files: [])
+#     puts "SqlRender#initialize"
+#     puts "  format: #{format.inspect}"
+#     puts "  ids: #{ids.inspect}"
+#     puts "  delete: #{delete.inspect}"
+#     puts "  files: #{files.inspect}"
+      constrain database, Database
+      constrain ids, String => Integer
+      @database = database
+      self.format = format
+      (@ids = ids.dup).default = 0
+      @delete = delete
+      @files = files
+
+      @tables = database.schemas.map(&:tables).flatten.sort
+      @insert_tables = []
+      @update_tables = []
+      @insert_records = {}
+      @update_records = []
+      @tables.each { |table|
+        next if table.empty?
+        @insert_tables << table if table.max_id > @ids[table.uid]
+        @update_tables << table if table.ids.min || 0 <= @ids[table.uid]
+        inserts, updates = table.records.partition { |record| record.id > @ids[table.uid] }
+        @insert_records[table] = inserts if !inserts.empty?
+        @update_records += updates
+      }
+      @table_uids = @tables.select { |table| !table.empty? }.map(&:uid)
+      @materialized_views = @tables.map(&:type).map(&:depending_materialized_views).flatten.uniq
+    end
+
+    def to_a(files = @files)
+      case format
+        when :sql; to_sql.flatten
+        when :exec; to_exec.flatten
+        when :psql; to_psql(files).flatten.compact
+      end
+    end
+
+    def to_s(files = @files)
+      case format
+        when :sql; to_a.join("\n")
+        when :exec; to_a.join("\n")
+        when :psql; to_psql(files).map { |group| group.join("\n") }.join("\n\n") 
+      end
+    end
+
+    def to_h
+      @to_h ||= {
+        disable: render_triggers(:disable),
+        delete: render_deletes(delete),
+        update: render_updates,
+        insert: render_inserts,
+        restart: render_restart_sequences,
+        enable: render_triggers(:enable),
+        refresh: render_refresh_materialized_views
+      }
+    end
+
+  protected
+    # Returns a single-element array of array of SQL statements
+    def to_sql
+      [to_h[:delete] + to_h[:update] + to_h[:insert]]
+    end
+
+    # Returns an array of non-empty arrays of SQL statements
+    def to_exec() to_h.values.reject(&:empty?) end
+
+    # Returns an array of arrays of SQL statements
+    def to_psql(files = [])
+      [render_psql_header(files), render_begin] + to_exec.select { |a| !a.empty? } + [render_commit]
+    end
+
+    def render_psql_header(files = [])
+      if files.empty?
+        files_text = ""
+      else
+        files_text = " from " + files.join(", ")
+      end
+      [
+          "-- Auto-generated by fox(1)" + files_text,
+          "",
+          "\\set QUIET",
+          "\\set ON_ERROR_STOP"
+      ]
+    end
+
+    def render_begin() 
+      [ "begin;" ]
+    end
+
+    def render_truncate()
+      @tables.empty? ? [] : [ "truncate #{@tables.map(&:uid).join(", ")} restart identity cascade;" ]
+    end
+
+    # :call-seq:
+    #   render_triggers(arg)
+    #
+    # +arg+ can be :disable or :enable
+    #
+    def render_triggers(arg)
+      [:disable, :enable].include?(arg) or raise Error, "Illegal value"
+      tables = @with_deletes ? @tables : @tables.reject(&:empty?)
+      tables.map { |table| "alter table #{table.uid} #{arg} trigger all;" }
+    end
+
+    def render_deletes(kind)
+      table_uids = 
+          case kind
+            when :none; []
+            when :touched; @tables.reject(&:empty?).map(&:uid)
+            when :recursive
+              tables = @tables.reject(&:empty?)
+              (tables.map(&:uid) + tables.map { |table| table.type.depending_tables.map(&:uid) }.flatten).uniq
+            when :all; @tables.map(&:uid)
+          else
+            raise ArgumentError
+          end
+      table_uids.map { |uid|
+        if !@ids.key?(uid)
+          "delete from #{uid};"
+        else
+          "delete from #{uid} where id > #{@ids[uid]};"
+        end
+      }
+    end
+
+    def render_updates
+      @update_records.map { |record|
+        "update #{record.table.uid} set " \
+            + record.value_columns
+                .select { |column| !column.type.primary_key? }
+                .map { |column| "#{column.name} = #{render_value(column)}" }
+                .join(", ") + " " \
+            + "where id = #{record.id};" 
+      }
+    end
+
+    def render_inserts
+      @insert_records.map { |table, records|
+        "insert into #{table.uid} (#{table.type.value_columns.map(&:name).join(', ')}) values " + \
+        records.sort_by(&:id).map { |record|
+          "(" +
+              record.type.value_columns.map { |column_type|
+                record.field?(column_type.name) ? render_value(record[column_type.name]) : 'DEFAULT'
+              }.join(", ") +
+          ")"
+        }.join(", ") + ";"
+      }
+    end
+
+    def render_literal(value, element_type = nil)
+      case value
+        when TrueClass, FalseClass; value.to_s
+        when Integer; value.to_s
+        when String; "'#{PG::Connection.escape_string(value.to_s)}'"
+        when NilClass; "NULL"
+        when Array
+          if value.empty?
+            "ARRAY[]::#{element_type}[]" # FIXME: Doesn't handle multidimensional arrays
+          else
+            "ARRAY[" + value.map { |v| render_literal(v) }.join(",") + "]"
+          end
+        when Hash
+          "'" + value.to_json + "'"
+        when Time
+          "'" + value.to_s + "'"
+      else
+        raise "Oops: got #{value.inspect} (#{value.class})"
+      end
+    end
+
+    def render_value(column)
+      type = column.value_type.type
+      if type.array?
+        render_literal(column.value, type.element_type)
+      else
+        render_literal(column.value)
+      end
+    end
+
+    def render_restart_sequences()
+      @tables.map { |table|
+        if table.type.subtable?
+          nil
+        elsif table.empty? && @with_deletes || !table.empty? && table.max_id > @ids[table.uid]
+          "alter table #{table.uid} alter column id restart with #{table.max_id+1};"
+        else
+          nil
+        end
+      }.compact
+    end
+
+    def render_refresh_materialized_views()
+      if @materialized_views.empty?
+        []
+      else
+        @materialized_views.map { |view| "refresh materialized view #{view.uid};" }
+      end
+    end
+
+    def render_commit() 
+      [ "commit;" ]
+    end
+  end
+end
+

data/lib/data/value.rb

@@ -0,0 +1,96 @@
+module PgGraph::Data
+  class Value < Node
+    attr_reader :referenced_object
+    def initialize(type, referenced_object, **opts)
+      super(type, **opts)
+      @referenced_object = referenced_object
+    end
+  end
+
+  # Fake class: RecordValue.new(table, records) simply returns the Record object
+  class RecordValue < Value
+    def self.new(table, records) 
+      constrain table, Table
+      constrain records, [Record]
+      records.first
+    end
+  end
+
+  class TableValue < Value
+    # Referenced table
+    alias_method :table, :referenced_object
+
+    # Forward to table
+    forward_to :table, :schema, :associations
+
+    # Forward to table implementation
+    forward_to :@impl, :records, :size, :empty?, :[], :id?, :ids, :data, :to_h
+
+    def initialize(table, records = [])
+      constrain table, Table
+      constrain records, [Record]
+      super(table.type, table, dimension: 2 )
+      @impl = Table.new(table.schema, table.type)
+      records.each { |r| @impl.send(:add_record, r) }
+    end
+  end
+
+  # MmTableValue is implemented as a hash from integer ID to non-empty array of
+  # identical Record objects
+  class MmTableValue < Value
+    # Referenced table
+    alias_method :table, :referenced_object
+
+    # Forward to table
+    forward_to :table, :schema, :associations
+
+    def initialize(table, records = [])
+      constrain records, [Record]
+      super(table.type, table, dimension: 3)
+      @impl = {}
+      records.each { |record| (@impl[record.id] ||= []) << record }
+    end
+    
+    # Number of records including duplicates
+    def size() records.size end
+
+    # True if table is empty
+    def empty?() @impl.empty? end
+
+    # #[] returns the unique record for the given key
+    def [](id) @impl[id]&.first end
+
+    # True if the table contains a record with the given ID
+    def id?(id) @impl.key?(id) end
+
+    # List of unique record IDs
+    def ids() @impl.keys end
+
+    # List of Record objects including duplicates
+    def records() @records ||= @impl.values.flatten end
+
+    # List of unique record objects
+    def unique_records() @unique_records ||= @impl.values.map(&:first) end
+
+    # Redefine #data to return a map from ID to array of (identical) records
+    def data() @impl.map { |k,records| [k, records.map(&:value)] }.to_h end
+
+    # Define #value to return a map from ID to the record with that ID
+    def value() @impl.map { |k,records| [k, records.first.value] }.to_h end
+
+    # Return the number of (duplicate) records for the given ID
+    def count(id) @imp[id]&.size || 0 end
+
+    def to_h()
+      @impl.map { |id, records| [id, records.map { |record| record.to_h }] }.to_h
+    end
+
+  protected
+    # Define #add_record to handle duplicate records
+    def add_record(record)
+      constrain record, Record
+      (@impl[record.id] ||= []) << record
+    end
+  end
+end
+

data/lib/ext/meta.rb

@@ -0,0 +1,56 @@
+ 
+require "pg_meta"
+
+# Extend MetaDb with a link table detection method
+module PgMeta
+  class Table
+    # True if table is a N:M link table. A N:M relation allows for multiple
+    # relations between two records
+    #
+    # A table is a N:M table if
+    #   o it has a primary key named 'id' as the first column
+    #   o has two reference fields using the link field naming convention
+    #   o and nothing else
+    #
+    def mm_table?
+      @mm_table ||= 
+          if columns.size != 3
+            false
+          elsif columns.values.first.name != "id" || columns.values.first != primary_key_column
+            false
+          else
+            referential_constraints.size == 2 &&
+            referential_constraints.values.map { |constraint|
+              constraint.referencing_columns
+            }.sort == [[columns.values[1]], [columns.values[2]]].sort
+          end
+    end
+
+    # True if table is a N:N link table. A N:N relation have at most one
+    # relation between two records. A N:N link table is also a M:M table
+    #
+    # A table is a N:N link table if
+    #   o it has a primary key named 'id' as the first column
+    #   o has two reference fields using the link field naming convention
+    #   o has a unique index on the two reference fields
+    #   o and nothing else
+    #
+    def nm_table?
+      @nm_table ||= @mm_table && begin
+        expected_columns = referential_constraints.values.map(&:columns).flatten.sort
+        unique_constraints.values.any? { |constraint|
+          expected_columns == constraint.columns.sort
+        }
+      end
+    end
+
+    def path
+      schema.name + "." + name
+    end
+  end
+
+  class Column
+    def path() table.path + "." + name end
+  end
+end
+

data/lib/ext/module.rb

@@ -0,0 +1,18 @@
+class Module
+  # Forward list of methods to object. The arguments should be strings or symbols
+  def forward_method(object, *methods)
+    forward_to(object, *methods)
+  end
+
+  # Same but with a better name
+  def forward_to(object, *methods)
+    for method in Array(methods).flatten
+      if method =~ /=$/
+        class_eval("def #{method}(arg) #{object}&.#{method}(arg) end")
+      else
+        class_eval("def #{method}(*args, &block) #{object}&.#{method}(*args, &block) end")
+      end
+    end
+  end
+end
+

data/lib/pg_graph/inflector.rb

@@ -0,0 +1,105 @@
+require "dry-inflector"
+
+module PgGraph
+  class Inflector #< Dry::Inflector
+    def initialize()
+      @inflector = Dry::Inflector.new
+    end
+
+    def pluralize(word)
+      return word if plural? word
+      result = @inflector.pluralize(word)
+      if result == word
+        word =~ /s$/ ? "#{word}es" : "#{word}s"
+      else
+        result
+      end
+    end
+
+    # Note that DryInflector::singularize handles the special PgGraph
+    # pluralization rules by default
+    def singularize(word)
+      @inflector.singularize(word)
+    end
+
+    # #plural? is defined using #singularize because #pluralize would cause
+    # endless recursion
+    def plural?(word)
+      singularize(word) != word
+    end
+
+    def singular?(word)
+      singularize(word) == word
+    end
+
+    def table2table_type(name)
+      record_type2table_type(table2record_type(name))
+    end
+
+    def table2record_type(name)
+      singularize(name)
+    end
+
+    def table_type2table(name)
+      record_type2table(table_type2record_type(name))
+    end
+    
+    def table_type2record_type(name)
+      name[1..-2]
+    end
+
+    def record_type2table(name)
+      if name =~ /s$/
+        name + "es"
+      else
+        pluralize(name)
+      end
+    end
+
+    def record_type2table_type(name)
+      type2array(name)
+    end
+
+    def type2array(name)
+      "[#{name}]"
+    end
+
+    # Convert a column name to a field name by removing a '_id' at the end
+    def column2field_name(column_name)
+#     column_name.sub(/_id$|_kind$/, "")
+      column_name.sub(/_id$/, "")
+    end
+
+    # Camelize string
+    def camelize(s)
+      s = s.sub(/^[a-z\d]*/) { |match| match.capitalize }
+      s.gsub(/(?:_|(\/))([a-z\d]*)/) { "#{$1}#{$2.capitalize}" }
+    end
+
+    # Remove module prefix from class names. Used in debugging
+    def klass_name(qualified_klass_name)
+      demodulize(qualified_klass_name)
+    end
+
+    # Types are both native postgres type and types from the information_schema
+    def postgres_type2ruby_class(name)
+      case name
+        when "character varying", "varchar", "text", "uuid"; String
+        when "smallint", "integer", "int4", "int2"; Integer
+        when "double precision", "float8", "numeric"; Float
+        when "bool", "boolean"; Boolean
+        when "json"; Hash
+        when "bytea"; String
+        when "timestamp", "timestamp without time zone"; Time
+        when /^_/; Array
+      else
+        raise "Unsupported postgres type: #{name.inspect} (FIXIT!)"
+      end
+    end
+
+    SUPPORTED_RUBY_CLASSES = [String, Integer, Float, Boolean, Hash, Time, NilClass, Array]
+  end
+
+  def self.inflector() @inflector ||= Inflector.new end
+end
+

data/lib/pg_graph/reflector.rb

@@ -0,0 +1,187 @@
+
+require 'constrain'
+require 'indented_io'
+
+module PgGraph
+  class Reflection
+    # Textual representation of match RE (String)
+    attr_reader :match
+
+    # Template for 'this' field name. Can be nil
+    attr_reader :this
+
+    # Template for 'that' field name or false if the field shouldn't be
+    # included in the model. Can be nil
+    attr_reader :that
+
+    # Don't pluralize the result of #that if false. Default false
+    attr_reader :pluralize
+
+    # RE corresponding to #match. #re always match a full UID
+    attr_reader :re 
+
+    # Number of name components (database, schema, table, column). It can be
+    # between one and four. By ordering reflections from highest to lowest
+    # number of components, specific matches will be tested before more general
+    # matches
+    attr_reader :components
+
+    # +this+ and +that+ are template strings, nil, or false
+    def initialize(match, this, that, pluralize = false)
+      constrain match, Regexp, String
+      constrain this, String, NilClass
+      constrain that, String, FalseClass, NilClass
+      constrain pluralize, TrueClass, FalseClass, NilClass
+      @match = match.is_a?(Regexp) ? match.source : match
+      if @match =~ /^\/(.*)\/$/
+        re = $1
+        @re = Regexp.new("^(?:\\w+\\.)*#{re}$")
+        @components = re.scan(/(?<!\\)\\(?:\\\\)*\./).count + 1
+      else
+        @re = Regexp.new("^(?:\\w+\\.)*#{@match}$")
+        @components = @match.count(".") + 1
+      end
+      (1..4).include?(@components) or raise "Illegal number of name components: #{@match}"
+      @this = this
+      @that = that
+      @pluralize = pluralize || pluralize.nil?
+    end
+
+    def to_yaml
+      { match: match, this: this, that: that, pluralize: pluralize }
+    end
+
+    def ==(other)
+      METHODS.all? { |method| self.send(method) == other.send(method) }
+    end
+
+    def inspect() "#<Reflection #{match}>" end
+
+    # +hash+ has the keys :match, :this, :that, and :pluralize. The keys can
+    # also be strings
+    def self.load_yaml(hash)
+      Reflection.new *METHODS.map { |key| 
+        value = hash[key].nil? ? hash[key.to_s] : hash[key] 
+        value == "nil" ? nil : value
+      }
+    end
+
+  private
+    METHODS = %w(match this that pluralize).map(&:to_sym)
+  end
+
+  class Reflector
+    # Reflections ordered from most-specific to least-specific and newest to oldest
+    def reflections()
+      # assumes @reflection keys are created in descending order
+      @reflection_list ||= @reflections.values.flatten
+    end
+
+    def initialize(reflections = [], default_reflections: Reflector.default_reflections)
+      constrain reflections, [Reflection]
+      # @reflection maps from number of reflection components to list of reflections. The
+      # keys are initially created in descending to ensure that #values return
+      # lists of components sorted from most to least specific. New reflections
+      # are inserted at the beginning of the lists to make it possible to
+      # override ealier reflections
+      @reflections = {}
+      (1..4).to_a.reverse.each { |components| @reflections[components] = [] }
+      add(default_reflections || [])
+      add(reflections)
+    end
+
+    def dup() Reflector.new(reflections.dup) end
+
+    # Return true if the Reflector has no reflections
+    def empty?() 
+      @reflection_list ? reflections.empty? : !@reflections.values.all?(:empty?)
+    end
+
+    # New reflections are inserted as a block at the head of the list of
+    # reflections
+    def add(*reflections)
+      @reflection_list = nil
+      # reverse makes it possible to use #insert but keep order:
+      Array(reflections).flatten.reverse.each { |reflection| 
+        @reflections[reflection.components].insert(0, reflection) 
+      }
+    end
+
+    # Find 'that' field name for the given UID by searching through reflections
+    # for a match. Returns nil if no match was found or if a matching
+    # reflection has #continue equal to false
+    def this(uid)
+      constrain uid, String
+      do_match(uid, :this)&.first
+    end
+
+    # Find 'that' field name for the given UID by searching through reflections
+    # for a match. The field name is pluralized unless +unique+ is true. The
+    # :table option can be used to override the table name in '$$' rules. This
+    # is used in N:M and M:M relations. Returns nil if no match was found or if
+    # a matching reflection has #continue equal to false
+    def that(uid, unique, table: nil)
+      constrain uid, String
+      if (name, pluralize = do_match(uid, :that, table: table))
+        pluralize != false && unique ? name : PgGraph.inflector.pluralize(name)
+      end
+    end
+
+    def to_yaml
+      reflections.map { |reflection| reflection.to_yaml }
+    end
+
+    def self.load_yaml(yaml_array, default_reflections: Reflector.default_reflections)
+      reflections = yaml_array.map { |hash| Reflection.load_yaml(hash) }
+      Reflector.new(reflections, default_reflections: default_reflections)
+    end
+
+    def self.load_file(file, default_reflections: Reflector.default_reflections)
+      load_yaml YAML.load(IO.read(file)), default_reflections: default_reflections
+    end
+
+    def self.default_reflections
+      @default_reflections ||= begin
+        initializers = [
+          {"match"=>"/parent_id/", "that"=>"child"},
+          {"match"=>"/child_id/", "that"=>"parent"}, 
+          {"match"=>"/parent_(\\w+)_id/", "that"=>"child_$1"}, 
+          {"match"=>"/child_(\\w+)_id/", "that"=>"parent_$1"}, 
+          {"match"=>"kind", "this"=>"kind", "that"=>"$$"}, 
+          {"match"=>"/(\\w+)_kind/", "this"=>"$1", "that"=>"$$"}, 
+          {"match"=>"/(\\w+)_by_id/", "this"=>"$1_by", "that"=>"$1_$$", pluralize: true},
+          {"match"=>"/(\\w+)_id/", "this"=>"$1", "that"=>"$$"},
+          {"match"=>"/(\\w+)/", "this"=>"$1", "that"=>"$$"}, # Kind fields w/o explicit 'kind'
+        ]
+        initializers.map { |initializer| Reflection.load_yaml(initializer) }
+      end
+    end
+
+    def self.default_reflections=(reflections)
+      @default_reflections = reflections
+    end
+
+  private
+    # +kind+ can be :this or :that
+    def do_match(uid, kind, table: nil)
+      reflections.find { |reflection|
+        match_data = reflection.re.match(uid) or next
+        template = reflection.send(kind).dup
+        if template == false
+          return nil
+        elsif template
+          table ||= uid.split(".")[-2]
+          template.gsub!(/\$\$/, table)
+          match_data.captures.each.with_index { |replacement, i| template.gsub!(/\$#{i+1}/, replacement) }
+          return [template, reflection.pluralize]
+        else
+          next
+        end
+      }
+    end
+  end
+end
+
+
+
+