rdbr 0.1.0

data/lib/rdbr/catalog/sqlite.rb

@@ -0,0 +1,182 @@
+module RDBr
+  module Catalog
+    class Sqlite < Reader
+      FOREIGN_KEY_ACTIONS = {
+        'NO ACTION' => :no_action,
+        'RESTRICT' => :restrict,
+        'CASCADE' => :cascade,
+        'SET NULL' => :set_null,
+        'SET DEFAULT' => :set_default
+      }.freeze
+      TYPE_MAPPINGS = {
+        /BOOL/ => :boolean, /JSON/ => :json, /DATETIME|TIMESTAMP/ => :timestamp, /DATE/ => :date,
+        /TIME/ => :time, /INT/ => :int8, /CHAR|CLOB|TEXT/ => :text, /REAL|FLOA|DOUB/ => :float8,
+        /\A\z|BLOB/ => :blob
+      }.freeze
+
+      def read
+        database_rows = connection.select_all('PRAGMA database_list')
+        namespaces = database_rows.filter_map{|row| build_namespace(row.fetch('name')) }
+        Metadata::Database.new(name: database_name(database_rows), adapter: :sqlite, namespaces: namespaces)
+      end
+
+      private
+
+      def build_namespace(name)
+        relations = relation_rows(name).map{|row| build_relation(name, row) }
+        return if relations.empty? && name == 'temp' && !include_system?
+
+        Metadata::Namespace.new(name: name, relations: relations)
+      end
+
+      def relation_rows(namespace)
+        schema = connection.quote_identifier(namespace)
+        system_clause = include_system? ? '' : " AND name NOT LIKE 'sqlite\\_%' ESCAPE '\\'"
+        connection.select_all(<<~SQL)
+          SELECT name, type, sql
+          FROM #{schema}.sqlite_schema
+          WHERE type IN ('table', 'view')#{system_clause}
+          ORDER BY name
+        SQL
+      end
+
+      def build_relation(namespace, row)
+        table_name = row.fetch('name')
+        columns = column_rows(namespace, table_name)
+        indexes = index_rows(namespace, table_name)
+        Metadata::Relation.new(
+          name: table_name,
+          kind: row.fetch('type').to_sym,
+          columns: build_columns(columns),
+          primary_key: build_primary_key(columns),
+          foreign_keys: build_foreign_keys(namespace, table_name),
+          unique_constraints: build_unique_constraints(namespace, indexes),
+          indexes: build_indexes(namespace, indexes)
+        )
+      end
+
+      def column_rows(namespace, table_name)
+        pragma(namespace, 'table_xinfo', table_name)
+      end
+
+      def build_columns(rows)
+        rows.sort_by{|row| Integer(row.fetch('cid')) }.map do |row|
+          sql_type = row.fetch('type').to_s
+          Metadata::Column.new(
+            name: row.fetch('name'),
+            logical_type: logical_type(sql_type),
+            sql_type: sql_type.empty? ? 'BLOB' : sql_type,
+            nullable: !truthy?(row.fetch('notnull')) && !Integer(row.fetch('pk')).positive?,
+            default: row['dflt_value'],
+            ordinal: Integer(row.fetch('cid')) + 1
+          )
+        end
+      end
+
+      def build_primary_key(rows)
+        columns = rows.select{|row| Integer(row.fetch('pk')).positive? }
+                      .sort_by{|row| Integer(row.fetch('pk')) }
+                      .map{|row| row.fetch('name') }
+        Metadata::PrimaryKey.new(columns: columns) unless columns.empty?
+      end
+
+      def build_foreign_keys(namespace, table_name)
+        pragma(namespace, 'foreign_key_list', table_name).group_by{|row| row.fetch('id') }.map do |id, rows|
+          ordered = rows.sort_by{|row| Integer(row.fetch('seq')) }
+          referenced_table = ordered.first.fetch('table')
+          referenced_columns = ordered.map{|row| row['to'] }
+          if referenced_columns.any?{|column| column.to_s.empty? }
+            referenced_columns = primary_key_columns(namespace, referenced_table)
+          end
+          Metadata::ForeignKey.new(
+            name: "sqlite_fk_#{table_name}_#{id}",
+            columns: ordered.map{|row| row.fetch('from') },
+            referenced_namespace: namespace,
+            referenced_relation: referenced_table,
+            referenced_columns: referenced_columns,
+            on_update: foreign_key_action(ordered.first.fetch('on_update')),
+            on_delete: foreign_key_action(ordered.first.fetch('on_delete'))
+          )
+        end
+      end
+
+      def index_rows(namespace, table_name)
+        pragma(namespace, 'index_list', table_name)
+      end
+
+      def build_unique_constraints(namespace, rows)
+        rows.select{|row| row.fetch('origin') == 'u' }.filter_map do |row|
+          columns = index_parts(namespace, row.fetch('name')).first
+          Metadata::UniqueConstraint.new(name: row.fetch('name'), columns: columns) unless columns.empty?
+        end
+      end
+
+      def build_indexes(namespace, rows)
+        rows.filter_map do |row|
+          columns, expression = index_parts(namespace, row.fetch('name'))
+          next if columns.empty? && expression.nil?
+
+          Metadata::Index.new(
+            name: row.fetch('name'),
+            columns: columns,
+            expressions: expression ? [ expression ] : [],
+            unique: truthy?(row.fetch('unique')),
+            where: index_predicate(namespace, row),
+            using: 'btree'
+          )
+        end
+      end
+
+      def index_parts(namespace, index_name)
+        rows = pragma(namespace, 'index_xinfo', index_name).select{|row| truthy?(row.fetch('key', 1)) }
+        columns = rows.filter_map{|row| row['name'] }
+        expression = index_sql(namespace, index_name) if columns.length < rows.length
+        [ columns, expression ]
+      end
+
+      def index_predicate(namespace, row)
+        return unless truthy?(row.fetch('partial'))
+
+        index_sql(namespace, row.fetch('name'))&.split(/\bWHERE\b/i, 2)&.last&.strip
+      end
+
+      def index_sql(namespace, index_name)
+        schema = connection.quote_identifier(namespace)
+        name = index_name.to_s.gsub("'", "''")
+        connection.select_all("SELECT sql FROM #{schema}.sqlite_schema WHERE type = 'index' AND name = '#{name}'")
+                  .first&.fetch('sql', nil)
+      end
+
+      def primary_key_columns(namespace, table_name)
+        column_rows(namespace, table_name).select{|row| Integer(row.fetch('pk')).positive? }
+                                          .sort_by{|row| Integer(row.fetch('pk')) }
+                                          .map{|row| row.fetch('name') }
+      end
+
+      def pragma(namespace, operation, object)
+        schema = connection.quote_identifier(namespace)
+        identifier = connection.quote_identifier(object)
+        connection.select_all("PRAGMA #{schema}.#{operation}(#{identifier})")
+      end
+
+      def database_name(rows)
+        main = rows.find{|row| row.fetch('name') == 'main' }
+        file = main&.fetch('file', '').to_s
+        file.empty? ? 'main' : File.basename(file)
+      end
+
+      def logical_type(sql_type)
+        normalized = sql_type.upcase
+        TYPE_MAPPINGS.find{|pattern, _type| normalized.match?(pattern) }&.last || :numeric
+      end
+
+      def foreign_key_action(value)
+        FOREIGN_KEY_ACTIONS.fetch(value.to_s.upcase, :no_action)
+      end
+
+      def truthy?(value)
+        value == true || value.to_i == 1
+      end
+    end
+  end
+end

data/lib/rdbr/catalog.rb

@@ -0,0 +1,20 @@
+require_relative 'catalog/errors'
+require_relative 'catalog/connection'
+require_relative 'catalog/serializer'
+require_relative 'catalog/reader'
+require_relative 'catalog/postgresql'
+require_relative 'catalog/sqlite'
+require_relative 'catalog/mysql'
+
+module RDBr
+  module Catalog
+    module_function
+
+    def read(database_url:, include_system: false)
+      connection = Connection.new(database_url)
+      Reader.for(connection.adapter_name).new(connection, include_system: include_system).read
+    ensure
+      connection&.disconnect
+    end
+  end
+end

data/lib/rdbr/cli.rb

@@ -0,0 +1,74 @@
+require 'rackup'
+
+module RDBr
+  module CLI
+    module_function
+
+    def run(
+      arguments,
+      output: $stdout,
+      error: $stderr,
+      **dependencies
+    )
+      catalog_reader = dependencies.fetch(:catalog_reader, Catalog.method(:read))
+      query_builder = dependencies.fetch(:query_builder, Query.method(:build))
+      server_starter = dependencies.fetch(:server_starter, Rackup::Server.method(:start))
+      browser_launcher = dependencies.fetch(:browser_launcher, BrowserLauncher)
+      services = { catalog_reader: catalog_reader, query_builder: query_builder,
+                   server_starter: server_starter, browser_launcher: browser_launcher }
+      command, options = CLIOptions.parse(arguments)
+      case command
+      when 'inspect'
+        output.puts Catalog::Serializer.pretty_json(read_catalog(options, catalog_reader))
+      when 'serve'
+        serve(options, services, output)
+      when 'version'
+        output.puts VERSION
+      else
+        error.puts "Unknown command: #{command}"
+        return 1
+      end
+      0
+    rescue Catalog::Error, Query::Error, OptionParser::ParseError, ArgumentError => e
+      error.puts e.message
+      1
+    end
+
+    def serve(options, services, output)
+      if options[:database_url]
+        query_service = configure_web(options, services.fetch(:catalog_reader), services.fetch(:query_builder))
+      end
+      if options[:open_browser]
+        services.fetch(:browser_launcher).open_when_ready(host: options[:host], port: options[:port], output: output)
+      end
+      services.fetch(:server_starter).call(app: Web::Application, Host: options[:host], Port: options[:port])
+    ensure
+      query_service.disconnect if query_service.respond_to?(:disconnect)
+    end
+
+    def read_catalog(options, catalog_reader)
+      catalog_reader.call(
+        database_url: options.fetch(:database_url),
+        include_system: options.fetch(:include_system)
+      )
+    end
+
+    def configure_web(options, catalog_reader, query_builder)
+      catalog = read_catalog(options, catalog_reader)
+      access_policy = Access::ReadOnly.new(statement_timeout: options.fetch(:statement_timeout))
+      query_service = query_builder.call(
+        database_url: options.fetch(:database_url),
+        catalog: catalog,
+        connection_options: options.slice(
+          :statement_timeout, :pool_size, :checkout_timeout, :max_value_bytes
+        ),
+        access_policy: access_policy,
+        max_results: options.fetch(:max_results)
+      )
+      Web::Application.set :catalog, catalog
+      Web::Application.set :query_service, query_service
+      Web::Application.set :access_policy, access_policy
+      query_service
+    end
+  end
+end

data/lib/rdbr/cli_options.rb

@@ -0,0 +1,63 @@
+require 'optparse'
+
+module RDBr
+  module CLIOptions
+    module_function
+
+    def parse(arguments)
+      command = arguments.first&.start_with?('-') ? 'serve' : (arguments.shift || 'serve')
+      options = defaults
+      parser(options).parse!(arguments)
+      raise ArgumentError, 'A database URL is required for inspect' if command == 'inspect' && !options[:database_url]
+
+      [ command, options ]
+    end
+
+    def defaults
+      {
+        host: '127.0.0.1',
+        port: 9292,
+        database_url: ENV['DATABASE_URL'],
+        include_system: false,
+        open_browser: true,
+        statement_timeout: ENV.fetch('RDBR_STATEMENT_TIMEOUT', 5000),
+        pool_size: ENV.fetch('RDBR_POOL_SIZE', 5),
+        checkout_timeout: ENV.fetch('RDBR_CHECKOUT_TIMEOUT', 5000),
+        max_value_bytes: ENV.fetch('RDBR_MAX_VALUE_BYTES', 1_048_576),
+        max_results: ENV.fetch('RDBR_MAX_RESULTS', 100)
+      }
+    end
+
+    def parser(options)
+      OptionParser.new do |parser|
+        parser.banner = 'Usage: rdbr [serve|inspect|version] [options]'
+        parser.on('--database-url URL', 'Relational database URL'){|url| options[:database_url] = url }
+        parser.on('--include-system', 'Include system namespaces'){ options[:include_system] = true }
+        parser.on('--host HOST', 'Host to bind (default: 127.0.0.1)'){|host| options[:host] = host }
+        parser.on('--port PORT', Integer, 'Port to bind (default: 9292)'){|port| options[:port] = port }
+        parser.on('--[no-]open', 'Open the browser when the server is ready (default: true)') do |open|
+          options[:open_browser] = open
+        end
+        add_query_options(parser, options)
+      end
+    end
+
+    def add_query_options(parser, options)
+      parser.on('--statement-timeout MS', Integer, 'Query timeout in milliseconds (default: 5000)') do |timeout|
+        options[:statement_timeout] = timeout
+      end
+      parser.on('--pool-size COUNT', Integer, 'Database connection pool size (default: 5)') do |size|
+        options[:pool_size] = size
+      end
+      parser.on('--checkout-timeout MS', Integer, 'Pool checkout timeout in milliseconds') do |timeout|
+        options[:checkout_timeout] = timeout
+      end
+      parser.on('--max-value-bytes BYTES', Integer, 'Maximum bytes returned for one value') do |bytes|
+        options[:max_value_bytes] = bytes
+      end
+      parser.on('--max-results COUNT', Integer, 'Maximum rows per page (default: 100)') do |limit|
+        options[:max_results] = limit
+      end
+    end
+  end
+end

data/lib/rdbr/metadata/column.rb

@@ -0,0 +1,44 @@
+module RDBr
+  module Metadata
+    Column = Data.define(
+      :name,
+      :logical_type,
+      :sql_type,
+      :nullable,
+      :default,
+      :generated_expression,
+      :identity,
+      :ordinal
+    ) do
+      def initialize(
+        name:,
+        logical_type: nil,
+        sql_type: nil,
+        nullable: true,
+        default: nil,
+        generated_expression: nil,
+        identity: nil,
+        ordinal: nil
+      )
+        super(
+          name: Values.name(name, field: 'column name'),
+          logical_type: Values.deep_copy_freeze(logical_type),
+          sql_type: Values.optional_name(sql_type, field: 'SQL type'),
+          nullable: Values.boolean(nullable, field: 'nullable'),
+          default: Values.deep_copy_freeze(default),
+          generated_expression: Values.optional_name(generated_expression, field: 'generated expression'),
+          identity: Values.deep_copy_freeze(identity),
+          ordinal: ordinal
+        )
+      end
+
+      def generated?
+        !generated_expression.nil?
+      end
+
+      def identity?
+        !identity.nil? && identity != false
+      end
+    end
+  end
+end

data/lib/rdbr/metadata/constraints.rb

@@ -0,0 +1,25 @@
+module RDBr
+  module Metadata
+    UniqueConstraint = Data.define(:name, :columns) do
+      def initialize(columns:, name: nil)
+        super(
+          name: Values.optional_name(name, field: 'unique constraint name'),
+          columns: Values.names(columns, field: 'unique constraint columns')
+        )
+      end
+
+      def composite?
+        columns.length > 1
+      end
+    end
+
+    CheckConstraint = Data.define(:name, :expression) do
+      def initialize(expression:, name: nil)
+        super(
+          name: Values.optional_name(name, field: 'check constraint name'),
+          expression: Values.name(expression, field: 'check constraint expression')
+        )
+      end
+    end
+  end
+end

data/lib/rdbr/metadata/database.rb

@@ -0,0 +1,37 @@
+module RDBr
+  module Metadata
+    Database = Data.define(:name, :adapter, :namespaces) do
+      include Relationships
+
+      def initialize(name:, namespaces: [], adapter: nil)
+        database_namespaces = Values.collection(namespaces, member: Namespace, field: 'namespaces')
+        Values.unique_names!(database_namespaces, field: 'namespaces')
+        @relationship_candidates = build_relationship_candidates(database_namespaces).freeze
+        super(
+          name: Values.name(name, field: 'database name'),
+          adapter: Values.deep_copy_freeze(adapter),
+          namespaces: database_namespaces
+        )
+      end
+
+      def namespace(name)
+        namespaces.find{|namespace| namespace.name == name.to_s }
+      end
+
+      def relations
+        namespaces.flat_map(&:relations).freeze
+      end
+
+      def relation(name, namespace: nil)
+        return self.namespace(namespace)&.relation(name) if namespace
+
+        matches = relations.select{|relation| relation.name == name.to_s }
+        raise InvalidMetadataError, "relation #{name.inspect} is ambiguous; specify a namespace" if matches.length > 1
+
+        matches.first
+      end
+    end
+
+    Catalog = Database
+  end
+end

data/lib/rdbr/metadata/index.rb

@@ -0,0 +1,30 @@
+module RDBr
+  module Metadata
+    Index = Data.define(:name, :columns, :expressions, :unique, :where, :using) do
+      def initialize(name:, columns: [], expressions: [], unique: false, where: nil, using: nil)
+        index_columns = Values.names(columns, field: 'index columns', allow_empty: true)
+        index_expressions = Values.names(expressions, field: 'index expressions', allow_empty: true)
+        if index_columns.empty? && index_expressions.empty?
+          raise InvalidMetadataError, 'index must contain columns or expressions'
+        end
+
+        super(
+          name: Values.name(name, field: 'index name'),
+          columns: index_columns,
+          expressions: index_expressions,
+          unique: Values.boolean(unique, field: 'index unique'),
+          where: Values.optional_name(where, field: 'index predicate'),
+          using: Values.optional_name(using, field: 'index method')
+        )
+      end
+
+      def partial?
+        !where.nil?
+      end
+
+      def expression?
+        !expressions.empty?
+      end
+    end
+  end
+end

data/lib/rdbr/metadata/keys.rb

@@ -0,0 +1,56 @@
+module RDBr
+  module Metadata
+    PrimaryKey = Data.define(:name, :columns) do
+      def initialize(columns:, name: nil)
+        super(
+          name: Values.optional_name(name, field: 'primary key name'),
+          columns: Values.names(columns, field: 'primary key columns')
+        )
+      end
+
+      def composite?
+        columns.length > 1
+      end
+    end
+
+    ForeignKey = Data.define(
+      :name,
+      :columns,
+      :referenced_namespace,
+      :referenced_relation,
+      :referenced_columns,
+      :on_update,
+      :on_delete
+    ) do
+      def initialize(
+        columns:,
+        referenced_relation:,
+        referenced_columns:,
+        name: nil,
+        referenced_namespace: nil,
+        on_update: nil,
+        on_delete: nil
+      )
+        local_columns = Values.names(columns, field: 'foreign key columns')
+        remote_columns = Values.names(referenced_columns, field: 'referenced columns')
+        if local_columns.length != remote_columns.length
+          raise InvalidMetadataError, 'foreign key column counts must match'
+        end
+
+        super(
+          name: Values.optional_name(name, field: 'foreign key name'),
+          columns: local_columns,
+          referenced_namespace: Values.optional_name(referenced_namespace, field: 'referenced namespace'),
+          referenced_relation: Values.name(referenced_relation, field: 'referenced relation'),
+          referenced_columns: remote_columns,
+          on_update: Values.deep_copy_freeze(on_update),
+          on_delete: Values.deep_copy_freeze(on_delete)
+        )
+      end
+
+      def composite?
+        columns.length > 1
+      end
+    end
+  end
+end

data/lib/rdbr/metadata/namespace.rb

@@ -0,0 +1,26 @@
+module RDBr
+  module Metadata
+    Namespace = Data.define(:name, :relations) do
+      def initialize(name:, relations: [])
+        namespace_relations = Values.collection(relations, member: Relation, field: 'relations')
+        Values.unique_names!(namespace_relations, field: 'relations')
+        super(
+          name: Values.name(name, field: 'namespace name'),
+          relations: namespace_relations
+        )
+      end
+
+      def relation(name)
+        relations.find{|relation| relation.name == name.to_s }
+      end
+
+      def tables
+        relations.select(&:table?).freeze
+      end
+
+      def views
+        relations.select(&:view?).freeze
+      end
+    end
+  end
+end

data/lib/rdbr/metadata/relation.rb

@@ -0,0 +1,104 @@
+module RDBr
+  module Metadata
+    Relation = Data.define(
+      :name,
+      :kind,
+      :comment,
+      :columns,
+      :primary_key,
+      :foreign_keys,
+      :unique_constraints,
+      :check_constraints,
+      :indexes
+    ) do
+      def initialize(
+        name:,
+        kind:,
+        columns:,
+        comment: nil,
+        primary_key: nil,
+        foreign_keys: [],
+        unique_constraints: [],
+        check_constraints: [],
+        indexes: []
+      )
+        relation_columns = Values.collection(columns, member: Column, field: 'columns')
+        Values.unique_names!(relation_columns, field: 'columns')
+        validate_primary_key!(primary_key)
+        relation_foreign_keys = metadata_collection(foreign_keys, ForeignKey, 'foreign keys')
+        relation_unique_constraints = metadata_collection(unique_constraints, UniqueConstraint, 'unique constraints')
+        relation_check_constraints = metadata_collection(check_constraints, CheckConstraint, 'check constraints')
+        relation_indexes = metadata_collection(indexes, Index, 'indexes')
+        validate_local_columns!(
+          relation_columns, primary_key, relation_foreign_keys, relation_unique_constraints, relation_indexes
+        )
+
+        super(
+          name: Values.name(name, field: 'relation name'),
+          kind: Values.deep_copy_freeze(kind.to_sym),
+          comment: Values.optional_string(comment, field: 'relation comment'),
+          columns: relation_columns,
+          primary_key: primary_key,
+          foreign_keys: relation_foreign_keys,
+          unique_constraints: relation_unique_constraints,
+          check_constraints: relation_check_constraints,
+          indexes: relation_indexes
+        )
+      end
+
+      def column(name)
+        columns.find{|column| column.name == name.to_s }
+      end
+
+      def table?
+        kind == :table
+      end
+
+      def view?
+        [ :view, :materialized_view ].include?(kind)
+      end
+
+      def keyless?
+        primary_key.nil?
+      end
+
+      def uniquely_constrained_column?(name)
+        column_name = name.to_s
+        return true if primary_key&.columns == [ column_name ]
+        return true if unique_constraints.any?{|constraint| constraint.columns == [ column_name ] }
+
+        indexes.any? do |index|
+          index.unique && !index.partial? && !index.expression? && index.columns == [ column_name ]
+        end
+      end
+
+      private
+
+      def metadata_collection(value, member, field)
+        collection = Values.collection(value, member: member, field: field)
+        Values.unique_names!(collection, field: field)
+        collection
+      end
+
+      def validate_primary_key!(primary_key)
+        return if primary_key.nil? || primary_key.is_a?(PrimaryKey)
+
+        raise InvalidMetadataError, 'primary key must be a PrimaryKey or nil'
+      end
+
+      def validate_local_columns!(columns, primary_key, foreign_keys, unique_constraints, indexes)
+        known_names = columns.map(&:name)
+        referenced_names = [
+          primary_key&.columns,
+          *foreign_keys.map(&:columns),
+          *unique_constraints.map(&:columns),
+          *indexes.map(&:columns)
+        ].compact.flatten.uniq
+        unknown_names = referenced_names - known_names
+        return if unknown_names.empty?
+
+        raise InvalidMetadataError, "unknown local columns: #{unknown_names.join(', ')}"
+      end
+    end
+  end
+end