dbagent 3.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 120c9311649f81fdc84a37b73f207105c4ad560008750418f0f1ea1e581d3b3f
+  data.tar.gz: de68e2c53e6119d7df4099e7ae0aafa7beaed5ad1aeb656c2eca90a0e12d9bd0
+SHA512:
+  metadata.gz: 5f7e584a37a4c74612bf15fd95c0218fd6e908fd85b8e913410eaaa7c1c22a360ad71033aeb71dfb72563e68bb0f237659fc45756f5a6542b440ccd4eb5d7590
+  data.tar.gz: 6636ba62ddecbd1f39c731b36870b4908a0480d4e3d39da6a495947ba66d12802663c4626ae8d1f964652fce4450ad9a45f2b2162924ff62b4253adf6f8edde7

data/Gemfile

@@ -0,0 +1,2 @@
+source 'http://rubygems.org'
+gemspec

data/LICENSE.md

@@ -0,0 +1,22 @@
+# The MIT Licence
+
+    Copyright (c) 2019 - Enspirit SPRL (Bernard Lambeau)
+
+    Permission is hereby granted, free of charge, to any person obtaining
+    a copy of this software and associated documentation files (the
+    "Software"), to deal in the Software without restriction, including
+    without limitation the rights to use, copy, modify, merge, publish,
+    distribute, sublicense, and/or sell copies of the Software, and to
+    permit persons to whom the Software is furnished to do so, subject to
+    the following conditions:
+
+    The above copyright notice and this permission notice shall be
+    included in all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+    NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+    LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+    OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+    WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,117 @@
+# DbAgent, a ruby tool to migrate, spy and seed relational databases
+
+DbAgent helps managing a relational database lifecyle through three main tools:
+
+* Migrations: powered by [Sequel](http://sequel.jeremyevans.net/), migrate as simply as `rake db:migrate`. Supports both superuser and normal user migrations.
+
+* Spy: using [Shemaspy](http://schemaspy.sourceforge.net/), get your database schema browsable at any moment, through a simple web interface.
+
+* Seed: maintain, install and flush database content as datasets, organized hierarchically in .json files. Very handy for automated tests, for instance.
+
+## Get started using Docker
+
+DbAgent is expected to be used as its Docker agent, available as `enspirit/dbagent`. Simply mount migrations and data folders, and you're ready to go.
+
+See the examples folder for details.
+
+## Available environment variables
+
+* `DBAGENT_ROOT_FOLDER`     Main folder where data, migrations and viewpoints can be found
+* `DBAGENT_LOGLEVEL`        Log level to use for dbagent messages (defaults to `WARN`)
+* `DBAGENT_LOGSQL`          Low Sequel's SQL queries (defaults to `no`)
+* `DBAGENT_ADAPTER`         Sequel's adapter (defaults to `postgres`)
+* `DBAGENT_HOST`            Database server host (defaults to `localhost`)
+* `DBAGENT_PORT`            Database server port (defaults to `5432`)
+* `DBAGENT_DB`              Database name (defaults to `suppliers-and-parts`)
+* `DBAGENT_USER`            Database user (defaults to `dbagent`)
+* `DBAGENT_PASSWORD`        Database password (defaults to `dbagent`)
+* `DBAGENT_SOCKET`          Database server socket (if host/port is not used)
+* `DBAGENT_SUPER_USER`      Superuser name (postgres only)
+* `DBAGENT_SUPER_DB`        Superuser database (postgres only)
+* `DBAGENT_SUPER_PASSWORD`  Superuser password (postgres only)
+* `DBAGENT_VIEWPOINT`       Bmg viewpoint (class name) when using db:flush
+
+## Available rake tasks
+
+The following rake tasks helps you managing the database. They must typically be executed on the docker container.
+
+```
+rake db:check-seeds      # Checks that all seeds can be installed correctly
+rake db:create           # Creates an fresh new user & database (USE WITH CARE)
+rake db:drop             # Drops the user & database (USE WITH CARE)
+rake db:flush[to]        # Flushes the database as a particular data set
+rake db:migrate          # Runs migrations on the current database
+rake db:ping             # Pings the database, making sure everything's ready for migration
+rake db:rebuild          # Rebuilds the database from scratch (USE WITH CARE)
+rake db:repl             # Opens a database REPL
+rake db:seed[from]       # Seeds the database with a particular data set
+rake db:spy              # Dumps the schema documentation into database/schema
+rake db:backup           # Makes a database backup to the backups folder
+rake db:restore[match]   # Restore the last matching database backup file from backups folder
+rake db:revive           # Shortcut for both db:restore and db:migrate
+rake db:tables           # List tables with those with fewer dependencies first
+rake db:dependencies[of] # List tables that depend of a given one
+```
+
+## Available webservices
+
+```
+GET  /schema/                # Browser the database schema (requires a former `rake db:spy`)
+POST /seeds/install?id=...   # Install a particular dataset, id is the name of a folder in `data` folder
+POST /seeds/flush?id=...     # Flushes the current database content as a named dataset
+```
+
+## Hacking on dbagent
+
+### Installing the library
+
+```
+bundle install
+```
+
+### Preparing your computer
+
+The tests require a valid PostgreSQL installation with the suppliers-and-parts
+database installed. A `dbagent` user would be needed on the PostgreSQL installation
+to bootstrap the process.
+
+```
+sudo su postgres -c 'createuser --createdb dbagent -P'
+```
+
+DbAgent tries to connect to the suppliers-and-parts with a dbagent/dbagent user/password
+pair by default. If you change the database name, user, or password please adapt the
+environment variables accordingly in the commands below.
+
+### Installing the example database
+
+```
+DBAGENT_ROOT_FOLDER=examples/suppliers-and-parts bundle exec rake db:create db:migrate db:seed['base']
+```
+### Running test
+
+To run the test you need to have `Docker` on your computer.
+
+Run:
+```
+make test
+```
+
+Don't forget to delete created ressources for the tests bun running:
+```
+make clean
+```
+
+## Contribute
+
+Please use github issues and pull requests for all questions, bug reports,
+and contributions. Don't hesitate to get in touch with us with an early code
+spike if you plan to add non trivial features.
+
+## Licence
+
+This software is distributed by Enspirit SRL under a MIT Licence. Please
+contact Bernard Lambeau (blambeau@gmail.com) with any question.
+
+Enspirit (https://enspirit.be) and Klaro App (https://klaro.cards) are both
+actively using and contributing to the library.

data/Rakefile

@@ -0,0 +1,16 @@
+require 'path'
+
+def shell(*cmds)
+  cmd = cmds.join("\n")
+  puts cmd
+  system cmd
+end
+
+#
+# Install all tasks found in tasks folder
+#
+# See .rake files there for complete documentation.
+#
+Dir["tasks/*.rake"].each do |taskfile|
+  load taskfile
+end

data/lib/db_agent.rb

@@ -0,0 +1,80 @@
+require 'path'
+require 'logger'
+require 'sequel'
+require 'sinatra'
+require 'bmg'
+require 'bmg/sequel'
+module DbAgent
+
+  # Current version of DbAgent
+  VERSION = "3.0.0"
+
+  # Simply checks that a path exists of raise an error
+  def self._!(path)
+    Path(path).tap do |p|
+      raise "Missing #{p.basename}." unless p.exists?
+    end
+  end
+
+  # Root folder of the project structure
+  ROOT_FOLDER = if ENV['DBAGENT_ROOT_FOLDER']
+    _!(ENV['DBAGENT_ROOT_FOLDER'])
+  else
+    Path.backfind('.[Gemfile]') or raise("Missing Gemfile")
+  end
+
+  # Logger instance to use
+  LOGGER = Logger.new(STDOUT)
+  LOGGER.level = Logger.const_get(ENV['DBAGENT_LOGLEVEL'] || 'WARN')
+
+  # What database configuration to use for normal access
+  def self.default_config
+    cfg = {
+      adapter:  ENV['DBAGENT_ADAPTER']  || 'postgres',
+      port:     ENV['DBAGENT_PORT']     || 5432,
+      database: ENV['DBAGENT_DB']       || 'suppliers-and-parts',
+      user:     ENV['DBAGENT_USER']     || 'dbagent',
+      password: ENV['DBAGENT_PASSWORD'] || 'dbagent',
+      test:     false
+    }
+
+    # Favor a socket approach if specified, otherwise fallback to
+    # host with default to postgres
+    if socket = ENV['DBAGENT_SOCKET']
+      cfg[:socket] = socket
+    else
+      cfg[:host] = ENV['DBAGENT_HOST'] || 'localhost'
+    end
+
+    # Set a logger if explicitly requested
+    if ENV['DBAGENT_LOGSQL'] == 'yes'
+      cfg[:loggers] = [LOGGER]
+    end
+
+    cfg
+  end
+
+  # What database configuration to use for superuser access
+  def self.default_superconfig
+    cfg = default_config
+    cfg.merge({
+      user:     ENV['DBAGENT_SUPER_USER']     || cfg[:user],
+      database: ENV['DBAGENT_SUPER_DB']       || cfg[:database],
+      password: ENV['DBAGENT_SUPER_PASSWORD'] || cfg[:password]
+    })
+  end
+
+  def self.default_handler
+    DbHandler.factor({
+      config: default_config,
+      superconfig: default_superconfig,
+      root: ROOT_FOLDER
+    })
+  end
+
+end # module DbAgent
+require 'db_agent/viewpoint'
+require 'db_agent/seeder'
+require 'db_agent/table_orderer'
+require 'db_agent/db_handler'
+require 'db_agent/webapp'

data/lib/db_agent/db_handler.rb

@@ -0,0 +1,130 @@
+module DbAgent
+  class DbHandler
+
+    def initialize(options)
+      @config = options[:config]
+      @superconfig = options[:superconfig]
+      @root_folder = options[:root]
+      @backup_folder = options[:backup] || options[:root]/'backups'
+      @schema_folder = options[:schema] || options[:root]/'schema'
+      @migrations_folder = options[:migrations] || options[:root]/'migrations'
+      @data_folder = options[:data] || options[:root]/'data'
+      @viewpoints_folder = options[:viewpoints] || options[:root]/'viewpoints'
+      require_viewpoints!
+    end
+    attr_reader :config, :superconfig
+    attr_reader :backup_folder, :schema_folder, :migrations_folder
+    attr_reader :data_folder, :viewpoints_folder
+
+    def ping
+      puts "Using #{config}"
+      sequel_db.test_connection
+      puts "Everything seems fine!"
+    end
+
+    def create
+      raise NotImplementedError
+    end
+
+    def drop
+      raise NotImplementedError
+    end
+
+    def backup
+      raise NotImplementedError
+    end
+
+    def repl
+      raise NotImplementedError
+    end
+
+    def wait_server
+      require 'net/ping'
+      raise "No host found" unless config[:host]
+      check = Net::Ping::External.new(config[:host])
+      puts "Trying to ping `#{config[:host]}`"
+      15.downto(0) do |i|
+        print "."
+        if check.ping?
+          print "\nServer found.\n"
+          break
+        elsif i == 0
+          print "\n"
+          raise "Server not found, I give up."
+        else
+          sleep(1)
+        end
+      end
+    end
+
+    def wait
+      15.downto(0) do |i|
+        begin
+          puts "Using #{config}"
+          sequel_db.test_connection
+          puts "Database is there. Great."
+          break
+        rescue Sequel::Error
+          raise if i==0
+          sleep(1)
+        end
+      end
+    end
+
+    def restore(t, args)
+      raise NotImplementedError
+    end
+
+    def migrate(version = nil)
+      Sequel.extension :migration
+      if (sf = migrations_folder/'superuser').exists?
+        Sequel::Migrator.run(sequel_superdb, migrations_folder/'superuser', table: 'superuser_migrations', target: version)
+      end
+      Sequel::Migrator.run(sequel_db, migrations_folder, target: version)
+    end
+
+    def repl
+      raise NotImplementedError
+    end
+
+    def spy
+      raise NotImplementedError
+    end
+
+    def self.factor(options)
+      case options[:config][:adapter]
+      when 'postgres'
+        PostgreSQL.new(options)
+      when 'mssql'
+        MSSQL.new(options)
+      when 'mysql'
+        MySQL.new(options)
+      else
+        PostgreSQL.new(options)
+      end
+    end
+
+    def sequel_db
+      @sequel_db ||= ::Sequel.connect(config)
+    end
+
+    def sequel_superdb
+      raise "No superconfig set" if superconfig.nil?
+      @sequel_superdb ||= ::Sequel.connect(superconfig)
+    end
+
+    def system(cmd, *args)
+      puts cmd
+      ::Kernel.system(cmd, *args)
+    end
+
+    def require_viewpoints!
+      f = viewpoints_folder.expand_path
+      Path.require_tree(f) if f.directory?
+    end
+
+  end # class DbHandler
+end # module DbAgent
+require_relative 'db_handler/postgresql'
+require_relative 'db_handler/mssql'
+require_relative 'db_handler/mysql'

data/lib/db_agent/db_handler/mssql.rb

@@ -0,0 +1,31 @@
+module DbAgent
+  class DbHandler
+    class MSSQL < DbHandler
+
+      def create
+        raise
+      end
+
+      def drop
+        raise
+      end
+
+      def backup
+        raise
+      end
+
+      def repl
+        raise
+      end
+
+      def spy
+        jdbc_jar = (Path.dir.parent/'vendor').glob('mssql*.jar').first
+        system %Q{java -jar vendor/schemaSpy_5.0.0.jar -dp #{jdbc_jar} -t mssql05 -host #{config[:host]} -u #{config[:user]} -p #{config[:password]} -db #{config[:database]} -port #{config[:port]} -s dbo -o #{schema_folder}/spy}
+        system %Q{open #{schema_folder}/spy/index.html}
+      end
+
+      def restore(t, args)
+      end
+    end # MSSQL
+  end # DbHandler
+end # DbAgent

data/lib/db_agent/db_handler/mysql.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module DbAgent
+  class DbHandler
+    class MySQL < DbHandler
+      def create
+        raise
+      end
+
+      def drop
+        raise
+      end
+
+      def backup
+        datetime = Time.now.strftime('%Y%m%dT%H%M%S')
+        shell mysqldump(config[:database], "> #{backup_folder}/backup-#{datetime}.sql")
+      end
+
+      def repl
+        shell mysql(config[:database])
+      end
+
+      def spy
+        jdbc_jar = (Path.dir.parent / 'vendor').glob('mysql*.jar').first
+        system %(java -jar vendor/schemaSpy_5.0.0.jar -dp #{jdbc_jar} -t mysql -host #{config[:host]} -u #{config[:user]} -p #{config[:password]} -db #{config[:database]} -s public -o #{schema_folder}/spy)
+        system %(open #{schema_folder}/spy/index.html)
+      end
+
+      def restore(_t, args)
+        candidates = backup_folder.glob('*.sql').sort
+        if args[:pattern] && rx = Regexp.new(args[:pattern])
+          candidates = candidates.select { |f| f.basename.to_s =~ rx }
+        end
+        file = candidates.last
+        shell mysql(config[:database], '<', file.to_s)
+      end
+
+      private
+
+      def mysql_cmd(cmd, *args)
+        conf = config
+        %(#{cmd} -h #{config[:host]} --password=#{config[:password]} -P #{config[:port]} -u #{config[:user]} #{args.join(' ')})
+      end
+
+      def mysql(*args)
+        mysql_cmd('mysql', *args)
+      end
+
+      def mysqldump(*args)
+        mysql_cmd('mysqldump', *args)
+      end
+    end # MySQL
+  end # DbHandler
+end # DbAgent

data/lib/db_agent/db_handler/postgresql.rb

@@ -0,0 +1,66 @@
+module DbAgent
+  class DbHandler
+    class PostgreSQL < DbHandler
+
+      def create
+        shell pg_cmd("createuser","--no-createdb","--no-createrole","--no-superuser","--no-password",config[:user]),
+        pg_cmd("createdb","--owner=#{config[:user]}", config[:database])
+      end
+
+      def drop
+        shell pg_cmd("dropdb", config[:database]),
+        pg_cmd("dropuser", config[:user])
+      end
+
+      def backup
+        datetime = Time.now.strftime("%Y%m%dT%H%M%S")
+        shell pg_dump("--clean", config[:database], "> #{backup_folder}/backup-#{datetime}.sql")
+      end
+
+      def repl
+        shell pg_cmd('psql', config[:database])
+      end
+
+      def spy
+        spy_jar = DbAgent._!('vendor').glob('schema*.jar').first
+        jdbc_jar = DbAgent._!('vendor').glob('postgresql*.jar').first
+        cmd = ""
+        cmd << %Q{java -jar #{spy_jar} -dp #{jdbc_jar} -t pgsql}
+        cmd << %Q{ -host #{config[:host]}}
+        cmd << %Q{ -port #{config[:port]}} if config[:port]
+        cmd << %Q{ -u #{config[:user]}}
+        cmd << %Q{ -p #{config[:password]}} if config[:password]
+        cmd << %Q{ -db #{config[:database]} -s public -o #{schema_folder}/spy}
+        system(cmd)
+        system %Q{open #{schema_folder}/spy/index.html}
+      end
+
+      def restore(t, args)
+        candidates = backup_folder.glob("*.sql").sort
+        if args[:pattern] && rx = Regexp.new(args[:pattern])
+          candidates = candidates.select{|f| f.basename.to_s =~ rx }
+        end
+        file = candidates.last
+        shell pg_cmd('psql', config[:database], '<', file.to_s)
+      end
+
+    private
+
+      def pg_cmd(cmd, *args)
+        %Q{#{cmd} -h #{config[:host]} -p #{config[:port]} -U #{config[:user]} #{args.join(' ')}}
+      end
+
+      def psql(*args)
+        cmd = "psql"
+        cmd = "PGPASSWORD=#{config[:password]} #{cmd}" if config[:password]
+        pg_cmd(cmd, *args)
+      end
+
+      def pg_dump(*args)
+        cmd = "pg_dump"
+        cmd = "PGPASSWORD=#{config[:password]} #{cmd}" if config[:password]
+        pg_cmd(cmd, *args)
+      end
+    end # class PostgreSQL
+  end # module DbHandler
+end # module DbAgent

data/lib/db_agent/seeder.rb

@@ -0,0 +1,125 @@
+module DbAgent
+  class Seeder
+
+    def initialize(handler)
+      @handler = handler
+    end
+    attr_reader :handler
+
+    def install(from)
+      handler.sequel_db.transaction do
+        folder = handler.data_folder/from
+
+        # load files in order
+        pairs = merged_data(from)
+        names = pairs.keys.sort{|p1,p2|
+          pairs[p1].basename <=> pairs[p2].basename
+        }
+
+        # Truncate tables then fill them
+        names.reverse.each do |name|
+          LOGGER.info("Emptying table `#{name}`")
+          handler.sequel_db[name.to_sym].delete
+        end
+        names.each do |name|
+          LOGGER.info("Filling table `#{name}`")
+          file = pairs[name]
+          handler.sequel_db[name.to_sym].multi_insert(file.load)
+        end
+      end
+    end
+
+    def flush_empty(to = "empty")
+      target = (handler.data_folder/to).rm_rf.mkdir_p
+      (target/"metadata.json").write <<-JSON.strip
+        {}
+      JSON
+      TableOrderer.new(handler).tsort.each_with_index do |table_name, index|
+        (target/"#{(index*10).to_s.rjust(5,"0")}-#{table_name}.json").write("[]")
+      end
+    end
+
+    def flush(to)
+      target = (handler.data_folder/to).rm_rf.mkdir_p
+      source = (handler.data_folder/"empty")
+      (target/"metadata.json").write <<-JSON.strip
+        { "inherits": "empty" }
+      JSON
+      seed_files(source).each do |f|
+        flush_seed_file(f, to)
+      end
+    end
+
+    def flush_seed_file(f, to)
+      target = (handler.data_folder/to)
+      table = file2table(f)
+      flush_table(table, target, f.basename, true)
+    end
+
+    def flush_table(table_name, target_folder, file_name, skip_empty)
+      data = viewpoint.send(table_name.to_sym).to_a
+      if data.empty? && skip_empty
+        LOGGER.info("Skipping table `#{table_name}` since empty")
+      else
+        LOGGER.info("Flushing table `#{table_name}`")
+        json = JSON.pretty_generate(data)
+        (target_folder/file_name).write(json)
+      end
+    end
+
+    def each_seed(install = true)
+      handler.data_folder.glob('**/*') do |file|
+        next unless file.directory?
+        next unless (file/"metadata.json").exists?
+
+        base = file.relative_to(handler.data_folder)
+        begin
+          Seeder.new(handler).install(base)
+          puts "#{base} OK"
+          yield(self, file) if block_given?
+        rescue => ex
+          puts "KO on #{file}"
+          puts ex.message
+        end if install
+      end
+    end
+
+  private
+
+    def merged_data(from)
+      folder = handler.data_folder/from
+      data   = {}
+
+      # load metadata and install parent dataset if any
+      metadata = (folder/"metadata.json").load
+      if parent = metadata["inherits"]
+        data = merged_data(parent)
+      end
+
+      seed_files(folder).each do |f|
+        data[file2table(f)] = f
+      end
+
+      data
+    end
+
+    def seed_files(folder)
+      folder
+        .glob("*.json")
+        .reject{|f| f.basename.to_s =~ /^metadata/ }
+    end
+
+    def file2table(f)
+      f.basename.rm_ext.to_s[/^\d+-(.*)/, 1]
+    end
+
+    def viewpoint
+      @viewpoint ||= if vp = ENV['DBAGENT_VIEWPOINT']
+        Kernel.const_get(vp).new(handler.sequel_db)
+      else
+        Viewpoint::Base.new(handler.sequel_db)
+      end
+    end
+
+  end # class Seeder
+end # module DbAgent

data/lib/db_agent/table_orderer.rb

@@ -0,0 +1,94 @@
+require 'tsort'
+module DbAgent
+  class TableOrderer
+
+    def initialize(handler)
+      @handler = handler
+    end
+    attr_reader :handler
+
+    def db
+      handler.sequel_db
+    end
+
+    def tsort
+      @tsort ||= TSortComputation.new(db).to_a
+    end
+
+    def graph
+      @graph ||= TSortComputation.new(db).graph
+    end
+
+    def dependencies(table)
+      _dependencies(table, ds = {})
+      ds
+        .inject([]){|memo,(_,plus)| (memo + plus).uniq }
+        .sort{|t1,t2| tsort.index(t1) - tsort.index(t2) }
+        .reject{|x| x == table }
+    end
+
+    def _dependencies(table, ds)
+      return ds if ds.has_key?(table)
+      ds[table] = graph[table]
+      ds[table].each do |child|
+        _dependencies(child, ds)
+      end
+    end
+    private :_dependencies
+
+    class TSortComputation
+      include TSort
+
+      def initialize(db, except = [])
+        @db = db
+        @except = except
+      end
+      attr_reader :db, :except
+
+      def graph
+        g = Hash.new{|h,k| h[k] = [] }
+        tsort_each_node.each do |table|
+          tsort_each_child(table) do |child|
+            g[child] << table
+          end
+        end
+        g
+      rescue TSort::Cyclic
+        raise unless killed = to_kill
+        TSortComputation.new(db, except + [killed]).graph
+      end
+
+      def to_a
+        tsort.to_a
+      rescue TSort::Cyclic
+        raise unless killed = to_kill
+        TSortComputation.new(db, except + [killed]).to_a
+      end
+
+      def tsort_each_node(&bl)
+        db.tables.each(&bl)
+      end
+
+      def tsort_each_child(table, &bl)
+        db.foreign_key_list(table)
+          .map{|fk| fk[:table] }
+          .reject{|t|
+            except.any?{|killed| killed.first == table && killed.last == t }
+          }
+          .each(&bl)
+      end
+
+    private
+
+      def to_kill
+        each_strongly_connected_component
+          .select{|scc| scc.size > 1 }
+          .sort_by{|scc| scc.size }
+          .first
+      end
+
+    end # class TSortComputation
+
+  end # class TableOrderer
+end # module Dbagent
+

data/lib/db_agent/viewpoint.rb

@@ -0,0 +1,3 @@
+require_relative 'viewpoint/base'
+require_relative 'viewpoint/delegate'
+require_relative 'viewpoint/typecheck'

data/lib/db_agent/viewpoint/base.rb

@@ -0,0 +1,18 @@
+module DbAgent
+  module Viewpoint
+    # Factors relations on top of a Sequel database.
+    class Base
+
+      def initialize(db)
+        @db = db
+      end
+      attr_reader :db
+
+      def method_missing(name, *args, &bl)
+        return super unless args.empty? and bl.nil?
+        Bmg.sequel(name, db)
+      end
+
+    end # class Base
+  end # module Viewpoint
+end # module DbAgent

data/lib/db_agent/viewpoint/delegate.rb

@@ -0,0 +1,13 @@
+module DbAgent
+  module Viewpoint
+    # Delegates all relation accesses to `child`.
+    module Delegate
+
+      def method_missing(name, *args, &bl)
+        return super unless args.empty? and bl.nil?
+        child.send(name)
+      end
+
+    end # module Delegate
+  end # module Viewpoint
+end # module DbAgent

data/lib/db_agent/viewpoint/typecheck.rb

@@ -0,0 +1,19 @@
+module DbAgent
+  module Viewpoint
+    # Forces typechecking on all child relations.
+    class TypeCheck
+
+      def initialize(db, child = nil)
+        @db = db
+        @child = child || DbAgent::Viewpoint::Base.new(db)
+      end
+      attr_reader :db, :child
+
+      def method_missing(name, *args, &bl)
+        return super unless args.empty? && bl.nil?
+        child.send(name).with_typecheck
+      end
+
+    end # class TypeCheck
+  end # module Viewpoint
+end # module DbAgent

data/lib/db_agent/webapp.rb

@@ -0,0 +1,35 @@
+module DbAgent
+  class Webapp < Sinatra::Base
+
+    set :raise_errors, true
+    set :show_exceptions, false
+    set :dump_errors, true
+    set :db_handler, DbAgent.default_handler
+
+    get '/ping' do
+      settings.db_handler.sequel_db.test_connection
+      status 200
+      "ok"
+    end
+
+    get %r{/schema/?} do
+      send_file(settings.db_handler.schema_folder/'spy/index.html')
+    end
+
+    get '/schema/*' do |url|
+      send_file(settings.db_handler.schema_folder/'spy'/url)
+    end
+
+    post '/seeds/install' do
+      Seeder.new(settings.db_handler.sequel_db).install(request["id"])
+      "ok"
+    end
+
+    post '/seeds/flush' do
+      seed_name = request["id"]
+      Seeder.new(settings.db_handler.sequel_db).flush(request["id"])
+      "ok"
+    end
+
+  end
+end

data/tasks/db.rake

@@ -0,0 +1,108 @@
+namespace :db do
+
+  task :require do
+    $:.unshift File.expand_path('../../lib', __FILE__)
+    require 'db_agent'
+    include DbAgent
+  end
+
+  def db_handler
+    @db_handler ||= DbAgent.default_handler
+  end
+
+  desc "Pings the database, making sure everything's ready for migration"
+  task :ping => :require do
+    db_handler.ping
+  end
+
+  desc "Drops the user & database (USE WITH CARE)"
+  task :drop => :require do
+    db_handler.drop
+  end
+
+  desc "Creates an fresh new user & database (USE WITH CARE)"
+  task :create => :require do
+    db_handler.create
+  end
+
+  desc "Waits for the database server to ping, up to 15 seconds"
+  task :wait_server => :require do
+    db_handler.wait_server
+  end
+
+  desc "Waits for the database to ping, up to 15 seconds"
+  task :wait => :require do
+    db_handler.wait
+  end
+
+  desc "Dump a database backup"
+  task :backup => :require do
+    db_handler.backup
+  end
+
+  desc "Restore from the last database backup"
+  task :restore, :pattern do |t,args|
+    db_handler.restore(t, args)
+  end
+  task :restore => :require
+
+  desc "Runs migrations on the current database"
+  task :migrate, [:version] => :require do |_,args|
+    version = args[:version].to_i if args[:version]
+    db_handler.migrate(version)
+  end
+
+  desc "Opens a database REPL"
+  task :repl => :require do
+    db_handler.repl
+  end
+
+  desc "Dumps the schema documentation into database/schema"
+  task :spy => :require do
+    db_handler.spy
+  end
+
+  desc "Rebuilds the database from scratch (USE WITH CARE)"
+  task :rebuild => [ :drop, :create, :migrate ]
+
+  desc "Revive the database from the last backup"
+  task :revive => [ :restore, :migrate ]
+
+
+  desc "Checks that all seeds can be installed correctly"
+  task :"check-seeds" do
+    Seeder.new(db_handler).each_seed(true)
+  end
+  task :"check-seeds" => :require
+
+  desc "Seeds the database with a particular data set"
+  task :seed, :from do |t,args|
+    Seeder.new(db_handler).install(args[:from] || 'empty')
+  end
+  task :seed => :require
+
+  desc "Flushes the database as a particular data set"
+  task :flush, :to do |t,args|
+    Seeder.new(db_handler).flush(args[:to] || Time.now.strftime("%Y%M%d%H%M%S").to_s)
+  end
+  task :flush => :require
+
+  desc "Flushes the initial empty files as a data set"
+  task :flush_empty, :to do |t,args|
+    Seeder.new(db_handler).flush_empty(args[:to] || Time.now.strftime("%Y%M%d%H%M%S").to_s)
+  end
+  task :flush_empty => :require
+
+  desc "Shows what tables depend on a given one"
+  task :dependencies, :of do |t,args|
+    puts TableOrderer.new(db_handler).dependencies(args[:of].to_sym).reverse
+  end
+  task :dependencies => :require
+
+  desc "Shows all tables in order"
+  task :tables do |t|
+    puts TableOrderer.new(db_handler).tsort
+  end
+  task :tables => :require
+
+end

data/tasks/gem.rake

@@ -0,0 +1,39 @@
+require 'rubygems/package_task'
+
+# Dynamically load the gem spec
+gemspec_file = File.expand_path('../../dbagent.gemspec', __FILE__)
+gemspec      = Kernel.eval(File.read(gemspec_file))
+
+Gem::PackageTask.new(gemspec) do |t|
+
+  # Name of the package
+  t.name = gemspec.name
+
+  # Version of the package
+  t.version = gemspec.version
+
+  # Directory used to store the package files
+  t.package_dir = "pkg"
+
+  # True if a gzipped tar file (tgz) should be produced
+  t.need_tar = false
+
+  # True if a gzipped tar file (tar.gz) should be produced
+  t.need_tar_gz = false
+
+  # True if a bzip2'd tar file (tar.bz2) should be produced
+  t.need_tar_bz2 = false
+
+  # True if a zip file should be produced (default is false)
+  t.need_zip = false
+
+  # List of files to be included in the package.
+  t.package_files = gemspec.files
+
+  # Tar command for gzipped or bzip2ed archives.
+  t.tar_command = "tar"
+
+  # Zip command for zipped archives.
+  t.zip_command = "zip"
+
+end

data/tasks/test.rake

@@ -0,0 +1,11 @@
+namespace :test do
+
+  desc %q{Run all RSpec tests}
+  task :unit do
+    require 'rspec'
+    RSpec::Core::Runner.run(%w[-I. -Ilib -Ispec --pattern=spec/**/test_*.rb --color .])
+  end
+
+  task :all => :"unit"
+end
+task :test => :"test:all"

metadata

@@ -0,0 +1,215 @@
+--- !ruby/object:Gem::Specification
+name: dbagent
+version: !ruby/object:Gem::Version
+  version: 3.0.0
+platform: ruby
+authors:
+- Bernard Lambeau
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2021-08-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13'
+- !ruby/object:Gem::Dependency
+  name: path
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: sinatra
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: bmg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.18'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.18'
+- !ruby/object:Gem::Dependency
+  name: net-ping
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: predicate
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+- !ruby/object:Gem::Dependency
+  name: rack-test
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+description: A tool to migrate, spy and seed relational databases
+email: blambeau@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- LICENSE.md
+- README.md
+- Rakefile
+- lib/db_agent.rb
+- lib/db_agent/db_handler.rb
+- lib/db_agent/db_handler/mssql.rb
+- lib/db_agent/db_handler/mysql.rb
+- lib/db_agent/db_handler/postgresql.rb
+- lib/db_agent/seeder.rb
+- lib/db_agent/table_orderer.rb
+- lib/db_agent/viewpoint.rb
+- lib/db_agent/viewpoint/base.rb
+- lib/db_agent/viewpoint/delegate.rb
+- lib/db_agent/viewpoint/typecheck.rb
+- lib/db_agent/webapp.rb
+- tasks/db.rake
+- tasks/gem.rake
+- tasks/test.rake
+homepage: http://github.com/enspirit/dbagent
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.8
+signing_key: 
+specification_version: 4
+summary: A tool to migrate, spy and seed relational databases.
+test_files: []