dbagent 3.3.0 → 3.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: eb16765f4f64bec5eff152e6db516bb5b672348a1b909229ce46e024b37132ab
4
- data.tar.gz: 48b7d28235340c73b316aabba304a20ccf854cdea919eecca1c7f19b1f594487
3
+ metadata.gz: 487cd25271375faf22bf6f47cc79b839c825cb3a377a16fc330b01ca78954078
4
+ data.tar.gz: dc64a2f4519365470a2e2641c788b6a5ea04b6fbc0d1406f7955e97ba9920218
5
5
  SHA512:
6
- metadata.gz: 61ee06b3ca6734ed61889878e2806eaa5774d08cd864b3ca2a1b90acaf22d505d830fbfb01b96c50340e2c1b27462e658efe2a7db54a2e4243873f912ddb3762
7
- data.tar.gz: 786ada66e5e8ea0e584e8ef61a36f71134df08fca60b50b3da6fd35e06db76c904c2814db1b2f21a3beb921485983840cd8a34b218dd558e23e0ce839f724f47
6
+ metadata.gz: 9185c3155c82d38a668f1ccbaf857ef07f847787f6d4b4dd56620d7b97d5e42cf834658b2fda29577c81fc4f6d30bc4c533cd510d5e9b91c453f42c3b8745106
7
+ data.tar.gz: 58bea5156caa6c070cb5582396d1ead8583485e8055ea067c837a6a7a187771e32f0a7d489eda5ecba99e6bde573387087cbd4dd62dad3648c855709d889ab49
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: dbagent
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.3.0
4
+ version: 3.6.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Bernard Lambeau
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2023-04-17 00:00:00.000000000 Z
11
+ date: 2023-10-17 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: sequel
@@ -189,32 +189,12 @@ email: blambeau@gmail.com
189
189
  executables: []
190
190
  extensions: []
191
191
  extra_rdoc_files: []
192
- files:
193
- - Gemfile
194
- - LICENSE.md
195
- - README.md
196
- - Rakefile
197
- - lib/db_agent.rb
198
- - lib/db_agent/db_handler.rb
199
- - lib/db_agent/db_handler/mssql.rb
200
- - lib/db_agent/db_handler/mysql.rb
201
- - lib/db_agent/db_handler/postgresql.rb
202
- - lib/db_agent/seeder.rb
203
- - lib/db_agent/table_orderer.rb
204
- - lib/db_agent/version.rb
205
- - lib/db_agent/viewpoint.rb
206
- - lib/db_agent/viewpoint/base.rb
207
- - lib/db_agent/viewpoint/delegate.rb
208
- - lib/db_agent/viewpoint/typecheck.rb
209
- - lib/db_agent/webapp.rb
210
- - tasks/db.rake
211
- - tasks/gem.rake
212
- - tasks/test.rake
192
+ files: []
213
193
  homepage: http://github.com/enspirit/dbagent
214
194
  licenses:
215
195
  - MIT
216
196
  metadata: {}
217
- post_install_message:
197
+ post_install_message:
218
198
  rdoc_options: []
219
199
  require_paths:
220
200
  - lib
@@ -229,8 +209,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
229
209
  - !ruby/object:Gem::Version
230
210
  version: '0'
231
211
  requirements: []
232
- rubygems_version: 3.1.6
233
- signing_key:
212
+ rubygems_version: 3.3.26
213
+ signing_key:
234
214
  specification_version: 4
235
215
  summary: A tool to migrate, spy and seed relational databases.
236
216
  test_files: []
data/Gemfile DELETED
@@ -1,2 +0,0 @@
1
- source 'http://rubygems.org'
2
- gemspec
data/LICENSE.md DELETED
@@ -1,22 +0,0 @@
1
- # The MIT Licence
2
-
3
- Copyright (c) 2019 - Enspirit SPRL (Bernard Lambeau)
4
-
5
- Permission is hereby granted, free of charge, to any person obtaining
6
- a copy of this software and associated documentation files (the
7
- "Software"), to deal in the Software without restriction, including
8
- without limitation the rights to use, copy, modify, merge, publish,
9
- distribute, sublicense, and/or sell copies of the Software, and to
10
- permit persons to whom the Software is furnished to do so, subject to
11
- the following conditions:
12
-
13
- The above copyright notice and this permission notice shall be
14
- included in all copies or substantial portions of the Software.
15
-
16
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
- EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
- MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
- NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
- LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
- OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
- WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md DELETED
@@ -1,119 +0,0 @@
1
- [![Integration](https://github.com/enspirit/dbagent/actions/workflows/integration.yml/badge.svg?branch=master)](https://github.com/enspirit/dbagent/actions/workflows/integration.yml)
2
-
3
- # DbAgent, a ruby tool to migrate, spy and seed relational databases
4
-
5
- DbAgent helps managing a relational database lifecyle through three main tools:
6
-
7
- * Migrations: powered by [Sequel](http://sequel.jeremyevans.net/), migrate as simply as `rake db:migrate`. Supports both superuser and normal user migrations.
8
-
9
- * Spy: using [Shemaspy](http://schemaspy.sourceforge.net/), get your database schema browsable at any moment, through a simple web interface.
10
-
11
- * Seed: maintain, install and flush database content as datasets, organized hierarchically in .json files. Very handy for automated tests, for instance.
12
-
13
- ## Get started using Docker
14
-
15
- DbAgent is expected to be used as its Docker agent, available as `enspirit/dbagent`. Simply mount migrations and data folders, and you're ready to go.
16
-
17
- See the examples folder for details.
18
-
19
- ## Available environment variables
20
-
21
- * `DBAGENT_ROOT_FOLDER` Main folder where data, migrations and viewpoints can be found
22
- * `DBAGENT_LOGLEVEL` Log level to use for dbagent messages (defaults to `WARN`)
23
- * `DBAGENT_LOGSQL` Low Sequel's SQL queries (defaults to `no`)
24
- * `DBAGENT_ADAPTER` Sequel's adapter (defaults to `postgres`)
25
- * `DBAGENT_HOST` Database server host (defaults to `localhost`)
26
- * `DBAGENT_PORT` Database server port (defaults to `5432`)
27
- * `DBAGENT_DB` Database name (defaults to `suppliers-and-parts`)
28
- * `DBAGENT_USER` Database user (defaults to `dbagent`)
29
- * `DBAGENT_PASSWORD` Database password (defaults to `dbagent`)
30
- * `DBAGENT_SOCKET` Database server socket (if host/port is not used)
31
- * `DBAGENT_SUPER_USER` Superuser name (postgres only)
32
- * `DBAGENT_SUPER_DB` Superuser database (postgres only)
33
- * `DBAGENT_SUPER_PASSWORD` Superuser password (postgres only)
34
- * `DBAGENT_VIEWPOINT` Bmg viewpoint (class name) when using db:flush
35
-
36
- ## Available rake tasks
37
-
38
- The following rake tasks helps you managing the database. They must typically be executed on the docker container.
39
-
40
- ```
41
- rake db:check-seeds # Checks that all seeds can be installed correctly
42
- rake db:create # Creates an fresh new user & database (USE WITH CARE)
43
- rake db:drop # Drops the user & database (USE WITH CARE)
44
- rake db:flush[to] # Flushes the database as a particular data set
45
- rake db:migrate # Runs migrations on the current database
46
- rake db:ping # Pings the database, making sure everything's ready for migration
47
- rake db:rebuild # Rebuilds the database from scratch (USE WITH CARE)
48
- rake db:repl # Opens a database REPL
49
- rake db:seed[from] # Seeds the database with a particular data set
50
- rake db:spy # Dumps the schema documentation into database/schema
51
- rake db:backup # Makes a database backup to the backups folder
52
- rake db:restore[match] # Restore the last matching database backup file from backups folder
53
- rake db:revive # Shortcut for both db:restore and db:migrate
54
- rake db:tables # List tables with those with fewer dependencies first
55
- rake db:dependencies[of] # List tables that depend of a given one
56
- ```
57
-
58
- ## Available webservices
59
-
60
- ```
61
- GET /schema/ # Browser the database schema (requires a former `rake db:spy`)
62
- POST /seeds/install?id=... # Install a particular dataset, id is the name of a folder in `data` folder
63
- POST /seeds/flush?id=... # Flushes the current database content as a named dataset
64
- ```
65
-
66
- ## Hacking on dbagent
67
-
68
- ### Installing the library
69
-
70
- ```
71
- bundle install
72
- ```
73
-
74
- ### Preparing your computer
75
-
76
- The tests require a valid PostgreSQL installation with the suppliers-and-parts
77
- database installed. A `dbagent` user would be needed on the PostgreSQL installation
78
- to bootstrap the process.
79
-
80
- ```
81
- sudo su postgres -c 'createuser --createdb dbagent -P'
82
- ```
83
-
84
- DbAgent tries to connect to the suppliers-and-parts with a dbagent/dbagent user/password
85
- pair by default. If you change the database name, user, or password please adapt the
86
- environment variables accordingly in the commands below.
87
-
88
- ### Installing the example database
89
-
90
- ```
91
- DBAGENT_ROOT_FOLDER=examples/suppliers-and-parts bundle exec rake db:create db:migrate db:seed['base']
92
- ```
93
- ### Running test
94
-
95
- To run the test you need to have `Docker` on your computer.
96
-
97
- Run:
98
- ```
99
- make test
100
- ```
101
-
102
- Don't forget to delete created ressources for the tests bun running:
103
- ```
104
- make clean
105
- ```
106
-
107
- ## Contribute
108
-
109
- Please use github issues and pull requests for all questions, bug reports,
110
- and contributions. Don't hesitate to get in touch with us with an early code
111
- spike if you plan to add non trivial features.
112
-
113
- ## Licence
114
-
115
- This software is distributed by Enspirit SRL under a MIT Licence. Please
116
- contact Bernard Lambeau (blambeau@gmail.com) with any question.
117
-
118
- Enspirit (https://enspirit.be) and Klaro App (https://klaro.cards) are both
119
- actively using and contributing to the library.
data/Rakefile DELETED
@@ -1,16 +0,0 @@
1
- require 'path'
2
-
3
- def shell(*cmds)
4
- cmd = cmds.join("\n")
5
- puts cmd
6
- system cmd
7
- end
8
-
9
- #
10
- # Install all tasks found in tasks folder
11
- #
12
- # See .rake files there for complete documentation.
13
- #
14
- Dir["tasks/*.rake"].each do |taskfile|
15
- load taskfile
16
- end
@@ -1,31 +0,0 @@
1
- module DbAgent
2
- class DbHandler
3
- class MSSQL < DbHandler
4
-
5
- def create
6
- raise
7
- end
8
-
9
- def drop
10
- raise
11
- end
12
-
13
- def backup
14
- raise
15
- end
16
-
17
- def repl
18
- raise
19
- end
20
-
21
- def spy
22
- jdbc_jar = (Path.dir.parent/'vendor').glob('mssql*.jar').first
23
- system %Q{java -jar vendor/schemaSpy_5.0.0.jar -dp #{jdbc_jar} -t mssql05 -host #{config[:host]} -u #{config[:user]} -p #{config[:password]} -db #{config[:database]} -port #{config[:port]} -s dbo -o #{schema_folder}/spy}
24
- system %Q{open #{schema_folder}/spy/index.html}
25
- end
26
-
27
- def restore(t, args)
28
- end
29
- end # MSSQL
30
- end # DbHandler
31
- end # DbAgent
@@ -1,54 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module DbAgent
4
- class DbHandler
5
- class MySQL < DbHandler
6
- def create
7
- raise
8
- end
9
-
10
- def drop
11
- raise
12
- end
13
-
14
- def backup
15
- datetime = Time.now.strftime('%Y%m%dT%H%M%S')
16
- shell mysqldump(config[:database], "> #{backup_folder}/backup-#{datetime}.sql")
17
- end
18
-
19
- def repl
20
- shell mysql(config[:database])
21
- end
22
-
23
- def spy
24
- jdbc_jar = (Path.dir.parent / 'vendor').glob('mysql*.jar').first
25
- system %(java -jar vendor/schemaSpy_5.0.0.jar -dp #{jdbc_jar} -t mysql -host #{config[:host]} -u #{config[:user]} -p #{config[:password]} -db #{config[:database]} -s public -o #{schema_folder}/spy)
26
- system %(open #{schema_folder}/spy/index.html)
27
- end
28
-
29
- def restore(_t, args)
30
- candidates = backup_folder.glob('*.sql').sort
31
- if args[:pattern] && rx = Regexp.new(args[:pattern])
32
- candidates = candidates.select { |f| f.basename.to_s =~ rx }
33
- end
34
- file = candidates.last
35
- shell mysql(config[:database], '<', file.to_s)
36
- end
37
-
38
- private
39
-
40
- def mysql_cmd(cmd, *args)
41
- conf = config
42
- %(#{cmd} -h #{config[:host]} --password=#{config[:password]} -P #{config[:port]} -u #{config[:user]} #{args.join(' ')})
43
- end
44
-
45
- def mysql(*args)
46
- mysql_cmd('mysql', *args)
47
- end
48
-
49
- def mysqldump(*args)
50
- mysql_cmd('mysqldump', *args)
51
- end
52
- end # MySQL
53
- end # DbHandler
54
- end # DbAgent
@@ -1,66 +0,0 @@
1
- module DbAgent
2
- class DbHandler
3
- class PostgreSQL < DbHandler
4
-
5
- def create
6
- shell pg_cmd("createuser","--no-createdb","--no-createrole","--no-superuser","--no-password",config[:user]),
7
- pg_cmd("createdb","--owner=#{config[:user]}", config[:database])
8
- end
9
-
10
- def drop
11
- shell pg_cmd("dropdb", config[:database]),
12
- pg_cmd("dropuser", config[:user])
13
- end
14
-
15
- def backup
16
- datetime = Time.now.strftime("%Y%m%dT%H%M%S")
17
- shell pg_dump("--clean", config[:database], "> #{backup_folder}/backup-#{datetime}.sql")
18
- end
19
-
20
- def repl
21
- shell pg_cmd('psql', config[:database])
22
- end
23
-
24
- def spy
25
- spy_jar = DbAgent._!('vendor').glob('schema*.jar').first
26
- jdbc_jar = DbAgent._!('vendor').glob('postgresql*.jar').first
27
- cmd = ""
28
- cmd << %Q{java -jar #{spy_jar} -dp #{jdbc_jar} -t pgsql}
29
- cmd << %Q{ -host #{config[:host]}}
30
- cmd << %Q{ -port #{config[:port]}} if config[:port]
31
- cmd << %Q{ -u #{config[:user]}}
32
- cmd << %Q{ -p #{config[:password]}} if config[:password]
33
- cmd << %Q{ -db #{config[:database]} -s public -o #{schema_folder}/spy}
34
- system(cmd)
35
- system %Q{open #{schema_folder}/spy/index.html}
36
- end
37
-
38
- def restore(t, args)
39
- candidates = backup_folder.glob("*.sql").sort
40
- if args[:pattern] && rx = Regexp.new(args[:pattern])
41
- candidates = candidates.select{|f| f.basename.to_s =~ rx }
42
- end
43
- file = candidates.last
44
- shell pg_cmd('psql', config[:database], '<', file.to_s)
45
- end
46
-
47
- private
48
-
49
- def pg_cmd(cmd, *args)
50
- %Q{#{cmd} -h #{config[:host]} -p #{config[:port]} -U #{config[:user]} #{args.join(' ')}}
51
- end
52
-
53
- def psql(*args)
54
- cmd = "psql"
55
- cmd = "PGPASSWORD=#{config[:password]} #{cmd}" if config[:password]
56
- pg_cmd(cmd, *args)
57
- end
58
-
59
- def pg_dump(*args)
60
- cmd = "pg_dump"
61
- cmd = "PGPASSWORD=#{config[:password]} #{cmd}" if config[:password]
62
- pg_cmd(cmd, *args)
63
- end
64
- end # class PostgreSQL
65
- end # module DbHandler
66
- end # module DbAgent
@@ -1,130 +0,0 @@
1
- module DbAgent
2
- class DbHandler
3
-
4
- def initialize(options)
5
- @config = options[:config]
6
- @superconfig = options[:superconfig]
7
- @root_folder = options[:root]
8
- @backup_folder = options[:backup] || options[:root]/'backups'
9
- @schema_folder = options[:schema] || options[:root]/'schema'
10
- @migrations_folder = options[:migrations] || options[:root]/'migrations'
11
- @data_folder = options[:data] || options[:root]/'data'
12
- @viewpoints_folder = options[:viewpoints] || options[:root]/'viewpoints'
13
- require_viewpoints!
14
- end
15
- attr_reader :config, :superconfig
16
- attr_reader :backup_folder, :schema_folder, :migrations_folder
17
- attr_reader :data_folder, :viewpoints_folder
18
-
19
- def ping
20
- puts "Using #{config}"
21
- sequel_db.test_connection
22
- puts "Everything seems fine!"
23
- end
24
-
25
- def create
26
- raise NotImplementedError
27
- end
28
-
29
- def drop
30
- raise NotImplementedError
31
- end
32
-
33
- def backup
34
- raise NotImplementedError
35
- end
36
-
37
- def repl
38
- raise NotImplementedError
39
- end
40
-
41
- def wait_server
42
- require 'net/ping'
43
- raise "No host found" unless config[:host]
44
- check = Net::Ping::External.new(config[:host])
45
- puts "Trying to ping `#{config[:host]}`"
46
- 15.downto(0) do |i|
47
- print "."
48
- if check.ping?
49
- print "\nServer found.\n"
50
- break
51
- elsif i == 0
52
- print "\n"
53
- raise "Server not found, I give up."
54
- else
55
- sleep(1)
56
- end
57
- end
58
- end
59
-
60
- def wait
61
- 15.downto(0) do |i|
62
- begin
63
- puts "Using #{config}"
64
- sequel_db.test_connection
65
- puts "Database is there. Great."
66
- break
67
- rescue Sequel::Error
68
- raise if i==0
69
- sleep(1)
70
- end
71
- end
72
- end
73
-
74
- def restore(t, args)
75
- raise NotImplementedError
76
- end
77
-
78
- def migrate(version = nil)
79
- Sequel.extension :migration
80
- if (sf = migrations_folder/'superuser').exists?
81
- Sequel::Migrator.run(sequel_superdb, migrations_folder/'superuser', table: 'superuser_migrations', target: version)
82
- end
83
- Sequel::Migrator.run(sequel_db, migrations_folder, target: version)
84
- end
85
-
86
- def repl
87
- raise NotImplementedError
88
- end
89
-
90
- def spy
91
- raise NotImplementedError
92
- end
93
-
94
- def self.factor(options)
95
- case options[:config][:adapter]
96
- when 'postgres'
97
- PostgreSQL.new(options)
98
- when 'mssql'
99
- MSSQL.new(options)
100
- when /mysql/
101
- MySQL.new(options)
102
- else
103
- PostgreSQL.new(options)
104
- end
105
- end
106
-
107
- def sequel_db
108
- @sequel_db ||= ::Sequel.connect(config)
109
- end
110
-
111
- def sequel_superdb
112
- raise "No superconfig set" if superconfig.nil?
113
- @sequel_superdb ||= ::Sequel.connect(superconfig)
114
- end
115
-
116
- def system(cmd, *args)
117
- puts cmd
118
- ::Kernel.system(cmd, *args)
119
- end
120
-
121
- def require_viewpoints!
122
- f = viewpoints_folder.expand_path
123
- Path.require_tree(f) if f.directory?
124
- end
125
-
126
- end # class DbHandler
127
- end # module DbAgent
128
- require_relative 'db_handler/postgresql'
129
- require_relative 'db_handler/mssql'
130
- require_relative 'db_handler/mysql'
@@ -1,167 +0,0 @@
1
- module DbAgent
2
- class Seeder
3
-
4
- def initialize(handler)
5
- @handler = handler
6
- end
7
- attr_reader :handler
8
-
9
- def install(from)
10
- handler.sequel_db.transaction do
11
- before_seeding!
12
-
13
- folder = handler.data_folder/from
14
-
15
- # load files in order
16
- pairs = merged_data(from)
17
- names = pairs.keys.sort{|p1,p2|
18
- pairs[p1].basename <=> pairs[p2].basename
19
- }
20
-
21
- # Truncate tables
22
- names.reverse.each do |name|
23
- LOGGER.info("Emptying table `#{name}`")
24
- handler.sequel_db[name.to_sym].delete
25
- end
26
-
27
- # Fill them
28
- names.each do |name|
29
- LOGGER.info("Filling table `#{name}`")
30
- file = pairs[name]
31
- handler.sequel_db[name.to_sym].multi_insert(file.load)
32
- end
33
-
34
- after_seeding!(folder)
35
- end
36
- end
37
-
38
- def insert_script(from)
39
- folder = handler.data_folder/from
40
-
41
- # load files in order
42
- pairs = merged_data(from)
43
- names = pairs.keys.sort{|p1,p2|
44
- pairs[p1].basename <=> pairs[p2].basename
45
- }
46
-
47
- # Fill them
48
- names.each do |name|
49
- file = pairs[name]
50
- data = file.load
51
- next if data.empty?
52
-
53
- keys = data.first.keys
54
- values = data.map{|t|
55
- keys.map{|k| t[k] }
56
- }
57
- puts handler.sequel_db[name.to_sym].multi_insert_sql(keys, values)
58
- end
59
- end
60
-
61
- def flush_empty(to = "empty")
62
- target = (handler.data_folder/to).rm_rf.mkdir_p
63
- (target/"metadata.json").write <<-JSON.strip
64
- {}
65
- JSON
66
- TableOrderer.new(handler).tsort.each_with_index do |table_name, index|
67
- (target/"#{(index*10).to_s.rjust(5,"0")}-#{table_name}.json").write("[]")
68
- end
69
- end
70
-
71
- def flush(to)
72
- target = (handler.data_folder/to).rm_rf.mkdir_p
73
- source = (handler.data_folder/"empty")
74
- (target/"metadata.json").write <<-JSON.strip
75
- { "inherits": "empty" }
76
- JSON
77
- seed_files(source).each do |f|
78
- flush_seed_file(f, to)
79
- end
80
- end
81
-
82
- def flush_seed_file(f, to)
83
- target = (handler.data_folder/to)
84
- table = file2table(f)
85
- flush_table(table, target, f.basename, true)
86
- end
87
-
88
- def flush_table(table_name, target_folder, file_name, skip_empty)
89
- data = viewpoint.send(table_name.to_sym).to_a
90
- if data.empty? && skip_empty
91
- LOGGER.info("Skipping table `#{table_name}` since empty")
92
- else
93
- LOGGER.info("Flushing table `#{table_name}`")
94
- json = JSON.pretty_generate(data)
95
- (target_folder/file_name).write(json)
96
- end
97
- end
98
-
99
- def each_seed(install = true)
100
- handler.data_folder.glob('**/*') do |file|
101
- next unless file.directory?
102
- next unless (file/"metadata.json").exists?
103
-
104
- base = file.relative_to(handler.data_folder)
105
- begin
106
- Seeder.new(handler).install(base)
107
- puts "#{base} OK"
108
- yield(self, file) if block_given?
109
- rescue => ex
110
- puts "KO on #{file}"
111
- puts ex.message
112
- end if install
113
- end
114
- end
115
-
116
- private
117
-
118
- def before_seeding!
119
- file = handler.data_folder/"before_seeding.sql"
120
- return unless file.exists?
121
-
122
- handler.sequel_db.execute(file.read)
123
- end
124
-
125
- def after_seeding!(folder)
126
- file = folder/"after_seeding.sql"
127
- handler.sequel_db.execute(file.read) if file.exists?
128
- after_seeding!(folder.parent) unless folder == handler.data_folder
129
- end
130
-
131
- def merged_data(from)
132
- folder = handler.data_folder/from
133
- data = {}
134
-
135
- # load metadata and install parent dataset if any
136
- metadata = (folder/"metadata.json").load
137
- if parent = metadata["inherits"]
138
- data = merged_data(parent)
139
- end
140
-
141
- seed_files(folder).each do |f|
142
- data[file2table(f)] = f
143
- end
144
-
145
- data
146
- end
147
-
148
- def seed_files(folder)
149
- folder
150
- .glob("*.json")
151
- .reject{|f| f.basename.to_s =~ /^metadata/ }
152
- end
153
-
154
- def file2table(f)
155
- f.basename.rm_ext.to_s[/^\d+-(.*)/, 1]
156
- end
157
-
158
- def viewpoint
159
- @viewpoint ||= if vp = ENV['DBAGENT_VIEWPOINT']
160
- Kernel.const_get(vp).new(handler.sequel_db)
161
- else
162
- Viewpoint::Base.new(handler.sequel_db)
163
- end
164
- end
165
-
166
- end # class Seeder
167
- end # module DbAgent
@@ -1,94 +0,0 @@
1
- require 'tsort'
2
- module DbAgent
3
- class TableOrderer
4
-
5
- def initialize(handler)
6
- @handler = handler
7
- end
8
- attr_reader :handler
9
-
10
- def db
11
- handler.sequel_db
12
- end
13
-
14
- def tsort
15
- @tsort ||= TSortComputation.new(db).to_a
16
- end
17
-
18
- def graph
19
- @graph ||= TSortComputation.new(db).graph
20
- end
21
-
22
- def dependencies(table)
23
- _dependencies(table, ds = {})
24
- ds
25
- .inject([]){|memo,(_,plus)| (memo + plus).uniq }
26
- .sort{|t1,t2| tsort.index(t1) - tsort.index(t2) }
27
- .reject{|x| x == table }
28
- end
29
-
30
- def _dependencies(table, ds)
31
- return ds if ds.has_key?(table)
32
- ds[table] = graph[table]
33
- ds[table].each do |child|
34
- _dependencies(child, ds)
35
- end
36
- end
37
- private :_dependencies
38
-
39
- class TSortComputation
40
- include TSort
41
-
42
- def initialize(db, except = [])
43
- @db = db
44
- @except = except
45
- end
46
- attr_reader :db, :except
47
-
48
- def graph
49
- g = Hash.new{|h,k| h[k] = [] }
50
- tsort_each_node.each do |table|
51
- tsort_each_child(table) do |child|
52
- g[child] << table
53
- end
54
- end
55
- g
56
- rescue TSort::Cyclic
57
- raise unless killed = to_kill
58
- TSortComputation.new(db, except + [killed]).graph
59
- end
60
-
61
- def to_a
62
- tsort.to_a
63
- rescue TSort::Cyclic
64
- raise unless killed = to_kill
65
- TSortComputation.new(db, except + [killed]).to_a
66
- end
67
-
68
- def tsort_each_node(&bl)
69
- db.tables.each(&bl)
70
- end
71
-
72
- def tsort_each_child(table, &bl)
73
- db.foreign_key_list(table)
74
- .map{|fk| fk[:table] }
75
- .reject{|t|
76
- except.any?{|killed| killed.first == table && killed.last == t }
77
- }
78
- .each(&bl)
79
- end
80
-
81
- private
82
-
83
- def to_kill
84
- each_strongly_connected_component
85
- .select{|scc| scc.size > 1 }
86
- .sort_by{|scc| scc.size }
87
- .first
88
- end
89
-
90
- end # class TSortComputation
91
-
92
- end # class TableOrderer
93
- end # module Dbagent
94
-
@@ -1,6 +0,0 @@
1
- module DbAgent
2
-
3
- # Current version of DbAgent
4
- VERSION = "3.3.0"
5
-
6
- end
@@ -1,18 +0,0 @@
1
- module DbAgent
2
- module Viewpoint
3
- # Factors relations on top of a Sequel database.
4
- class Base
5
-
6
- def initialize(db)
7
- @db = db
8
- end
9
- attr_reader :db
10
-
11
- def method_missing(name, *args, &bl)
12
- return super unless args.empty? and bl.nil?
13
- Bmg.sequel(name, db)
14
- end
15
-
16
- end # class Base
17
- end # module Viewpoint
18
- end # module DbAgent
@@ -1,13 +0,0 @@
1
- module DbAgent
2
- module Viewpoint
3
- # Delegates all relation accesses to `child`.
4
- module Delegate
5
-
6
- def method_missing(name, *args, &bl)
7
- return super unless args.empty? and bl.nil?
8
- child.send(name)
9
- end
10
-
11
- end # module Delegate
12
- end # module Viewpoint
13
- end # module DbAgent
@@ -1,19 +0,0 @@
1
- module DbAgent
2
- module Viewpoint
3
- # Forces typechecking on all child relations.
4
- class TypeCheck
5
-
6
- def initialize(db, child = nil)
7
- @db = db
8
- @child = child || DbAgent::Viewpoint::Base.new(db)
9
- end
10
- attr_reader :db, :child
11
-
12
- def method_missing(name, *args, &bl)
13
- return super unless args.empty? && bl.nil?
14
- child.send(name).with_typecheck
15
- end
16
-
17
- end # class TypeCheck
18
- end # module Viewpoint
19
- end # module DbAgent
@@ -1,3 +0,0 @@
1
- require_relative 'viewpoint/base'
2
- require_relative 'viewpoint/delegate'
3
- require_relative 'viewpoint/typecheck'
@@ -1,35 +0,0 @@
1
- module DbAgent
2
- class Webapp < Sinatra::Base
3
-
4
- set :raise_errors, true
5
- set :show_exceptions, false
6
- set :dump_errors, true
7
- set :db_handler, DbAgent.default_handler
8
-
9
- get '/ping' do
10
- settings.db_handler.sequel_db.test_connection
11
- status 200
12
- "ok"
13
- end
14
-
15
- get %r{/schema/?} do
16
- send_file(settings.db_handler.schema_folder/'spy/index.html')
17
- end
18
-
19
- get '/schema/*' do |url|
20
- send_file(settings.db_handler.schema_folder/'spy'/url)
21
- end
22
-
23
- post '/seeds/install' do
24
- Seeder.new(settings.db_handler).install(request["id"])
25
- "ok"
26
- end
27
-
28
- post '/seeds/flush' do
29
- seed_name = request["id"]
30
- Seeder.new(settings.db_handler).flush(request["id"])
31
- "ok"
32
- end
33
-
34
- end
35
- end
data/lib/db_agent.rb DELETED
@@ -1,78 +0,0 @@
1
- require 'path'
2
- require 'logger'
3
- require 'sequel'
4
- require 'sinatra'
5
- require 'bmg'
6
- require 'bmg/sequel'
7
- module DbAgent
8
- require_relative 'db_agent/version'
9
-
10
- # Simply checks that a path exists of raise an error
11
- def self._!(path)
12
- Path(path).tap do |p|
13
- raise "Missing #{p.basename}." unless p.exists?
14
- end
15
- end
16
-
17
- # Root folder of the project structure
18
- ROOT_FOLDER = if ENV['DBAGENT_ROOT_FOLDER']
19
- _!(ENV['DBAGENT_ROOT_FOLDER'])
20
- else
21
- Path.backfind('.[Gemfile]') or raise("Missing Gemfile")
22
- end
23
-
24
- # Logger instance to use
25
- LOGGER = Logger.new(STDOUT)
26
- LOGGER.level = Logger.const_get(ENV['DBAGENT_LOGLEVEL'] || 'WARN')
27
-
28
- # What database configuration to use for normal access
29
- def self.default_config
30
- cfg = {
31
- adapter: ENV['DBAGENT_ADAPTER'] || 'postgres',
32
- port: ENV['DBAGENT_PORT'] || 5432,
33
- database: ENV['DBAGENT_DB'] || 'suppliers-and-parts',
34
- user: ENV['DBAGENT_USER'] || 'dbagent',
35
- password: ENV['DBAGENT_PASSWORD'] || 'dbagent',
36
- test: false
37
- }
38
-
39
- # Favor a socket approach if specified, otherwise fallback to
40
- # host with default to postgres
41
- if socket = ENV['DBAGENT_SOCKET']
42
- cfg[:socket] = socket
43
- else
44
- cfg[:host] = ENV['DBAGENT_HOST'] || 'localhost'
45
- end
46
-
47
- # Set a logger if explicitly requested
48
- if ENV['DBAGENT_LOGSQL'] == 'yes'
49
- cfg[:loggers] = [LOGGER]
50
- end
51
-
52
- cfg
53
- end
54
-
55
- # What database configuration to use for superuser access
56
- def self.default_superconfig
57
- cfg = default_config
58
- cfg.merge({
59
- user: ENV['DBAGENT_SUPER_USER'] || cfg[:user],
60
- database: ENV['DBAGENT_SUPER_DB'] || cfg[:database],
61
- password: ENV['DBAGENT_SUPER_PASSWORD'] || cfg[:password]
62
- })
63
- end
64
-
65
- def self.default_handler
66
- DbHandler.factor({
67
- config: default_config,
68
- superconfig: default_superconfig,
69
- root: ROOT_FOLDER
70
- })
71
- end
72
-
73
- end # module DbAgent
74
- require 'db_agent/viewpoint'
75
- require 'db_agent/seeder'
76
- require 'db_agent/table_orderer'
77
- require 'db_agent/db_handler'
78
- require 'db_agent/webapp'
data/tasks/db.rake DELETED
@@ -1,114 +0,0 @@
1
- namespace :db do
2
-
3
- task :require do
4
- $:.unshift File.expand_path('../../lib', __FILE__)
5
- require 'db_agent'
6
- include DbAgent
7
- end
8
-
9
- def db_handler
10
- @db_handler ||= DbAgent.default_handler
11
- end
12
-
13
- desc "Pings the database, making sure everything's ready for migration"
14
- task :ping => :require do
15
- db_handler.ping
16
- end
17
-
18
- desc "Drops the user & database (USE WITH CARE)"
19
- task :drop => :require do
20
- db_handler.drop
21
- end
22
-
23
- desc "Creates an fresh new user & database (USE WITH CARE)"
24
- task :create => :require do
25
- db_handler.create
26
- end
27
-
28
- desc "Waits for the database server to ping, up to 15 seconds"
29
- task :wait_server => :require do
30
- db_handler.wait_server
31
- end
32
-
33
- desc "Waits for the database to ping, up to 15 seconds"
34
- task :wait => :require do
35
- db_handler.wait
36
- end
37
-
38
- desc "Dump a database backup"
39
- task :backup => :require do
40
- db_handler.backup
41
- end
42
-
43
- desc "Restore from the last database backup"
44
- task :restore, :pattern do |t,args|
45
- db_handler.restore(t, args)
46
- end
47
- task :restore => :require
48
-
49
- desc "Runs migrations on the current database"
50
- task :migrate, [:version] => :require do |_,args|
51
- version = args[:version].to_i if args[:version]
52
- db_handler.migrate(version)
53
- end
54
-
55
- desc "Opens a database REPL"
56
- task :repl => :require do
57
- db_handler.repl
58
- end
59
-
60
- desc "Dumps the schema documentation into database/schema"
61
- task :spy => :require do
62
- db_handler.spy
63
- end
64
-
65
- desc "Rebuilds the database from scratch (USE WITH CARE)"
66
- task :rebuild => [ :drop, :create, :migrate ]
67
-
68
- desc "Revive the database from the last backup"
69
- task :revive => [ :restore, :migrate ]
70
-
71
-
72
- desc "Checks that all seeds can be installed correctly"
73
- task :"check-seeds" do
74
- Seeder.new(db_handler).each_seed(true)
75
- end
76
- task :"check-seeds" => :require
77
-
78
- desc "Seeds the database with a particular data set"
79
- task :seed, :from do |t,args|
80
- Seeder.new(db_handler).install(args[:from] || 'empty')
81
- end
82
- task :seed => :require
83
-
84
- desc "Prints an INSERT script for a particular data set"
85
- task :insert_script, :from do |t,args|
86
- Seeder.new(db_handler).insert_script(args[:from] || 'empty')
87
- end
88
- task :insert_script => :require
89
-
90
- desc "Flushes the database as a particular data set"
91
- task :flush, :to do |t,args|
92
- Seeder.new(db_handler).flush(args[:to] || Time.now.strftime("%Y%M%d%H%M%S").to_s)
93
- end
94
- task :flush => :require
95
-
96
- desc "Flushes the initial empty files as a data set"
97
- task :flush_empty, :to do |t,args|
98
- Seeder.new(db_handler).flush_empty(args[:to] || Time.now.strftime("%Y%M%d%H%M%S").to_s)
99
- end
100
- task :flush_empty => :require
101
-
102
- desc "Shows what tables depend on a given one"
103
- task :dependencies, :of do |t,args|
104
- puts TableOrderer.new(db_handler).dependencies(args[:of].to_sym).reverse
105
- end
106
- task :dependencies => :require
107
-
108
- desc "Shows all tables in order"
109
- task :tables do |t|
110
- puts TableOrderer.new(db_handler).tsort
111
- end
112
- task :tables => :require
113
-
114
- end
data/tasks/gem.rake DELETED
@@ -1,39 +0,0 @@
1
- require 'rubygems/package_task'
2
-
3
- # Dynamically load the gem spec
4
- gemspec_file = File.expand_path('../../dbagent.gemspec', __FILE__)
5
- gemspec = Kernel.eval(File.read(gemspec_file))
6
-
7
- Gem::PackageTask.new(gemspec) do |t|
8
-
9
- # Name of the package
10
- t.name = gemspec.name
11
-
12
- # Version of the package
13
- t.version = gemspec.version
14
-
15
- # Directory used to store the package files
16
- t.package_dir = "pkg"
17
-
18
- # True if a gzipped tar file (tgz) should be produced
19
- t.need_tar = false
20
-
21
- # True if a gzipped tar file (tar.gz) should be produced
22
- t.need_tar_gz = false
23
-
24
- # True if a bzip2'd tar file (tar.bz2) should be produced
25
- t.need_tar_bz2 = false
26
-
27
- # True if a zip file should be produced (default is false)
28
- t.need_zip = false
29
-
30
- # List of files to be included in the package.
31
- t.package_files = gemspec.files
32
-
33
- # Tar command for gzipped or bzip2ed archives.
34
- t.tar_command = "tar"
35
-
36
- # Zip command for zipped archives.
37
- t.zip_command = "zip"
38
-
39
- end
data/tasks/test.rake DELETED
@@ -1,11 +0,0 @@
1
- namespace :test do
2
-
3
- desc %q{Run all RSpec tests}
4
- task :unit do
5
- require 'rspec'
6
- RSpec::Core::Runner.run(%w[-I. -Ilib -Ispec --pattern=spec/**/test_*.rb --color .])
7
- end
8
-
9
- task :all => :"unit"
10
- end
11
- task :test => :"test:all"