dbagent 3.3.0 → 3.6.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: eb16765f4f64bec5eff152e6db516bb5b672348a1b909229ce46e024b37132ab
4
- data.tar.gz: 48b7d28235340c73b316aabba304a20ccf854cdea919eecca1c7f19b1f594487
3
+ metadata.gz: 487cd25271375faf22bf6f47cc79b839c825cb3a377a16fc330b01ca78954078
4
+ data.tar.gz: dc64a2f4519365470a2e2641c788b6a5ea04b6fbc0d1406f7955e97ba9920218
5
5
  SHA512:
6
- metadata.gz: 61ee06b3ca6734ed61889878e2806eaa5774d08cd864b3ca2a1b90acaf22d505d830fbfb01b96c50340e2c1b27462e658efe2a7db54a2e4243873f912ddb3762
7
- data.tar.gz: 786ada66e5e8ea0e584e8ef61a36f71134df08fca60b50b3da6fd35e06db76c904c2814db1b2f21a3beb921485983840cd8a34b218dd558e23e0ce839f724f47
6
+ metadata.gz: 9185c3155c82d38a668f1ccbaf857ef07f847787f6d4b4dd56620d7b97d5e42cf834658b2fda29577c81fc4f6d30bc4c533cd510d5e9b91c453f42c3b8745106
7
+ data.tar.gz: 58bea5156caa6c070cb5582396d1ead8583485e8055ea067c837a6a7a187771e32f0a7d489eda5ecba99e6bde573387087cbd4dd62dad3648c855709d889ab49
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: dbagent
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.3.0
4
+ version: 3.6.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Bernard Lambeau
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2023-04-17 00:00:00.000000000 Z
11
+ date: 2023-10-17 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: sequel
@@ -189,32 +189,12 @@ email: blambeau@gmail.com
189
189
  executables: []
190
190
  extensions: []
191
191
  extra_rdoc_files: []
192
- files:
193
- - Gemfile
194
- - LICENSE.md
195
- - README.md
196
- - Rakefile
197
- - lib/db_agent.rb
198
- - lib/db_agent/db_handler.rb
199
- - lib/db_agent/db_handler/mssql.rb
200
- - lib/db_agent/db_handler/mysql.rb
201
- - lib/db_agent/db_handler/postgresql.rb
202
- - lib/db_agent/seeder.rb
203
- - lib/db_agent/table_orderer.rb
204
- - lib/db_agent/version.rb
205
- - lib/db_agent/viewpoint.rb
206
- - lib/db_agent/viewpoint/base.rb
207
- - lib/db_agent/viewpoint/delegate.rb
208
- - lib/db_agent/viewpoint/typecheck.rb
209
- - lib/db_agent/webapp.rb
210
- - tasks/db.rake
211
- - tasks/gem.rake
212
- - tasks/test.rake
192
+ files: []
213
193
  homepage: http://github.com/enspirit/dbagent
214
194
  licenses:
215
195
  - MIT
216
196
  metadata: {}
217
- post_install_message:
197
+ post_install_message:
218
198
  rdoc_options: []
219
199
  require_paths:
220
200
  - lib
@@ -229,8 +209,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
229
209
  - !ruby/object:Gem::Version
230
210
  version: '0'
231
211
  requirements: []
232
- rubygems_version: 3.1.6
233
- signing_key:
212
+ rubygems_version: 3.3.26
213
+ signing_key:
234
214
  specification_version: 4
235
215
  summary: A tool to migrate, spy and seed relational databases.
236
216
  test_files: []
data/Gemfile DELETED
@@ -1,2 +0,0 @@
1
- source 'http://rubygems.org'
2
- gemspec
data/LICENSE.md DELETED
@@ -1,22 +0,0 @@
1
- # The MIT Licence
2
-
3
- Copyright (c) 2019 - Enspirit SPRL (Bernard Lambeau)
4
-
5
- Permission is hereby granted, free of charge, to any person obtaining
6
- a copy of this software and associated documentation files (the
7
- "Software"), to deal in the Software without restriction, including
8
- without limitation the rights to use, copy, modify, merge, publish,
9
- distribute, sublicense, and/or sell copies of the Software, and to
10
- permit persons to whom the Software is furnished to do so, subject to
11
- the following conditions:
12
-
13
- The above copyright notice and this permission notice shall be
14
- included in all copies or substantial portions of the Software.
15
-
16
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
- EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
- MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
- NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
- LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
- OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
- WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md DELETED
@@ -1,119 +0,0 @@
1
- [![Integration](https://github.com/enspirit/dbagent/actions/workflows/integration.yml/badge.svg?branch=master)](https://github.com/enspirit/dbagent/actions/workflows/integration.yml)
2
-
3
- # DbAgent, a ruby tool to migrate, spy and seed relational databases
4
-
5
- DbAgent helps managing a relational database lifecyle through three main tools:
6
-
7
- * Migrations: powered by [Sequel](http://sequel.jeremyevans.net/), migrate as simply as `rake db:migrate`. Supports both superuser and normal user migrations.
8
-
9
- * Spy: using [Shemaspy](http://schemaspy.sourceforge.net/), get your database schema browsable at any moment, through a simple web interface.
10
-
11
- * Seed: maintain, install and flush database content as datasets, organized hierarchically in .json files. Very handy for automated tests, for instance.
12
-
13
- ## Get started using Docker
14
-
15
- DbAgent is expected to be used as its Docker agent, available as `enspirit/dbagent`. Simply mount migrations and data folders, and you're ready to go.
16
-
17
- See the examples folder for details.
18
-
19
- ## Available environment variables
20
-
21
- * `DBAGENT_ROOT_FOLDER` Main folder where data, migrations and viewpoints can be found
22
- * `DBAGENT_LOGLEVEL` Log level to use for dbagent messages (defaults to `WARN`)
23
- * `DBAGENT_LOGSQL` Low Sequel's SQL queries (defaults to `no`)
24
- * `DBAGENT_ADAPTER` Sequel's adapter (defaults to `postgres`)
25
- * `DBAGENT_HOST` Database server host (defaults to `localhost`)
26
- * `DBAGENT_PORT` Database server port (defaults to `5432`)
27
- * `DBAGENT_DB` Database name (defaults to `suppliers-and-parts`)
28
- * `DBAGENT_USER` Database user (defaults to `dbagent`)
29
- * `DBAGENT_PASSWORD` Database password (defaults to `dbagent`)
30
- * `DBAGENT_SOCKET` Database server socket (if host/port is not used)
31
- * `DBAGENT_SUPER_USER` Superuser name (postgres only)
32
- * `DBAGENT_SUPER_DB` Superuser database (postgres only)
33
- * `DBAGENT_SUPER_PASSWORD` Superuser password (postgres only)
34
- * `DBAGENT_VIEWPOINT` Bmg viewpoint (class name) when using db:flush
35
-
36
- ## Available rake tasks
37
-
38
- The following rake tasks helps you managing the database. They must typically be executed on the docker container.
39
-
40
- ```
41
- rake db:check-seeds # Checks that all seeds can be installed correctly
42
- rake db:create # Creates an fresh new user & database (USE WITH CARE)
43
- rake db:drop # Drops the user & database (USE WITH CARE)
44
- rake db:flush[to] # Flushes the database as a particular data set
45
- rake db:migrate # Runs migrations on the current database
46
- rake db:ping # Pings the database, making sure everything's ready for migration
47
- rake db:rebuild # Rebuilds the database from scratch (USE WITH CARE)
48
- rake db:repl # Opens a database REPL
49
- rake db:seed[from] # Seeds the database with a particular data set
50
- rake db:spy # Dumps the schema documentation into database/schema
51
- rake db:backup # Makes a database backup to the backups folder
52
- rake db:restore[match] # Restore the last matching database backup file from backups folder
53
- rake db:revive # Shortcut for both db:restore and db:migrate
54
- rake db:tables # List tables with those with fewer dependencies first
55
- rake db:dependencies[of] # List tables that depend of a given one
56
- ```
57
-
58
- ## Available webservices
59
-
60
- ```
61
- GET /schema/ # Browser the database schema (requires a former `rake db:spy`)
62
- POST /seeds/install?id=... # Install a particular dataset, id is the name of a folder in `data` folder
63
- POST /seeds/flush?id=... # Flushes the current database content as a named dataset
64
- ```
65
-
66
- ## Hacking on dbagent
67
-
68
- ### Installing the library
69
-
70
- ```
71
- bundle install
72
- ```
73
-
74
- ### Preparing your computer
75
-
76
- The tests require a valid PostgreSQL installation with the suppliers-and-parts
77
- database installed. A `dbagent` user would be needed on the PostgreSQL installation
78
- to bootstrap the process.
79
-
80
- ```
81
- sudo su postgres -c 'createuser --createdb dbagent -P'
82
- ```
83
-
84
- DbAgent tries to connect to the suppliers-and-parts with a dbagent/dbagent user/password
85
- pair by default. If you change the database name, user, or password please adapt the
86
- environment variables accordingly in the commands below.
87
-
88
- ### Installing the example database
89
-
90
- ```
91
- DBAGENT_ROOT_FOLDER=examples/suppliers-and-parts bundle exec rake db:create db:migrate db:seed['base']
92
- ```
93
- ### Running test
94
-
95
- To run the test you need to have `Docker` on your computer.
96
-
97
- Run:
98
- ```
99
- make test
100
- ```
101
-
102
- Don't forget to delete created ressources for the tests bun running:
103
- ```
104
- make clean
105
- ```
106
-
107
- ## Contribute
108
-
109
- Please use github issues and pull requests for all questions, bug reports,
110
- and contributions. Don't hesitate to get in touch with us with an early code
111
- spike if you plan to add non trivial features.
112
-
113
- ## Licence
114
-
115
- This software is distributed by Enspirit SRL under a MIT Licence. Please
116
- contact Bernard Lambeau (blambeau@gmail.com) with any question.
117
-
118
- Enspirit (https://enspirit.be) and Klaro App (https://klaro.cards) are both
119
- actively using and contributing to the library.
data/Rakefile DELETED
@@ -1,16 +0,0 @@
1
- require 'path'
2
-
3
- def shell(*cmds)
4
- cmd = cmds.join("\n")
5
- puts cmd
6
- system cmd
7
- end
8
-
9
- #
10
- # Install all tasks found in tasks folder
11
- #
12
- # See .rake files there for complete documentation.
13
- #
14
- Dir["tasks/*.rake"].each do |taskfile|
15
- load taskfile
16
- end
@@ -1,31 +0,0 @@
1
- module DbAgent
2
- class DbHandler
3
- class MSSQL < DbHandler
4
-
5
- def create
6
- raise
7
- end
8
-
9
- def drop
10
- raise
11
- end
12
-
13
- def backup
14
- raise
15
- end
16
-
17
- def repl
18
- raise
19
- end
20
-
21
- def spy
22
- jdbc_jar = (Path.dir.parent/'vendor').glob('mssql*.jar').first
23
- system %Q{java -jar vendor/schemaSpy_5.0.0.jar -dp #{jdbc_jar} -t mssql05 -host #{config[:host]} -u #{config[:user]} -p #{config[:password]} -db #{config[:database]} -port #{config[:port]} -s dbo -o #{schema_folder}/spy}
24
- system %Q{open #{schema_folder}/spy/index.html}
25
- end
26
-
27
- def restore(t, args)
28
- end
29
- end # MSSQL
30
- end # DbHandler
31
- end # DbAgent
@@ -1,54 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module DbAgent
4
- class DbHandler
5
- class MySQL < DbHandler
6
- def create
7
- raise
8
- end
9
-
10
- def drop
11
- raise
12
- end
13
-
14
- def backup
15
- datetime = Time.now.strftime('%Y%m%dT%H%M%S')
16
- shell mysqldump(config[:database], "> #{backup_folder}/backup-#{datetime}.sql")
17
- end
18
-
19
- def repl
20
- shell mysql(config[:database])
21
- end
22
-
23
- def spy
24
- jdbc_jar = (Path.dir.parent / 'vendor').glob('mysql*.jar').first
25
- system %(java -jar vendor/schemaSpy_5.0.0.jar -dp #{jdbc_jar} -t mysql -host #{config[:host]} -u #{config[:user]} -p #{config[:password]} -db #{config[:database]} -s public -o #{schema_folder}/spy)
26
- system %(open #{schema_folder}/spy/index.html)
27
- end
28
-
29
- def restore(_t, args)
30
- candidates = backup_folder.glob('*.sql').sort
31
- if args[:pattern] && rx = Regexp.new(args[:pattern])
32
- candidates = candidates.select { |f| f.basename.to_s =~ rx }
33
- end
34
- file = candidates.last
35
- shell mysql(config[:database], '<', file.to_s)
36
- end
37
-
38
- private
39
-
40
- def mysql_cmd(cmd, *args)
41
- conf = config
42
- %(#{cmd} -h #{config[:host]} --password=#{config[:password]} -P #{config[:port]} -u #{config[:user]} #{args.join(' ')})
43
- end
44
-
45
- def mysql(*args)
46
- mysql_cmd('mysql', *args)
47
- end
48
-
49
- def mysqldump(*args)
50
- mysql_cmd('mysqldump', *args)
51
- end
52
- end # MySQL
53
- end # DbHandler
54
- end # DbAgent
@@ -1,66 +0,0 @@
1
- module DbAgent
2
- class DbHandler
3
- class PostgreSQL < DbHandler
4
-
5
- def create
6
- shell pg_cmd("createuser","--no-createdb","--no-createrole","--no-superuser","--no-password",config[:user]),
7
- pg_cmd("createdb","--owner=#{config[:user]}", config[:database])
8
- end
9
-
10
- def drop
11
- shell pg_cmd("dropdb", config[:database]),
12
- pg_cmd("dropuser", config[:user])
13
- end
14
-
15
- def backup
16
- datetime = Time.now.strftime("%Y%m%dT%H%M%S")
17
- shell pg_dump("--clean", config[:database], "> #{backup_folder}/backup-#{datetime}.sql")
18
- end
19
-
20
- def repl
21
- shell pg_cmd('psql', config[:database])
22
- end
23
-
24
- def spy
25
- spy_jar = DbAgent._!('vendor').glob('schema*.jar').first
26
- jdbc_jar = DbAgent._!('vendor').glob('postgresql*.jar').first
27
- cmd = ""
28
- cmd << %Q{java -jar #{spy_jar} -dp #{jdbc_jar} -t pgsql}
29
- cmd << %Q{ -host #{config[:host]}}
30
- cmd << %Q{ -port #{config[:port]}} if config[:port]
31
- cmd << %Q{ -u #{config[:user]}}
32
- cmd << %Q{ -p #{config[:password]}} if config[:password]
33
- cmd << %Q{ -db #{config[:database]} -s public -o #{schema_folder}/spy}
34
- system(cmd)
35
- system %Q{open #{schema_folder}/spy/index.html}
36
- end
37
-
38
- def restore(t, args)
39
- candidates = backup_folder.glob("*.sql").sort
40
- if args[:pattern] && rx = Regexp.new(args[:pattern])
41
- candidates = candidates.select{|f| f.basename.to_s =~ rx }
42
- end
43
- file = candidates.last
44
- shell pg_cmd('psql', config[:database], '<', file.to_s)
45
- end
46
-
47
- private
48
-
49
- def pg_cmd(cmd, *args)
50
- %Q{#{cmd} -h #{config[:host]} -p #{config[:port]} -U #{config[:user]} #{args.join(' ')}}
51
- end
52
-
53
- def psql(*args)
54
- cmd = "psql"
55
- cmd = "PGPASSWORD=#{config[:password]} #{cmd}" if config[:password]
56
- pg_cmd(cmd, *args)
57
- end
58
-
59
- def pg_dump(*args)
60
- cmd = "pg_dump"
61
- cmd = "PGPASSWORD=#{config[:password]} #{cmd}" if config[:password]
62
- pg_cmd(cmd, *args)
63
- end
64
- end # class PostgreSQL
65
- end # module DbHandler
66
- end # module DbAgent
@@ -1,130 +0,0 @@
1
- module DbAgent
2
- class DbHandler
3
-
4
- def initialize(options)
5
- @config = options[:config]
6
- @superconfig = options[:superconfig]
7
- @root_folder = options[:root]
8
- @backup_folder = options[:backup] || options[:root]/'backups'
9
- @schema_folder = options[:schema] || options[:root]/'schema'
10
- @migrations_folder = options[:migrations] || options[:root]/'migrations'
11
- @data_folder = options[:data] || options[:root]/'data'
12
- @viewpoints_folder = options[:viewpoints] || options[:root]/'viewpoints'
13
- require_viewpoints!
14
- end
15
- attr_reader :config, :superconfig
16
- attr_reader :backup_folder, :schema_folder, :migrations_folder
17
- attr_reader :data_folder, :viewpoints_folder
18
-
19
- def ping
20
- puts "Using #{config}"
21
- sequel_db.test_connection
22
- puts "Everything seems fine!"
23
- end
24
-
25
- def create
26
- raise NotImplementedError
27
- end
28
-
29
- def drop
30
- raise NotImplementedError
31
- end
32
-
33
- def backup
34
- raise NotImplementedError
35
- end
36
-
37
- def repl
38
- raise NotImplementedError
39
- end
40
-
41
- def wait_server
42
- require 'net/ping'
43
- raise "No host found" unless config[:host]
44
- check = Net::Ping::External.new(config[:host])
45
- puts "Trying to ping `#{config[:host]}`"
46
- 15.downto(0) do |i|
47
- print "."
48
- if check.ping?
49
- print "\nServer found.\n"
50
- break
51
- elsif i == 0
52
- print "\n"
53
- raise "Server not found, I give up."
54
- else
55
- sleep(1)
56
- end
57
- end
58
- end
59
-
60
- def wait
61
- 15.downto(0) do |i|
62
- begin
63
- puts "Using #{config}"
64
- sequel_db.test_connection
65
- puts "Database is there. Great."
66
- break
67
- rescue Sequel::Error
68
- raise if i==0
69
- sleep(1)
70
- end
71
- end
72
- end
73
-
74
- def restore(t, args)
75
- raise NotImplementedError
76
- end
77
-
78
- def migrate(version = nil)
79
- Sequel.extension :migration
80
- if (sf = migrations_folder/'superuser').exists?
81
- Sequel::Migrator.run(sequel_superdb, migrations_folder/'superuser', table: 'superuser_migrations', target: version)
82
- end
83
- Sequel::Migrator.run(sequel_db, migrations_folder, target: version)
84
- end
85
-
86
- def repl
87
- raise NotImplementedError
88
- end
89
-
90
- def spy
91
- raise NotImplementedError
92
- end
93
-
94
- def self.factor(options)
95
- case options[:config][:adapter]
96
- when 'postgres'
97
- PostgreSQL.new(options)
98
- when 'mssql'
99
- MSSQL.new(options)
100
- when /mysql/
101
- MySQL.new(options)
102
- else
103
- PostgreSQL.new(options)
104
- end
105
- end
106
-
107
- def sequel_db
108
- @sequel_db ||= ::Sequel.connect(config)
109
- end
110
-
111
- def sequel_superdb
112
- raise "No superconfig set" if superconfig.nil?
113
- @sequel_superdb ||= ::Sequel.connect(superconfig)
114
- end
115
-
116
- def system(cmd, *args)
117
- puts cmd
118
- ::Kernel.system(cmd, *args)
119
- end
120
-
121
- def require_viewpoints!
122
- f = viewpoints_folder.expand_path
123
- Path.require_tree(f) if f.directory?
124
- end
125
-
126
- end # class DbHandler
127
- end # module DbAgent
128
- require_relative 'db_handler/postgresql'
129
- require_relative 'db_handler/mssql'
130
- require_relative 'db_handler/mysql'
@@ -1,167 +0,0 @@
1
- module DbAgent
2
- class Seeder
3
-
4
- def initialize(handler)
5
- @handler = handler
6
- end
7
- attr_reader :handler
8
-
9
- def install(from)
10
- handler.sequel_db.transaction do
11
- before_seeding!
12
-
13
- folder = handler.data_folder/from
14
-
15
- # load files in order
16
- pairs = merged_data(from)
17
- names = pairs.keys.sort{|p1,p2|
18
- pairs[p1].basename <=> pairs[p2].basename
19
- }
20
-
21
- # Truncate tables
22
- names.reverse.each do |name|
23
- LOGGER.info("Emptying table `#{name}`")
24
- handler.sequel_db[name.to_sym].delete
25
- end
26
-
27
- # Fill them
28
- names.each do |name|
29
- LOGGER.info("Filling table `#{name}`")
30
- file = pairs[name]
31
- handler.sequel_db[name.to_sym].multi_insert(file.load)
32
- end
33
-
34
- after_seeding!(folder)
35
- end
36
- end
37
-
38
- def insert_script(from)
39
- folder = handler.data_folder/from
40
-
41
- # load files in order
42
- pairs = merged_data(from)
43
- names = pairs.keys.sort{|p1,p2|
44
- pairs[p1].basename <=> pairs[p2].basename
45
- }
46
-
47
- # Fill them
48
- names.each do |name|
49
- file = pairs[name]
50
- data = file.load
51
- next if data.empty?
52
-
53
- keys = data.first.keys
54
- values = data.map{|t|
55
- keys.map{|k| t[k] }
56
- }
57
- puts handler.sequel_db[name.to_sym].multi_insert_sql(keys, values)
58
- end
59
- end
60
-
61
- def flush_empty(to = "empty")
62
- target = (handler.data_folder/to).rm_rf.mkdir_p
63
- (target/"metadata.json").write <<-JSON.strip
64
- {}
65
- JSON
66
- TableOrderer.new(handler).tsort.each_with_index do |table_name, index|
67
- (target/"#{(index*10).to_s.rjust(5,"0")}-#{table_name}.json").write("[]")
68
- end
69
- end
70
-
71
- def flush(to)
72
- target = (handler.data_folder/to).rm_rf.mkdir_p
73
- source = (handler.data_folder/"empty")
74
- (target/"metadata.json").write <<-JSON.strip
75
- { "inherits": "empty" }
76
- JSON
77
- seed_files(source).each do |f|
78
- flush_seed_file(f, to)
79
- end
80
- end
81
-
82
- def flush_seed_file(f, to)
83
- target = (handler.data_folder/to)
84
- table = file2table(f)
85
- flush_table(table, target, f.basename, true)
86
- end
87
-
88
- def flush_table(table_name, target_folder, file_name, skip_empty)
89
- data = viewpoint.send(table_name.to_sym).to_a
90
- if data.empty? && skip_empty
91
- LOGGER.info("Skipping table `#{table_name}` since empty")
92
- else
93
- LOGGER.info("Flushing table `#{table_name}`")
94
- json = JSON.pretty_generate(data)
95
- (target_folder/file_name).write(json)
96
- end
97
- end
98
-
99
- def each_seed(install = true)
100
- handler.data_folder.glob('**/*') do |file|
101
- next unless file.directory?
102
- next unless (file/"metadata.json").exists?
103
-
104
- base = file.relative_to(handler.data_folder)
105
- begin
106
- Seeder.new(handler).install(base)
107
- puts "#{base} OK"
108
- yield(self, file) if block_given?
109
- rescue => ex
110
- puts "KO on #{file}"
111
- puts ex.message
112
- end if install
113
- end
114
- end
115
-
116
- private
117
-
118
- def before_seeding!
119
- file = handler.data_folder/"before_seeding.sql"
120
- return unless file.exists?
121
-
122
- handler.sequel_db.execute(file.read)
123
- end
124
-
125
- def after_seeding!(folder)
126
- file = folder/"after_seeding.sql"
127
- handler.sequel_db.execute(file.read) if file.exists?
128
- after_seeding!(folder.parent) unless folder == handler.data_folder
129
- end
130
-
131
- def merged_data(from)
132
- folder = handler.data_folder/from
133
- data = {}
134
-
135
- # load metadata and install parent dataset if any
136
- metadata = (folder/"metadata.json").load
137
- if parent = metadata["inherits"]
138
- data = merged_data(parent)
139
- end
140
-
141
- seed_files(folder).each do |f|
142
- data[file2table(f)] = f
143
- end
144
-
145
- data
146
- end
147
-
148
- def seed_files(folder)
149
- folder
150
- .glob("*.json")
151
- .reject{|f| f.basename.to_s =~ /^metadata/ }
152
- end
153
-
154
- def file2table(f)
155
- f.basename.rm_ext.to_s[/^\d+-(.*)/, 1]
156
- end
157
-
158
- def viewpoint
159
- @viewpoint ||= if vp = ENV['DBAGENT_VIEWPOINT']
160
- Kernel.const_get(vp).new(handler.sequel_db)
161
- else
162
- Viewpoint::Base.new(handler.sequel_db)
163
- end
164
- end
165
-
166
- end # class Seeder
167
- end # module DbAgent
@@ -1,94 +0,0 @@
1
- require 'tsort'
2
- module DbAgent
3
- class TableOrderer
4
-
5
- def initialize(handler)
6
- @handler = handler
7
- end
8
- attr_reader :handler
9
-
10
- def db
11
- handler.sequel_db
12
- end
13
-
14
- def tsort
15
- @tsort ||= TSortComputation.new(db).to_a
16
- end
17
-
18
- def graph
19
- @graph ||= TSortComputation.new(db).graph
20
- end
21
-
22
- def dependencies(table)
23
- _dependencies(table, ds = {})
24
- ds
25
- .inject([]){|memo,(_,plus)| (memo + plus).uniq }
26
- .sort{|t1,t2| tsort.index(t1) - tsort.index(t2) }
27
- .reject{|x| x == table }
28
- end
29
-
30
- def _dependencies(table, ds)
31
- return ds if ds.has_key?(table)
32
- ds[table] = graph[table]
33
- ds[table].each do |child|
34
- _dependencies(child, ds)
35
- end
36
- end
37
- private :_dependencies
38
-
39
- class TSortComputation
40
- include TSort
41
-
42
- def initialize(db, except = [])
43
- @db = db
44
- @except = except
45
- end
46
- attr_reader :db, :except
47
-
48
- def graph
49
- g = Hash.new{|h,k| h[k] = [] }
50
- tsort_each_node.each do |table|
51
- tsort_each_child(table) do |child|
52
- g[child] << table
53
- end
54
- end
55
- g
56
- rescue TSort::Cyclic
57
- raise unless killed = to_kill
58
- TSortComputation.new(db, except + [killed]).graph
59
- end
60
-
61
- def to_a
62
- tsort.to_a
63
- rescue TSort::Cyclic
64
- raise unless killed = to_kill
65
- TSortComputation.new(db, except + [killed]).to_a
66
- end
67
-
68
- def tsort_each_node(&bl)
69
- db.tables.each(&bl)
70
- end
71
-
72
- def tsort_each_child(table, &bl)
73
- db.foreign_key_list(table)
74
- .map{|fk| fk[:table] }
75
- .reject{|t|
76
- except.any?{|killed| killed.first == table && killed.last == t }
77
- }
78
- .each(&bl)
79
- end
80
-
81
- private
82
-
83
- def to_kill
84
- each_strongly_connected_component
85
- .select{|scc| scc.size > 1 }
86
- .sort_by{|scc| scc.size }
87
- .first
88
- end
89
-
90
- end # class TSortComputation
91
-
92
- end # class TableOrderer
93
- end # module Dbagent
94
-
@@ -1,6 +0,0 @@
1
- module DbAgent
2
-
3
- # Current version of DbAgent
4
- VERSION = "3.3.0"
5
-
6
- end
@@ -1,18 +0,0 @@
1
- module DbAgent
2
- module Viewpoint
3
- # Factors relations on top of a Sequel database.
4
- class Base
5
-
6
- def initialize(db)
7
- @db = db
8
- end
9
- attr_reader :db
10
-
11
- def method_missing(name, *args, &bl)
12
- return super unless args.empty? and bl.nil?
13
- Bmg.sequel(name, db)
14
- end
15
-
16
- end # class Base
17
- end # module Viewpoint
18
- end # module DbAgent
@@ -1,13 +0,0 @@
1
- module DbAgent
2
- module Viewpoint
3
- # Delegates all relation accesses to `child`.
4
- module Delegate
5
-
6
- def method_missing(name, *args, &bl)
7
- return super unless args.empty? and bl.nil?
8
- child.send(name)
9
- end
10
-
11
- end # module Delegate
12
- end # module Viewpoint
13
- end # module DbAgent
@@ -1,19 +0,0 @@
1
- module DbAgent
2
- module Viewpoint
3
- # Forces typechecking on all child relations.
4
- class TypeCheck
5
-
6
- def initialize(db, child = nil)
7
- @db = db
8
- @child = child || DbAgent::Viewpoint::Base.new(db)
9
- end
10
- attr_reader :db, :child
11
-
12
- def method_missing(name, *args, &bl)
13
- return super unless args.empty? && bl.nil?
14
- child.send(name).with_typecheck
15
- end
16
-
17
- end # class TypeCheck
18
- end # module Viewpoint
19
- end # module DbAgent
@@ -1,3 +0,0 @@
1
- require_relative 'viewpoint/base'
2
- require_relative 'viewpoint/delegate'
3
- require_relative 'viewpoint/typecheck'
@@ -1,35 +0,0 @@
1
- module DbAgent
2
- class Webapp < Sinatra::Base
3
-
4
- set :raise_errors, true
5
- set :show_exceptions, false
6
- set :dump_errors, true
7
- set :db_handler, DbAgent.default_handler
8
-
9
- get '/ping' do
10
- settings.db_handler.sequel_db.test_connection
11
- status 200
12
- "ok"
13
- end
14
-
15
- get %r{/schema/?} do
16
- send_file(settings.db_handler.schema_folder/'spy/index.html')
17
- end
18
-
19
- get '/schema/*' do |url|
20
- send_file(settings.db_handler.schema_folder/'spy'/url)
21
- end
22
-
23
- post '/seeds/install' do
24
- Seeder.new(settings.db_handler).install(request["id"])
25
- "ok"
26
- end
27
-
28
- post '/seeds/flush' do
29
- seed_name = request["id"]
30
- Seeder.new(settings.db_handler).flush(request["id"])
31
- "ok"
32
- end
33
-
34
- end
35
- end
data/lib/db_agent.rb DELETED
@@ -1,78 +0,0 @@
1
- require 'path'
2
- require 'logger'
3
- require 'sequel'
4
- require 'sinatra'
5
- require 'bmg'
6
- require 'bmg/sequel'
7
- module DbAgent
8
- require_relative 'db_agent/version'
9
-
10
- # Simply checks that a path exists of raise an error
11
- def self._!(path)
12
- Path(path).tap do |p|
13
- raise "Missing #{p.basename}." unless p.exists?
14
- end
15
- end
16
-
17
- # Root folder of the project structure
18
- ROOT_FOLDER = if ENV['DBAGENT_ROOT_FOLDER']
19
- _!(ENV['DBAGENT_ROOT_FOLDER'])
20
- else
21
- Path.backfind('.[Gemfile]') or raise("Missing Gemfile")
22
- end
23
-
24
- # Logger instance to use
25
- LOGGER = Logger.new(STDOUT)
26
- LOGGER.level = Logger.const_get(ENV['DBAGENT_LOGLEVEL'] || 'WARN')
27
-
28
- # What database configuration to use for normal access
29
- def self.default_config
30
- cfg = {
31
- adapter: ENV['DBAGENT_ADAPTER'] || 'postgres',
32
- port: ENV['DBAGENT_PORT'] || 5432,
33
- database: ENV['DBAGENT_DB'] || 'suppliers-and-parts',
34
- user: ENV['DBAGENT_USER'] || 'dbagent',
35
- password: ENV['DBAGENT_PASSWORD'] || 'dbagent',
36
- test: false
37
- }
38
-
39
- # Favor a socket approach if specified, otherwise fallback to
40
- # host with default to postgres
41
- if socket = ENV['DBAGENT_SOCKET']
42
- cfg[:socket] = socket
43
- else
44
- cfg[:host] = ENV['DBAGENT_HOST'] || 'localhost'
45
- end
46
-
47
- # Set a logger if explicitly requested
48
- if ENV['DBAGENT_LOGSQL'] == 'yes'
49
- cfg[:loggers] = [LOGGER]
50
- end
51
-
52
- cfg
53
- end
54
-
55
- # What database configuration to use for superuser access
56
- def self.default_superconfig
57
- cfg = default_config
58
- cfg.merge({
59
- user: ENV['DBAGENT_SUPER_USER'] || cfg[:user],
60
- database: ENV['DBAGENT_SUPER_DB'] || cfg[:database],
61
- password: ENV['DBAGENT_SUPER_PASSWORD'] || cfg[:password]
62
- })
63
- end
64
-
65
- def self.default_handler
66
- DbHandler.factor({
67
- config: default_config,
68
- superconfig: default_superconfig,
69
- root: ROOT_FOLDER
70
- })
71
- end
72
-
73
- end # module DbAgent
74
- require 'db_agent/viewpoint'
75
- require 'db_agent/seeder'
76
- require 'db_agent/table_orderer'
77
- require 'db_agent/db_handler'
78
- require 'db_agent/webapp'
data/tasks/db.rake DELETED
@@ -1,114 +0,0 @@
1
- namespace :db do
2
-
3
- task :require do
4
- $:.unshift File.expand_path('../../lib', __FILE__)
5
- require 'db_agent'
6
- include DbAgent
7
- end
8
-
9
- def db_handler
10
- @db_handler ||= DbAgent.default_handler
11
- end
12
-
13
- desc "Pings the database, making sure everything's ready for migration"
14
- task :ping => :require do
15
- db_handler.ping
16
- end
17
-
18
- desc "Drops the user & database (USE WITH CARE)"
19
- task :drop => :require do
20
- db_handler.drop
21
- end
22
-
23
- desc "Creates an fresh new user & database (USE WITH CARE)"
24
- task :create => :require do
25
- db_handler.create
26
- end
27
-
28
- desc "Waits for the database server to ping, up to 15 seconds"
29
- task :wait_server => :require do
30
- db_handler.wait_server
31
- end
32
-
33
- desc "Waits for the database to ping, up to 15 seconds"
34
- task :wait => :require do
35
- db_handler.wait
36
- end
37
-
38
- desc "Dump a database backup"
39
- task :backup => :require do
40
- db_handler.backup
41
- end
42
-
43
- desc "Restore from the last database backup"
44
- task :restore, :pattern do |t,args|
45
- db_handler.restore(t, args)
46
- end
47
- task :restore => :require
48
-
49
- desc "Runs migrations on the current database"
50
- task :migrate, [:version] => :require do |_,args|
51
- version = args[:version].to_i if args[:version]
52
- db_handler.migrate(version)
53
- end
54
-
55
- desc "Opens a database REPL"
56
- task :repl => :require do
57
- db_handler.repl
58
- end
59
-
60
- desc "Dumps the schema documentation into database/schema"
61
- task :spy => :require do
62
- db_handler.spy
63
- end
64
-
65
- desc "Rebuilds the database from scratch (USE WITH CARE)"
66
- task :rebuild => [ :drop, :create, :migrate ]
67
-
68
- desc "Revive the database from the last backup"
69
- task :revive => [ :restore, :migrate ]
70
-
71
-
72
- desc "Checks that all seeds can be installed correctly"
73
- task :"check-seeds" do
74
- Seeder.new(db_handler).each_seed(true)
75
- end
76
- task :"check-seeds" => :require
77
-
78
- desc "Seeds the database with a particular data set"
79
- task :seed, :from do |t,args|
80
- Seeder.new(db_handler).install(args[:from] || 'empty')
81
- end
82
- task :seed => :require
83
-
84
- desc "Prints an INSERT script for a particular data set"
85
- task :insert_script, :from do |t,args|
86
- Seeder.new(db_handler).insert_script(args[:from] || 'empty')
87
- end
88
- task :insert_script => :require
89
-
90
- desc "Flushes the database as a particular data set"
91
- task :flush, :to do |t,args|
92
- Seeder.new(db_handler).flush(args[:to] || Time.now.strftime("%Y%M%d%H%M%S").to_s)
93
- end
94
- task :flush => :require
95
-
96
- desc "Flushes the initial empty files as a data set"
97
- task :flush_empty, :to do |t,args|
98
- Seeder.new(db_handler).flush_empty(args[:to] || Time.now.strftime("%Y%M%d%H%M%S").to_s)
99
- end
100
- task :flush_empty => :require
101
-
102
- desc "Shows what tables depend on a given one"
103
- task :dependencies, :of do |t,args|
104
- puts TableOrderer.new(db_handler).dependencies(args[:of].to_sym).reverse
105
- end
106
- task :dependencies => :require
107
-
108
- desc "Shows all tables in order"
109
- task :tables do |t|
110
- puts TableOrderer.new(db_handler).tsort
111
- end
112
- task :tables => :require
113
-
114
- end
data/tasks/gem.rake DELETED
@@ -1,39 +0,0 @@
1
- require 'rubygems/package_task'
2
-
3
- # Dynamically load the gem spec
4
- gemspec_file = File.expand_path('../../dbagent.gemspec', __FILE__)
5
- gemspec = Kernel.eval(File.read(gemspec_file))
6
-
7
- Gem::PackageTask.new(gemspec) do |t|
8
-
9
- # Name of the package
10
- t.name = gemspec.name
11
-
12
- # Version of the package
13
- t.version = gemspec.version
14
-
15
- # Directory used to store the package files
16
- t.package_dir = "pkg"
17
-
18
- # True if a gzipped tar file (tgz) should be produced
19
- t.need_tar = false
20
-
21
- # True if a gzipped tar file (tar.gz) should be produced
22
- t.need_tar_gz = false
23
-
24
- # True if a bzip2'd tar file (tar.bz2) should be produced
25
- t.need_tar_bz2 = false
26
-
27
- # True if a zip file should be produced (default is false)
28
- t.need_zip = false
29
-
30
- # List of files to be included in the package.
31
- t.package_files = gemspec.files
32
-
33
- # Tar command for gzipped or bzip2ed archives.
34
- t.tar_command = "tar"
35
-
36
- # Zip command for zipped archives.
37
- t.zip_command = "zip"
38
-
39
- end
data/tasks/test.rake DELETED
@@ -1,11 +0,0 @@
1
- namespace :test do
2
-
3
- desc %q{Run all RSpec tests}
4
- task :unit do
5
- require 'rspec'
6
- RSpec::Core::Runner.run(%w[-I. -Ilib -Ispec --pattern=spec/**/test_*.rb --color .])
7
- end
8
-
9
- task :all => :"unit"
10
- end
11
- task :test => :"test:all"