db_meta 0.1.1 → 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +2 -0
  3. data/CHANGELOG.md +16 -0
  4. data/Gemfile.lock +4 -4
  5. data/README.md +52 -2
  6. data/db_meta.gemspec +1 -1
  7. data/lib/db_meta/abstract.rb +13 -5
  8. data/lib/db_meta/constant.rb +45 -0
  9. data/lib/db_meta/logger.rb +15 -0
  10. data/lib/db_meta/oracle/base.rb +54 -0
  11. data/lib/db_meta/oracle/connection.rb +39 -0
  12. data/lib/db_meta/oracle/helper.rb +37 -0
  13. data/lib/db_meta/oracle/objects.rb +204 -0
  14. data/lib/db_meta/oracle/oracle.rb +172 -3
  15. data/lib/db_meta/oracle/types/column.rb +70 -0
  16. data/lib/db_meta/oracle/types/comment.rb +20 -0
  17. data/lib/db_meta/oracle/types/constraint.rb +91 -0
  18. data/lib/db_meta/oracle/types/constraint_collection.rb +46 -0
  19. data/lib/db_meta/oracle/types/database_link.rb +33 -0
  20. data/lib/db_meta/oracle/types/function.rb +30 -0
  21. data/lib/db_meta/oracle/types/grant.rb +46 -0
  22. data/lib/db_meta/oracle/types/grant_collection.rb +42 -0
  23. data/lib/db_meta/oracle/types/index.rb +43 -0
  24. data/lib/db_meta/oracle/types/lob.rb +13 -0
  25. data/lib/db_meta/oracle/types/materialized_view.rb +65 -0
  26. data/lib/db_meta/oracle/types/package.rb +42 -0
  27. data/lib/db_meta/oracle/types/package_body.rb +13 -0
  28. data/lib/db_meta/oracle/types/procedure.rb +30 -0
  29. data/lib/db_meta/oracle/types/sequence.rb +41 -0
  30. data/lib/db_meta/oracle/types/synonym.rb +42 -0
  31. data/lib/db_meta/oracle/types/synonym_collection.rb +42 -0
  32. data/lib/db_meta/oracle/types/table.rb +164 -0
  33. data/lib/db_meta/oracle/types/table_data_collection.rb +85 -0
  34. data/lib/db_meta/oracle/types/trigger.rb +64 -0
  35. data/lib/db_meta/oracle/types/type.rb +41 -0
  36. data/lib/db_meta/oracle/types/type_body.rb +13 -0
  37. data/lib/db_meta/oracle/types/view.rb +60 -0
  38. data/lib/db_meta/version.rb +1 -1
  39. data/lib/db_meta.rb +18 -6
  40. data/todo.txt +43 -0
  41. metadata +36 -5
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 8887a183a25b6272667070306efb1984a7801a51
4
- data.tar.gz: 65ff06a084c42b17c3298808543a7309ac83ece8
3
+ metadata.gz: bb34635584c1a09d42f2c6f81da1b4a7e7d2cf9c
4
+ data.tar.gz: acbaa14b19d76975da4060d2055e5d3b5436ca5f
5
5
  SHA512:
6
- metadata.gz: db74f4213879ce983a1f85a1dc7e0eed34f1005493520ba78714399ec17a80265e8f55374514ddd7d54cbad770f6158eb3f428f2e1d96c6c944186e13663f698
7
- data.tar.gz: c051dc8ee2b358d3d7953f3dc692233873c1abb8271afadd7dacab65eb26975eb121be68e5c5ec9cdde6f8adbeb4bed45c9f4e71acbf673d7efd3db3e9cbb6fd
6
+ metadata.gz: 7accbae0b13625e3d37a1a6b20305036b932540937ae1d60691598376dd0c50d15573d49c18bfad2687ce8c7c054b0dbc8dcf8ec4b995518fae1139e5dd6a8fe
7
+ data.tar.gz: 5c6c67e5bc788880b54d5dbe48bcef0377e8ae7a6d4b63abd87d27895d1c74094b352ae12d415955813f481dbd01e3c4048f1427677a6cfe86ecfd4f5ec86441
data/.gitignore CHANGED
@@ -34,3 +34,5 @@ build/
34
34
 
35
35
  # unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
36
36
  .rvmrc
37
+
38
+ examples/
data/CHANGELOG.md ADDED
@@ -0,0 +1,16 @@
1
+ ## 0.1.3 (2017-08-29)
2
+
3
+ * very early version where most of the core oracle types are supported
4
+ * ability to extract core data
5
+
6
+ ## 0.1.2 (2017-08-03)
7
+
8
+ * another very early version
9
+
10
+ ## 0.1.1 (2016-05-10)
11
+
12
+ * very early development version
13
+
14
+ ## 0.1.0 (2016-05-10)
15
+
16
+ * initial setup
data/Gemfile.lock CHANGED
@@ -1,8 +1,8 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- db_meta (0.1.1)
5
- ruby-oci8 (~> 2.2)
4
+ db_meta (0.1.3)
5
+ ruby-oci8 (~> 2.2.4.1)
6
6
 
7
7
  GEM
8
8
  remote: https://rubygems.org/
@@ -22,7 +22,7 @@ GEM
22
22
  diff-lcs (>= 1.2.0, < 2.0)
23
23
  rspec-support (~> 3.4.0)
24
24
  rspec-support (3.4.1)
25
- ruby-oci8 (2.2.2)
25
+ ruby-oci8 (2.2.4.1)
26
26
 
27
27
  PLATFORMS
28
28
  ruby
@@ -34,4 +34,4 @@ DEPENDENCIES
34
34
  rspec (~> 3.0)
35
35
 
36
36
  BUNDLED WITH
37
- 1.12.3
37
+ 1.15.3
data/README.md CHANGED
@@ -1,2 +1,52 @@
1
- # db_meta
2
- Database meta and core data extraction
1
+ [![Gem Version](https://badge.fury.io/rb/db_meta.svg)](https://badge.fury.io/rb/db_meta)
2
+ [![Dependency Status](https://gemnasium.com/badges/github.com/thomis/db_meta.svg)](https://gemnasium.com/github.com/thomis/db_meta)
3
+
4
+ # Welcome to db_meta
5
+ Database meta and core data extraction.
6
+
7
+ ## Is it production ready?
8
+
9
+ Well, I would not say, but I am using it already for my database development work where the gem covers my needs. Be careful and check carefully. Please create an issue when you think that someting is work or missing.
10
+
11
+ ## Installation
12
+ via Gemfile
13
+ ```
14
+ gem 'db_meta'
15
+ ```
16
+
17
+ via command prompt
18
+ ```
19
+ gem install db_meta
20
+ ```
21
+
22
+ ## Example
23
+ ```
24
+ require 'rubygems'
25
+ require 'db_meta'
26
+
27
+ meta = DbMeta::DbMeta.new(username: 'a_username', password: 'a_password', instance: 'an_instance')
28
+ meta.fetch
29
+ meta.extract
30
+ ```
31
+
32
+ ## Supported Databases
33
+ - Oracle
34
+
35
+ ### Supported Oracle object types
36
+ - Table (including Trigger, Constraint, Index)
37
+ - View and Materialized Views
38
+ - Grant
39
+ - Function, Procedures, Packages
40
+ - Type
41
+ - Synonym
42
+ - Database Link
43
+ - more to come...
44
+
45
+ ## Planned Features
46
+ - Storage and tablespace clause
47
+
48
+ ## Contributing
49
+ ...
50
+
51
+ ## License
52
+ db_meta is released under [Apache License, Version 2.0](https://opensource.org/licenses/Apache-2.0)
data/db_meta.gemspec CHANGED
@@ -22,5 +22,5 @@ Gem::Specification.new do |spec|
22
22
  spec.add_development_dependency "rake", "~> 10.0"
23
23
  spec.add_development_dependency "rspec", "~> 3.0"
24
24
 
25
- spec.add_dependency "ruby-oci8", "~> 2.2"
25
+ spec.add_dependency "ruby-oci8", "~> 2.2.4.1"
26
26
  end
@@ -8,26 +8,34 @@ module DbMeta
8
8
  TYPES[type] = self
9
9
  end
10
10
 
11
- def self.from_type(type, args)
12
- raise "type [#{type}] is unknown" unless TYPES.keys.include?(type)
11
+ def self.from_type(type, args={})
12
+ raise "Abstract type [#{type}] is unknown" unless TYPES.keys.include?(type)
13
13
  TYPES[type].new(args)
14
14
  end
15
15
 
16
- def initialize(**args)
16
+ def initialize(args={})
17
17
  @username = args[:username]
18
18
  @password = args[:password]
19
19
  @instance = args[:instance]
20
+ @worker = args[:worker] || 10
21
+
22
+ @include = args[:include]
23
+
24
+ @objects = []
25
+ @invalid_objects = Hash.new([])
26
+
27
+ @base_folder = args[:base_folder] || File.expand_path(File.join(Dir.pwd,"/#{@username}@#{@instance}"))
20
28
 
21
29
  raise 'username is mandatory, pass a username argument during initialization' if @username.nil?
22
30
  raise 'password is mandatory, pass a password argument during initialization' if @password.nil?
23
31
  raise 'instance is mandatory, pass a instance argument during initialization' if @instance.nil?
24
32
  end
25
33
 
26
- def fetch(**args)
34
+ def fetch(args={})
27
35
  raise 'Needs to be implemented in derived class'
28
36
  end
29
37
 
30
- def extract(**args)
38
+ def extract(args={})
31
39
  raise 'Needs to be implemented in derived class'
32
40
  end
33
41
 
@@ -0,0 +1,45 @@
1
+ module DbMeta
2
+
3
+ SUMMARY_COLUMN_FORMAT_NAME = "%-40s"
4
+ SUMMARY_COLUMN_FORMAT_NAME_RIGHT = "%40s"
5
+
6
+ TYPE_SEQUENCE = {
7
+ 'SUMMARY' => 0,
8
+ 'CREATE' => 1,
9
+ 'DROP' => 1,
10
+
11
+ 'DATABASE LINK' => 2,
12
+ 'SEQUENCE' => 3,
13
+ 'TABLE' => 4,
14
+ 'LOB' => 4.1,
15
+ 'VIEW' => 5,
16
+ 'MATERIALIZED VIEW' => 6,
17
+ 'FUNCTION' => 7,
18
+ 'PROCEDURE' => 8,
19
+ 'PACKAGE' => 9,
20
+ 'PACKAGE BODY' => 9.1,
21
+ 'SYNONYM' => 10,
22
+ 'TRIGGER' => 11,
23
+ 'GRANT' => 12,
24
+ 'GRANT EXTERNAL' => 13,
25
+ 'TYPE' => 14,
26
+ 'INDEX' => 22,
27
+ 'DATA' => 40,
28
+ 'CONSTRAINT' => 60
29
+ }
30
+
31
+ EXTRACT_FORMATS = [:sql]
32
+
33
+ OBJECT_QUERY = "
34
+ select * from (
35
+ select OBJECT_TYPE, OBJECT_NAME, STATUS from user_objects
36
+ union all
37
+ select 'CONSTRAINT' as OBJECT_TYPE, CONSTRAINT_NAME as OBJECT_NAME, STATUS from user_constraints
38
+ union all
39
+ select 'GRANT' as OBJECT_TYPE, grantee || ',' || owner || ',' || table_name || ',' || grantor || ',' || privilege || ',' || grantable as object_name, 'VALID' as status from user_tab_privs
40
+ ) order by object_type, object_name
41
+ "
42
+
43
+ OBJECT_FILTER = ['LOB', 'PACKAGE BODY', 'CONSTRAINT', 'GRANT']
44
+
45
+ end
@@ -0,0 +1,15 @@
1
+ require 'logger'
2
+
3
+ class Logger
4
+
5
+ class Formatter
6
+ Format2 = "%s\t%s\t%s\n"
7
+
8
+ def call(severity, time, progname, msg)
9
+ time_in_string = "#{time.strftime("%Y-%m-%d %H:%M:%S")}.#{"%03d" % (time.usec/1000)}"
10
+ Format2 % [time_in_string,severity,msg]
11
+ end
12
+
13
+ end
14
+
15
+ end
@@ -0,0 +1,54 @@
1
+ module DbMeta
2
+ module Oracle
3
+ class Base
4
+ include DbMeta::Oracle::Helper
5
+
6
+ TYPES = {}
7
+
8
+ attr_accessor :type, :status, :name, :extract_type
9
+
10
+ def self.register_type(type)
11
+ TYPES[type] = self
12
+ end
13
+
14
+ def self.from_type(args={})
15
+ type = args['OBJECT_TYPE']
16
+
17
+ # return instance of known type
18
+ return TYPES[type].new(args) if TYPES.keys.include?(type)
19
+
20
+ # There is no implementation for this type yet. Let's just use Base
21
+ Log.warn("Don't know how to handle oracle type [#{type}] yet")
22
+ Base.new(args)
23
+ end
24
+
25
+ def initialize(args={})
26
+ @type = args['OBJECT_TYPE']
27
+ @name = args['OBJECT_NAME']
28
+
29
+ @status = :unknown
30
+ @status = args['STATUS'].downcase.to_sym if args['STATUS']
31
+
32
+ @extract_type = :default # :default, :embedded, :merged
33
+ end
34
+
35
+
36
+ def fetch
37
+ end
38
+
39
+ def extract(args={})
40
+ 'needs to be implemented'
41
+ end
42
+
43
+ def ddl_drop
44
+ "DROP #{@type} #{@name};"
45
+ end
46
+
47
+ def system_object?
48
+ # true if there is a '$' in the object name
49
+ return @name =~ /\$/i
50
+ end
51
+
52
+ end
53
+ end
54
+ end
@@ -0,0 +1,39 @@
1
+ require 'singleton'
2
+
3
+ module DbMeta
4
+ module Oracle
5
+ class Connection
6
+ include Singleton
7
+
8
+ attr_accessor :username, :password, :database_instance
9
+ attr_reader :pool
10
+ attr_reader :worker
11
+
12
+ def set(username, password, database_instance, worker)
13
+ @username = username
14
+ @password = password
15
+ @database_instance = database_instance
16
+ @worker = worker
17
+ end
18
+
19
+ def get
20
+ unless @pool
21
+ # create connection pool
22
+ @pool = ::OCI8::ConnectionPool.new(2, @worker, 2, @username, @password, @database_instance)
23
+ Log.info("Connected to #{@username}@#{@database_instance}")
24
+ end
25
+
26
+ # create and return logical connection. It creates physical connection as needed.
27
+ return ::OCI8.new(@username, @password, @pool)
28
+ end
29
+
30
+ def disconnect
31
+ return unless @pool
32
+ @pool.destroy
33
+ Log.info("Logged off from #{@username}@#{@database_instance}")
34
+ @pool = nil
35
+ end
36
+
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,37 @@
1
+ module DbMeta
2
+ module Oracle
3
+ module Helper
4
+
5
+ def block(title, size = 80)
6
+ line = '-- ' + ('-' * (size-3))
7
+ buffer = [line, "-- #{title}", line].join("\n")
8
+ end
9
+
10
+ def type_sequence(type)
11
+ return TYPE_SEQUENCE[type] || 99
12
+ end
13
+
14
+ def write_buffer_to_file(buffer, file)
15
+ buffer = buffer.join("\n") if buffer.is_a?(Array)
16
+ File.open(file.downcase.gsub(' ', '_'), 'w') do |output|
17
+ output.write(buffer)
18
+ end
19
+ end
20
+
21
+ def remove_folder(folder)
22
+ FileUtils.rm_rf(folder)
23
+ end
24
+
25
+ def create_folder(folder)
26
+ Dir.mkdir(folder.downcase.gsub(' ', '_'))
27
+ rescue
28
+ end
29
+
30
+ def pluralize(n, singular, plural=nil)
31
+ return singular if n == 1
32
+ return (plural || (singular + 's'))
33
+ end
34
+
35
+ end
36
+ end
37
+ end
@@ -0,0 +1,204 @@
1
+ module DbMeta
2
+ module Oracle
3
+ class Objects
4
+ include DbMeta::Oracle::Helper
5
+ extend DbMeta::Oracle::Helper
6
+
7
+ attr_reader :summary_system_object
8
+
9
+ def initialize
10
+ @data = Hash.new{ |h, type| h[type] = {} }
11
+ @worker_queue = Queue.new
12
+ @types_with_object_status_default = []
13
+
14
+ @summary = Hash.new{ |h, type| h[type] = 0 }
15
+ @summary_system_object = Hash.new{ |h, type| h[type] = 0 }
16
+ @invalids = Hash.new{ |h, type| h[type] = [] }
17
+ end
18
+
19
+ def <<(object)
20
+ @data[object.type][object.name] = object
21
+ @worker_queue << object
22
+
23
+ @summary[object.type] += 1
24
+ @summary_system_object[object.type] += 1 if object.system_object?
25
+ @invalids[object.type] << object if [:invalid, :disabled].include?(object.status)
26
+ end
27
+
28
+ def fetch(args={})
29
+ # fetch details in parallel
30
+ # start as many worker threads as max physical connections
31
+ worker = (1..Connection.instance.worker).map do
32
+ Thread.new do
33
+ begin
34
+ while object = @worker_queue.pop(true)
35
+ Log.info(" - #{object.type} - #{object.name}")
36
+ object.fetch
37
+ end
38
+ rescue ThreadError
39
+ end
40
+ end
41
+ end
42
+ worker.map(&:join) # wait until all are done
43
+ end
44
+
45
+ def merge_synonyms
46
+ Log.info("Merging synonyms...")
47
+ synonym_collection = SynonymCollection.new(type: 'SYNONYM', name: 'ALL')
48
+
49
+ @data['SYNONYM'].values.each do |object|
50
+ synonym_collection << object
51
+ end
52
+
53
+ return if synonym_collection.empty?
54
+
55
+ self << synonym_collection
56
+ @summary['SYNONYM'] -= 1 # no need to count collection object
57
+ end
58
+
59
+ def merge_grants
60
+ Log.info("Merging grants...")
61
+ grant_collection = GrantCollection.new(type: 'GRANT', name: 'ALL')
62
+
63
+ @data['GRANT'].values.sort_by{ |o| o.sort_value }.each do |object|
64
+ grant_collection << object
65
+ end
66
+
67
+ return if grant_collection.empty?
68
+
69
+ self << grant_collection
70
+ @summary['GRANT'] -= 1 # no need to count collection object
71
+ end
72
+
73
+ def embed_indexes
74
+ Log.info("Embedding indexes...")
75
+
76
+ @data['INDEX'].values.each do |object|
77
+ @data['TABLE'][object.table_name].add_object(object)
78
+ end
79
+ end
80
+
81
+ def embed_constraints
82
+ Log.info("Embedding constraints...")
83
+
84
+ @data["CONSTRAINT"].values.each do |constraint|
85
+ @data['TABLE'][constraint.table_name].add_object(constraint)
86
+ end
87
+ end
88
+
89
+ def embed_triggers
90
+ Log.info("Embedding triggers...")
91
+
92
+ @data["TRIGGER"].values.each do |object|
93
+ table_object = @data['TABLE'][object.table_name]
94
+
95
+ if table_object
96
+ table_object.add_object(object)
97
+ else
98
+ # if there is no table relation, just extract as default
99
+ object.extract_type = :default
100
+ end
101
+ end
102
+ end
103
+
104
+ def merge_constraints
105
+ Log.info("Merging constraints...")
106
+ constraint_collection = ConstraintCollection.new(type: 'CONSTRAINT', name: 'ALL FOREIGN KEYS')
107
+
108
+ @data['CONSTRAINT'].values.each do |object|
109
+ next unless object.extract_type == :merged
110
+ constraint_collection << object
111
+ end
112
+
113
+ return if constraint_collection.empty?
114
+
115
+ self << constraint_collection
116
+ @summary['CONSTRAINT'] -= 1 # no need to count collection object
117
+ end
118
+
119
+ def handle_table_data(args)
120
+ Log.info("Handling table data...")
121
+
122
+ @exclude_data = args[:exclude_data] if args[:exclude_data]
123
+ @include_data = args[:include_data] if args[:include_data]
124
+
125
+ tables = []
126
+ @data['TABLE'].values.each do |table|
127
+ next if table.system_object?
128
+ next if table.name =~ @exclude_data if @exclude_data
129
+ next unless table.name =~ @include_data if @include_data
130
+ tables << table
131
+ end
132
+
133
+ self << TableDataCollection.new(name: 'ALL CORE DATA', type: 'DATA', tables: tables)
134
+ @summary['DATA'] -= 1 # no need to count DATA object
135
+ end
136
+
137
+ def default_each
138
+ @data.keys.sort_by{ |type| type_sequence(type) }.each do |type|
139
+ @data[type].keys.sort.each do |name|
140
+ object = @data[type][name]
141
+ next if object.system_object?
142
+ next unless object.extract_type == :default
143
+ yield(object)
144
+ end
145
+ end
146
+ end
147
+
148
+ def reverse_default_each
149
+ @data.keys.sort_by{ |type| type_sequence(type) }.reverse_each do |type|
150
+ @data[type].keys.sort.each do |name|
151
+ object = @data[type][name]
152
+ next if object.system_object?
153
+ next unless object.extract_type == :default
154
+ yield object
155
+ end
156
+ end
157
+ end
158
+
159
+ def summary_each
160
+ @summary.each_pair do |type, count|
161
+ yield type, count
162
+ end
163
+ end
164
+
165
+ def invalids?
166
+ @invalids.keys.size > 0
167
+ end
168
+
169
+ def invalid_each
170
+ @invalids.each_pair do |type, objects|
171
+ yield type, objects
172
+ end
173
+ end
174
+
175
+
176
+ def self.all
177
+ objects = []
178
+
179
+ # get all objects as a hash containing name, type, status
180
+ items = []
181
+ types = []
182
+ connection = Connection.instance.get
183
+ cursor = connection.exec(OBJECT_QUERY)
184
+ cursor.fetch_hash do |item|
185
+ items << item
186
+ types << item['OBJECT_TYPE']
187
+ end
188
+ cursor.close
189
+
190
+ # sort items and make an object instance
191
+ items.sort_by{ |i| [ type_sequence(i['OBJECT_TYPE']), i['OBJECT_NAME']]}.each do |item|
192
+ objects << Base.from_type(item)
193
+ end
194
+
195
+ Log.info("Objects: #{items.size}, Object Types: #{types.uniq.size}")
196
+
197
+ objects
198
+ ensure
199
+ connection.logoff if connection # closes logical connection
200
+ end
201
+
202
+ end
203
+ end
204
+ end