db_meta 0.1.1 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8887a183a25b6272667070306efb1984a7801a51
-  data.tar.gz: 65ff06a084c42b17c3298808543a7309ac83ece8
+  metadata.gz: bb34635584c1a09d42f2c6f81da1b4a7e7d2cf9c
+  data.tar.gz: acbaa14b19d76975da4060d2055e5d3b5436ca5f
 SHA512:
-  metadata.gz: db74f4213879ce983a1f85a1dc7e0eed34f1005493520ba78714399ec17a80265e8f55374514ddd7d54cbad770f6158eb3f428f2e1d96c6c944186e13663f698
-  data.tar.gz: c051dc8ee2b358d3d7953f3dc692233873c1abb8271afadd7dacab65eb26975eb121be68e5c5ec9cdde6f8adbeb4bed45c9f4e71acbf673d7efd3db3e9cbb6fd
+  metadata.gz: 7accbae0b13625e3d37a1a6b20305036b932540937ae1d60691598376dd0c50d15573d49c18bfad2687ce8c7c054b0dbc8dcf8ec4b995518fae1139e5dd6a8fe
+  data.tar.gz: 5c6c67e5bc788880b54d5dbe48bcef0377e8ae7a6d4b63abd87d27895d1c74094b352ae12d415955813f481dbd01e3c4048f1427677a6cfe86ecfd4f5ec86441

data/.gitignore

@@ -34,3 +34,5 @@ build/
 
 # unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
 .rvmrc
+
+examples/

data/CHANGELOG.md

@@ -0,0 +1,16 @@
+## 0.1.3 (2017-08-29)
+
+* very early version where most of the core oracle types are supported
+* ability to extract core data
+
+## 0.1.2 (2017-08-03)
+
+* another very early version
+
+## 0.1.1 (2016-05-10)
+
+* very early development version
+
+## 0.1.0 (2016-05-10)
+
+* initial setup

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    db_meta (0.1.1)
-      ruby-oci8 (~> 2.2)
+    db_meta (0.1.3)
+      ruby-oci8 (~> 2.2.4.1)
 
 GEM
   remote: https://rubygems.org/
@@ -22,7 +22,7 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.4.0)
     rspec-support (3.4.1)
-    ruby-oci8 (2.2.2)
+    ruby-oci8 (2.2.4.1)
 
 PLATFORMS
   ruby
@@ -34,4 +34,4 @@ DEPENDENCIES
   rspec (~> 3.0)
 
 BUNDLED WITH
-   1.12.3
+   1.15.3

data/README.md

@@ -1,2 +1,52 @@
-# db_meta
-Database meta and core data extraction
+[![Gem Version](https://badge.fury.io/rb/db_meta.svg)](https://badge.fury.io/rb/db_meta)
+[![Dependency Status](https://gemnasium.com/badges/github.com/thomis/db_meta.svg)](https://gemnasium.com/github.com/thomis/db_meta)
+
+# Welcome to db_meta
+Database meta and core data extraction.
+
+## Is it production ready?
+
+Well, I would not say, but I am using it already for my database development work where the gem covers my needs. Be careful and check carefully. Please create an issue when you think that someting is work or missing.
+
+## Installation
+via Gemfile
+```
+gem 'db_meta'
+```
+
+via command prompt
+```
+gem install db_meta
+```
+
+## Example
+```
+require 'rubygems'
+require 'db_meta'
+
+meta = DbMeta::DbMeta.new(username: 'a_username', password: 'a_password', instance: 'an_instance')
+meta.fetch
+meta.extract
+```
+
+## Supported Databases
+- Oracle
+
+### Supported Oracle object types
+- Table (including Trigger, Constraint, Index)
+- View and Materialized Views
+- Grant
+- Function, Procedures, Packages
+- Type
+- Synonym
+- Database Link
+- more to come...
+
+## Planned Features
+- Storage and tablespace clause
+
+## Contributing
+...
+
+## License
+db_meta is released under [Apache License, Version 2.0](https://opensource.org/licenses/Apache-2.0)

data/db_meta.gemspec

@@ -22,5 +22,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec", "~> 3.0"
 
-  spec.add_dependency "ruby-oci8", "~> 2.2"
+  spec.add_dependency "ruby-oci8", "~> 2.2.4.1"
 end

data/lib/db_meta/abstract.rb

@@ -8,26 +8,34 @@ module DbMeta
       TYPES[type] = self
     end
 
-    def self.from_type(type, args)
-      raise "type [#{type}] is unknown" unless TYPES.keys.include?(type)
+    def self.from_type(type, args={})
+      raise "Abstract type [#{type}] is unknown" unless TYPES.keys.include?(type)
       TYPES[type].new(args)
     end
 
-    def initialize(**args)
+    def initialize(args={})
       @username = args[:username]
       @password = args[:password]
       @instance = args[:instance]
+      @worker = args[:worker] || 10
+
+      @include = args[:include]
+
+      @objects = []
+      @invalid_objects = Hash.new([])
+
+      @base_folder = args[:base_folder] || File.expand_path(File.join(Dir.pwd,"/#{@username}@#{@instance}"))
 
       raise 'username is mandatory, pass a username argument during initialization' if @username.nil?
       raise 'password is mandatory, pass a password argument during initialization' if @password.nil?
       raise 'instance is mandatory, pass a instance argument during initialization' if @instance.nil?
     end
 
-    def fetch(**args)
+    def fetch(args={})
       raise 'Needs to be implemented in derived class'
     end
 
-    def extract(**args)
+    def extract(args={})
       raise 'Needs to be implemented in derived class'
     end
 

data/lib/db_meta/constant.rb

@@ -0,0 +1,45 @@
+module DbMeta
+
+  SUMMARY_COLUMN_FORMAT_NAME = "%-40s"
+  SUMMARY_COLUMN_FORMAT_NAME_RIGHT = "%40s"
+
+  TYPE_SEQUENCE = {
+    'SUMMARY' => 0,
+    'CREATE' => 1,
+    'DROP' => 1,
+
+    'DATABASE LINK' => 2,
+    'SEQUENCE' => 3,
+    'TABLE' => 4,
+    'LOB' => 4.1,
+    'VIEW' => 5,
+    'MATERIALIZED VIEW' => 6,
+    'FUNCTION' => 7,
+    'PROCEDURE' => 8,
+    'PACKAGE' => 9,
+    'PACKAGE BODY' => 9.1,
+    'SYNONYM' => 10,
+    'TRIGGER' => 11,
+    'GRANT' => 12,
+    'GRANT EXTERNAL' => 13,
+    'TYPE' => 14,
+    'INDEX' => 22,
+    'DATA' => 40,
+    'CONSTRAINT' => 60
+  }
+
+  EXTRACT_FORMATS = [:sql]
+
+  OBJECT_QUERY = "
+    select * from (
+      select OBJECT_TYPE, OBJECT_NAME, STATUS from user_objects
+      union all
+      select 'CONSTRAINT' as OBJECT_TYPE, CONSTRAINT_NAME as OBJECT_NAME, STATUS from user_constraints
+      union all
+      select 'GRANT' as OBJECT_TYPE, grantee || ',' || owner || ',' || table_name || ',' || grantor || ',' || privilege || ',' || grantable as object_name, 'VALID' as status from user_tab_privs
+    ) order by object_type, object_name
+  "
+
+  OBJECT_FILTER = ['LOB', 'PACKAGE BODY', 'CONSTRAINT', 'GRANT']
+
+end

data/lib/db_meta/logger.rb

@@ -0,0 +1,15 @@
+require 'logger'
+
+class Logger
+
+  class Formatter
+    Format2 = "%s\t%s\t%s\n"
+
+    def call(severity, time, progname, msg)
+      time_in_string = "#{time.strftime("%Y-%m-%d %H:%M:%S")}.#{"%03d" % (time.usec/1000)}"
+      Format2 % [time_in_string,severity,msg]
+    end
+
+  end
+
+end

data/lib/db_meta/oracle/base.rb

@@ -0,0 +1,54 @@
+module DbMeta
+  module Oracle
+    class Base
+      include DbMeta::Oracle::Helper
+
+      TYPES = {}
+
+      attr_accessor :type, :status, :name, :extract_type
+
+      def self.register_type(type)
+        TYPES[type] = self
+      end
+
+      def self.from_type(args={})
+        type = args['OBJECT_TYPE']
+
+        # return instance of known type
+        return TYPES[type].new(args) if TYPES.keys.include?(type)
+
+        # There is no implementation for this type yet. Let's just use Base
+        Log.warn("Don't know how to handle oracle type [#{type}] yet")
+        Base.new(args)
+      end
+
+      def initialize(args={})
+        @type = args['OBJECT_TYPE']
+        @name = args['OBJECT_NAME']
+
+        @status = :unknown
+        @status = args['STATUS'].downcase.to_sym if args['STATUS']
+
+        @extract_type = :default # :default, :embedded, :merged
+      end
+
+
+      def fetch
+      end
+
+      def extract(args={})
+        'needs to be implemented'
+      end
+
+      def ddl_drop
+        "DROP #{@type} #{@name};"
+      end
+
+      def system_object?
+        # true if there is a '$' in the object name
+        return @name =~ /\$/i
+      end
+
+    end
+  end
+end

data/lib/db_meta/oracle/connection.rb

@@ -0,0 +1,39 @@
+require 'singleton'
+
+module DbMeta
+  module Oracle
+    class Connection
+      include Singleton
+
+      attr_accessor :username, :password, :database_instance
+      attr_reader :pool
+      attr_reader :worker
+
+      def set(username, password, database_instance, worker)
+        @username = username
+        @password = password
+        @database_instance = database_instance
+        @worker = worker
+      end
+
+      def get
+        unless @pool
+          # create connection pool
+          @pool = ::OCI8::ConnectionPool.new(2, @worker, 2, @username, @password, @database_instance)
+          Log.info("Connected to #{@username}@#{@database_instance}")
+        end
+
+        # create and return logical connection. It creates physical connection as needed.
+        return ::OCI8.new(@username, @password, @pool)
+      end
+
+      def disconnect
+        return unless @pool
+        @pool.destroy
+        Log.info("Logged off from #{@username}@#{@database_instance}")
+        @pool = nil
+      end
+
+    end
+  end
+end

data/lib/db_meta/oracle/helper.rb

@@ -0,0 +1,37 @@
+module DbMeta
+  module Oracle
+    module Helper
+
+      def block(title, size = 80)
+        line = '-- ' + ('-' * (size-3))
+        buffer = [line, "-- #{title}", line].join("\n")
+      end
+
+      def type_sequence(type)
+        return TYPE_SEQUENCE[type] || 99
+      end
+
+      def write_buffer_to_file(buffer, file)
+        buffer = buffer.join("\n") if buffer.is_a?(Array)
+        File.open(file.downcase.gsub(' ', '_'), 'w') do |output|
+          output.write(buffer)
+        end
+      end
+
+      def remove_folder(folder)
+        FileUtils.rm_rf(folder)
+      end
+
+      def create_folder(folder)
+        Dir.mkdir(folder.downcase.gsub(' ', '_'))
+      rescue
+      end
+
+      def pluralize(n, singular, plural=nil)
+        return singular if n == 1
+        return (plural || (singular + 's'))
+      end
+
+    end
+  end
+end

data/lib/db_meta/oracle/objects.rb

@@ -0,0 +1,204 @@
+module DbMeta
+  module Oracle
+    class Objects
+      include DbMeta::Oracle::Helper
+      extend DbMeta::Oracle::Helper
+
+      attr_reader :summary_system_object
+
+      def initialize
+        @data = Hash.new{ |h, type|  h[type] = {} }
+        @worker_queue = Queue.new
+        @types_with_object_status_default = []
+
+        @summary = Hash.new{ |h, type| h[type] = 0 }
+        @summary_system_object = Hash.new{ |h, type| h[type] = 0 }
+        @invalids = Hash.new{ |h, type| h[type] = [] }
+      end
+
+      def <<(object)
+        @data[object.type][object.name] = object
+        @worker_queue << object
+
+        @summary[object.type] += 1
+        @summary_system_object[object.type] += 1 if object.system_object?
+        @invalids[object.type] << object if [:invalid, :disabled].include?(object.status)
+      end
+
+      def fetch(args={})
+        # fetch details in parallel
+        # start as many worker threads as max physical connections
+        worker = (1..Connection.instance.worker).map do
+          Thread.new do
+            begin
+              while object = @worker_queue.pop(true)
+                Log.info(" - #{object.type} - #{object.name}")
+                object.fetch
+              end
+            rescue ThreadError
+            end
+          end
+        end
+        worker.map(&:join) # wait until all are done
+      end
+
+      def merge_synonyms
+        Log.info("Merging synonyms...")
+        synonym_collection = SynonymCollection.new(type: 'SYNONYM', name: 'ALL')
+
+        @data['SYNONYM'].values.each do |object|
+          synonym_collection << object
+        end
+
+        return if synonym_collection.empty?
+
+        self << synonym_collection
+        @summary['SYNONYM'] -= 1    # no need to count collection object
+      end
+
+      def merge_grants
+        Log.info("Merging grants...")
+        grant_collection = GrantCollection.new(type: 'GRANT', name: 'ALL')
+
+        @data['GRANT'].values.sort_by{ |o| o.sort_value }.each do |object|
+          grant_collection << object
+        end
+
+        return if grant_collection.empty?
+
+        self << grant_collection
+        @summary['GRANT'] -= 1    # no need to count collection object
+      end
+
+      def embed_indexes
+        Log.info("Embedding indexes...")
+
+        @data['INDEX'].values.each do |object|
+          @data['TABLE'][object.table_name].add_object(object)
+        end
+      end
+
+      def embed_constraints
+        Log.info("Embedding constraints...")
+
+        @data["CONSTRAINT"].values.each do |constraint|
+          @data['TABLE'][constraint.table_name].add_object(constraint)
+        end
+      end
+
+      def embed_triggers
+        Log.info("Embedding triggers...")
+
+        @data["TRIGGER"].values.each do |object|
+          table_object = @data['TABLE'][object.table_name]
+
+          if table_object
+            table_object.add_object(object)
+          else
+            # if there is no table relation, just extract as default
+            object.extract_type = :default
+          end
+        end
+      end
+
+      def merge_constraints
+        Log.info("Merging constraints...")
+        constraint_collection = ConstraintCollection.new(type: 'CONSTRAINT', name: 'ALL FOREIGN KEYS')
+
+        @data['CONSTRAINT'].values.each do |object|
+          next unless object.extract_type == :merged
+          constraint_collection << object
+        end
+
+        return if constraint_collection.empty?
+
+        self << constraint_collection
+        @summary['CONSTRAINT'] -= 1    # no need to count collection object
+      end
+
+      def handle_table_data(args)
+        Log.info("Handling table data...")
+
+        @exclude_data = args[:exclude_data] if args[:exclude_data]
+        @include_data = args[:include_data] if args[:include_data]
+
+        tables = []
+        @data['TABLE'].values.each do |table|
+          next if table.system_object?
+          next if table.name =~ @exclude_data if @exclude_data
+          next unless table.name =~ @include_data if @include_data
+          tables << table
+        end
+
+        self << TableDataCollection.new(name: 'ALL CORE DATA', type: 'DATA', tables: tables)
+        @summary['DATA'] -= 1 # no need to count DATA object
+      end
+
+      def default_each
+        @data.keys.sort_by{ |type| type_sequence(type) }.each do |type|
+          @data[type].keys.sort.each do |name|
+            object = @data[type][name]
+            next if object.system_object?
+            next unless object.extract_type == :default
+            yield(object)
+          end
+        end
+      end
+
+      def reverse_default_each
+        @data.keys.sort_by{ |type| type_sequence(type) }.reverse_each do |type|
+          @data[type].keys.sort.each do |name|
+            object = @data[type][name]
+            next if object.system_object?
+            next unless object.extract_type == :default
+            yield object
+          end
+        end
+      end
+
+      def summary_each
+        @summary.each_pair do |type, count|
+          yield type, count
+        end
+      end
+
+      def invalids?
+        @invalids.keys.size > 0
+      end
+
+      def invalid_each
+        @invalids.each_pair do |type, objects|
+          yield type, objects
+        end
+      end
+
+
+      def self.all
+        objects = []
+
+        # get all objects as a hash containing name, type, status
+        items = []
+        types = []
+        connection = Connection.instance.get
+        cursor = connection.exec(OBJECT_QUERY)
+        cursor.fetch_hash do |item|
+          items << item
+          types << item['OBJECT_TYPE']
+        end
+        cursor.close
+
+        # sort items and make an object instance
+        items.sort_by{ |i| [ type_sequence(i['OBJECT_TYPE']), i['OBJECT_NAME']]}.each do |item|
+          objects << Base.from_type(item)
+        end
+
+        Log.info("Objects: #{items.size}, Object Types: #{types.uniq.size}")
+
+        objects
+      ensure
+        connection.logoff if connection # closes logical connection
+      end
+
+    end
+  end
+end