dbtools 0.5.2

data/lib/dbtools/converter/csv2rdf_converter.rb

@@ -0,0 +1,68 @@
+require 'csv'
+require 'rdf'
+
+module Dbtools::Converter
+  class Csv2rdf_converter
+
+    # Constructor for the csv2rdf converter.
+    # @param filename
+    #   Filename of the csv file that needs to be converted.
+    # @param uri
+    #   RDF URI for the subject. This will be prepended with the row number.
+    #   Example:
+    #     uri = 'http://example.org/fileid'
+    #     <http://example.org/fileid#123> <predicate> "value"
+    # @param default_vocabulary
+    #   Base vocabulary for the column names.
+    #   Example:
+    #     default_vocabulary = "http://geophy.io/"
+    #     <subject> <http://geophy.io/column1> "value"
+    def initialize(filename, uri, default_vocabulary: "http://geophy.io/", options: {})
+      @uri = uri
+      @default_vocabulary = default_vocabulary
+      delimiter = options[:col_sep]
+      delimiter ||= guess_delimiter(filename)
+      opts = { :headers => true,
+               :header_converters => :symbol,
+               :converters => :all,
+               :col_sep => delimiter,
+               :skip_blanks => true
+      }.merge(options)
+      @csv = CSV.open(filename, opts)
+    end
+
+    # Converts the current row to rdf triples.
+    def each_triple
+      @csv.each do |row|
+        lineno = @csv.lineno
+        #print a triple with the row id
+        rdf = RDF::Statement({ subject:   RDF::URI.new("#{@uri}##{lineno}"),
+                               predicate: RDF::URI.new("#{@default_vocabulary}rid"),
+                               object:    lineno
+                             })
+        yield rdf.to_ntriples
+        row.each do |colname, colvalue|
+          next if colvalue.nil? or colvalue.to_s.empty?
+          rdf = RDF::Statement({ subject:   RDF::URI.new("#{@uri}##{lineno}"),
+                                 predicate: RDF::URI.new(File.join(@default_vocabulary, colname.to_s)),
+                                 object:    colvalue
+                               })
+          yield rdf.to_ntriples
+          # yield "#{subject} #{predicate} #{object} ."
+        end
+      end
+    end
+
+    # Attempt to guess delimiter based on occurrence in the header.
+    def guess_delimiter(filename)
+      delimiters = [',', '|', "\t", ';']
+      lines = File.foreach("#{filename}").first(10).join
+      delimiters_count = delimiters.map { |x| [x, lines.count(x)] }.to_h
+      # Key is the delimiter, value is the occurence.
+      most_likely_delimiter = delimiters_count.max_by { |k, v| v }
+      # Check if the occurrence is not zero.
+      raise "No delimiter detected. " if most_likely_delimiter[1].zero?
+      return most_likely_delimiter.first
+    end
+  end
+end

data/lib/dbtools/converter/csv_importer.rb

@@ -0,0 +1,107 @@
+require 'csv'
+require 'time'
+
+module Dbtools::Converter
+  class Csv_importer
+    attr_reader :tablename, :delimiter
+    def initialize(filename, delimiter: '', tablename: '')
+      @delimiter = guess_delimiter(filename) if delimiter.empty?
+      options = { :headers => true,
+                  :header_converters => :symbol,
+                  :converters => :all,
+                  :col_sep => @delimiter
+      }
+      csv = CSV.open(filename, options)
+      arr = Array.new
+      csv.take(10000).each do |row|
+        arr.push(row)
+      end
+      @data = CSV::Table.new(arr)
+      @tablename = tablename
+      @tablename = File.basename(filename, '.csv').gsub(/[^0-9a-zA-Z_]/,'_').to_sym if tablename.empty?
+      @types = Hash.new
+    end
+
+    # Try to infer the type of the columns, and store them.
+    def infer_type_of_columns
+      @data.by_col!.each do |colName, rows|
+        # Count all the types.
+        count = Hash.new
+        rows.each do |entry|
+          type = infer_type(entry)
+          count[type] = count[type].nil? ? 1 : count[type] + 1
+        end
+        # Set the type to the most occurring type.
+        most_occurring_type = count.sort_by(&:last).last
+        type = most_occurring_type.first
+
+        # Let float take precedence over integers if it occurred.
+        type = Float if type == Fixnum && !count[Float].nil?
+        # Let string take precedence over other types if it occurred.
+        type = String unless count[String].nil?
+        @types[colName] = type
+      end
+    end
+
+    # Infer the type of a value by using Ruby's internal type inference system.
+    def infer_type(entry)
+      type = (Time.parse(entry) rescue nil)
+      type = entry if type.nil?
+      return type.class
+    end
+
+    # Converts a ruby class to a string representing a SQL type.
+    def class_to_sql_type(klass)
+      # There's probably a better way to detect the type.. 
+      if klass == Fixnum
+        'BIGINT'
+      elsif klass == Float
+        'FLOAT'
+      #elsif klass == Time
+        #'DATE'
+      else
+        'VARCHAR(255)'
+      end
+    end
+
+    # Returns a sql schema script of the csv file.
+    def to_sql_schema_script
+      infer_type_of_columns if @types.empty?
+      output = %{CREATE TABLE IF NOT EXISTS "#{@tablename}" ( \n}
+      columns = @data.by_col!.map.with_index do |data, index|
+        colName, rows = data
+        # Use column position if no header is defined. 
+        colName = "col_#{index}" if (colName.nil? || colName.empty?)
+        # Check if column can be null.
+        nullable = rows.all? { |entry| !entry.to_s.gsub(/\s/, '').empty? }
+
+        sql_type = class_to_sql_type(@types[colName])
+
+        result = "\t#{colName.downcase} #{sql_type}" 
+        result << "\tNOT NULL" if nullable
+        result
+      end.join(", \n")
+      output << columns << "\n);\n"
+    end
+
+    # Writes the script to a file.
+    def output_schema_to_file(filename)
+      open(filename, 'w') { |f| f << to_sql_schema_script}
+      return filename
+    end
+
+    # Attempt to guess delimiter based on occurrence in the header.
+    def guess_delimiter(filename)
+      delimiters = [',', '|', "\t", ';']
+      lines = File.foreach("#{filename}").first(10).join
+      delimiters_count = delimiters.map { |x| [x, lines.count(x)] }.to_h
+      puts delimiters_count
+      
+      # Key is the delimiter, value is the occurence. 
+      most_likely_delimiter = delimiters_count.max_by { |k, v| v }
+      # Check if the occurrence is not zero. 
+      raise "No delimiter detected. " if most_likely_delimiter[1].zero?
+      return most_likely_delimiter.first
+    end
+  end
+end

data/lib/dbtools/converter/excel2csv_converter.rb

@@ -0,0 +1,40 @@
+require 'roo'
+require 'roo-xls'
+require 'csv'
+require 'fileutils'
+
+module Dbtools::Converter
+  class Excel2csv_converter
+
+    # Initialize the roo excel library.
+    def initialize(filename)
+      @excel = Roo::Spreadsheet.open(filename)
+      @excel_filename = File.basename(filename)
+    end
+
+    # Output all sheets in the excel to csv. Set the sheetname as the filename.
+    def output(folder)
+      FileUtils.mkdir_p(folder)
+      h = Hash.new
+      @excel.each_with_pagename do |sheetname, sheet|
+        filename = @excel_filename.gsub(/[^0-9a-zA-Z_-]/,'_') + "_" + sheetname.gsub(/[^0-9a-zA-Z_-]/,'_') + ".csv"
+        path = File.join(folder, filename)
+        output = File.open(path, "w") do |f|
+          f.write(sheet.to_csv)
+        end
+        h[sheetname] = path
+      end
+      @excel.close
+      return h
+    end
+
+    # Convert an excel sheet to csv, given the index.
+    def sheet2csv(sheet_index)
+      begin
+        @excel.sheet(sheet_index).to_csv
+      rescue ArgumentError => e
+        puts e.message
+      end
+    end
+  end
+end

data/lib/dbtools/converter/google_drive2_rdf_converter.rb

@@ -0,0 +1,97 @@
+require 'spira'
+require 'dbtools/google_drive/google_drive_folder'
+require 'dbtools/google_drive/google_drive_file'
+
+module Dbtools::Converter
+  class GoogleDrive2RDFConverter
+    def initialize
+      Spira.repository = RDF::Repository.new
+    end
+
+    # Converts a google drive file instance to RDF statements
+    # @param [Google::Apis::DriveV3::File] google_drive_file
+    #   Google drive file that will be converted.
+    # @return [String]
+    #   RDF ntriples in string format.
+    def drivefile2rdf(google_drive_file)
+      file = google_drive_file
+      result = []
+      Spira.repository ||= RDF::Repository.new
+
+      if file.mime_type == 'application/vnd.google-apps.folder'
+        # Google Folder specific attributes
+        uri = RDF::URI.new("https://drive.google.com/drive/folders/#{file.id}")
+        drive_file = uri.as(Dbtools::Google_Drive::GoogleDriveFolder)
+      else
+        # Google File specific attributes
+        uri = RDF::URI.new(Dbtools::Google_Drive::Google_drive_api.get_url_from_id(file.id))
+        drive_file = uri.as(Dbtools::Google_Drive::GoogleDriveFile)
+        drive_file.file_extension = file.file_extension
+        drive_file.web_content_link = file.web_content_link
+      end
+      # Shared attributes
+      drive_file.name = file.name
+      drive_file.identifier = file.id
+      drive_file.created_time = file.created_time
+      drive_file.mime_type = file.mime_type
+      drive_file.size = file.size
+      drive_file.modified_time = file.modified_time
+      drive_file.icon_link = file.icon_link
+      drive_file.description = file.description
+      drive_file.web_view_link = file.web_view_link
+      drive_file.trashed = file.trashed
+
+      # Assign all key-value pairs from properties attribute to the Spira resource.
+      # Could probably be done for all attributes above..
+      # Untested so commented to prevent errors
+      # file.properties.each do |key, value|
+      #   m = "#{key}="
+      #   drive_file.send(m, value) if drive_file.respond_to?(m)
+      # end if file.properties
+
+      # Add bi-directional relation for parents-children.
+      drive_file.parents = file.parents.map do |parent_id|
+        parent_uri = RDF::URI.new("https://drive.google.com/drive/folders/#{parent_id}")
+        parent_drive_folder = parent_uri.as(Dbtools::Google_Drive::GoogleDriveFolder)
+        parent_drive_folder.children << drive_file
+        result << parent_drive_folder
+        parent_drive_folder
+      end if file.parents
+
+      result << drive_file
+      return result
+    end
+
+    # Serializes a list of files to RDF statements.
+    # Yields rdf ntriples for every file.
+    # @param files
+    #     List of files to be converted. Default is all files.
+    # @param verbose
+    #     Prints progress if true
+    def serialize_as_rdf(files, verbose: true)
+      if verbose
+        total = files.size
+        count = 0
+      end
+      files.each do |file|
+        if verbose
+          count += 1
+          STDERR.puts("Converting file to rdf: #{count}/#{total}\t\r")
+        end
+        # get tree method returns a hash with [id, file].
+        file = file[1] if files.is_a?(Hash)
+        yield drivefile2rdf(file).map(&:to_ntriples).join("\n")
+      end
+    end
+
+    private
+    def get_variables(google_drive_file)
+      attributes = {}
+      google_drive_file.instance_variables.each do |var|
+        attributes[var.to_s.delete('@')] = google_drive_file.instance_variable_get(var)
+      end
+      return attributes
+    end
+
+  end
+end

data/lib/dbtools/database/database_data.rb

@@ -0,0 +1,146 @@
+require 'dbtools/constants'
+
+module Dbtools::Database
+  class DatabaseData
+    attr_reader :name, :tables
+    def initialize(name)
+      @name = name
+      @tables = Hash.new
+    end
+
+    # Add table if it doesn't exist yet.
+    def add_table(table_name, schema)
+      key = "#{schema}.#{table_name}"
+      @tables[key] = Table.new(table_name, schema) unless @tables.include?(table_name)
+      return @tables[key]
+    end
+
+    def to_s
+      output = "#{@name}: \n"
+      @tables.each do |k, v|
+        output << v.to_s << "\n"
+      end
+      return output
+    end
+  end
+
+  class Table
+    attr_reader :name, :schema, :columns
+    def initialize(name, schema)
+      @name = name
+      @schema = schema
+      @columns = Hash.new
+    end
+
+    # Put quotes around name to avoid casing problems.
+    def name
+      return "\"#{@name}\""
+    end
+
+    # Put quotes around name to avoid casing problems.
+    def schema
+      return "\"#{@schema}\""
+    end
+
+    # Add column if it doesn't exist yet.
+    def add_column(column_name, data_type)
+      @columns[column_name] = Column.new(column_name, name, schema, data_type) unless @columns.include?(column_name)
+      return @columns[column_name]
+    end
+
+    # Create a query to count all records that are empty.
+    def query_empty_records
+      query_columns = columns.values.map do |col|
+        %{SUM(CASE WHEN #{col.not_empty} THEN 1 ELSE 0 END) AS #{col.name}}
+      end.join(", \n")
+      query = unless query_columns.empty?
+                %{SELECT #{query_columns} FROM #{schema}.#{name}}
+              else
+                ''
+              end
+      return query
+    end
+
+    # Create a query to count all the records.
+    def query_total_records
+      return %{SELECT COUNT(*) FROM #{schema}.#{name}}
+    end
+
+    # Create a query to count all distinct lowercased values per column.
+    def query_distinct_lowercased_entries
+      # Skip unless column is a string column
+      query_columns = columns.values.map do |col|
+        %{count(distinct(lower(#{col.full_name}))) AS #{col.name}} if Dbtools::Constants::STRING_COLUMNS.include?(col.data_type)
+      end
+      # Remove nulls caused by skipping
+      query_columns.compact!
+      query_columns = query_columns.join(", \n")
+      query = unless query_columns.empty?
+                %{SELECT #{query_columns} FROM #{schema}.#{name}}
+              else
+                ''
+              end
+      return query
+    end
+
+    # Create a query to count all distinct values per column.
+    def query_distinct_entries
+      # Skip unless column is a string column
+      query_columns = columns.values.map do |col|
+        %{count(distinct(#{col.full_name})) AS #{col.name}} if Dbtools::Constants::STRING_COLUMNS.include?(col.data_type)
+      end
+      # Remove nulls caused by skipping
+      query_columns.compact!
+      query_columns = query_columns.join(", \n")
+
+      query = unless query_columns.empty?
+                %{SELECT #{query_columns} FROM #{schema}.#{name}}
+              else
+                ''
+              end
+      return query
+    end
+
+    def to_s
+      output = "+ #{@name}: \n"
+      @columns.each do |k, v|
+        output << v.to_s << "\n"
+      end
+      return output
+    end
+  end
+
+  class Column
+    attr_reader :name, :full_name, :data_type
+    attr_accessor :total_entries, :missing_entries, :distinct_entries, :distinct_lower_entries
+
+    def initialize(name, table_name, schema_name, data_type)
+      @name = name
+      @full_name = "#{schema_name}.#{table_name}.#{self.name}"
+      @total_entries = 0
+      @missing_entries = 0
+      @distinct_entries = 0
+      @distinct_lower_entries = 0
+      @data_type = data_type
+    end
+
+    def not_empty
+      statement = case @data_type
+                    when 'character varying', 'varchar'
+                      "#{full_name} IS NULL OR #{full_name} = \'\'"
+                    else
+                      "#{full_name} IS NULL"
+                  end
+      return statement
+    end
+
+    # Put quotes around name to avoid casing problems.
+    def name
+      return "\"#{@name}\""
+    end
+
+    def to_s
+      return "  - #{@name.ljust(40)}: \t #{missing_entries}, #{total_entries}"
+    end
+  end
+end