dbtools 0.5.2

data/lib/tasks/import.rb

@@ -0,0 +1,142 @@
+require 'yaml'
+require 'thor'
+require 'find'
+require 'dbtools/rdf/rdf_reader'
+require 'dbtools/constants'
+require 'dbtools/converter/csv_importer'
+require 'dbtools/converter/excel2csv_converter'
+require 'dbtools/database/mysql_connection'
+require 'dbtools/database/postgresql_connection'
+require 'dbtools/google_drive/google_drive_api'
+require 'fileutils'
+
+module Dbtools
+  class Import < Thor
+    package_name "dbtools"
+
+    def initialize(*args)
+      super
+      load_config
+    end
+
+    # Load a database dump into Postgres.
+    desc 'postgres_dump [database_name. dump_file]', 'Load a database dump into Postgres. Database will be created if it doesn\'t exist' 
+    def postgres_dump(database_name, file)
+      system "createdb #{@postgres_connection_options} #{database_name}"
+      system "#{@postgres_connection_command} --dbname=#{database_name} --file=#{file}"
+      return "#{@postgres_connection_url}#{database_name}"
+    end
+
+    desc 'mysql_dump [database_name. dump_file]', 'Load a database dump into MySQL. Database will be created if it doesn\'t exist' 
+    def mysql_dump(database_name, file)
+      system "#{@mysql_connection_command} -e 'CREATE DATABASE #{database_name}'"
+      system "#{@mysql_connection_command} < #{file}"
+      return "#{@mysql_connection_url}#{database_name}"
+    end
+
+    desc 'convert_to_utf8 [file]', 'Converts a file to UTF-8 encoding. Replaces the original file.' 
+    def convert_to_utf8(filename)
+      puts "Converting #{filename} to utf8"
+
+      #filename_utf8_encoded = "#{filename}_utf8"
+      #system "iconv -f utf-8 -t utf-8 -c < #{filename} > #{filename_utf8_encoded}"
+
+      filename_tmp = filename + '.tmp'
+      filename_old = filename + '.old'
+
+      begin
+        File.open(filename_tmp, 'w') do |file_out|
+          File.foreach(filename) do |line_input|
+            file_out.puts(line_input.encode!('UTF-8', 'UTF-8', :invalid => :replace))
+          end
+        end
+        # Swap filenames
+        File.rename(filename, filename_old)
+        File.rename(filename_tmp, filename)
+        File.rename(filename_old, filename_tmp)
+      rescue Exception => e
+        raise "Failed to convert encoding of file #{filename} to UTF-8 because #{e.message}. "
+      ensure 
+        File.delete(filename_tmp)
+      end
+    end
+
+    desc 'create_schema [csv_file, table_name]', 'Creates a schema based on the csv headers and values in the table' 
+    def create_schema(csv_file, table_name)
+      csv = Dbtools::Converter::Csv_importer.new(csv_file, tablename: table_name)
+      puts csv.to_sql_schema_script
+    end
+
+    desc 'csv_in postgres [csv_file, database_name, table_name]', 'Imports a csv file into a postgres database. Automatically creates the database. '
+    def csv_in_postgres(csv_file, database_name, table_name)
+      # Converts file to utf8
+      convert_to_utf8(csv_file)
+      # Sanitize table name
+      table_name = table_name.downcase.gsub(/[^0-9a-zA-Z_]/,'_')
+
+      # Creates a table schema based on the csv.
+      csv = Dbtools::Converter::Csv_importer.new(csv_file, tablename: table_name)
+      csv_schema_command = csv.to_sql_schema_script
+      #copy_csv_command = "\\COPY \"#{table_name}\" FROM '#{csv_file}' DELIMITER '#{csv.delimiter}' CSV HEADER;"
+
+      #system "createdb #{@postgres_connection_options} #{database_name}"
+
+      puts "Importing #{csv_file} into #{database_name}.#{table_name}"
+      # Create the table and copy the csv contents into the table. 
+      begin
+        psql = Dbtools::Database::PostgresqlConnection.new("#{@postgres_connection_url}")
+        psql.create_database(database_name)
+        psql.close 
+      # Create the database
+
+        psql = Dbtools::Database::PostgresqlConnection.new("#{@postgres_connection_url}#{database_name}")
+        psql.execute_query(csv_schema_command)
+        psql.copy_from_file(csv_file, table_name, csv.delimiter)
+        psql.execute_files(Dbtools::Constants::PLSQL_FUNCTIONS_DIR)
+      ensure
+        psql.close
+      end
+      return "#{@postgres_connection_url}#{database_name}"
+    end
+
+    desc 'excel [database_name, file]', 'Loads an excel file as a csv into the database. '
+    def excel(database_name, file)
+      folder = File.join("/tmp", database_name)
+      # Convert spreadsheets to Excel. 
+      files = excel2csv(file, folder)
+      result = ''
+      files.each do |sheet_name, csv_file|
+        begin
+          # Load every file in Postgres
+          table_name = "#{File.basename(csv_file, File.extname(csv_file))}_#{sheet_name}"
+          result = csv_in_postgres(csv_file, database_name, table_name)
+        rescue Exception => e
+          puts "Could not load #{csv_file} into #{database_name} because #{e.message}. "
+        end
+      end
+      FileUtils.remove_entry_secure(folder)
+      return result
+    end
+
+    desc 'excel2csv [file, target_dir]', 'Converts an Excel file to CSV files. Every spreadsheet will be a CSV file.'
+    def excel2csv(file, target_dir)
+      converter = Dbtools::Converter::Excel2csv_converter.new(file)
+      files = converter.output(target_dir)
+      return files
+    end
+
+
+    private
+    # Loads all configurations needed
+    def load_config
+      config = YAML.load(File.read(Dbtools::Constants::DB_TARGET_CONFIG_PATH))
+      postgres_config = config['postgres']
+      mysql_config = config['mysql']
+      @postgres_connection_url = "postgres://#{postgres_config['username']}:#{postgres_config['password']}@#{postgres_config['host']}/"
+      @mysql_connection_url = "mysql2://#{mysql_config['username']}:#{mysql_config['password']}@#{mysql_config['host']}/"
+      @postgres_connection_options = "--username=#{postgres_config['username']} --host=#{postgres_config['host']} --port=#{postgres_config['port']}"
+      @postgres_connection_command = "psql #{@postgres_connection_options}"
+      @mysql_connection_command = "mysql -u #{mysql_config['username']} -p#{mysql_config['password']} -h #{mysql_config['host']}"
+    end
+  end
+end

data/lib/tasks/postgres.rb

@@ -0,0 +1,29 @@
+require 'dbtools/database/postgresql_connection'
+require 'thor'
+require 'dbtools/constants'
+
+module Dbtools
+  class Postgres < Thor
+    package_name "dbtools"
+
+    desc 'load_functions [database_url]', 'Deploys the catalog of pl/sql functions in the specified database.'
+    def load_functions(database_url)
+      db = check_adapter(database_url)
+      # All functions are located in this directory. 
+      db.execute_files(Dbtools::Constants::PLSQL_FUNCTIONS_DIR)
+      db.close
+    end
+
+    private
+    # Check if the url is a postgres or mysql connection.
+    def check_adapter(url)
+      adapter = url.match("^([a-zA-Z0-9]+):\/\/(.+)@(.+)\/(.*)").captures[0]
+      if adapter == "postgres"
+        return Dbtools::Database::PostgresqlConnection.new(url)
+      else
+        puts "Invalid url"
+        return nil
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,307 @@
+--- !ruby/object:Gem::Specification
+name: dbtools
+version: !ruby/object:Gem::Version
+  version: 0.5.2
+platform: ruby
+authors:
+- Hung Nguyen
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-09-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rdf-reasoner
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.1
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.19.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.19.1
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+- !ruby/object:Gem::Dependency
+  name: activemodel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+- !ruby/object:Gem::Dependency
+  name: google-api-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: linkeddata
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: whenever
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: roo
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
+- !ruby/object:Gem::Dependency
+  name: roo-xls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: ru_bee
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: mysql2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.18'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.18'
+- !ruby/object:Gem::Dependency
+  name: spira
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+description: 
+email:
+- h.nguyen@geophy.com
+executables:
+- dbtools
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- Thorfile
+- bin/dbtools
+- config/client_secret_dbtools.json
+- config/config.yml
+- config/database_config.yml
+- config/databases.txt
+- config/schedule.rb
+- dbtools.gemspec
+- lib/dbtools.rb
+- lib/dbtools/constants.rb
+- lib/dbtools/converter/csv2rdf_converter.rb
+- lib/dbtools/converter/csv_importer.rb
+- lib/dbtools/converter/excel2csv_converter.rb
+- lib/dbtools/converter/google_drive2_rdf_converter.rb
+- lib/dbtools/database/database_data.rb
+- lib/dbtools/database/db_connection.rb
+- lib/dbtools/database/mysql_connection.rb
+- lib/dbtools/database/postgresql_connection.rb
+- lib/dbtools/database/violation.rb
+- lib/dbtools/google_drive/google_drive_api.rb
+- lib/dbtools/google_drive/google_drive_entity.rb
+- lib/dbtools/google_drive/google_drive_file.rb
+- lib/dbtools/google_drive/google_drive_folder.rb
+- lib/dbtools/plsql_functions/connect_server.sql
+- lib/dbtools/plsql_functions/link.sql
+- lib/dbtools/plsql_functions/unlink.sql
+- lib/dbtools/rdf/rdf_reader.rb
+- lib/dbtools/version.rb
+- lib/rdf/geophy.rb
+- lib/tasks/aws.rb
+- lib/tasks/backup.rb
+- lib/tasks/check.rb
+- lib/tasks/ckan.rb
+- lib/tasks/convert.rb
+- lib/tasks/dump.rb
+- lib/tasks/googledrivetool.rb
+- lib/tasks/import.rb
+- lib/tasks/postgres.rb
+homepage: 
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.10
+signing_key: 
+specification_version: 4
+summary: Tool for database maintenance.
+test_files: []