jungle_path 0.0.0

data/lib/jungle_path/db_access/import/db_dir.rb

@@ -0,0 +1,74 @@
+module JunglePath
+	module DBAccess
+		module Import
+			require 'jungle_path/db_access/import/delete'
+			require 'jungle_path/db_access/import/insert'
+			require 'jungle_path/db_access/import/schema'
+			require 'jungle_path/db_access/import/select'
+			
+			class DBDir
+		    attr_reader :config
+		    attr_reader :path
+		    attr_reader :log_path
+		    attr_reader :postgresql
+		    attr_reader :select
+		    attr_reader :insert
+		    attr_reader :delete
+		    attr_reader :schema
+
+		    def initialize(config, logger=nil)
+		      @logger = logger
+		      @config = config
+		      @path = config.path
+		      @log_path = config.log_path
+		      @postgresql = config.postgresql
+		      Dir.mkdir(path) unless Dir.exist?(path)
+		      clear_import_files
+		      @select = JunglePath::DBAccess::Import::Select.new @config, @logger
+		      @insert = JunglePath::DBAccess::Import::Insert.new @config, @logger
+		      @delete = JunglePath::DBAccess::Import::Delete.new @config, @logger
+		      @schema = JunglePath::DBAccess::Import::Schema.new @config, @logger
+		    end
+
+		    def reset_sequence_for_table(table_name)
+		      @insert.db.reset_sequence_for_table(table_name)
+		    end
+
+		    def transaction
+		      begin_transaction
+		      yield
+		      commit_transaction
+		    rescue
+		      rollback_transaction
+		      raise
+		    end
+
+		    def clear_import_files
+		      if Dir.exist?(path)
+		        data_files = Dir.glob(File.join(path, "*.dat"))
+		        data_files.each do |file_name|
+		          File.delete file_name
+		        end
+		      end
+		    end
+
+		    private
+
+		    def begin_transaction
+		      # not implemented :)
+		    end
+
+		    def commit_transaction
+		      @insert.close_files
+		      @insert.write_to_db
+		      @insert.reset
+		    end
+
+		    def rollback_transaction
+		      @insert.close_files
+		      @insert.reset
+		    end
+		  end
+		end
+	end
+end

data/lib/jungle_path/db_access/import/delete.rb

@@ -0,0 +1,30 @@
+module JunglePath
+	module DBAccess
+		require 'jungle_path/db_access/io/db'
+
+		module Import
+			require 'jungle_path/db_access/import'
+
+			class Delete
+		    def initialize(config, logger=nil)
+		      @logger = logger
+		      @path = config.path
+		      @db = nil
+		      if config.postgresql
+		        @db = JunglePath::DBAccess::IO::DB.new(config.postgresql, @logger)
+		      end
+		    end
+
+		    def _models(model, force=false)
+		      if force
+		        file_name = JunglePath::DBAccess::Import.data_file_name(@path, model.class.table_name)
+		        File.delete file_name if File.exist?(file_name)
+		        if @db
+		          @db.delete._models(model, force)
+		        end
+		      end
+		    end
+		  end
+		end
+	end
+end

data/lib/jungle_path/db_access/import/insert.rb

@@ -0,0 +1,168 @@
+module JunglePath
+	module DBAccess
+		require 'jungle_path/db_access/io/db'
+
+		module Import
+			require 'jungle_path/db_access/import'
+			
+			class Insert
+		    attr_reader :db
+		    def initialize(config, logger=nil)
+		      @logger = logger
+		      @path = config.path
+		      @db = nil
+		      if config.postgresql
+		        @db = JunglePath::DBAccess::IO::DB.new(config.postgresql, logger)
+		      end
+		      reset
+		    end
+
+		    def _model(model)
+		      table_name = model.class.table_name
+		      file = open_file(model)
+		      @counters[table_name] += 1
+		      if model.respond_to?(:id) and model.id == nil
+		        model.id = @counters[table_name]
+		      end
+		      file.puts model_values_to_string(model)
+		      puts "    #{table_name} rows processed count: #{@counters[table_name]}." if @counters[table_name] % 1000 == 0
+		      model
+		    end
+
+		    def open_file(model)
+		      table_name = model.class.table_name
+		      file = @files[table_name]
+		      unless file
+		        file = File.open(JunglePath::DBAccess::Import.data_file_name(@path, table_name), 'a')
+		        @models[table_name] = model
+		        @files[table_name] = file
+		        @counters[table_name] = 0
+		        puts "writing file: #{table_name}."
+		      end
+		      file
+		    end
+
+		    def close_files
+		      @files.each do |key, file|
+		        unless file.closed?
+		          file.fsync
+		          file.close
+		          puts "    #{key} rows processed count: #{@counters[key]}."
+		        end
+		      end
+		    end
+
+		    def reset
+		      @models = {}
+		      @files = {}
+		      @counters = {}
+		    end
+
+		    def write_to_db
+		      if @db
+		        @models.each do |key, model|
+		          file_name = JunglePath::DBAccess::Import.data_file_name(@path, model.class.table_name)
+		          @db.copy.into_table model.class, file_name
+		          File.delete file_name
+		        end
+		      end
+		    end
+
+		    def model_values_to_string(model)
+		      values = []
+		      model._columns.each do |key, column|
+		        value = model._values[key]
+		        if column.base_type == :json or column.base_type == :jsonb
+		          values << json_dump(value)
+		        else
+		          values << value_to_string(value)
+		        end
+		      end
+		      values.join("\t")
+		    end
+
+		    def string_fix(value)
+		      v = value.gsub('\\', '\\\\\\\\')
+		      v = v.gsub('"', '\\\\\"')
+		      v = v.gsub("\n", '\n')
+		      v = v.gsub("\r", '\r')
+		      v = v.gsub("\t", '\t')
+		    end
+
+		    def json_dump(value)
+		      if value.class == Hash
+		        json_dump_hash(value)
+		      elsif value.class == Array
+		        json_dump_array(value)
+		      else
+		        value_to_string(value, true)
+		      end
+		    end
+
+		    def json_dump_hash(hash)
+		      a = []
+		      hash.each do |key, value|
+		        if value.class == Hash
+		          #a << json_dump_hash(value)
+		          a << "\"#{key}\":#{json_dump_hash(value)}"
+		        elsif value.class == Array
+		          #a << json_dump_array(value)
+		          a << "\"#{key}\":#{json_dump_array(value)}"
+		        else
+		          a << "\"#{key}\":#{value_to_string(value, true)}"
+		        end
+		      end
+		      "{#{a.join(',')}}"
+		    end
+
+		    def json_dump_array(array)
+		      values = []
+		      array.each do |value|
+		        if value.class == Hash
+		          #a << json_dump_hash(value)
+		          a << "\"#{key}\":#{json_dump_hash(value)}"
+		        elsif value.class == Array
+		          #a << json_dump_array(value)
+		          a << "\"#{key}\":#{json_dump_array(value)}"
+		        else
+		          values << "#{value_to_string(value, true)}"
+		        end
+		      end
+		      "[#{values.join(',')}]"
+		    end
+
+		    def value_to_string value, json=false
+		      v = nil
+		      if value == nil
+		        v = '\N'
+		      elsif value.class == String
+		        v = string_fix(value)
+		        v = "\"#{v}\"" if json
+		      elsif value.class == DateTime
+		        v = value.strftime("%Y-%m-%d %H:%M:%S.%6N")
+		        v = "\"#{v}\"" if json
+		      elsif value.class == Date
+		        v = value.strftime("%Y-%m-%d")
+		        v = "\"#{v}\"" if json
+		      elsif value.class == Fixnum
+		        v = "#{value}"
+		      elsif value.class == Float
+		        v = "#{value}"
+		      elsif value.class == Integer
+		        v = "#{value}"
+		      elsif value.class == TrueClass
+		        v = "t"
+		        v = "\"#{v}\"" if json
+		      elsif value.class == FalseClass
+		        v = "f"
+		        v = "\"#{v}\"" if json
+		      else
+		        v = "#{value}"
+		        v = "\"#{v}\"" if json
+		      end
+		      v
+		    end
+		  end
+		end
+	end
+end

data/lib/jungle_path/db_access/import/schema.rb

@@ -0,0 +1,34 @@
+module JunglePath
+	module DBAccess
+		require 'jungle_path/db_access/io/db'
+
+		module Import
+			require 'jungle_path/db_access/import'
+
+			class Schema
+				def initialize(config, logger=nil)
+					@logger = logger
+					@path = config.path
+					@db = nil
+					if config.postgresql
+						@db = JunglePath::DBAccess::IO::DB.new(config.postgresql, @logger)
+					end
+				end
+
+				def drop_table(table_class)
+					file_name = JunglePath::DBAccess::Import.data_file_name(@path, table_class.table_name)
+					File.delete file_name if File.exist?(file_name)
+					if @db
+						@db.schema.drop_table(table_class)
+					end
+				end
+
+				def create_table(table_class)
+					if @db
+						@db.schema.create_table(table_class)
+					end
+				end
+			end
+		end
+	end
+end

data/lib/jungle_path/db_access/import/select.rb

@@ -0,0 +1,68 @@
+module JunglePath
+	require 'jungle_path/controller/base'
+
+	module DBAccess
+		require 'jungle_path/db_access/io/db'
+
+		module Import
+			require 'jungle_path/db_access/import'
+
+			class Select
+				def initialize(config, logger=nil)
+					@logger = logger
+					@path = config.path
+					@db = nil
+					if config.postgresql
+						@db = JunglePath::DBAccess::IO::DB.new(config.postgresql, logger)
+					end
+				end
+
+				def _model(model)
+					nil
+				end
+
+				def _model_by_any(model)
+					nil
+				end
+
+				def _models(model)
+					if @db
+						@db.select._models(model)
+					else
+						from_file(model)
+					end
+				end
+
+				def from_file(model)
+					# select all of a given model.
+					file_name = JunglePath::DBAccess::Import.data_file_name(@path, model.class.table_name)
+					puts "file_name: #{file_name}."
+					models = []
+					lines = File.readlines(file_name)
+					puts "lines.count: #{lines.count}."
+					lines.each do |line|
+						values = line.split("\t")
+						hash = to_hash(model, values)
+						params = JunglePath::Controller::Base.transform(hash, model.class.columns)
+						models << model.class.new(params, false)
+					end
+					models
+				end
+
+				def to_hash(model, values)
+					hash = {}
+					i = 0
+					model._columns.keys.each do |key|
+						value = values[i]
+						if value == '\N'
+							value = nil
+						end
+						hash[key] = value
+						i += 1
+					end
+					hash
+				end
+			end
+		end
+	end
+end

data/lib/jungle_path/db_access/import.rb

@@ -0,0 +1,15 @@
+#require 'pry-byebug'
+module JunglePath
+	module DBAccess
+		module Import
+			require 'jungle_path/db_access/import/db_dir'
+			require 'jungle_path/db_access/import/delete'
+			require 'jungle_path/db_access/import/insert'
+			require 'jungle_path/db_access/import/schema'
+			require 'jungle_path/db_access/import/select'
+		  def self.data_file_name(path, table_name)
+		    File.join(path, "#{table_name}.dat")
+		  end
+		end
+	end
+end

data/lib/jungle_path/db_access/io/chunked_file_reader.rb

@@ -0,0 +1,62 @@
+module JunglePath
+	module DBAccess
+		module IO
+			class ChunkedFileReader
+				# yield up chunks of lines. treat entire chunk as good or bad.
+				# if chunk size is 1 line and it is bad move start_at forward by 1 (chunk size).
+				# if chunk was good move start_at forward by chunk size.
+				attr_reader :start_at, :chunk_size, :data_file, :done, :line
+				def initialize data_file
+					@data_file = data_file
+					@start_at = 1
+					@chunk_size = 1000
+					@done = false
+					@line = nil
+				end
+
+				def was_bad_chunk
+					if @chunk_size == 1
+						@start_at = @start_at + 1
+					else
+						@chunk_size = @chunk_size / 2
+						if @chunk_size < 1
+							@chunk_size = 1
+						end
+					end
+				end
+
+				def was_good_chunk
+					@start_at = @start_at + @chunk_size
+					@chunk_size = @chunk_size * 2
+					if @chunk_size > 1000000
+						@chunk_size = 1000000
+					end
+				end
+
+				def each
+					@line = nil
+					pointer = 0
+					yield_size = 0
+					File.open(@data_file, 'r') do |file|
+						puts "reading: #{data_file}."
+						while line = file.gets
+							pointer += 1
+							if pointer == @start_at
+								puts "starting chunk at #{pointer}."
+							end
+							puts "    at line number #{pointer}." if pointer % 10000 == 0
+							if pointer >= @start_at and yield_size < @chunk_size
+								@line = line
+								yield_size += 1
+								yield line
+							end
+						end
+						if yield_size == 0
+							@done = true
+						end
+					end
+				end
+			end
+		end
+	end
+end

data/lib/jungle_path/db_access/io/config.rb

@@ -0,0 +1,19 @@
+module JunglePath
+	module DBAccess
+		module IO
+			class Config
+				attr_accessor :name, :type, :user_name, :password, :host, :extensions, :port, :options
+				def initialize name: nil, type: nil, user_name: nil, password: nil, host: nil, extensions: [], port: nil, options: nil
+					@name = name
+					@type = type
+					@user_name = user_name
+					@password = password
+					@host = host
+					@extensions = extensions
+					@port =  port
+					@options = options
+				end
+			end
+		end
+	end
+end

data/lib/jungle_path/db_access/io/copy.rb

@@ -0,0 +1,73 @@
+require 'sequel'
+
+module JunglePath
+	module DBAccess
+		module IO
+			require 'jungle_path/db_access/io/chunked_file_reader'
+			require 'jungle_path/db_access/io/init_db'
+
+			class Copy
+				include JunglePath::DBAccess::IO::InitDB
+
+				def from_table model_class
+					nil
+				end
+
+				def into_table model_class, data_file
+					done = false
+					loops = 0
+					until done
+						loops +=1
+						puts "loops: #{loops}."
+						error = nil
+						file_reader = JunglePath::DBAccess::IO::ChunkedFileReader.new data_file unless file_reader # must work with chunks of lines since we won't know which exact line was bad if there is a postgresql error.
+						begin
+							@db.copy_into(model_class.table_name, :data => file_reader)
+						rescue Sequel::ForeignKeyConstraintViolation => e
+							error = e
+						rescue Sequel::UniqueConstraintViolation => e
+							error = e
+						rescue Sequel::DatabaseError => e
+							error = e
+						rescue Exception => e
+							error = e
+						end
+						if error
+							if file_reader.chunk_size == 1
+								log_error "bad data: exception: #{error.class}."
+								log_error "  message: #{error.message.gsub("\n", '\n').gsub("\r", '\r')}."
+								log_error "  was_bad_chunk: start_at: #{file_reader.start_at}, chunk_size: #{file_reader.chunk_size}."
+								line = file_reader.line
+								log_error "  line: #{line.gsub("\n", '\n').gsub("\r", '\r')}."
+								parts = line.split("\t")
+								parts.each_with_index do |part, index|
+									log_error "    #{index}: part: #{part}."
+								end
+							end
+							file_reader.was_bad_chunk
+						else
+							puts "was_good_chunk: start_at: #{file_reader.start_at}, chunk_size: #{file_reader.chunk_size}."
+							file_reader.was_good_chunk
+						end
+						done = file_reader.done
+						if done
+							log "done with #{model_class.table_name}."
+						end
+					end
+				end
+
+				def log_error(msg)
+					#binding.pry
+					puts "log error: #{msg}"
+					@logger.error(msg)
+				end
+
+				def log(msg)
+					#binding.pry
+					puts "log info: #{msg}"
+					@logger.info(msg) if @logger
+				end
+			end
+		end
+	end
+end

data/lib/jungle_path/db_access/io/db.rb

@@ -0,0 +1,82 @@
+module JunglePath
+	module DBAccess
+		module IO
+			require 'jungle_path/db_access/io'
+			require 'jungle_path/db_access/io/select'
+			require 'jungle_path/db_access/io/insert'
+			require 'jungle_path/db_access/io/update'
+			require 'jungle_path/db_access/io/delete'
+			require 'jungle_path/db_access/io/schema'
+			require 'jungle_path/db_access/io/copy'
+
+			class DB
+				attr_reader :select
+				attr_reader :insert
+				attr_reader :update
+				attr_reader :delete
+				attr_reader :schema
+				attr_reader :copy
+				attr_reader :base
+				attr_reader :database_name
+				attr_reader :config
+				attr_reader :postgresql
+
+				def initialize(config, logger=nil)
+					@logger = logger
+					@config = config
+					@postgresql = config
+					@db = JunglePath::DBAccess::IO.connection(
+						database_type: config.type,
+						user_name: config.user_name,
+						database_name: config.name,
+						host: config.host,
+						extensions: config.extensions,
+						password: config.password,
+						port: config.port,
+						options: config.options
+					)
+					@select = JunglePath::DBAccess::IO::Select.new @db, @logger
+					@insert = JunglePath::DBAccess::IO::Insert.new @db, @logger
+					@update = JunglePath::DBAccess::IO::Update.new @db, @logger
+					@delete = JunglePath::DBAccess::IO::Delete.new @db, @logger
+					@schema = JunglePath::DBAccess::IO::Schema.new @db, @logger
+					@copy = JunglePath::DBAccess::IO::Copy.new @db, @logger
+					@base = @db
+					@database_name = config.name
+				end
+
+				def reset_sequence_for_table(table_name)
+					#max = @db[table_name.to_sym].max(:id) + 1
+					#@db.run "alter sequence #{table_name}_id_seq restart with #{max}"
+					@db.run "select setval('#{table_name}_id_seq', (select max(id)+1 from \"#{table_name}\"), false)"
+				end
+
+				def get_max_id_for_table(table_name, id_column_name=:id)
+					ds = @db["select max(#{id_column_name}) as max_id from \"#{table_name}\""]
+					result = ds.first
+					max_id = result[:max_id]
+				end
+
+				def transaction
+					@db.transaction do
+						yield
+					end
+				end
+
+				private
+
+				def begin_transaction
+					# not implemented :)
+				end
+
+				def commit_transaction
+					# not implemented :)
+				end
+
+				def rollback_transaction
+					# not implemented :)
+				end
+			end
+		end
+	end
+end

data/lib/jungle_path/db_access/io/delete.rb

@@ -0,0 +1,23 @@
+module JunglePath
+	module DBAccess
+		module IO
+			require 'jungle_path/db_access/io/init_db'
+
+			class Delete
+				include JunglePath::DBAccess::IO::InitDB
+
+				def _model(model)
+					ds = @db[model._table_name].where(model._primary_key)
+					#puts ds.sql
+					ds.delete
+				end
+
+				def _models(model, confirm=false)
+					ds = @db[model._table_name].where(handle_json_columns(model, model._has_value_hash))
+					return ds.delete if confirm
+					return ds.sql
+				end
+			end
+		end
+	end
+end

data/lib/jungle_path/db_access/io/init_db.rb

@@ -0,0 +1,39 @@
+require 'sequel'
+
+module JunglePath
+	module DBAccess
+		module IO
+			module InitDB
+				def initialize(db, logger=nil)
+					@db = db
+					@logger = logger
+				end
+
+				def handle_json_columns(model, hash)
+					new_hash = {}
+					columns = model.class.columns
+					hash.each do |key, value|
+						if columns[key].type == :json
+							unless value == nil
+								#if value.class == String # must convert to hash... (assuming this is a string of json...)
+								#  value = JsonWrap.load value, {:mode => :compat, :symbol_keys => true, :indent=>0 }
+								#end
+								new_hash[key] = Sequel.pg_json(value)
+							end
+						elsif columns[key].type == :jsonb
+							unless value == nil
+								#if value.class == String # must convert to hash... (assuming this is a string of json...)
+								#  value = JsonWrap.load value, {:mode => :compat, :symbol_keys => true, :indent=>0 }
+								#end
+								new_hash[key] = Sequel.pg_jsonb(value)
+							end
+						else
+							new_hash[key] = value unless value == nil
+						end
+					end
+					new_hash
+				end
+			end
+		end
+	end
+end