one-hot-dictionary 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 9220f9fc81b30d224a1393ccc7fcd6a2ed103e2725a7cd1d3167c05db377448a
+  data.tar.gz: ffe5c90869236274f1d50824a9b9df9ceb0182e237fcb142e1f66fa5406d1e75
+SHA512:
+  metadata.gz: 2bbb25ea5ffa11ab111193e827ecc093f5665c85b235971fe31e98044b429eccb0346b71d9d9110773fae3edd91e939781cbb9b61703aee5d4f67d70e6e03dd6
+  data.tar.gz: 44488a304307b8f845aba24739eac3c95b9d5bf1a5c5d10ba28d01f627116ef591a8cfe8fac1478a22bd196612ee2d4ef2baeae5a3cf912b74a5608e2e0403b3

data/lib/DICTIONARY.rb

@@ -0,0 +1,123 @@
+#!/usr/bin/env ruby
+=begin
+Created: 	14/09/2023.
+Version: 	1.0.0
+=end
+require 'numo/narray'
+include Numo
+=begin 
+Dictionary
+=end
+class DICTIONARY < ENCODER
+	def init(string, x_dim)
+		@x_dim = x_dim
+		@frequencyHash = wordFrequency(string)
+		@stringToEncoding, @encodingToString = hotEncodeVocabulary(@frequencyHash, x_dim)
+	end
+	def getFrequencyHash()
+		return @frequencyHash
+	end
+	def getEncodingHashs()
+		return @stringToEncoding, @encodingToString
+	end
+	def wordFrequency(words)
+		if words.is_a?(String)
+			words = words.split
+		end
+		frequency = {}
+		entries = 0
+		for i in 0...words.length()
+			if frequency[words[i]] == nil
+				frequency[words[i]] = 1
+				entries += 1
+			else
+				frequency[words[i]] = frequency[words[i]] + 1
+			end
+		end
+		return frequency.sort_by {|k, v| v}.reverse
+	end
+	def hotEncodeVocabulary(hash, maxEntries)
+		stringToEncoding = {}
+		encodingToString = {}
+		binary = 0b1
+		hash.each do |key, value|
+			stringToEncoding["#{key}"] = binary
+			encodingToString[binary] = "#{key}"
+			binary = binary << 1
+			if (binary) == (0b1 << maxEntries)
+				break;
+			end
+		end
+		return stringToEncoding, encodingToString
+	end
+	def readFileDataArray(fileDataArray, fileIndex, start, input_len, predict_len)
+		fileWordArray = (fileDataArray[fileIndex]).split
+		return fileWordArray[start, input_len], fileWordArray[(start+predict_len), (input_len)]
+	end
+	def encodeArray(wordArray, hash=@stringToEncoding)
+		if wordArray.instance_of? String
+			wordArray = wordArray.split()
+		end 
+		array = Int32.zeros(wordArray.length(), @x_dim)
+		for i in 0...wordArray.length()
+			word = wordArray[i]
+			encoding = hash[word]
+			if encoding != nil
+				binaryArray = binaryToArray(encoding, @x_dim)
+				array[i, true] = binaryArray
+			end
+		end
+		return array
+	end
+	def binaryToArray(binary, x_dim)
+		binaryFormatted = ((0b1 << (x_dim+1) ^ binary*2)).to_s(2)[1, x_dim]
+		binaryArray = DFloat.zeros(1, x_dim)
+		binaryArray = binaryFormatted.split("").map(&:to_i)
+		return binaryArray
+	end
+	def arrayToBinary(array)
+		binaryString = array.to_a.join("")
+		binary = binaryString.to_i(2)
+		return binary
+	end
+	def decodeArray(encodedArray, hash=@encodingToString)
+		output = ""
+		for i in 0...encodedArray.shape()[0]
+			decodedWord = hash[arrayToBinary(encodedArray[i, true])]
+			if decodedWord != nil
+				output += decodedWord + " "
+			else 
+				output += "@ "
+			end
+		end
+		return output
+	end
+	def decodeArrayByMaximum(encodedArray, hash=@encodingToString)
+		output = ""
+		for i in 0..encodedArray.shape()[0]-1
+			if encodedArray[i, true].to_a.max != 0
+				max_element = encodedArray[i, true].to_a.each_with_index.max[1]
+				encodedArrayBW = Int32.zeros(encodedArray[i, true].shape())
+				encodedArrayBW[max_element] = 1
+				decodedWord = hash[arrayToBinary(encodedArrayBW)]
+				if decodedWord != nil
+					output += decodedWord + " "
+				else 
+					output += "@ "
+				end
+			else 
+				output += "$ "
+			end
+		end
+		return output
+	end
+	def viewHash(hash)
+		puts "key -> value"
+		entries = 0
+		hash.each do |key, value|
+			puts key.to_s + " -> " + value.to_s
+			entries += 1
+		end
+		puts "Entires in hash: " + entries.to_s
+	end
+end

data/lib/ENCODER.rb

@@ -0,0 +1,132 @@
+#!/usr/bin/env ruby
+=begin
+██╗░░░░░░██████╗████████╗███╗░░░███╗
+██║░░░░░██╔════╝╚══██╔══╝████╗░████║
+██║░░░░░╚█████╗░░░░██║░░░██╔████╔██║
+██║░░░░░░╚═══██╗░░░██║░░░██║╚██╔╝██║
+███████╗██████╔╝░░░██║░░░██║░╚═╝░██║
+╚══════╝╚═════╝░░░░╚═╝░░░╚═╝░░░░░╚═╝
+Created: 	14/09/2023.
+Version: 	1.0.0
+Author: 	Ryan May
+This is a text generator LSTM developed in Ruby. 
+In this project I gain more of an understanding of the training and creation of LSTM networks.
+This LSTM network utilises the Numo::Narray library to calculate cell states and the cell network.
+Expansion on this project will be utilised in my Honours project. 
+=end
+require 'matrix' 		# https://www.rubyguides.com/2019/01/ruby-matrix/
+require 'numo/narray'
+include Numo
+=begin 
+███████╗███╗░░██╗░█████╗░░█████╗░██████╗░███████╗██████╗░
+██╔════╝████╗░██║██╔══██╗██╔══██╗██╔══██╗██╔════╝██╔══██╗
+█████╗░░██╔██╗██║██║░░╚═╝██║░░██║██║░░██║█████╗░░██████╔╝
+██╔══╝░░██║╚████║██║░░██╗██║░░██║██║░░██║██╔══╝░░██╔══██╗
+███████╗██║░╚███║╚█████╔╝╚█████╔╝██████╔╝███████╗██║░░██║
+╚══════╝╚═╝░░╚══╝░╚════╝░░╚════╝░╚═════╝░╚══════╝╚═╝░░╚═╝
+The encoder class is used to translate input characters and sentances into one-hot encoded vectors. 
+Additionally, the encoder class contains functions to handle file reading, regex filtering, and conversions
+between 'Matrix' and 'NArray' types.
+=end
+class ENCODER
+	def init(charmatrix = nil)
+		if charmatrix == nil
+			@Length = 56
+			@SelectionMatrix = Matrix.build(1,56) {0} # 1 row 32 columns
+			@CharMatrix = Array['.',',',"\s",'!',
+								'a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z',
+								'A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P','Q','R','S','T','U','V','W','X','Y','Z']
+		else 
+			@Length = charmatrix.length()
+			@SelectionMatrix = Matrix.build(1, charmatrix.length()) { 0 }
+			@CharMatrix = charmatrix
+		end
+	end
+	def matrixToNArray(matrix)
+		nArray = DFloat[*matrix.to_a]
+		return nArray
+	end
+	def nArrayToMatrix(nArray)
+		#puts nArray.to_a.map(&:inspect)
+		matrix = Matrix[*nArray.to_a]
+		#puts matrix.to_a.map(&:inspect)
+		return matrix
+	end
+	def listFilesInDirectory(path)
+		fileNameArray = Dir.entries(path).reject {|f| File.directory?(f) || f[0].include?('.')}
+		return fileNameArray
+	end
+	def readPlaintextFilesInDirectory(path)
+		fileDataArray = Array.new(0) { "" }
+		# Get names of all files in a folder. 
+		fileNameArray = Dir.entries(path).reject {|f| File.directory?(f) || f[0].include?('.')}
+		# itterate through all files in Array
+		for index in 0...fileNameArray.length()
+			puts "reading file " + path + "/" + fileNameArray[index]
+			fileData = readPlaintextfile(path + "/" + fileNameArray[index])
+			fileDataArray = fileDataArray << fileData
+		end
+		return fileDataArray
+	end
+	def readFileDataArray(fileDataArray, index, start, input_len, predict_len)
+		combination = hotEncodeSentance(filter(fileDataArray[index])[start,predict_len+input_len])
+		return combination
+	end
+	def readPlaintextfile(fileName)
+		file = File.open(fileName)
+		file_data = file.read
+		file.close
+		return file_data
+	end
+	def filter(sentance, regex=nil)
+		if regex == nil
+			sentance = sentance.gsub(/[^A-Za-z\., ]/, '')
+		else 
+			sentance = sentance.gsub(regex, '')
+		end
+		return sentance
+	end
+	def hotEncodeSentance(sentance)
+		letters = sentance.split(//)
+		matrix = Matrix.build(0, 0){0}
+		for index in 0...letters.length()
+			charvector = hotEncodeCharacter(letters[index])
+			if charvector != nil
+				matrix = Matrix.rows(matrix.to_a << charvector.to_a)
+			end
+		end
+		return matrix
+	end
+	def hotDecodeSentance(matrix)
+		sentance = ""
+		for index in 0...matrix.row_count()
+			charvector = matrix.row(index)
+			char = hotDecodeCharacter(charvector)
+			sentance = sentance + char 
+		end
+		return sentance
+	end
+	def hotEncodeCharacter(char)
+		# reset selection matrix
+		@SelectionMatrix = Matrix.build(1,@Length) {0}
+		index = @CharMatrix.index char
+		@SelectionMatrix[0,index] = 1
+		return @SelectionMatrix.row(0)
+	end
+	def hotDecodeCharacter(vector)
+		# convert vector to array so we can search for the up bit
+		vectorArray = vector.to_a
+		index = vectorArray.each_with_index.max[1] 
+		if index != nil
+			return @CharMatrix[index]
+		else 
+			return '@'
+		end
+	end
+	def stringDifferencePercent(a, b)
+	  longer = [a.size, b.size].max
+	  same = a.each_char.zip(b.each_char).count { |a,b| a == b }
+	  similarity = (longer - same) / a.size.to_f
+	  return similarity
+	end
+end

metadata

@@ -0,0 +1,44 @@
+--- !ruby/object:Gem::Specification
+name: one-hot-dictionary
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Ryan May
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2023-06-29 00:00:00.000000000 Z
+dependencies: []
+description:
+email: 19477774@student.curtin.edu.au
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/DICTIONARY.rb
+- lib/ENCODER.rb
+homepage: https://github.com/ryan-n-may/Ruby_One_Hot_Dictionary
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.15
+signing_key:
+specification_version: 4
+summary: A dictionary gem that one-hot encoddes strings.
+test_files: []