LinkGrouper 1.0.1-universal-linux

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: df01bc2b149a8f1e15377f457b633157c444acd0
+  data.tar.gz: cad193cef4ee4aaa27969a34c1bc3c83c3fec7fb
+SHA512:
+  metadata.gz: 4e96a95a5377c5dc25ec77a76b6c5e7216ecfac290109f79d4af3e056aaadf892f8b6154bf5a967246c6adf006a9c925d3d400e2e7340eb82548efb67d8552e1
+  data.tar.gz: 44d584956cf46b0e8f9cc78d45a86d1715fd2bf4b84b2db55908b1f0dbf0cf108e37496437bc696cedcd4a686abdc77a65d6d79d14cbbd5716997aaa52e989af

data/README

@@ -0,0 +1,39 @@
+Given a list of links (as a text file), this module will group similar links (or find unique links) and write them to separate files. 
+e.g. URL list -- 
+/level1/uid8
+/level1/level2/gid6
+/status
+/level1/uid10
+/level1/level2/gid0
+/xyz
+/level1
+/level1
+/level1
+/level1/level2
+
+e.g. output (given groupsize of 2)
+Then this will list the following in file 1 (URLs which could not be grouped) -- 
+/status
+/xyz
+This will exist in file 2 -- 
+/level1/uid8
+/level1/uid10
+Contents of file 3 -- 
+/level1/level2/gid0
+/level1/level2/gid6
+Contents of file 4 -- 
+/level1/level2
+conents of file 5 -- 
+/level1
+/level1
+/level1
+
+Use LinkGrouper.uniqcounter method as the entry point.
+1st arg -- The directory in which files will be made each of which represents a group.
+2nd arg -- A file which contains a \n separated list of absolute URLs
+3rd arg -- The 'groupsize'. A identified category/class/group of URLs must be of this size to be considered a category/class/group.
+
+LinkGrouper_app.rb has been made which is an app using the module.
+# First arg -- Will write into a directory (plist) with it's path supplied to argument. This contains files each of which represents a class/category/group of log entries. The file is named with a random UUID.
+# 2nd arg -- A file which contains a \n separated list of absolute URLs
+# 3rd arg -- The 'groupsize'. A identified category/class/group of URLs must be of this size to be considered a category/class/group.

data/bin/LinkGrouper_app.rb

@@ -0,0 +1,7 @@
+#! /usr/bin/ruby
+# An app using the module.
+# First arg -- Will write into a directory (plist) with it's path supplied to argument. This contains files each of which represents a class/category/group of log entries. The file is named with a random UUID.
+# 2nd arg -- A file which contains a \n separated list of absolute URLs
+# 3rd arg -- The 'groupsize'. A identified category/class/group of URLs must be of this size to be considered a category/class/group.
+require "LinkGrouper.rb"
+LinkGrouper.uniqcounter(ARGV[0], ARGV[1], ARGV[2].to_i) 

data/lib/LinkGrouper.rb

@@ -0,0 +1,308 @@
+#! /usr/bin/ruby
+# Licence: GPLv3
+module LinkGrouper
+	require "fileutils.rb"
+	# Takes as argument a file which contains a \n separated list of URLs
+	# uniqcounter is the endpoint function.
+	# Will write into a directory (plist) with it's name supplied to argument. This contains files each of which represents a class/catagory/group of log entries.
+	# takes in as argument the groupsize
+	# e.g. URL list -- 
+	# /level1/uid8
+	# /level1/level2/gid6
+	# /status
+	# /level1/uid10
+	# /level1/level2/gid0
+	# /xyz
+	# /level1
+	# /level1
+	# /level1
+	# /level1/level2
+	# Then this will list the following in file 1 (URLs which could not be grouped) -- 
+	# /status
+	# /xyz
+	# This will exist in file 2 -- 
+	# /level1/uid8
+	# /level1/uid10
+	# Contents of file 3 -- 
+	# /level1/level2/gid0
+	# /level1/level2/gid6
+	# Contents of file 4 -- 
+	# /level1/level2
+	# conents of file 5 -- 
+	# /level1
+	# /level1
+	# /level1
+	# 'groupsize' -- A identified category of URLs must be of this size to be considered as a group.
+	# level1 and level1/ are 2 different URLs. bundle
+
+	# uniqcounter returns a hash as a number (a kind of status code) and it's value which is a human readable description of the status
+
+	# Algo (see unique_link_classifier.txt in solution process to see the method) -- 
+	# 1) Sort the file using previously packaged code. The sorted version of the file (named sorted_links) will be placed in plist[uniqcounter calls sorted_links which uses unix commands to achieve the same].
+	# 2) Start creating the bundle[uniqcounter calls nextBundle]. In case you've redirected from 3), the next bundle will include the last line of the previous bundle [nextBundle call nextBundleCalculate(true)]. For each line being added to the bundle, check if no. of / is the same (procedures as stated in unique_link_classifier.txt)[nextBundle calls bundleSlashCount using the array returned by bundleLines]; if the no. of / are different, move to 3). If this's not the first time 2) is called, move to the next bundle in the same way as discussed in unique_link_classifier.txt [uniqcounter calls nextBundle with no arguments and then calls bundleLines]. If 10) was executed, the next bundle will start from the lines after the lines that were written [uniqcounter calls to_slIncrement() with arguments as the no. of successive lines detected in the bundle. to_slIncrement is called (with argument 0) even when there were no successive lines added. This has been done for proper behavior when nextBundleCalculate would return false] and restart from the beginning of 2). If a bundle cannot be formed because of missing lines, terminate program [uniqcounter exists program if nextBundle returns false] after writing all remaining lines to separate files [nextBundleCalculate would return nil because of missing lines, and before that it'll write the remaining lines to separate files].
+	# 3) in case a bundle couldn't be formed, write all these links except the last link to separate files [nextBundle calls writeLink] and move to 2).
+	# 5) Identify common parts in the bundle.[uniqcounter calls bundleIdnetifyGroup]
+	# 6) If a common part is found [bundleIdnetifyGroup returns true], move to 7), otherwise write each of the links to different files [uniqcounter calls writeLink passing the array containing the lines of the bundle] and move to 2).
+	# 7) Expand the bundle size by 1 [uniqcounter calls bundleLines with argument x], if there are no new lines, terminate program after writing the bundle [if bundleLines returns nil, it calls writeLinks with arguments as returned by the last successful call to bundleLines and terminates program].
+	# 8) Check if the newly added line in the bundle has same no of / [uniqcounter calls bundleSlashCount with value returned by bundleLines just above]. If not, move to 10 [if bundleSlashCount retruns false]
+	# 9) Check if the newly added line in the bundle has the same prefix as other lines of the bundle[uniqcounter calls bundleIdnetifyGroup with the array returned by bundleLines], in other words check if the new line can be a part of the group. If so, move to 7[if bundleIdnetifyGroup returns true add value of x to it's previous value +1.  Initial value of x is 0.], otherwise move to 10[bundleIdnetifyGroup returned false].
+	# 10) Write the bundle to a file if a group was made off the bundle without increasing x [uniqcounter calls writeLinks with an array as an argument which did not make bundleIdnetifyGroup return false] and move to 2. If no group could be formed in the 1st place, write the bundle to individual files[uniqcounter calls writeLink with the the bundled array as argument].
+
+	# function/variable list 
+	# $from_sl -- Points to the line from which the current bundle starts in sorted_links. First line is 1. Initial value is 1.
+	# $to_sl -- Points to the line till which the current bundle end in sorted_links. Initial value is 1
+	# $slinksIO -- IO object which opens sorted_links ro.
+	# $plist
+	# $slNo -- no. of lines in in sorted_links
+	# $groupsize
+	# $to_slCaller -- 
+	# sorted_links -- Writes file sorted_links. Takes a single argument of the original list.
+	# to_slIncrement(<int>) -- Will increase the size of $to_sl by (int), which default to 1. Will update $to_slCaller to point to the last function which called this function.
+	# boolean bundleSlashCount(<Array>) -- Takes in an array and returns true only if the no. of / in them are identical otherwise it returns false.
+	# bool nextBundle() -- Calls nextBundleCalculate to check if no. of lines to create the next bundle exists; if it returns nil, this will return false, if it returns true, uses it to create the next bundle. Value of from_sl and to_sl will be increased 1 at a time via to_slIncrement and for each addition, the corresponding lines will be made an array off (using bundleLines) and sent to bundleSlashCount; if bundleSlashCount returns true, will add another line to it (using bundleLines). This process will happen until the target/new from_sl and to_sl is reached. In case bundleSlashCount returned false at any time, will call writeLink with an array from index from_sl to to_sl - 1, then call nextBundleCalculate(true) (i.e. increment new bundle with an overlap) and then restart from the beginning.
+	# Array nextBundleCalculate(<boolean>) -- It'll increase to_sl by to_sl+<int>, then set from_sl to to_sl+1, and then sets to_sl to to_sl+1+<groupsize> after checking if line no. to_sl+1+<groupsize> exists. If it does not, will return nil, otherwise will return an array [from_sl, to_sl] without increase them actually. The algorithm differs if $from_sl and $to_sl are at their initial values; it'll just increase to_sl to accommodate the $groupsize. If <boolean> is set to true, the array returned will be an 'overlap' from the previous bundle, i.e. the first line will be common between the previous bundle and this new target bundle.
+		# 	Whenever this returns nil because there are not enough lines, each remaining link will be written to a file via function writeEnd() which determines what to write.
+	#writeEnd -- Writes remaining lines of sorted_links based on certain condition; if nextBundleCalculate was called because the no. of / were different for the lines which were being added to the under-formation bundle (by nextBundle), then we need to write all lines from $to_sl onwards. If this function was called after a successful bundle was identified (and $to_sl was increased by uniqcounter instead via function to_slIncrement), then we need to write all lines after $to_sl. Look at the value of $to_slCaller to determine who incremented $to_sl last (either uniqcounter or nextBundle).
+	# Array bundleLines($slinksIO,<int>) -- Returns an array which contains all lines between and including from_sl to to_sl, if the argument is set to nil (default arg). If not, will return an array which contains lines exceeding to_sl by <int>. When <int> is not nil, will also check if these excess lines (<int>) exists or not. If it does not, will return nil; when using <int>, will not update to_sl.
+	# writeLinks(<Array>, <string>) -- Will generate a uuid and write a file named as the generated uuid if that file does not exist in directory as specified by <string> (which will defaults to plist). If the file named uuid exists, will exit the program with a message.
+	# writeLink(<Array>, <string>) -- For each element of <Array> will call writeLinks passing a single element array as argument. If <string> is not present (default arguments nil), will execute writeLinks without a second argument.
+	# <string> bundleIdnetifyGroup(<Array>, prefix=nil) -- Will try and find common parts in the <Array> (the same one as returned by bundleLines) and return nil if there are no common parts; if a common part is found, will return the common parts. The second argument, if present, will check if all the elements of the array has the same prefix, if so will return the prefix itself, otherwise return nil.
+	# uniqcounter -- Coordinator function, calls various functions with the correct arguments. Working within the algo itself.
+
+	require "securerandom"
+
+	$from_sl = 1
+	$to_sl = 1
+
+	def LinkGrouper.to_slIncrement(incby)
+		$to_sl = $to_sl+incby
+		$to_slCaller = caller(1,1)[0].sub(/.*`([a-zA-Z]+)'$/, "\\1")
+	end
+
+	def LinkGrouper.sorted_links(flist)
+		unless system("sort -r #{flist} > #{$plist + '/sorted_links'}")
+			return false
+		end
+		return true
+	end
+
+	def LinkGrouper.bundleSlashCount(links)
+		counter = nil
+		links.each {
+			|i|
+			unless counter
+				counter = i.count('/')
+			end
+			return false if i.count('/') != counter
+		}
+		true
+	end
+
+	# TODO: Existing file detection not implemented.
+	def LinkGrouper.writeLinks(links, plist=$plist)
+		IO.write("#{plist}/#{SecureRandom.uuid}", links.join("\n"))
+	end
+
+	# TODO:Plist 2nd argument not handled.
+	def LinkGrouper.writeLink(links, plist=nil)
+		links.each {
+			|line|
+			writeLinks([line])
+		}
+	end
+
+	def LinkGrouper.nextBundle
+	# 	Increment $to_sl 1 by 1 until we reach the targets. Retrieving targets.
+		targets = nextBundleCalculate
+	# 	no. of lines not sufficient.
+		return false unless targets
+		
+		ctr = 1
+		$from_sl, $to_sl = targets[0], targets[0]
+	# 	for each added line, check if bundleSlashCount returns true for the under-creating bundle.
+		while $to_sl < targets[1]
+			to_slIncrement(1)
+			bundledLines = bundleLines($slinksIO)
+	# 		if no. of / in the newly added line differs...
+			unless bundleSlashCount(bundledLines)
+				writeBundledLines = Array.new
+				ctr = 1
+	# 			Write all the lines returned by bundleLines except the last.
+				bundledLines.each {
+					|i|
+					while ctr < bundledLines.length
+						writeBundledLines.push(i)
+						ctr += 1
+					end
+				}
+				writeLink(writeBundledLines)
+				targets = nextBundleCalculate(true)
+				return false unless targets
+				$from_sl, $to_sl = targets[0], targets[0]
+			end
+		end
+		true
+	end
+
+	def LinkGrouper.bundleLines(linksIO, extra=nil)
+		if extra && (extra + $to_sl > $slNo)
+			return nil
+		end
+		linesArray = Array.new
+		linksIO.pos = 0
+		linksIO.lineno = 0
+		linescount=($to_sl - $from_sl) + 1
+		linksIO.each_line {
+			|line|
+			line = line.chomp
+			if linksIO.lineno > $to_sl + (extra == nil ? 0 : extra)
+				break
+			end
+			if linksIO.lineno >= $from_sl
+				linesArray.push(line)
+			end
+		}
+		linesArray
+	end
+
+	def LinkGrouper.writeEnd
+		if $to_slCaller == "uniqcounter"
+			$from_sl = $to_sl + 1
+			if $from_sl <= $slNo
+				$to_sl = $slNo
+				writeLink(bundleLines($slinksIO))
+			end
+		else
+			$from_sl = $to_sl
+			if $from_sl <= $slNo
+				$to_sl = $slNo
+				writeLink(bundleLines($slinksIO))
+			end
+		end
+	end
+
+	def LinkGrouper.nextBundleCalculate(overlap=nil)
+	# 	check if there are enough lines remaining. If not, write the remaining lines.
+		if !overlap
+			if ($to_sl+1)+($groupsize - 1) > $slNo
+				writeEnd
+				return nil
+			end
+		else
+			if ($to_sl)+($groupsize - 1) > $slNo
+				writeEnd
+				return nil
+			end
+		end
+	# 	if it's not the initial value
+		if ($from_sl != 1 || $to_sl != 1)
+			from_slNext = $to_sl+1
+			to_slNext = from_slNext + ($groupsize - 1)
+	# 		if it is...
+		else
+			from_slNext = $from_sl
+			to_slNext = $to_sl + ($groupsize - 1)
+		end
+		unless overlap
+				return [from_slNext, to_slNext]
+		end
+	# 	For the initial values of from_sl and to_sl, this will return 0, 0 which is invalid.
+		if ($from_sl != 1 || $to_sl != 1)
+			return [from_slNext-1, to_slNext-1]
+		else
+			return nil
+		end
+	end
+
+	def LinkGrouper.bundleIdnetifyGroup(links, prefix=nil)
+	# 	Algo
+	# 1) Identify everything before the 1st separator for the first log entry. Call this string D.
+	# 2) Search for D in other logs in the same position. If even one of the log entries dont have the 1st separator, return false. If this's not the 1st time 2) is called, return the D that was common to all log entries.
+	# 3) Identify the next separator, call it D and move to 2)
+		if prefix
+# 			TODO: If we search only the last element of links for the prefix, it'll be a lot faster. We dont need to search other links.
+			return nil if links.grep(/^#{prefix}/).length != links.length
+			return prefix
+		end
+		dir = nil
+	# 	loop till we reach the end of the first link; i.e. everything in the 1st link was found to be common.
+		while ( (dir == nil ? 0 : dir.length) != links[0].length )
+	# 		identify next directory/prefix
+			direscaped = nil
+			direscaped = Regexp.escape(dir) if dir
+			dir = links[0].sub(/^(#{direscaped}\/*[^\/]*\/*).*/, "\\1")
+	# 		break if this prefix was not found in even a single link.
+			(links.grep(/^#{dir}/).length != links.length) ? break : predir = dir
+		end
+	# 	this was the last prefix which was common for all the links.
+		predir
+	end
+# signature -- plist (explained above), flist (input file), groupsize
+	def LinkGrouper.uniqcounter(plist, flist, groupsize)
+		if groupsize < 2
+			return {3 => "We dont deal with groupsize below 2."}
+		end
+		
+		unless Dir.exist?(plist)
+			return {4 => "The write directory does not exist."}
+		end
+		
+		plist = plist.chomp('/')
+		
+		$groupsize = groupsize
+		$plist = plist
+		
+		linectr = 0
+		IO.foreach(flist) {
+			|line|
+			unless line =~ /^\//
+				return {1 => "Found line(s)(#{line}) which are not absolute links. Sanitize file."}
+			end
+			linectr += 1
+		}
+		$slNo = linectr
+		
+		unless sorted_links(flist)
+# 			TODO -- Force this.
+			FileUtils.rm "#{$plist}/sorted_links"
+			return {2 => "sort command was not successful"}
+		end
+		
+		$slinksIO = flistio = IO.new(IO.sysopen($plist+'/sorted_links', 'r'))
+		
+		while nextBundle
+			bundledLines = bundleLines($slinksIO)
+			bundledLinesExtra = bundledLines
+			workingPrefix = nil
+			testnext = 0
+# 			TODO -- optimization, first call bundleSlashCount, then call bundleIdnetifyGroup, possibally both of them at parallel
+			while (prefix = bundleIdnetifyGroup(bundledLinesExtra, workingPrefix)) && bundleSlashCount(bundledLinesExtra)
+	# bundledLines always contains lines in which a group was found. bundledLinesExtra contains the extra line which needs to be tested for an already identified prefix for the group.
+				workingPrefix = prefix
+				bundledLines = bundledLinesExtra
+				testnext += 1
+				bundledLinesExtra = bundleLines($slinksIO, testnext)
+	# 			logs ran out... exit
+				unless bundledLinesExtra
+					writeLinks(bundledLines)
+					$slinksIO.close
+					FileUtils.rm "#{$plist}/sorted_links"
+					return {0 => "Logs ran out while a group was identified"}
+				end
+			end
+	#Write only if a group was found for the bundle. Also increase $to_sl to the last successfully added line.
+			if workingPrefix
+				writeLinks(bundledLines)
+				to_slIncrement(testnext - 1)
+			else
+	# 			Write to individual files
+				writeLink(bundledLines)
+	# 			for proper functioning of writeEnd/nextBundleCalculate
+				to_slIncrement(0)
+			end
+		end
+		$slinksIO.close
+		FileUtils.rm "#{$plist}/sorted_links"
+		return {0 => "All done."}
+	end
+end

metadata

@@ -0,0 +1,50 @@
+--- !ruby/object:Gem::Specification
+name: LinkGrouper
+version: !ruby/object:Gem::Version
+  version: 1.0.1
+platform: universal-linux
+authors:
+- dE
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-03-25 00:00:00.000000000 Z
+dependencies: []
+description: Given a text file as input, will group similar links (or find unique
+  links) and write them to separate files. Read the README of the gem install directory
+  for the usage
+email: de.techno@gmail.com
+executables:
+- LinkGrouper_app.rb
+extensions: []
+extra_rdoc_files: []
+files:
+- README
+- bin/LinkGrouper_app.rb
+- lib/LinkGrouper.rb
+homepage: http://delogics.blogspot.com
+licenses:
+- Apache License, Version 2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements:
+- The sort command in the OS search path
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Library and application to group similar logs
+test_files: []