Dendrograms 0.0.1

data/Gemfile

@@ -0,0 +1,13 @@
+source :gemcutter
+# Add dependencies required to use your gem here.
+# Example:
+#   gem "activesupport", ">= 2.3.5"
+
+# Add dependencies to develop your gem here.
+# Include everything needed to run rake, tests, features, etc.
+group :development do
+  gem "bundler", ">= 1.0.0"
+  gem "jeweler"
+end
+
+gem "progressbar"

data/Gemfile.lock

@@ -0,0 +1,22 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    git (1.2.5)
+    jeweler (1.8.3)
+      bundler (~> 1.0)
+      git (>= 1.2.5)
+      rake
+      rdoc
+    json (1.6.6)
+    progressbar (0.10.0)
+    rake (0.9.2.2)
+    rdoc (3.12)
+      json (~> 1.4)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (>= 1.0.0)
+  jeweler
+  progressbar

data/README.markdown

@@ -0,0 +1,3 @@
+# Dendrograms
+
+A pure Ruby implementation of Clauset's Hierarchical Random Graph tools. Installs two CLI applications, `fitHRG` and `consensusHRG`.

data/Rakefile

@@ -0,0 +1,29 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "Dendrograms"
+  gem.homepage = "http://github.com/doches/Dendrograms"
+  gem.license = "MIT"
+  gem.summary = %Q{Ruby implementation of Clauset's Hierarchical Random Graphs}
+  gem.description = %Q{Ruby implementation of Clauset's Hierarchical Random Graphs}
+  gem.email = "trevor@texasexpat.net"
+  gem.authors = ["Trevor Fountain"]
+  gem.version = "0.0.1"
+  # Include your dependencies below. Runtime dependencies are required when using your gem,
+  # and development dependencies are only needed for development (ie running rake tasks, tests, etc)
+  #  gem.add_runtime_dependency 'jabber4r', '> 0.1'
+  #  gem.add_development_dependency 'rspec', '> 1.2.3'
+  gem.add_runtime_dependency 'progressbar'
+end
+Jeweler::RubygemsDotOrgTasks.new

data/bin/consensusHRG

@@ -0,0 +1,75 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift File.join(File.dirname(__FILE__),"..","lib")
+
+Description = "Re-implementation, basically, of Clauset's consensusHRG tool\nTakes a .dendro file and a wordmap; outputs a consensus hierarchy"
+Usage = "ruby #{$0} file.dendro file.wordmap > file.consensus.dot"
+Num_Args = 2
+
+require 'Dendrograms'
+include Dendrograms
+require 'progressbar'
+
+verbose = check_flag("-v","--verbose")
+samples = check_opt("-s","--samples","300").to_i
+spread = check_opt("-S","--spread","100").to_i
+
+if ARGV.size != Num_Args
+  STDERR.puts Description
+  STDERR.puts " "
+  STDERR.puts "Usage: #{Usage}"
+  exit(1)
+end
+
+STDERR.puts "#{samples} samples with a spread of #{spread} \n  -> #{samples*spread} resamples"
+
+dendro_file = ARGV.shift
+graph_file = dendro_file.gsub("-best.dendro",".pairs")
+wordmap_file = ARGV.shift
+@wordmap = {}
+IO.foreach(wordmap_file) do |line|
+  word,index = *(line.strip.split(/\s+/))
+  @wordmap[index] = word
+end
+
+graph = Graph.new(graph_file)
+dendrogram = Dendrogram.new(graph, dendro_file)
+
+progress = ProgressBar.new("Sampling",samples) if not verbose
+clusters = {}
+sample_index = 0
+STDERR.puts ["MCMC STEPS","LIKELIHOOD","TIME"].join("\t") if verbose
+start = Time.now.to_i
+while sample_index < samples
+  spread.times { dendrogram.sample! }
+  dclusters = dendrogram.clusters.map { |cluster| cluster.reject { |x| x.nil? }.sort.join("_") }.uniq
+  dclusters.each do |cluster|
+    clusters[cluster] ||= 0
+    clusters[cluster] += 1
+  end
+  STDERR.puts [dendrogram.mcmc_steps, dendrogram.likelihood,"#{Time.now.to_i-start}s"].join("\t") if verbose
+
+  sample_index += 1
+  progress.inc if not verbose
+end
+progress.finish if not verbose
+
+clusters.reject! { |k,v| v <= samples/2.0 }
+#clusters.map { |k,v| [k,v] }.sort { |a,b| a[1] <=> b[1] }.each { |k,v| STDERR.puts "#{v}:\t#{k.gsub('_',", ")}" }
+
+keep = clusters.map { |pair| pair[0].split("_").map { |x| x.to_i } }.sort { |b,a| a.size <=> b.size }
+keep.unshift keep.flatten.uniq
+keep.uniq!
+
+hnodes = [ConsensusNode.new(keep.shift)]
+while keep.size > 0
+  cluster = keep.shift
+  lca = hnodes.reject { |x| not x.contains(cluster) }.sort { |a,b| a.size <=> b.size }[0]
+  new_node = ConsensusNode.new(cluster)
+  lca.add_child(new_node)
+  hnodes.push new_node
+end
+
+puts "graph {"
+hnodes[0].to_dot(@wordmap)
+puts "}"

data/bin/fitHRG

@@ -0,0 +1,51 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift File.join(File.dirname(__FILE__),"..","lib")
+
+Description = "Re-implementation, basically, of Clauset's fitHRG tool. Takes a .pairs file and fits a HRG over the graph\nProduces a .dendro file with the fit HRG and a .info file with information about the run; these are updated as higher-likelihood dendrograms are found.\n\nIf you pass in an optional partial dendrogra, fitHRG will continue sampling from that saved point."
+Usage = "ruby #{$0} file.pairs [file.dendro]"
+Num_Args = 1
+
+require 'Dendrograms'
+include Dendrograms
+
+verbose = check_flag("-v","--verbose")
+verbose_saved = check_flag("-s", "--saved")
+
+if ARGV.size < Num_Args
+  STDERR.puts Description
+  STDERR.puts " "
+  STDERR.puts "Usage: #{Usage}"
+  exit(1)
+end
+
+pairs_file = ARGV.shift
+dendrogram_file = pairs_file.gsub(/\.pairs$/,"-best.dendro")
+info_file = pairs_file.gsub(/\.pairs$/,"-best.info")
+
+graph = Graph.new(pairs_file)
+dendrogram = nil
+if ARGV.empty?
+  dendrogram = Dendrogram.new(graph)
+else
+  dendrogram = Dendrogram.new(graph, ARGV.shift)
+end
+
+best_likelihood = dendrogram.sample!
+best_steps = 0
+start = Time.now.to_i
+STDERR.puts ["MCMC","LIKELIHOOD","BEST LIKEL.","AT MCMC","TIME"].join("\t") if verbose
+while true
+  saved = false
+  likelihood = dendrogram.sample!
+  if likelihood > best_likelihood
+    best_likelihood = likelihood
+    dendrogram.save(dendrogram_file,info_file)
+    best_steps = dendrogram.mcmc_steps
+    saved = true
+  end
+  
+  if (saved and verbose_saved) or dendrogram.mcmc_steps % 1000 == 0
+    STDERR.puts [dendrogram.mcmc_steps, dendrogram.likelihood, best_likelihood, best_steps, "#{Time.now.to_i-start}s"].join("\t") if verbose
+  end
+end

data/lib/Dendrograms.rb

@@ -0,0 +1,4 @@
+require 'Dendrograms/Graph'
+require 'Dendrograms/Dendrogram'
+require 'Dendrograms/Consensus'
+require 'Dendrograms/cli'

data/lib/Dendrograms/Consensus.rb

@@ -0,0 +1,50 @@
+module Dendrograms
+
+class ConsensusNode
+  attr_reader :index
+  @@index = 0
+  def initialize(children)
+    @children = children
+    @index = @@index
+    @@index += 1
+    @leaves = children.dup
+  end
+  
+  @@leaves = {}
+  def ConsensusNode.leaf(leaf)
+    @@leaves[leaf] ||= @@leaves.size
+    return @@leaves[leaf]
+  end
+  
+  def children
+    @children.map { |x| x.is_a?(ConsensusNode) ? x.children : x }.flatten
+  end
+  
+  def add_child(node)
+    @children = @children - node.children
+    @children.push node
+  end
+  
+  def contains(set)
+    (@leaves & set).size == set.size
+  end
+  
+  def size
+    @leaves.size
+  end
+  
+  def to_dot(wordmap)
+    puts "\tINTERNAL_#{@index} [shape=point, label=\"\"];"
+    @children.each do |child|
+      if child.is_a?(ConsensusNode)
+        puts "\tINTERNAL_#{@index} -- INTERNAL_#{child.index};"
+        child.to_dot(wordmap)
+      else
+        puts "\tLEAF_#{ConsensusNode.leaf(child)} [shape=none, label=\"#{wordmap[child.to_s]}\"];"
+        puts "\tINTERNAL_#{@index} -- LEAF_#{ConsensusNode.leaf(child)};"
+      end
+    end
+  end
+end
+
+end

data/lib/Dendrograms/Dendrogram.rb

@@ -0,0 +1,337 @@
+module Dendrograms
+
+# Represents a single node in a dendrogram. Provides methods for transformation, and for computing likelihood
+class DendrogramNode
+  attr_accessor :index, :left, :right
+  @@index = 0
+  
+  Epsilon = 0.00000000001
+  
+  def initialize(left, right)
+    @left = left
+    @right = right
+    @index = @@index
+    
+    @@index += 1
+    @child_cache = nil
+  end
+  
+  def to_s
+    [@index, 
+     (@left.is_a?(DendrogramNode) ? "#{@left.index} (D)" : "#{@left} (G)"),
+     (@right.is_a?(DendrogramNode) ? "#{@right.index} (D)" : "#{@right} (G)")].join("\t")
+  end
+  
+  @@leaves = {}
+  def DendrogramNode.resetLeaves
+    @@leaves = {}
+  end
+    
+  def DendrogramNode.linkToLeaf(node, leaf, wordmap)
+    dot = []
+    if @@leaves[leaf].nil?
+      @@leaves[leaf] = @@leaves.size
+      label = wordmap.nil? ? leaf : wordmap[leaf.to_s]
+      dot.push "LEAF_#{@@leaves[leaf]} [shape=none, label=\"#{label}\"];"
+    end
+    dot.push "#{node} -- LEAF_#{@@leaves[leaf]};"
+    return dot
+  end
+  
+  def to_dot(graph, wordmap=nil, likelihood=false)
+    dot = self.dot_node(graph,wordmap, likelihood)
+    
+    [@left, @right].each do |child|
+      if child.is_a?(DendrogramNode)
+        dot.push "INTERNAL_#{@index} -- INTERNAL_#{child.index};"
+      else
+        DendrogramNode.linkToLeaf("INTERNAL_#{@index}",child,wordmap).each { |x| dot.push x }
+      end
+    end
+    
+    return "\t#{dot.join("\n\t")}"
+  end
+  
+  def dot_node(graph, wordmap=nil, likelihood=false, decorate=true)
+    label = "\"\""
+    shape = "point"
+    color = "black"
+    if likelihood != false
+      theta = self.connectedness(graph)[0]
+      theta = (theta*100).to_i/100.0
+      shape = "none"
+      label = "\"#{theta}\""
+      color = theta > likelihood ? "blue" : "red"
+    end
+    if decorate
+      return ["INTERNAL#{@index} [shape=#{shape},label=#{label},fontcolor=#{color},color=red];"]
+    else
+      return ["INTERNAL#{@index} [shape=point, label=\"\"];"]
+    end
+  end
+  
+  def hierarchy_dot(graph, wordmap, likelihood)
+    dot = self.dot_node(graph,wordmap,likelihood,false)
+    theta = self.connectedness(graph)[0]
+    
+    if theta < likelihood
+      if @left.is_a?(DendrogramNode)
+        dot.push "INTERNAL#{@index} -- INTERNAL#{@left.index};"
+        dot.push @left.hierarchy_dot(graph,wordmap,likelihood)
+      else  
+        dot.push "INTERNAL#{@index} -- LEAF#{@left};"
+        dot.push "LEAF#{@left} [shape=none, label=\"#{wordmap[@left.to_s]}\"];"
+      end
+      if @right.is_a?(DendrogramNode)
+        dot.push "INTERNAL#{@index} -- INTERNAL#{@right.index};"
+        dot.push @right.hierarchy_dot(graph,wordmap,likelihood)
+      else  
+        dot.push "INTERNAL#{@index} -- LEAF#{@right};"
+        dot.push "LEAF#{@right} [shape=none, label=\"#{wordmap[@right.to_s]}\"];"
+      end
+    else
+      dot.push self.children.map { |x| ["INTERNAL#{@index} -- LEAF#{x};","LEAF#{x} [shape=none, label=\"#{wordmap[x.to_s]}\"];"] }
+#      dot.push self.children.map { |x| "LEAF#{x} [shape=none, label=\"#{wordmap[x.to_s]}\"];" }
+#      dot.push self.children.map { |x| "INTERNAL#{@index} -- LEAF#{x};" }
+    end
+    
+    dot.flatten
+  end
+  
+  def children(force = false)
+    if force or @child_cache.nil?
+      @child_cache = [@left.is_a?(DendrogramNode) ? @left.children() : @left, 
+                      @right.is_a?(DendrogramNode) ? @right.children() : @right].flatten
+    end
+    
+    return @child_cache
+  end
+  
+  def connectedness(graph)
+    left_children = @left.is_a?(DendrogramNode) ? @left.children : [@left]
+    right_children = @right.is_a?(DendrogramNode) ? @right.children : [@right]
+    
+    links = graph.edges_between(left_children, right_children).to_f
+    max_links = (left_children.size * right_children.size)
+    theta = links / max_links.to_f
+    
+    return [theta, max_links]
+  end
+  
+  def likelihood(graph)
+    theta,max_links = *self.connectedness(graph)
+    theta = Epsilon if theta <= 0.0
+    theta = 1.0-Epsilon if theta >= 1.0
+#    l = (theta**links) * (1-theta)**(max_links-links)
+    h = -theta*Math.log(theta) - (1-theta)*Math.log(1-theta)
+    return -h * max_links
+  end
+  
+  def mutable?
+    @left.is_a?(DendrogramNode) or @right.is_a?(DendrogramNode)
+  end
+  
+  def get_mutation
+    # Are we swapping children with the left or the right child?if rand > 0.5
+    child = nil
+    if @left.is_a?(DendrogramNode)
+      child = @left
+    else
+      child = @right
+    end
+    
+    # Are we swapping the child's left or right child?
+    do_left = false
+    if rand > 0.5
+      do_left = true
+    end
+    
+    return {:child => child, :do_left => do_left, :local_child => (child == @left ? @right : @left)}
+  end
+  
+  def mutate!(mutation = nil)
+    mutation ||= self.get_mutation
+    
+    if mutation[:do_left]
+      temp = mutation[:child].left
+      mutation[:child].left = mutation[:local_child]
+      if mutation[:local_child] == @left
+        @left = temp
+      else
+        @right = temp
+      end
+      mutation[:local_child] = temp
+    else
+      temp = mutation[:child].right
+      mutation[:child].right = mutation[:local_child]
+      if mutation[:local_child] == @left
+        @left = temp
+      else
+        @right = temp
+      end
+      mutation[:local_child] = temp
+    end
+    
+    mutation[:child].children(true)
+    self.children(true)
+    return mutation
+  end
+end
+
+# Takes a Graph, builds a dendrogram, and provides methods to sample, compute likelihood, and save (with optional info)
+class Dendrogram
+  attr_reader :graph, :likelihood, :mcmc_steps, :root
+  
+  def initialize(graph, tree_file=nil)
+    @graph = graph
+    @nodes = []
+    @likelihoods = []
+    @likelihood = 0
+    @mcmc_steps = 0
+    
+    if tree_file
+      index_map = {}
+      IO.foreach(tree_file) do |line|
+        if line =~ /^(\d+)\t(\d+) \(([D|G])\)\t(\d+) \(([D|G])\)/
+          index, left, ltype, right, rtype = $1.to_i, $2.to_i, $3, $4.to_i, $5
+          
+          node = DendrogramNode.new(left, right)
+          node.index = index
+          @nodes.push node
+          index_map[node.index] = node
+          
+          node.left = [left] if ltype == "D"
+          node.right = [right] if rtype == "D"
+        end
+      end
+      # Update mappings
+      @nodes.each do |node|
+        node.left = index_map[node.left[0]] if node.left.is_a?(Array)
+        node.right = index_map[node.right[0]] if node.right.is_a?(Array)
+        node.index = @nodes.index(node)
+      end
+      # Find root
+      @root = @nodes.sort { |b,a| a.children.size <=> b.children.size }[0]
+      
+      # Update MCMC, if possible
+      begin
+        info_file = tree_file.gsub(".dendro",".info")
+        if File.exists?(info_file)
+          status = YAML.load_file(info_file)
+          @mcmc_steps = status[:mcmc]
+        end
+      rescue
+        STDERR.puts "Unable to load MCMC status from .info; carrying on."
+      end
+    else
+      # Incrementally construct a balanced dendrogram
+      remaining = graph.nodes.dup.sort_by { rand }
+    
+      while remaining.size > 1
+        a = remaining.pop
+        b = remaining.shift
+      
+        node = DendrogramNode.new(a,b)
+        @nodes.push node
+        remaining.push(node)
+        remaining = remaining.sort_by { rand }
+      end
+    
+      # Hold on to the last remaining node; it's the root
+      @root = remaining.shift
+    end
+    
+    # Initialise likelihoods
+    @nodes.each_with_index { |node, index| @likelihoods[index] = node.likelihood(@graph) }
+    # Compute starting likelihood
+    @likelihood = @likelihoods.inject(0) { |s,x| s += x }
+  end
+  
+  # Returns the mean node likelihood
+  def mean_likelihood
+    mean = @likelihoods.map { |x| Math.exp(x) }.inject(0) { |s,x| s += x } / @likelihoods.size.to_f
+    STDERR.puts "Mean likelihood: #{mean}"
+    return mean
+  end
+  
+  def mean_theta
+    @nodes.map { |x| x.connectedness(@graph)[0] }.inject(0) { |s,x| s += x } / @nodes.size.to_f
+  end
+  
+  # Returns the median node connectednes
+  def median_theta
+    v = @nodes.map { |x| x.connectedness(@graph)[0] }
+    return v[v.size/2]
+  end
+  
+  def sample!
+    mutate = nil
+    while true
+      node = @nodes[(rand*@nodes.size).to_i]
+      if node.mutable?
+        mutate = node
+        break
+      end
+    end
+    
+    # Mutate tree
+    mutation = mutate.mutate!
+    
+    old_likelihood = @likelihood
+    self.update_likelihood([mutate, mutate.left, mutate.right])
+    
+    if not (@likelihood > old_likelihood or Math.log(rand) < @likelihood - old_likelihood)
+      mutate.mutate!(mutation)
+      self.update_likelihood([mutate, mutate.left, mutate.right])
+    end
+    @mcmc_steps += 1
+    
+    return @likelihood
+  end
+  
+  def clusters
+    @nodes.map { |node| node.children }
+  end
+  
+  # Update the likelihood given two modified nodes
+  def update_likelihood(nodes)
+    # Compute new likelihood
+    nodes.each do |node|
+      if node.is_a?(DendrogramNode)
+        @likelihood -= @likelihoods[node.index]
+        @likelihoods[node.index] = node.likelihood(@graph)
+        @likelihood += @likelihoods[node.index]
+      end
+    end
+  end
+  
+  def save(tree_file, info_file)
+    fout = File.open(tree_file,'w')
+    fout.puts @nodes.map { |node| node.to_s }.join("\n")
+    fout.close
+    
+    fout = File.open(info_file, 'w')
+    fout.puts({:likelihood => @likelihood, :mcmc => @mcmc_steps}.to_yaml)
+    fout.close
+    
+    self.to_dot(tree_file.gsub(/\.[^\.]+$/,".dot"))
+  end
+  
+  def get_dot(wordmap=nil, likelihood=false)
+    DendrogramNode.resetLeaves
+    ["graph {",@nodes.map { |node| node.to_dot(@graph,wordmap,likelihood) }.join("\n"),"}"].join("\n")
+  end
+  
+  def get_hierarchy_dot(wordmap=nil, likelihood=false)
+    DendrogramNode.resetLeaves
+    ["graph {", @root.hierarchy_dot(@graph, wordmap, likelihood).map { |x| "\t#{x}" }.join("\n"), "}"].join("\n")
+  end
+  
+  def to_dot(dot_file, wordmap=nil, likelihood=false)
+    fout = File.open(dot_file,'w')
+    fout.puts self.get_dot(wordmap,likelihood)
+    fout.close
+  end
+end
+
+end

data/lib/Dendrograms/Graph.rb

@@ -0,0 +1,40 @@
+module Dendrograms
+
+# Loads a graph from a .pairs file, and computes the number of edges between sets of nodes
+
+class Graph
+  attr_reader :nodes
+  
+  # Load a .pairs file
+  def initialize(pairs)
+    @nodes = {}
+    @edges = {}
+    
+    IO.foreach(pairs) do |line|
+      from, to = *(line.strip.split(/\s+/).map { |x| x.to_i })
+      @nodes[from] = true
+      @nodes[to] = true
+      @edges[edge_key(from,to)] = true
+      @edges[edge_key(to, from)] = true
+    end
+    @nodes = @nodes.keys
+  end
+  
+  # Compute the number of edges between two sets of nodes
+  def edges_between(set_a, set_b)
+    count = 0
+    set_a.each do |a|
+      set_b.each do |b|
+        count += 1 if @edges[edge_key(a,b)]
+      end
+    end
+    return count
+  end
+  
+  # Return a unique key for an edge between A and B
+  def edge_key(a,b)
+    "#{a}_#{b}".to_sym
+  end
+end
+
+end

data/lib/Dendrograms/cli.rb

@@ -0,0 +1,35 @@
+module Dendrograms
+
+def check_flag(short,long)
+  present = false
+  [short,long].each do |opt|
+    present = true if ARGV.include?(opt)
+    ARGV.reject! { |x| x == opt }
+  end
+  return present
+end
+
+def check_opt(short,long,default)
+	value = default
+	[short, long].each do |opt|
+		if ARGV.include?(opt)
+			index = ARGV.index(opt)
+			value = ARGV[index+1]
+			ARGV.delete_at(index)
+			ARGV.delete_at(index)
+			break
+		end
+	end
+	return value
+end
+
+def usage(description, usage, args)
+  if ARGV.size != args
+    STDERR.puts description
+    STDERR.puts " "
+    STDERR.puts "Usage: #{usage}"
+    exit(1)
+  end
+end
+
+end

data/scripts/apply_wordmap.rb

@@ -0,0 +1,79 @@
+#!/usr/bin/env ruby
+#
+# Apply a wordmap to all files with a given prefix
+#
+# Usage: apply_wordmap path/to/wordmap prefix/files
+
+require 'progressbar'
+
+if ARGV.size < 2
+    STDERR.puts "Apply a wordmap to all files with a given prefix"
+    STDERR.puts ""
+    STDERR.puts "Usage: #{$0} path/to/wordmap prefix/files [-w|--warn]"
+    exit(0)
+end
+
+@debug = ARGV.include?("-w") or ARGV.include?("--warn")
+ARGV.reject! { |x| x == "-w" or x == "--warn" }
+
+wordmap_path = ARGV.shift
+prefix = ARGV.shift
+
+def process(file)
+	ext = file.split(".").pop
+	
+	txt = IO.readlines(file).join("").strip
+	case ext
+		when "dot"
+			STDERR.puts "#{file} (DOT)"
+			progress = ProgressBar.new("Applying",@wordmap.size)
+			@wordmap.each_pair { |index,word| txt.gsub!("\"#{index}\"","\"#{word}\"") if not word.nil?; progress.inc }
+			progress.finish
+			return txt
+=begin			
+		when "matrix"
+			STDERR.puts "#{file} (MATRIX)"
+			progress = ProgressBar.new("Applying",@wordmap.size)
+			@wordmap.each_pair { |index,word| txt.gsub!(/^#{index}\:/,"#{word}:") if not word.nil?; progress.inc }
+			progress.finish
+			return txt
+		when "txt"
+			STDERR.puts "#{file} (TXT)"
+			progress = ProgressBar.new("Applying",@wordmap.size)
+			@wordmap.each_pair { |index,word| txt.gsub!("<#{index}>",word) if not word.nil?; progress.inc }
+			progress.finish
+			return txt
+		when "graph"
+			STDERR.puts "#{file} (GRAPH)"
+			progress = ProgressBar.new("Applying",@wordmap.size*2)
+			@wordmap.each_pair { |index,word| txt.gsub!(/^#{index}\t/,"#{word}\t") if not word.nil?; progress.inc }
+			@wordmap.each_pair { |index,word| txt.gsub!(/\t#{index}\t/,"\t#{word}\t") if not word.nil?; progress.inc }
+			progress.finish
+			return txt
+=end
+		else
+			STDERR.puts "Unrecognised file extension \"#{ext}\"" if @debug
+			return nil
+	end
+end
+
+@wordmap = {}
+IO.foreach(wordmap_path) do |line|
+	word,index = *(line.strip.split(/\s+/))
+	index = index.to_i
+	@wordmap[index] = word if @wordmap[index].nil?
+end
+
+Dir.glob("#{prefix}*").each do |file|
+	new_filename = file.split(".").reject { |x| x == "human" }
+	ext = new_filename.pop
+	new_filename = new_filename.push("human").push(ext).join(".")
+	if not File.exists?(new_filename)
+	  txt = process(file)
+	  if not txt.nil?
+		  fout = File.open(new_filename,'w')
+		  fout.puts txt
+		  fout.close
+		end
+	end
+end

data/scripts/dendro2clusters.rb

@@ -0,0 +1,37 @@
+require 'lib/cli'
+require 'lib/Dendrogram'
+require 'lib/Graph'
+
+usage(
+  "Takes a .dendro and outputs a clusterval-friendly yaml flat clustering",
+  "ruby #{$0} file.dendro > file.yaml",
+  1
+)
+
+dendro_f = ARGV.shift
+graph_f = Dir.glob(File.join(File.dirname(dendro_f),"*.pairs"))[0]
+wordmap_f = Dir.glob(File.join(File.dirname(dendro_f),"*.wordmap"))[0]
+
+@graph = Graph.new(graph_f)
+@dendrogram = Dendrogram.new(@graph, dendro_f)
+@wordmap = {}
+IO.foreach(wordmap_f) do |line|
+  word,index = *(line.strip.split(/\s+/))
+  @wordmap[index] = word
+end
+
+@min = @dendrogram.mean_theta
+
+@clusters = {}
+def identify_subtree_clusters(node)
+  return @clusters[@clusters.size.to_s] = [@wordmap[node.to_s]] if node.is_a?(Fixnum)
+  theta = node.connectedness(@graph)[0]
+  if theta > @min
+    @clusters[@clusters.size.to_s] = node.children.map { |x| @wordmap[x.to_s] }
+  else
+    return [identify_subtree_clusters(node.left), identify_subtree_clusters(node.right)]
+  end
+end
+
+identify_subtree_clusters(@dendrogram.root)
+puts @clusters.to_yaml

data/scripts/dendro2hdot.rb

@@ -0,0 +1,28 @@
+Description = "Takes a .dendro and outputs a .dot file"
+Usage = "ruby #{$0} file.dendro > file.dot"
+Num_Args = 1
+
+if ARGV.size != Num_Args
+  STDERR.puts Description
+  STDERR.puts " "
+  STDERR.puts "Usage: #{Usage}"
+  exit(1)
+end
+
+require 'lib/Dendrogram'
+require 'lib/Graph'
+
+dendro_f = ARGV.shift
+graph_f = Dir.glob(File.join(File.dirname(dendro_f),"*.pairs"))[0]
+wordmap_f = Dir.glob(File.join(File.dirname(dendro_f),"*.wordmap"))[0]
+@min = ARGV.shift.to_f
+
+@graph = Graph.new(graph_f)
+@dendrogram = Dendrogram.new(@graph, dendro_f)
+@wordmap = {}
+IO.foreach(wordmap_f) do |line|
+  word,index = *(line.strip.split(/\s+/))
+  @wordmap[index] = word
+end
+
+puts @dendrogram.get_dot(@wordmap, @dendrogram.mean_theta)

data/scripts/hrg2dendro.rb

@@ -0,0 +1,32 @@
+Description = "Takes a .hrg and a -names.lut and outputs a .dendro"
+Usage = "ruby #{$0} file.hrg file.lut > file.dendro"
+Num_Args = 2
+
+if ARGV.size != Num_Args
+  STDERR.puts Description
+  STDERR.puts " "
+  STDERR.puts "Usage: #{Usage}"
+  exit(1)
+end
+
+hrg_f = ARGV.shift
+lut_f = ARGV.shift
+names = nil
+IO.foreach(lut_f) do |line|
+  if names.nil?
+    names = {}
+  else
+    virtual, real = *(line.strip.split(/\s+/))
+    names[virtual] = real
+  end
+end
+
+#[ 0 ] L= 46 (D) R= 29 (D) p= 0 e= 0 n= 49
+IO.foreach(hrg_f) do |line|
+  if line =~ /\[ (\d+) \] L= (\d+) \(([D|G])\) R= (\d+) \(([D|G])\)/
+    index, left, left_type, right, right_type = $1, $2, $3, $4, $5
+    left = names[left] if left_type == "G"
+    right = names[right] if right_type == "G"
+    puts "#{index}\t#{left} (#{left_type})\t#{right} (#{right_type})"
+  end
+end

data/scripts/hrg2dot.rb

@@ -0,0 +1,64 @@
+# Takes a .hrg (and an optional wordmap) and outputs a .dot
+#
+# Usage: ruby #{$0} file.hrg [file.wordmap] > file.dot
+
+hrg = ARGV.shift
+wordmap_f = ARGV.empty? ? nil : ARGV.shift
+
+@map_names = true
+@names = {}
+IO.foreach(hrg.gsub("_best-dendro.hrg","-names.lut")) do |line|
+  if not line =~ /virtual/
+    virtual, real = *(line.strip.split(/\s+/))
+    @names[virtual] = real
+  end
+end
+
+@wordmap = nil
+if not wordmap_f.nil?
+  @wordmap = {}
+  IO.foreach(wordmap_f) do |line|
+    word, index = *(line.strip.split(/\s+/))
+    @wordmap[index] = word
+  end
+end
+
+@nodes = {}
+def internal(index)
+  if @nodes[index].nil?
+    @nodes[index] = @nodes.size
+    puts "\tINTERNAL#{@nodes[index]} [shape=point,label=\"\"];"
+  end
+  
+  "INTERNAL#{@nodes[index]}"
+end
+
+@leaves = {}
+def leaf(index)
+  index = @names[index] if @map_names
+  if @leaves[index].nil?
+    @leaves[index] = @leaves.size
+    label = index
+    label = @wordmap[label] if not @wordmap.nil?
+    puts "\tLEAF#{@leaves[index]} [shape=none,label=\"#{label}\"];"
+  end
+  
+  "LEAF#{@leaves[index]}"
+end
+
+def node(type,index)
+  if type == "D"
+    return internal(index)
+  else
+    return leaf(index)
+  end
+end
+
+puts "graph {"
+IO.foreach(hrg) do |line|
+  if line =~ /\[ (\d+) \] L= (\d+) \((D|G)\) R= (\d+) \((D|G)\)/
+    dnode = node("D",$1)
+    [[$3, $2], [$5, $4]].map { |pair| node(pair[0],pair[1]) }.each { |x| puts "\t#{dnode} -- #{x};" }
+  end
+end
+puts "}"

data/scripts/render.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+rm $1/*-best.human.dot
+ruby scripts/apply_wordmap.rb $1/*.wordmap $1
+dot -Tpng $1/*-best.human.dot -o render.png
+open render.png

metadata

@@ -0,0 +1,139 @@
+--- !ruby/object:Gem::Specification 
+name: Dendrograms
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: 
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Trevor Fountain
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-04-03 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  type: :runtime
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  name: progressbar
+  version_requirements: *id001
+  prerelease: false
+- !ruby/object:Gem::Dependency 
+  type: :development
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 23
+        segments: 
+        - 1
+        - 0
+        - 0
+        version: 1.0.0
+  name: bundler
+  version_requirements: *id002
+  prerelease: false
+- !ruby/object:Gem::Dependency 
+  type: :development
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  name: jeweler
+  version_requirements: *id003
+  prerelease: false
+- !ruby/object:Gem::Dependency 
+  type: :runtime
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  name: progressbar
+  version_requirements: *id004
+  prerelease: false
+description: Ruby implementation of Clauset's Hierarchical Random Graphs
+email: trevor@texasexpat.net
+executables: 
+- fitHRG
+- consensusHRG
+extensions: []
+
+extra_rdoc_files: 
+- README.markdown
+files: 
+- Gemfile
+- Gemfile.lock
+- README.markdown
+- Rakefile
+- bin/consensusHRG
+- bin/fitHRG
+- lib/Dendrograms.rb
+- lib/Dendrograms/Consensus.rb
+- lib/Dendrograms/Dendrogram.rb
+- lib/Dendrograms/Graph.rb
+- lib/Dendrograms/cli.rb
+- scripts/apply_wordmap.rb
+- scripts/dendro2clusters.rb
+- scripts/dendro2hdot.rb
+- scripts/hrg2dendro.rb
+- scripts/hrg2dot.rb
+- scripts/render.sh
+homepage: http://github.com/doches/Dendrograms
+licenses: 
+- MIT
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.8.15
+signing_key: 
+specification_version: 3
+summary: Ruby implementation of Clauset's Hierarchical Random Graphs
+test_files: []
+