clusta 0.0.1 → 0.0.2

data/README.rdoc

@@ -0,0 +1,66 @@
+= Clusta
+
+Clusta is a Ruby gem for network analysis built on top of
+Wukong[http://github.com/mrflip/wukong].
+
+Wukong lets you write Ruby scripts that run on your laptop as well as
+on a Hadoop cluster.
+
+Clusta is:
+
+- classes that make describing the geometry of networks easy
+- network algorithms written with these classes to use Wukong
+- a shim command-line program for running these algorithms
+
+Start with a file containing edges:
+
+  Edge	1	2
+  Edge	2	3
+  Edge	1	4
+  Edge	4	5
+  Edge	5	6
+  Edge	5	7
+  Edge	6	8
+  Edge	7	8
+  Edge	8	9
+
+Run this through a transformation named +edges_to_degrees+:
+
+  $ clusta --transform=edges_to_degrees /local/edges.tsv -
+  Degree	1	2
+  Degree	2	2
+  Degree	3	1
+  Degree	4	2
+  Degree	5	3
+  Degree	6	2
+  Degree	7	2
+  Degree	8	3
+  Degree	9	1
+
+Chain transformations together:
+
+  $ clusta --transform=edges_to_neighborhoods /local/edges.tsv - | clusta --transform=neighborhoods_to_degree_pairs - - | clusta --transform=degree_pairs_to_assortativities - -
+  Assortativity	1	2	1
+  Assortativity	1	3	1
+  Assortativity	2	1	1
+  Assortativity	2	2	4
+  Assortativity	2	3	5
+  Assortativity	3	1	1
+  Assortativity	3	2	5
+
+And then leverage Wukong when you're ready:
+
+  $ clusta --run=hadoop --transform=edges_to_neighborhoods /hdfs/edges.tsv /hdfs/neighborhoods.tsv
+  I, [2012-03-03T21:00:39.992750 #25835]  INFO -- :   Launching hadoop!
+  I, [2012-03-03T21:00:39.992979 #25835]  INFO -- : Running
+
+  /usr/lib/hadoop/bin/hadoop 	\
+    jar /usr/lib/hadoop/contrib/streaming/hadoop-*streaming*.jar 	\
+    -D mapred.job.name='clusta---spec/data/edges/undirected.unweighted.tsv----' 	\
+    -mapper  '/usr/bin/ruby1.9.1 clusta --map --log_interval=10000 --log_seconds=30 --transform=edges_to_degrees' 	\
+    -reducer '/usr/bin/ruby1.9.1 clusta --reduce --log_interval=10000 --log_seconds=30 --transform=edges_to_degrees' 	\
+    -input   'spec/data/edges/undirected.unweighted.tsv' 	\
+    -output  '-' 	\
+    -file    '/home/user/projects/networks/clusta/bin/clusta'
+  ...
+

data/VERSION

@@ -1 +1 @@
-0.0.1
+0.0.2

data/bin/clusta

@@ -4,32 +4,5 @@ $: << File.expand_path('../lib', File.dirname(__FILE__)) unless $:.include?(File
 
 require 'clusta'
 
-def usage
-  "usage: #{File.basename(__FILE__)} --transform=TRANSFORM_NAME [ARGS ...]"
-end
+Clusta::Runner.new(File.basename(__FILE__), ARGV.dup).run! if $0 == __FILE__
 
-def extract_transform_arg
-  transform_arg = ARGV.find_all { |arg| arg =~ Clusta::Transforms::ARG_REGEXP }.first
-  if transform_arg.nil?
-    $stderr.puts(usage)
-    exit(1)
-  end
-  # ARGV.delete_if { |arg| arg =~ Clusta::Transforms::ARG_REGEXP }
-  transform_arg
-end
-
-def add_default_run_arg
-  ARGV.unshift('--run=local') unless ARGV.detect { |arg| arg =~ /--run/ }
-end
-
-if $0 == __FILE__
-  begin
-    add_default_run_arg
-    transform = Clusta::Transforms.from_arg(extract_transform_arg)
-    script    = Clusta::Transforms.script_for(transform)
-    script.run
-  rescue Clusta::Error => e
-    $stderr.puts e.message
-    exit(1)
-  end
-end

data/lib/clusta.rb

@@ -14,11 +14,20 @@ module Clusta
     File.basename(path).gsub(/\.rb$/, '')
   end
 
+  def self.require_path path
+    File.join(File.dirname(path), File.basename(path).gsub(/\.rb$/, ''))
+  end
+  
   Error                     = Class.new(StandardError)
+  ArgumentError             = Class.new(Error)
   DirectednessMismatchError = Class.new(Error)
   AmbiguousArgumentsError   = Class.new(Error)
+  NotImplementedError       = Class.new(Error)
+  SortError                 = Class.new(Error)
 
-  autoload :Geometry,   'clusta/geometry'
-  autoload :Transforms, 'clusta/transforms'
-  
+  autoload :Geometry,      'clusta/geometry'
+  autoload :Transforms,    'clusta/transforms'
+  autoload :Runner,        'clusta/runner'
+  autoload :Schema,        'clusta/schema'
+  autoload :Serialization, 'clusta/serialization'
 end

data/lib/clusta/geometry.rb

@@ -2,24 +2,69 @@ module Clusta
   
   module Geometry
 
-    autoload :Element, 'clusta/geometry/element'
-
-    ELEMENTS = []
+    def self.names
+      @names ||= {}
+    end
 
-    def self.register_geometry name, path, geometries=nil
-      autoload name, path
-      self::ELEMENTS << name
+    def self.register_element klass, name=nil
+      if name
+        Wukong::RESOURCE_CLASS_MAP[name] = klass
+      else
+        klass.all_stream_names.each do |name|
+          Wukong::RESOURCE_CLASS_MAP[name] = klass
+        end
+      end
     end
 
+    def self.from_name name
+      begin
+        const_get(Clusta.classify(name))
+      rescue NameError => e
+        raise Error.new("No such transform: '#{name}'")
+      end
+    end
+    
     Dir[File.join(File.dirname(__FILE__), "geometry/*.rb")].each do |path|
       require_name = Clusta.require_name(path)
-      register_geometry Clusta.classify(require_name), "clusta/geometry/#{require_name}"
+      autoload Clusta.classify(require_name), "clusta/geometry/#{require_name}"
+      names[require_name] ||= {} unless require_name == 'all'
     end
 
     Dir[File.join(File.dirname(__FILE__), "geometry/directed/*.rb")].each do |path|
       require_name = Clusta.require_name(path)
-      register_geometry ("Directed" + Clusta.classify(require_name)), "clusta/geometry/directed/#{require_name}"
+      autoload ("Directed" + Clusta.classify(require_name)), "clusta/geometry/directed/#{require_name}"
+      names[require_name] ||= {}
+      names[require_name][:directed] = true
     end
 
+    def self.listing
+      [].tap do |out|
+        out << "Known geometries:"
+        out << ''
+        names.keys.sort.each do |element_name|
+          element = from_name(element_name)
+          if names[element_name][:directed]
+            directed_element = from_name("directed_#{element_name}")
+          else
+            directed_element = nil
+          end
+          
+          out << "  #{element}"
+          stream_names = element.all_stream_names.sort
+          stream_names.concat(directed_element.all_stream_names.sort) if directed_element
+          out << "    streams as: #{stream_names.uniq.join(', ')}"
+          out << ''
+        end
+      end.join("\n")
+    end
+
+    def self.load_from path
+      class_eval(File.read(path), path)
+      require_name = Clusta.require_name(path)
+      names[require_name] ||= {}
+      names[require_name][:directed] = true if require_name =~ /^directed_/
+    end
+    
+    
   end
 end

data/lib/clusta/geometry/all.rb

@@ -0,0 +1,3 @@
+Dir[File.join(File.dirname(__FILE__), '**/*.rb')].each do |path|
+  require Clusta.require_path path
+end

data/lib/clusta/geometry/assortativity.rb

@@ -4,8 +4,8 @@ module Clusta
 
     class Assortativity < Element
 
-      field :source_degree_value, :type => :int
-      field :target_degree_value, :type => :int
+      key   :source_degree_value, :type => :int
+      key   :target_degree_value, :type => :int
       field :count,               :type => :int
 
       def directed?

data/lib/clusta/geometry/degree.rb

@@ -3,7 +3,9 @@ module Clusta
     
     class Degree < Element
 
-      field :vertex_label
+      abbreviate 'D'
+
+      key   :vertex_label
       field :degree, :type => :int
       
       def directed?

data/lib/clusta/geometry/{edge_degree_pair.rb → degree_pair.rb}

@@ -1,9 +1,9 @@
 module Clusta
   module Geometry
 
-    class EdgeDegreePair < Element
-      field :source_label
-      field :target_label
+    class DegreePair < Element
+      key   :source_label
+      key   :target_label
       field :source_degree_value, :type => :int
       field :target_degree_value, :type => :int
 

data/lib/clusta/geometry/directed/degree.rb

@@ -3,7 +3,9 @@ module Clusta
     
     class DirectedDegree < Element
 
-      field :vertex_label
+      abbreviate 'DD'
+
+      key   :vertex_label
       field :in_degree,   :type => :int
       field :out_degree,  :type => :int
       

data/lib/clusta/geometry/directed/{edge_degree_pair.rb → degree_pair.rb}

@@ -1,9 +1,10 @@
 module Clusta
   module Geometry
 
-    class DirectedEdgeDegreePair < Element
-      field :source_label
-      field :target_label
+    class DirectedDegreePair < Element
+      
+      key   :source_label
+      key   :target_label
       field :source_in_degree_value,  :type => :int
       field :source_out_degree_value, :type => :int
       field :target_in_degree_value,  :type => :int

data/lib/clusta/geometry/directed/edge.rb

@@ -2,6 +2,8 @@ module Clusta
   module Geometry
     
     class DirectedEdge < Edge
+
+      abbreviate 'DE'
       
       def directed?
         true
@@ -15,8 +17,8 @@ module Clusta
         DirectedDegree.new(target_label, 1, 0)
       end
 
-      def arrow
-        DirectedArrow.new(target_label, weight)
+      def neighbor
+        DirectedNeighbor.new(target_label, weight)
       end
       
 

data/lib/clusta/geometry/directed/{arrow.rb → neighbor.rb}

@@ -1,7 +1,7 @@
 module Clusta
   module Geometry
 
-    class DirectedArrow < Arrow
+    class DirectedNeighbor < Neighbor
       def directed
         true
       end

data/lib/clusta/geometry/directed/neighborhood.rb

@@ -0,0 +1,31 @@
+module Clusta
+  module Geometry
+    
+    class DirectedNeighborhood < Neighborhood
+      
+      def directed?
+        true
+      end
+
+      def degree_pairs
+        neighbors.map do |neighbor|
+          # This vertex's in-degree is not known to us; we just have
+          # its out-degree based on the size of this neighborhood.
+          #
+          # We don't know anything about each neighbor's degree than
+          # its in-degree is at least 1 b/c it's in this vertex's
+          # neighborhood.
+          DirectedDegreePair.new(label, neighbor.label, 0, size, 1, 0)
+        end
+      end
+
+      def reversed_degree_pairs
+        neighbors.map do |neighbor|
+          DirectedDegreePair.new(neighbor.label, label, 1, 0, 0, size)
+        end
+      end
+      
+    end
+    
+  end
+end

data/lib/clusta/geometry/edge.rb

@@ -3,8 +3,10 @@ module Clusta
 
     class Edge < Element
 
-      field :source_label
-      field :target_label
+      abbreviate 'E'
+
+      key   :source_label
+      key   :target_label
       field :weight,      :optional => true
 
       def weighted?
@@ -48,8 +50,8 @@ module Clusta
         self.class.new(target_label, source_label, weight)
       end
 
-      def arrow
-        Arrow.new(target_label, weight)
+      def neighbor
+        Neighbor.new(target_label, weight)
       end
       
     end

data/lib/clusta/geometry/element.rb

@@ -3,129 +3,22 @@ module Clusta
 
     class Element
       
-      attr_accessor :input_fields
+      include Clusta::Schema
+      include Clusta::Serialization
 
-      @fields = []
-      class << self ; attr_reader :fields ; end
-
-      def self.inherited(subclass)
-        subclass.instance_variable_set("@fields", @fields.dup)
-        super
-      end
-
-      def self.field_names
-        @fields.map { |field| field[:name].to_s }
-      end
-
-      def self.has_optional_field?
-        @fields.any? { |field| field[:optional] }
-      end
-
-      def self.optional_field
-        @fields.detect { |field| field[:optional] }
-      end
-
-      def self.from_string string
-        return string unless string.is_a?(String)
-        args  = string.split(';')
-        klass_name = args.shift
-        raise ArgumentError.new("Elements instantiated from a string must match the format 'klass;[field1;[field2;]...]'") unless klass_name
-        Wukong.class_from_resource(klass_name).new(*args)
-      end
-      
-      def self.field name, options={}
-        raise AmbiguousArgumentsError.new("Cannot define a second optional field #{name} because field #{optional_field[:name]} is already optional.") if has_optional_field?
-        attr_reader name
-        case options[:type]
-        when :int
-          define_method "#{name}=" do |val|
-            instance_variable_set("@#{name}", val.to_i)
-          end
-        when :float
-          define_method "#{name}=" do |val|
-            instance_variable_set("@#{name}", val.to_f)
-          end
-        when :geometry
-          define_method "#{name}=" do |val|
-            instance_variable_set("@#{name}", self.class.from_string(val))
-          end
-        else
-          define_method "#{name}=" do |val|
-            instance_variable_set("@#{name}", val)
-          end
-        end
-        @fields << options.merge(:name => name)
-      end
-
-      def fields
-        self.class.fields
-      end
-
-      def self.input_fields name
-        alias_method name, :input_fields
+      if defined?(Settings) && Settings[:serialize] == 'json'
+        include Clusta::Serialization::JSON
+      else
+        include Clusta::Serialization::TSV
       end
 
-      def self.stream_name
-        if defined?(Settings) && Settings[:full_class_names]
-          to_s
-        else
-          to_s.split("::").last
-        end
-      end
-
-      def stream_name
-        self.class.stream_name
-      end
-
-      def initialize *args
-        self.class.fields.each_with_index do |field, index|
-          suffix = case index.to_s
-            when /1$/ then 'st'
-            when /2$/ then 'nd'
-            when /3$/ then 'rd'
-            else 'th'
-            end
-          case
-          when field[:optional]
-            self.send("#{field[:name]}=", args[index]) if args[index]
-          when args[index].nil?
-            raise ArgumentError.new("A #{self.class} requires a non-nil value for #{field[:name]} as its #{index}#{suffix} argument.")
-          else
-            self.send("#{field[:name]}=", args[index])
-          end
-        end
-        self.set_input_fields(*(args[self.class.fields.size..-1] || []))
-      end
-
-      def set_input_fields *input_fields
-        self.input_fields = input_fields.map do |field|
-          if field =~ /^[A-Z].*;/
-            self.class.from_string(field)
-          else
-            field
-          end
-        end
-      end
-
-      def output_fields
-        input_fields.map(&:to_s)
-      end
-
-      def to_flat
-        [stream_name].tap do |record|
-          fields.each do |field|
-            value  = send(field[:name])
-            record << value.to_s unless value.nil? && field[:optional]
-          end
-        end.concat(output_fields)
-      end
-
-      def to_s
-        to_flat.join(';')
+      def self.inherited subclass
+        Clusta::Geometry.register_element(subclass)
+        super
       end
       
     end
-
+      
   end
 end