clusta 0.0.1 → 0.0.2

data/lib/clusta/transforms.rb

@@ -2,25 +2,12 @@ module Clusta
 
   module Transforms
 
-    def self.register_streamable klass, aliases=nil
-      (aliases || [klass.to_s]).each do |klass_alias|
-        Wukong::RESOURCE_CLASS_MAP[klass_alias] = klass
-      end
-    end
-
-    def self.register_transform name, path
-      autoload name, path
-    end
-
-    Dir[File.join(File.dirname(__FILE__), "transforms/*.rb")].each do |path|
-      require_name = Clusta.require_name(path)
-      register_transform Clusta.classify(require_name), "clusta/transforms/#{require_name}"
+    def self.names
+      @names ||= []
     end
     
-    ARG_REGEXP = /--transform=[\w\d_]+/
-
-    def self.from_arg arg
-      from_name(arg.split('=').last)
+    def self.register_transform name
+      names << name
     end
 
     def self.from_name name
@@ -34,17 +21,63 @@ module Clusta
     def self.script_for transform
       mapper  = transform::Mapper  if defined?(transform::Mapper)
       reducer = transform::Reducer if defined?(transform::Reducer)
-      Wukong::Script.new(mapper, reducer)
+      options = (transform.respond_to?(:options) ? transform.options : {})
+      script  = defined?(transform::Script) ? transform::Script : default_script
+      script.new(mapper, reducer, options)
+    end
+
+    def self.default_script
+      Class.new(Wukong::Script).tap do |c|
+        c.class_eval do
+          def local_mode_sort_commandline
+            "sort -n -k2"
+          end
+        end
+      end
     end
-    
-  end
 
-  Geometry::ELEMENTS.each do |element_name|
-    Transforms.register_streamable Geometry.const_get(element_name), [
-       "Clusta::Geometry::#{element_name}",
-       "Geometry::#{element_name}",
-       element_name.to_s
-      ]
+    def self.has_mapper?(transform)
+      defined?(transform::Mapper)
+    end
+
+    def self.has_reducer?(transform)
+      defined?(transform::Reducer)
+    end
+
+    Dir[File.join(File.dirname(__FILE__), "transforms/*.rb")].each do |path|
+      require_name = Clusta.require_name(path)
+      autoload Clusta.classify(require_name), "clusta/transforms/#{require_name}"
+      register_transform require_name
+    end
+
+    def self.listing
+      [].tap do |out|
+        out << "Known transforms:"
+        out << ''
+        names.sort.each do |transform_name|
+          transform = from_name(transform_name)
+          name_suffix = case
+                        when has_mapper?(transform)     && has_reducer?(transform)     then ''
+                        when (! has_mapper?(transform)) && has_reducer?(transform)     then ' (reduce-only)'
+                        when has_mapper?(transform)     && (! has_reducer?(transform)) then ' (map-only)'
+                        when (! has_mapper?(transform)) && (! has_reducer?(transform)) then ' (nothing)'
+                        end
+          
+          out << "  #{transform_name}#{name_suffix}"
+          if transform.respond_to?(:help)
+            out << ''
+            out << "    #{transform.help}"
+          end
+          out <<  ''
+        end
+      end.join("\n")
+    end
+
+    def self.load_from path
+      class_eval(File.read(path), path)
+      register_transform(Clusta.require_name(path))
+    end
+    
   end
   
 end

data/lib/clusta/transforms/{edge_degree_pairs_to_assortativities.rb → degree_pairs_to_assortativities.rb}

@@ -1,13 +1,17 @@
+require 'clusta/geometry/degree_pair'
+require 'clusta/geometry/directed/degree_pair'
+require 'clusta/geometry/assortativity'
+
 module Clusta
 
   module Transforms
 
-    module EdgeDegreePairsToAssortativities
+    module DegreePairsToAssortativities
 
       class Mapper < Wukong::Streamer::StructStreamer
 
-        def process edge_degree_pair, *record
-          emit edge_degree_pair.assortativity
+        def process degree_pair, *record
+          emit degree_pair.assortativity
         end
         
       end

data/lib/clusta/transforms/edges_to_degrees.rb

@@ -1,3 +1,8 @@
+require 'clusta/geometry/edge'
+require 'clusta/geometry/directed/edge'
+require 'clusta/geometry/degree'
+require 'clusta/geometry/directed/degree'
+
 module Clusta
 
   module Transforms

data/lib/clusta/transforms/{edges_to_vertex_arrows.rb → edges_to_neighborhoods.rb}

@@ -1,8 +1,13 @@
+require 'clusta/geometry/edge'
+require 'clusta/geometry/directed/edge'
+require 'clusta/geometry/neighborhood'
+require 'clusta/geometry/directed/neighborhood'
+
 module Clusta
 
   module Transforms
 
-    module EdgesToVertexArrows
+    module EdgesToNeighborhoods
 
       class Mapper < Wukong::Streamer::StructStreamer
 
@@ -15,7 +20,7 @@ module Clusta
 
       class Reducer < Wukong::Streamer::AccumulatingReducer
 
-        attr_accessor :arrows, :directed
+        attr_accessor :neighbors, :directed
         
         include Wukong::Streamer::StructRecordizer
 
@@ -26,19 +31,19 @@ module Clusta
         end
 
         def start! new_edge, *record
-          self.arrows   = []
+          self.neighbors   = []
           self.directed = new_edge.directed?
         end
 
         def accumulate new_edge, *record
-          self.arrows << new_edge.arrow
+          self.neighbors << new_edge.neighbor
         end
 
         def finalize &block
           if directed
-            emit Geometry::DirectedVertexArrows.new(vertex_label, *arrows)
+            emit Geometry::DirectedNeighborhood.new(vertex_label, *neighbors)
           else
-            emit Geometry::VertexArrows.new(vertex_label, *arrows)
+            emit Geometry::Neighborhood.new(vertex_label, *neighbors)
           end
         end
         

data/lib/clusta/transforms/import.rb

@@ -1,9 +1,15 @@
+Settings.define :as, :required => true, :description => "Name of the Clusta class to import data as."
+
 module Clusta
 
   module Transforms
 
     module Import
 
+      def self.help
+        "Import data into the format expected by Clusta."
+      end
+
       class Mapper < Wukong::Streamer::Base
 
         def process *record

data/lib/clusta/transforms/neighborhoods_to_degree_pairs.rb

@@ -0,0 +1,70 @@
+require 'clusta/geometry/neighborhood'
+require 'clusta/geometry/directed/neighborhood'
+require 'clusta/geometry/neighbor'
+require 'clusta/geometry/directed/neighbor'
+require 'clusta/geometry/degree_pair'
+require 'clusta/geometry/directed/degree_pair'
+
+module Clusta
+
+  module Transforms
+
+    module NeighborhoodsToDegreePairs
+
+      class Mapper < Wukong::Streamer::StructStreamer
+
+        def process neighborhood, *record
+          neighborhood.reversed_degree_pairs.each { |degree_pair| emit(degree_pair) }
+        end
+        
+      end
+
+      class Reducer < Wukong::Streamer::AccumulatingReducer
+
+        attr_accessor :degree_pairs, :source_degree, :source_in_degree, :source_out_degree
+        
+        include Wukong::Streamer::StructRecordizer
+
+        def get_key new_degree_pair, *record
+          new_degree_pair.source_label
+        end
+
+        def start! new_degree_pair, *record
+          self.degree_pairs = []
+          if new_degree_pair.directed?
+            self.source_in_degree  = 0
+            self.source_out_degree = 0
+          else
+            self.source_degree = 0
+          end
+        end
+
+        def accumulate new_degree_pair, *record
+          self.degree_pairs      << new_degree_pair
+          if new_degree_pair.directed?
+            self.source_in_degree  += new_degree_pair.source_in_degree_value
+            self.source_out_degree += new_degree_pair.source_out_degree_value
+          else
+            self.source_degree += new_degree_pair.source_degree_value
+          end
+        end
+
+        def finalize &block
+          degree_pairs.each do |degree_pair|
+            if degree_pair.directed?
+              degree_pair.source_in_degree_value  = source_in_degree
+              degree_pair.source_out_degree_value = source_out_degree
+            else
+              degree_pair.source_degree_value = source_degree
+            end
+            emit degree_pair
+          end
+        end
+        
+      end
+      
+    end
+  end
+  
+end
+

data/lib/clusta/transforms/pm3d.rb

@@ -0,0 +1,46 @@
+module Clusta
+
+  module Transforms
+
+    module Pm3d
+
+      class Mapper < Wukong::Streamer::Base
+
+        def process *record
+          if record.first && record.first =~ /[^\d]/
+            emit record[1..-1]
+          else
+            emit record
+          end
+        end
+      end
+
+      class Reducer < Wukong::Streamer::AccumulatingReducer
+
+        attr_accessor :records
+
+        def start! *record
+          self.records = []
+        end
+
+        def accumulate *record
+          self.records << record
+        end
+
+        def finalize &block
+          records.each { |record| emit(record) }
+          emit []
+        end
+        
+      end
+
+      class Script < Wukong::Script
+        def local_mode_sort_commandline
+          "sort -n -k1 -k2"
+        end
+      end
+      
+    end
+  end
+  
+end

data/lib/clusta/transforms/prune_edges.rb

@@ -0,0 +1,34 @@
+require 'clusta/geometry/edge'
+require 'clusta/geometry/directed/edge'
+
+Settings.define :min_weight,            :type => Float,    :description => "Prune edges with weight less than this weight."
+Settings.define :max_weight,            :type => Float,    :description => "Prune edges with weight more than this weight."
+
+module Clusta
+
+  module Transforms
+
+    module PruneEdges
+
+      class Mapper < Wukong::Streamer::StructStreamer
+
+        def before_stream
+          raise ArgumentError.new("Must specify either a min_weight or a max_weight") if Settings[:min_weight].nil? && Settings[:max_weight].nil?
+        end
+
+        def within_weight_range? edge
+          return false if Settings[:min_weight] && Settings[:min_weight] > edge.weight.to_f
+          return false if Settings[:max_weight] && Settings[:max_weight] < edge.weight.to_f
+          true
+        end
+        
+        def process edge, *record
+          emit(edge) if edge.weighted? && within_weight_range?(edge)
+        end
+        
+      end
+      
+    end
+  end
+  
+end

data/spec/clusta/schema_spec.rb

@@ -0,0 +1,36 @@
+require 'spec_helper'
+
+describe Clusta::Schema do
+
+  before do
+    @root = Class.new
+    @root.send(:include, Clusta::Schema)
+  end
+
+  it "should not define any fields of its own" do
+    @root.fields.should == []
+  end
+
+  it "should allow a subclass to set its own fields without polluting the parent" do
+    subclass = Class.new(@root)
+    subclass.key :foo
+    @root.field_names.should_not include('foo')
+    subclass.field_names.should include('foo')
+  end
+
+  it "should allow a subclass of a subclass to set its own fields without polluting the parent" do
+    subclass1 = Class.new(@root)
+    subclass1.key :foo
+    subclass2 = Class.new(subclass1)
+    subclass2.key :bar
+    
+    @root.field_names.should_not include('foo')
+    @root.field_names.should_not include('bar')
+    
+    subclass1.field_names.should include('foo')
+    subclass1.field_names.should_not include('bar')
+
+    subclass2.field_names.should include('foo')
+    subclass2.field_names.should include('bar')
+  end
+end

data/spec/clusta/serialization/json_spec.rb

@@ -0,0 +1,133 @@
+require 'spec_helper'
+
+describe Clusta::Serialization::JSON do
+
+  def json_serializable_wrapper_class
+    Class.new.tap do |c|
+      c.send(:include, Clusta::Schema)
+      c.send(:include, Clusta::Serialization)
+      c.send(:include, Clusta::Serialization::JSON)
+    end
+  end
+
+  before do
+    @root = json_serializable_wrapper_class
+    @root.key :foo
+
+    @child = json_serializable_wrapper_class
+    @child.key :baz
+
+    @child.set_stream_name 'Child'
+  end
+
+  describe "processing inputs" do
+
+    it "should assign a single key field correctly" do
+      instance = @root.new("foovalue")
+      instance.foo.should == "foovalue"
+    end
+
+    it "should assign a multiple key fields correctly" do
+      @root.key :bar
+      instance = @root.new("foovalue", "barvalue")
+      instance.foo.should == "foovalue"
+      instance.bar.should == "barvalue"
+    end
+    
+    it "should assign and parse key and non-key fields when given as Ruby hashes" do
+      @root.field :bar
+      @root.field :baz
+      instance = @root.new("foovalue", {"bar" => "barvalue", "baz" =>  "bazvalue"})
+      instance.foo.should == "foovalue"
+      instance.bar.should == "barvalue"
+      instance.baz.should == "bazvalue"
+    end
+
+    it "should assign and parse key and non-key fields when given as a JSON string" do
+      @root.field :bar
+      @root.field :baz
+      instance = @root.new("foovalue", '{"bar":"barvalue", "baz":"bazvalue"}')
+      instance.foo.should == "foovalue"
+      instance.bar.should == "barvalue"
+      instance.baz.should == "bazvalue"
+    end
+    
+    it "should assign a value to an optional field only if it's present" do
+      @root.field :bar
+      @root.field :baz, :optional => true
+      instance = @root.new("foovalue", {"bar" => "barvalue"})
+      instance.foo.should == "foovalue"
+      instance.bar.should == "barvalue"
+      instance.baz.should be_nil
+
+      instance = @root.new("foovalue", {"bar" => "barvalue", "baz" => "bazvalue"})
+      instance.foo.should == "foovalue"
+      instance.bar.should == "barvalue"
+      instance.baz.should == "bazvalue"
+    end
+
+    it "should stash extra arguments it receives at initialization" do
+      @root.field :bar
+      instance = @root.new("foovalue", {"bar" => "barvalue"}, "extra1", "extra2")
+      instance.foo.should == "foovalue"
+      instance.bar.should == "barvalue"
+      instance.extra_inputs.size.should == 2
+      instance.extra_inputs[0].should == "extra1"
+      instance.extra_inputs[1].should == "extra2"
+    end
+
+  end
+  
+  describe "serializing" do
+
+    it "returns an array appropriate for Wukong with a single key field" do
+      output = @root.new("foovalue").to_flat
+      output[0].should == @root.stream_name
+      output[1].should == "foovalue"
+    end
+
+    it "returns an array appropriate for Wukong with a multiple key fields" do
+      @root.key :bar
+      output = @root.new("foovalue", "barvalue").to_flat
+      output[0].should == @root.stream_name
+      output[1].should == "foovalue"
+      output[2].should == "barvalue"
+    end
+
+    it "returns an array appropriate for Wukong with keys and non key fields" do
+      @root.field :bar
+      output = @root.new("foovalue", { 'bar' => 'barvalue'}).to_flat
+      output[0].should == @root.stream_name
+      output[1].should == "foovalue"
+      output[2].should match(/"bar" *: *"barvalue"/)
+    end
+    
+    it "returns an array with an optional field at the end only if it has a value" do
+      @root.field :bar, :optional => true
+      output1 = @root.new("foovalue").to_flat
+      output2 = @root.new("foovalue", {"bar" => "barvalue"}).to_flat
+
+      output1.size.should == 2
+      output1[0].should == @root.stream_name
+      output1[1].should == 'foovalue'
+      
+      output2.size.should == 3
+      output2[0].should == @root.stream_name
+      output2[1].should == 'foovalue'
+      output2[2].should match(/"bar" *: *"barvalue"/)
+    end
+
+    it "returns an array including any extra inputs it received as outputs" do
+      @root.field :bar
+      output = @root.new("foovalue", {"bar" => "barvalue"}, "extra1", "extra2").to_flat
+      output.size.should == 5
+      output[0].should == @root.stream_name
+      output[1].should == "foovalue"
+      output[2].should match(/"bar" *: *"barvalue"/)
+      output[3].should == 'extra1'
+      output[4].should == 'extra2'
+    end
+    
+  end
+
+end