clusta 0.0.1 → 0.0.2

data/lib/clusta/geometry/{arrow.rb → neighbor.rb}

@@ -1,8 +1,9 @@
 module Clusta
   module Geometry
 
-    class Arrow < Element
-      field :target_label
+    class Neighbor < Element
+      
+      key   :label
       field :weight, :optional => true
 
       def directed?
@@ -13,4 +14,3 @@ module Clusta
     
   end
 end
-

data/lib/clusta/geometry/neighborhood.rb

@@ -0,0 +1,41 @@
+module Clusta
+  module Geometry
+    
+    class Neighborhood < Vertex
+      
+      extra_inputs :neighbors
+
+      def joins? label
+        neighbors.detect { |neighbor| neighbor.label == label }
+      end
+
+      def directed?
+        false
+      end
+
+      def size
+        neighbors.size
+      end
+
+      def degree_pairs
+        neighbors.map do |neighbor|
+          # This vertex's degree is just the size of this
+          # neighborhood.
+          # 
+          # We don't know anything about each neighbor's degree other
+          # than it must be at least 1 b/c it's in this vertex's
+          # neighborhood.
+          DegreePair.new(label, neighbor.label, size, 1)
+        end
+      end
+
+      def reversed_degree_pairs
+        neighbors.map do |neighbor|
+          DegreePair.new(neighbor.label, label, 1, size)
+        end
+      end
+      
+    end
+    
+  end
+end

data/lib/clusta/geometry/vertex.rb

@@ -2,7 +2,10 @@ module Clusta
   module Geometry
 
     class Vertex < Element
-      field :label
+
+      abbreviate 'V'
+      
+      key :label
     end
     
   end

data/lib/clusta/runner.rb

@@ -1,15 +1,112 @@
+require 'configliere'
+
+Settings.use :commandline
+
+Settings.define :transform,       :description => "The name of the tranformation to run.",                                    :required => false
+Settings.define :list_transforms, :description => "List known transformations.",                                              :required => false, :type => :boolean, :default => false
+Settings.define :class_names,     :description => "The output format for class names, one of: 'long', 'medium', or 'short'.", :required => true,  :default => 'medium'
+Settings.define :serialize,       :description => "The serialization format for data, one of: 'json' or 'tsv'.",              :required => true,  :default => 'tsv'
+Settings.define :transforms_path, :description => "A colon-separated list of directories to require transform definitions.",  :required => false, :default => ''
+Settings.define :geometry_path,   :description => "A colon-separated list of directories to require geometry definitions.",   :required => false, :default => ''
+
 module Clusta
   
   class Runner
 
-    def initialize args
-      self.args = args
+    RUN_ARG_REGEXP = /--run=./
+    
+
+    def initialize name, argv
+      @name = name
+      @argv = argv
     end
 
     def run!
-      Settings.resolve!
+      begin
+        Settings.resolve!
+        case
+        when Settings[:list_transforms]
+          load_transforms!
+          list_transforms!
+        when Settings[:list_geometry]
+          load_geometry!
+          list_geometry!
+        when Settings[:transform]
+          load_transforms!
+          load_geometry!
+          run_transform!
+        when Settings[:map_command] || Settings[:reduce_command]
+          run_map_reduce!
+        else
+          print_help!
+        end
+      rescue Clusta::Error => e
+        $stderr.puts "ERROR: #{e.message}"
+        exit(1)
+      end
+    end
+
+    def load_transforms!
+      rb_files_within(:transforms_path) do |path|
+        Clusta::Transforms.load_from(path)
+      end
+    end
+
+    def load_geometry!
+      rb_files_within(:geometry_path) do |path|
+        Clusta::Geometry.load_from(path)
+      end
+    end
+
+    def rb_files_within key, &block
+      return if Settings[key].nil? || Settings[key].empty?
+      Settings[key].split(':').each do |dir|
+        expanded = File.expand_path(dir)
+        unless File.directory?(expanded)
+          $stderr.puts("WARNING: #{expanded} is not a directory")
+          next
+        end
+        Dir[File.join(expanded, '*.rb')].each do |path|
+          yield path
+        end
+      end
+    end
+    
+    def list_transforms!
+      puts Clusta::Transforms.listing
+    end
+
+    def list_geometry!
+      puts Clusta::Geometry.listing
+    end
+    
+    def run_transform!
+      transform = Clusta::Transforms.from_name(Settings[:transform])
+      ::ARGV.replace(@argv)
+      ::ARGV.push('--run=local') unless ARGV.any? { |arg| arg =~ self.class::RUN_ARG_REGEXP }
+      script    = Clusta::Transforms.script_for(transform)
+      script.run
+    end
+
+    def run_map_reduce!
+      ::ARGV.replace(@argv)
+      ::ARGV.push('--run=local') unless ARGV.any? { |arg| arg =~ self.class::RUN_ARG_REGEXP }
+      begin
+        s = Wukong::Script.new(nil, nil)
+      rescue RuntimeError => e
+        raise Error.new(e.message)
+      end
+      s.run
+    end
+
+    def print_help!
+      begin
+        s = Wukong::Script.new(nil, nil)
+      rescue RuntimeError => e
+        raise Error.new(e.message)
+      end
+      s.run
     end
     
   end
-  
 end

data/lib/clusta/schema.rb

@@ -0,0 +1,100 @@
+module Clusta
+  module Schema
+
+    def extra_inputs
+      @extra_inputs ||= []
+    end
+    attr_writer :extra_inputs
+    
+    def extra_outputs
+      extra_inputs.map(&:to_s)
+    end
+
+    def fields
+      self.class.fields
+    end
+
+    def keys
+      self.class.keys
+    end
+
+    def non_key_fields
+      self.class.non_key_fields
+    end
+    
+    def self.included klass
+      klass.extend(ClassMethods)
+      class << klass ; attr_reader :fields ; end
+      klass.instance_variable_set('@fields', [])
+    end
+
+    module ClassMethods
+
+      def extra_inputs name
+        alias_method name, :extra_inputs
+      end
+      
+      def inherited(subclass)
+        subclass.instance_variable_set("@fields", @fields.dup)
+        super
+      end
+      
+      def field_names
+        @fields.map { |field| field[:name].to_s }
+      end
+      
+      def has_optional_field?
+        @fields.any? { |field| field[:optional] }
+      end
+
+      def has_non_key_field?
+        @fields.any? { |field| ! field[:key] }
+      end
+      
+      def optional_field
+        @fields.detect { |field| field[:optional] }
+      end
+
+      def keys
+        @fields.find_all { |field| field[:key] }
+      end
+
+      def non_key_fields
+        @fields.find_all { |field| ! field[:key] }
+      end
+      
+      def field name, options={}
+        raise AmbiguousArgumentsError.new("Cannot define a second optional field #{name} because field #{optional_field[:name]} is already optional.") if has_optional_field?
+        
+        raise SortError.new("The first field defined must be a key that can be sorted on.")                             if @fields.empty? && (!options[:key])
+        raise SortError.new("Cannot define a key field #{name} because some non-key fields have already been defined.") if options[:key] && has_non_key_field?
+        raise SortError.new("Key fields (#{name}) cannot have type :geometry")                                          if options[:key] && options[:type] == :geometry
+        
+        attr_reader name
+        
+        case options[:type]
+        when :int
+          define_method "#{name}=" do |val|
+            instance_variable_set("@#{name}", val.to_i)
+          end
+        when :float
+          define_method "#{name}=" do |val|
+            instance_variable_set("@#{name}", val.to_f)
+          end
+        else
+          define_method "#{name}=" do |val|
+            instance_variable_set("@#{name}", val)
+          end
+        end
+        @fields << options.merge(:name => name)
+      end
+
+      def key name, options={}
+        field name, options.merge(:key => true)
+      end
+      
+    end
+    
+  end
+end
+

data/lib/clusta/serialization.rb

@@ -0,0 +1,63 @@
+module Clusta
+
+  # Defines methods that allow a class to (de)serialize itself in a
+  # way compatabile with Wukong.
+  module Serialization
+
+    autoload :TSV,  'clusta/serialization/tsv'
+    autoload :JSON, 'clusta/serialization/json'
+
+    def self.included klass
+      klass.extend(ClassMethods)
+    end
+
+    def stream_name
+      self.class.stream_name
+    end
+
+    def initialize *args
+      process_args(*args)
+    end
+
+    def process_args *args
+    end
+    
+    module ClassMethods
+
+      def set_stream_name string
+        Geometry.register_element self, string
+        @stream_name = string
+      end
+
+      def abbreviate string
+        Geometry.register_element self, string
+        @abbreviation = string
+      end
+
+      def abbreviation
+        @abbreviation
+      end
+
+      def all_stream_names
+        [stream_name].tap do |names|
+          names << abbreviation if abbreviation
+          names << to_s
+          names << to_s.split('::').last if respond_to?(:name) && name
+        end
+      end
+
+      def stream_name
+        return @stream_name if @stream_name
+        case
+        when defined?(Settings) && Settings[:class_names].to_s == 'short' && abbreviation
+          @stream_name = abbreviation
+        when defined?(Settings) && Settings[:class_names].to_s == 'long'
+          @stream_name = to_s
+        else
+          @stream_name = to_s.split("::").last
+        end
+      end
+    end
+
+  end
+end

data/lib/clusta/serialization/json.rb

@@ -0,0 +1,86 @@
+require 'json'
+
+module Clusta
+  module Serialization
+    module JSON
+
+      def to_hash
+        {}.tap do |json|
+          fields.each do |field|
+            value = send(field[:name])
+            value = value.to_hash if value.respond_to?(:to_hash)
+            json[field[:name]] = value
+          end
+        end
+      end
+
+      def to_flat
+        [stream_name].tap do |record|
+          keys.each do |key|
+            record << self.send(key[:name])
+          end
+          data = non_key_field_data
+          record << data.to_json unless data.empty?
+          record.concat(extra_outputs)
+        end
+      end
+
+      def non_key_field_data
+        {}.tap do |data|
+          non_key_fields.each do |field|
+            value = send(field[:name])
+            value = value.to_hash if value.respond_to?(:to_hash)
+            data[field[:name]] = value unless value.nil? && field[:optional]
+          end
+        end
+      end
+
+      def process_args *args
+        json_index = 0
+        self.class.keys.each_with_index do |key, index|
+          self.send("#{key[:name]}=", args[index])
+          json_index = index + 1
+        end
+
+        if args[json_index]
+          if args[json_index].is_a?(Hash)
+            data = args[json_index]
+          else
+            data = ::JSON.parse(args[json_index])
+          end
+        else
+          data = {}
+        end
+
+        non_key_fields.each do |field|
+          name = field[:name].to_s
+          case 
+          when field[:optional]
+            self.send("#{name}=", data[name]) if data.has_key?(name)
+          when (!data.has_key?(name))
+            raise ArgumentError.new("A #{self.class} requires a non-nil value for #{name}.")
+          when field[:type] == :geometry
+            self.send("#{name}=", data[name])
+          else
+            self.send("#{name}=", data[name])
+          end
+        end
+        
+        self.extra_inputs = (args[(json_index + 1)..-1] || [])
+      end
+
+      def self.included klass
+        klass.extend(ClassMethods)
+      end
+
+      module ClassMethods
+        
+        def from_json_component data
+          
+        end
+        
+      end
+      
+    end
+  end
+end

data/lib/clusta/serialization/tsv.rb

@@ -0,0 +1,81 @@
+module Clusta
+  module Serialization
+    module TSV
+
+      def to_flat
+        [stream_name].tap do |record|
+          fields.each do |field|
+            value = send(field[:name])
+            value = value.to_tsv_component if field[:type] == :geometry
+            record << value.to_s unless value.nil? && field[:optional]
+          end
+          record.concat(extra_outputs)
+        end
+      end
+
+      def suffix index
+        case index.to_s
+        when /1$/ then 'st'
+        when /2$/ then 'nd'
+        when /3$/ then 'rd'
+        else 'th'
+        end
+      end
+      
+      def process_args *args
+        self.class.fields.each_with_index do |field, index|
+          case
+          when field[:optional]
+            self.send("#{field[:name]}=", args[index]) if args[index]
+          when args[index].nil?
+            raise ArgumentError.new("A #{self.class} requires a non-nil value for #{field[:name]} as its #{index}#{suffix(index)} argument.")
+          when field[:type] == :geometry
+            self.send("#{field[:name]}=", self.class.from_tsv_component_string(args[index]))
+          else
+            self.send("#{field[:name]}=", args[index])
+          end
+        end
+        self.extra_inputs = (args[self.class.fields.size..-1] || [])
+      end
+
+      def extra_inputs= inputs
+        @extra_inputs = inputs.map do |input|
+          if input =~ /^[A-Z].*;/
+            self.class.from_tsv_component_string(input)
+          else
+            input
+          end
+        end
+      end
+
+      def extra_outputs
+        @extra_inputs.map do |input|
+          if input.respond_to?(:to_tsv_component)
+            input.to_tsv_component
+          else
+            input
+          end
+        end
+      end
+
+      def self.included klass
+        klass.extend(ClassMethods)
+      end
+
+      def to_tsv_component
+        to_flat.join(";")
+      end
+
+      module ClassMethods
+        
+        def from_tsv_component_string string
+          return string unless string.is_a?(String)
+          args  = string.split(';')
+          klass_name = args.shift
+          raise ArgumentError.new("Elements instantiated from a TSV component string must match the format 'klass;[field1;[field2;]...]'") unless klass_name
+          Wukong.class_from_resource(klass_name).new(*args)
+        end
+      end
+    end
+  end
+end