wukong 0.1.4 → 1.4.0

data/lib/wukong/and_pig/operators/evaluators.rb

@@ -1,7 +0,0 @@
-module Wukong
-  module AndPig
-    class PigVar
-
-    end
-  end
-end

data/lib/wukong/and_pig/operators/execution.rb

@@ -1,15 +0,0 @@
-module Wukong
-  module AndPig
-    class PigVar
-
-      # ===========================================================================
-      #
-      # STREAM
-      #
-      def stream options={}
-        new_in_chain klass, "STREAM #{relation}"
-      end
-    end
-  end
-end
-

data/lib/wukong/and_pig/operators/file_methods.rb

@@ -1,29 +0,0 @@
-module Wukong
-  module AndPig
-    class PigVar
-      # ===========================================================================
-      #
-      # Pig expressions
-      #
-
-      #
-      def dfs cmd, filename
-        # note == no '' on path
-        self.class.emit "%-23s\t           %s" % [cmd, filename]
-      end
-      #
-      # remove the stored file
-      #
-      def rmf! filename
-        dfs :rmf, filename
-      end
-
-      #
-      #
-      #
-      def mkdir filename
-        dfs :mkdir, filename
-      end
-    end
-  end
-end

data/lib/wukong/and_pig/operators/foreach.rb

@@ -1,98 +0,0 @@
-#
-# The FOREACH relational operator
-#
-module Wukong
-  module AndPig
-    class PigVar
-
-      # ===========================================================================
-      #
-      # FOREACH
-      #
-      def generate lval,  *field_specs
-        gen_clauses = field_specs.map{|field_spec| parse_gen_clause(field_spec)}.flatten
-        l_klass     = TypedStruct.new(* gen_clauses.map(&:name_type))
-        l_cmd       = "FOREACH  #{self.relation} GENERATE\n  #{gen_clauses.join(",\n  ")}"
-        new_in_chain(lval, l_klass, l_cmd)
-      end
-
-      #
-      # for a list of GENERATE args, we need
-      #
-      # * gen_clauses, the clause to stuff into the GENERATE line
-      #     gen_expr AS gen_field_name: gen_field_type
-      #
-      # * new_types, the resulting types for each
-      #
-      # gen_expr common cases include
-      #
-      #   field
-      #   Rel::field
-      #   Rel.(field)
-      #   "ComplicatedExpression"
-      #
-      #
-      # field_attrs
-      #
-      #
-      def parse_gen_clause field_spec
-        case field_spec
-        when AS
-          field_spec
-        when Symbol
-          AS[field_spec, field_spec, field_type(field_spec)];
-        when Array
-          alias_in, field_in, name, type = field_spec
-          name      ||= field_in
-          type        = alias_in.field_type(field_in)
-          AS[field_in, name, type, alias_in.relationize]
-        when Hash
-          field_spec.map do |field_in, field_out|
-            AS[field_in, field_out, field_type(field_in)]
-          end
-        else raise "Don't know how to specify type for #{field_specs.inspect}"
-        end
-      end
-    end
-  end
-end
-
-
-
-
-
-
-
-          # # when Array
-          # #   unless [2,3].include?(field_spec.length) then raise "Complex fields must be (field_spec, as_name) or (field_spec, as_name, as_type)" end
-          # #   field_expr, field_attr, field_type = field_spec
-          # #   field_as   = field_attr.is_a?(Array) ? "(#{field_attr.join(", ")})" : field_attr
-          # #   gen_clauses << "#{field_expr} AS #{field_as}"
-          # #   field_attrs << [field_attr, field_type || klass.members_types[field_expr]]
-
-      # def prelimify *field_specs
-      #   gen_clauses = []
-      #   field_attrs = []
-      #   field_specs.map do |field_spec|
-      #     unless field_spec.length == 2 then raise "Complex fields must be a pair (field_spec, as_name)" end
-      #     field_expr, field_attr = field_spec
-      #     gen_clauses << "#{field_expr}"
-      #     field_attrs += [field_attr].flatten
-      #   end
-      #   [ gen_clauses, field_attrs ]
-      # end
-      #
-      # # def generate *args
-      # #   gen_clauses, field_attrs = self.class.fieldify *args
-      # #   l_klass = TypedStruct.new(*field_attrs)
-      # #   new_in_chain l_klass, "FOREACH  #{relation} GENERATE\n    #{gen_clauses.join(",\n    ")}"
-      # # end
-      #
-      # def foreach *args
-      #   generate_clause = args.pop
-      #   prelim_exprs, prelim_attrs = prelimify *args
-      #   prelims = prelim_exprs.zip(prelim_attrs).map{|e,a| "#{a} = #{e}" }.join(";\n    ")+";"
-      #   gen_clauses, field_attrs   = fieldify *generate_clause
-      #   l_klass = TypedStruct.new(*field_attrs)
-      #   new_in_chain l_klass, %Q{FOREACH  #{relation} {\n    #{prelims}\n  GENERATE\n    #{gen_clauses.join(",\n    ")} ; } }
-      # end

data/lib/wukong/and_pig/operators/groupies.rb

@@ -1,212 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# The FOREACH relational operator
-#
-module Wukong
-  module AndPig
-    class PigVar
-
-      #===========================================================================
-      #
-      # GROUP and COGROUP
-      #
-
-      #
-      # COGROUP - Groups the data in two or more relations.
-      #
-      # == Syntax
-      #
-      #   alias  = COGROUP alias1 BY field_alias [INNER | OUTER],
-      #                    aliasN BY field_alias [INNER | OUTER] [PARALLEL n] ;
-      #
-      # == Structure
-      #
-      #   { group, <structure of alias1>, <structure of alias2>, ... }
-      #
-      # == Terms
-      #
-      # * alias         The name a relation.
-      #
-      # * field_alias The name of one or more fields in a relation.  If multiple
-      #                 fields are specified, separate with commas and enclose
-      #                 in parentheses. For example, X = COGROUP A BY (f1, f2);
-      #
-      #                 The number of fields specified in each BY clause must
-      #                 match. For example, X = COGROUP A BY (a1,a2,a3), B BY
-      #                 (b1,b2,b3);
-      #
-      # * BY            Keyword.
-      #
-      # * INNER         Eliminate NULLs on that grouping
-      # * OUTER         Do not eliminate NULLs on that grouping (default)
-      #
-      # * PARALLEL n -- Increase the parallelism of a job by specifying the
-      #                 number of reduce tasks, n. The optimal number of
-      #                 parallel tasks depends on the amount of memory on each
-      #                 node and the memory required by each of the tasks. To
-      #                 determine n, use the following as a general guideline:
-      #
-      #                     n = (nr_nodes - 1) * 0.45 * nr_GB
-      #
-      #                 where nr_nodes is the number of nodes used and nr_GB is
-      #                 the amount of physical memory on each node.
-      #
-      #                 Note the following:
-      #                 - Parallel only affects the number of reduce tasks. Map
-      #                   parallelism is determined by the input file, one map
-      #                   for each HDFS block.
-      #                 - If you don’t specify parallel, you still get the same
-      #                   map parallelism but only one reduce task.
-      #
-      # == Usage
-      #
-      # The COGOUP operator groups the data in two or more relations based on
-      # the common field values.
-      #
-      # Note: The COGROUP and JOIN operators perform similar functions. COGROUP
-      # creates a nested set of output tuples while JOIN creates a flat set of
-      # output tuples with NULLs eliminated.
-      #
-      # == Examples
-      #
-      # Suppose we have two relations, A and B.
-      #
-      # A: (owner:chararray, pet:chararray)
-      # ---------------
-      # (Alice, cat)
-      # (Alice, goldfish)
-      # (Alice, turtle)
-      # (Bob,   cat)
-      # (Bob,   dog)
-      #
-      # B: (friend1:chararray, friend2:charrarray)
-      # ---------------------
-      # (Cindy, Alice)
-      # (Mark, Alice)
-      # (Paul, Bob)
-      # (Paul, Jane)
-      #
-      # In this example tuples are co-grouped using field “owner” from relation
-      # A and field “friend2” from relation B as the key fields. The DESCRIBE
-      # operator shows the schema for relation X, which has two fields, "group"
-      # and "A" (for an explanation, see GROUP).
-      #
-      #   X = COGROUP A BY owner, B BY friend2;
-      #   DESCRIBE X;
-      #
-      #    X: {group: chararray,
-      #        A: {owner:   chararray,pet:     chararray},
-      #        B: {friend1: chararray,friend2: chararray}}
-      #
-      # Relation X looks like this. A tuple is created for each unique key
-      # field. The tuple includes the key field and two bags. The first bag is
-      # the tuples from the first relation with the matching key field. The
-      # second bag is the tuples from the second relation with the matching key
-      # field. If no tuples match the key field, the bag is empty.
-      #
-      #   (Alice, {(Alice, turtle), (Alice, goldfish), (Alice, cat)},
-      #           {(Cindy, Alice), (Mark, Alice)})
-      #   (Bob,   {(Bob, dog), (Bob, cat)},
-      #           {(Paul, Bob)})
-      #   (Jane,  {},
-      #           {(Paul, Jane)})
-      #
-      # In this example tuples are co-grouped and the INNER keyword is used to
-      # ensure that only bags with at least one tuple are returned.
-      #
-      #   X = COGROUP A BY owner INNER, B BY friend2 INNER;
-      #
-      # Relation X looks like this.
-      #
-      #   (Alice, {(Alice, turtle), (Alice, goldfish), (Alice, cat)},
-      #           {(Cindy, Alice), (Mark, Alice)})
-      #   (Bob,   {(Bob, dog), (Bob, cat)},
-      #           {(Paul, Bob)})
-      #
-      # In this example tuples are co-grouped and the INNER keyword is used
-      # asymmetrically on only one of the relations.
-      #
-      #   X = COGROUP A BY owner, B BY friend2 INNER;
-      #
-      # Relation X looks like this.
-      #
-      #   (Alice, {(Alice, turtle), (Alice, goldfish), (Alice, cat)},
-      #           {(Cindy, Alice), (Mark, Alice)})
-      #   (Bob,   {(Bob, dog), (Bob, cat)},
-      #           {(Paul, Bob)})
-      #   (Jane,  {},
-      #           {(Paul, Jane)})
-      #
-      #
-      def group group_by
-        l_klass   = l_klass_for_group group_by
-        by_clause = self.class.make_by_clause(group_by)
-        new_in_chain anon, l_klass, "GROUP    #{relation} #{by_clause}"
-      end
-
-      def self.make_by_clause by_spec
-        case by_spec
-        when Array      then 'BY ' + by_spec.join(", ")
-        when :all       then 'ALL'
-        when Symbol     then "BY #{by_spec}"
-        when String     then by_spec
-        when Hash       then make_by_clause(by_spec[:by])
-        else raise "Don't know how to group on #{by_spec.inspect}"
-        end
-      end
-      def types_for_fields field
-        klass.members_types[field]
-      end
-      def l_klass_for_group group_by
-        self.class.l_klass_for_group group_by, self
-      end
-      def self.l_klass_for_group group_by, *rels
-        TypedStruct.new(
-          [:group,       rels.first.types_for_fields(group_by)],
-          *rels.map{|rel| [rel.relation, rel.klass] }
-          )
-      end
-
-      #
-      # COGROUP pig expression:
-      #   UserPosts = COGROUP Posts BY user_id, Users BY user_id ;
-      #
-      def self.cogroup lval, *by
-        by_clause = by.map do |relation, group_by, as|
-          "%s %s" % [relation.relation, make_by_clause(group_by)]
-        end.join(", ")
-        l_klass  = l_klass_for_group by[0][1], *by.map(&:first)
-        rval = new l_klass, lval, "COGROUP    #{by_clause}"
-        set lval, rval
-      end
-
-      def cogroup *args
-        self.class.cogroup self, *args
-      end
-
-
-      # ===========================================================================
-      #
-      # JOIN
-      #
-      def self.klass_from_join by
-        klasses = by.map(&:first)
-        TypedStruct.new(*klasses.zip(klasses.map(&:klass)))
-      end
-
-      def self.join_by_clause by
-        by.map{|rel, field| "#{rel.relationize} BY #{field}" }.join(", ")
-      end
-
-      def self.join lval, by
-        parallel = by.delete(:parallel)
-        cmd  = "JOIN " + join_by_clause(by)
-        parallelize! cmd, :parallel => parallel
-        l_klass = klass_from_join(by)
-        rval = new(l_klass, lval, cmd)
-        set lval, rval
-      end
-
-    end
-  end
-end

data/lib/wukong/and_pig/operators/load_store.rb

@@ -1,65 +0,0 @@
-# == Load/StoreFunctions ==
-# BinaryDeserializer
-# BinarySerializer
-# BinStorage
-# PigStorage
-# PigDump
-# TextLoader
-
-module Wukong
-  module AndPig
-    class PigVar
-      #===========================================================================
-      #
-      # The "LOAD" pig expression:
-      #   MyRelation = LOAD 'my_relation.tsv' AS (attr_a: int, attr_b: chararray) ;
-      #
-      # The AS type spec is generated from klass
-      #
-      def self.pig_load rel, klass, options={ }
-        filename = options[:filename] || default_filename(rel)
-        self.set rel, self.new(klass, rel, "LOAD    '#{filename}' AS #{klass.typify(options[:has_rsrc])}")
-        if options[:has_rsrc]
-          lval = self[rel]
-          lval.generate lval, *lval.fields
-        end
-        rel
-      end
-
-      #===========================================================================
-      #
-      #
-      # The "STORE" pig imperative:
-      #   STORE Relation INTO 'filename'
-      # If no filename is given, the relation's name is used
-      #
-      def store filename=nil
-        filename ||= default_filename
-        self.class.emit "STORE %-19s INTO    '%s'" % [relation, filename]
-        self
-      end
-
-      # Store the relation, removing the existing file
-      def store! filename=nil
-        filename ||= default_filename
-        rmf!  filename
-        mkdir File.dirname(filename)
-        store filename
-      end
-
-      # Force a store to disk, then load (so all calculations proceed from there)
-      def checkpoint! options={}
-        options = options.reverse_merge :filename => default_filename
-        store!   options[:filename]
-        self.class.pig_load(self.name, klass, options)
-      end
-
-      def default_filename
-        self.class.default_filename self.name
-      end
-      def self.default_filename name
-        File.join(working_dir, name.to_s)
-      end
-    end
-  end
-end

data/lib/wukong/and_pig/operators/meta.rb

@@ -1,42 +0,0 @@
-# == DiagnosticOperators
-# describe
-# dump
-# explain
-# illustrate
-# == UDFStatements
-# define
-# register
-
-module Wukong
-  module AndPig
-    class PigVar
-      # DESCRIBE pig imperative
-      def describe
-        self.class.describe self
-      end
-      def self.describe rel
-        emit %Q{ -- PREDICTED: #{rel.klass.typify} }
-        simple_declaration :describe, rel.relationize
-        rel
-      end
-
-      # DUMP pig imperative
-      def dump()       simple_operation :dump        end
-
-      # EXPLAIN pig imperative
-      def explain()    simple_operation :explain     end
-
-      # ILLUSTRATE pig imperative
-      def illustrate() simple_operation :illustrate  end
-
-
-      def self.define pig_alias, *args
-        emit_imperative :DEFINE, pig_alias, args
-      end
-
-      def self.register path_to_jar
-        emit_imperative :REGISTER, path_to_jar
-      end
-    end
-  end
-end