zillabyte 0.9.48 → 0.9.49

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2d757597a1cc573c8e11bd13735efceb7bc03e73
-  data.tar.gz: d5c2a9a059791646d05d16c40fe906942522ba13
+  metadata.gz: 11c2e486f17f6b1e1198b6700e8519b4d1f9fecf
+  data.tar.gz: 7d57f06481ef828626ce8a960ff93e97dec78ee2
 SHA512:
-  metadata.gz: 6b109dc4ec9e2b27c610306c41bf4f35b9922431a1a47579c182326275263e7d6da63ab901719bde554e3d0f69ad23790109552d8e095b36f0803c5d501f3ea0
-  data.tar.gz: d5501fffe0acbb455dbecde048b382fd3642db7391c06eb23a80a9423e1555e6c905e72b5814ddc4c1442e04b002bc154010ca8a80fadeb74b72c7434da26cb0
+  metadata.gz: 7d9e61f6045437500441db54366e0e167da9fe108e8f5ccd086f3cd9f23445fa5d27b879a6d63828dd3dd4fcaa73b278ec27205ea9fa947dd6df65b54f271e2f
+  data.tar.gz: 9b0d9d9dc62671f262b35082a4a7bc85d71ca5833984c890d70bdee2dcfb68045f25ebf82268966101de7df445c11ad6b562c29dfae1daff6326fe9fae4b7611

data/ruby/lib/zillabyte/harness/app.rb

@@ -30,11 +30,14 @@ class Zillabyte::Harness::App < Zillabyte::Harness::Base
     _source_common(Zillabyte::Harness::Stream, *args, &block)
   end
   
+   def source_from_csv(*args, &block)
+    _source_from_csv_common(Zillabyte::Harness::Stream, *args)
+  end 
   
   def source_from_kinesis(*args, &block)
     _source_common(Zillabyte::Harness::Stream, "kinesis_demo_source")
   end
-  
+end
   
   
   

data/ruby/lib/zillabyte/harness/base.rb

@@ -35,5 +35,13 @@ class Zillabyte::Harness::Base
     return Zillabyte::Harness::StreamBuilder.new(stream)
   end 
   
-
+  def _source_from_csv_common(stream_class, *args, &block)
+    op = Zillabyte::Harness::OperationHandler.new(self, stream_class)
+    stream = op.build_multilang_operation("source_from_csv", *args, &block)
+      .add_operation_properties_to_info(:name, :type)
+      .handle_operation
+      .get_output_streams
+    return Zillabyte::Harness::StreamBuilder.new(stream)
+  end 
+ 
 end

data/ruby/lib/zillabyte/harness/helper.rb

@@ -90,10 +90,10 @@ class Zillabyte::Harness::Helper
     ee = "Error in \"source\": \n\t "
     pp = @@_print_check_source
     
-    rm = !source._relation.nil?
+    rm = !source._dataset.nil?
     mm = !source._matches.nil?
     if(rm or mm)
-      msg = "#{ee}A custom \"source\" may not contain a \"matches\" clause. To source from a relation, see the following. #{pp}"
+      msg = "#{ee}A custom \"source\" may not contain a \"matches\" clause. To source from a dataset, see the following. #{pp}"
       Zillabyte::Harness::Helper.print_error(msg)
     end
 
@@ -113,16 +113,31 @@ class Zillabyte::Harness::Helper
     pp = @@_print_check_source
 
     if(args.length != 1)
-      msg = "#{ee}Invalid number of arguments to \"source\". When sourcing from a relation, please supply a single SQL query-string or SXP query-array. #{pp}"
+      msg = "#{ee}Invalid number of arguments to \"source\". When sourcing from a dataset, please supply a single dataset name string. #{pp}"
       Zillabyte::Harness::Helper.print_error(msg)
     end
 
     if(!args[0].instance_of?(String) and !args[0].instance_of?(Array))
-      msg = "#{ee}Invalid argument to \"source\". When sourcing from a relation, please supply a single SQL query-string or SXP query-array. #{pp}"
+      msg = "#{ee}Invalid argument to \"source\". When sourcing from a dataset, please supply a single dataset name string. #{pp}"
       Zillabyte::Harness::Helper.print_error(msg)
     end
   end
 
+  def self.check_source_from_csv_args(args)
+    ee = "Error in \"source_from_csv\": \n\t "
+    pp = @@_print_check_source_from_csv
+
+    if(args.length > 2)
+      msg = "#{ee}Invalid number of arguments to \"source_from_csv\". When sourcing from a CSV, please supply a single CSV filename and optionally a header keyword argument as an array of header names. #{pp}"
+      Zillabyte::Harness::Helper.print_error(msg)
+    end
+    if(!args[0].instance_of?(String) and !args[0].instance_of?(Array))
+      msg = "#{ee}Invalid argument to \"source\". When sourcing from a CSV, please supply a single CSV filename and optionally a header keyword argument as an array of header names. #{pp}"
+      Zillabyte::Harness::Helper.print_error(msg)
+    end
+  end
+
+
   def self.check_emits(operation, emits, streams)
     if operation == "component"
       oo = "outputs"
@@ -302,13 +317,13 @@ class Zillabyte::Harness::Helper
     columns = sink._columns
 
     if(!name)
-      msg = "#{ee}Relation name must be specified! #{pp}"
+      msg = "#{ee}dataset name must be specified! #{pp}"
       Zillabyte::Harness::Helper.print_error(msg)
     end
     Zillabyte::Harness::Helper.check_name("sink", sink._name, {})
 
     if(columns.length == 0)
-      msg = "#{ee}Must be at least one output field to relation \"#{name}\". #{pp}"
+      msg = "#{ee}Must be at least one output field to dataset \"#{name}\". #{pp}"
       Zillabyte::Harness::Helper.print_error(msg)
     end
     Zillabyte::Harness::Helper.check_sink_columns(sink)
@@ -318,7 +333,7 @@ class Zillabyte::Harness::Helper
         next
       end
       if(s._name == name and s._columns != columns)
-        msg = "#{ee}The relation \"#{name}\" has already been specified and contains a different set of fields/types. #{pp}"
+        msg = "#{ee}The dataset \"#{name}\" has already been specified and contains a different set of fields/types. #{pp}"
         Zillabyte::Harness::Helper.print_error(msg)
       end
     end
@@ -334,19 +349,19 @@ class Zillabyte::Harness::Helper
     end
   end
 
-  def self.check_field_format(operation, pp, cname, ctype, relation_name)
+  def self.check_field_format(operation, pp, cname, ctype, dataset_name)
     ee = "Error in \"#{operation}\": \n\t "
 
     if(!(cname.is_a?(String) or cname.is_a?(Symbol)) or (cname =~ /^\w+$/).nil?)
-      msg = "#{ee}Field names must be non-empty STRINGS with only alphanumeric and underscore characters in \"#{relation_name}\". #{pp}"
+      msg = "#{ee}Field names must be non-empty STRINGS with only alphanumeric and underscore characters in \"#{dataset_name}\". #{pp}"
       Zillabyte::Harness::Helper.print_error(msg)
     end
     if(!ctype.instance_of?(Symbol))
-      msg = "#{ee}Field data types must be SYMBOLS in \"#{relation_name}\". #{pp}"
+      msg = "#{ee}Field data types must be SYMBOLS in \"#{dataset_name}\". #{pp}"
       Zillabyte::Harness::Helper.print_error(msg)
     end
     if(!ALLOWED_TYPES.member?(ctype))
-      msg = "#{ee}Invalid field data type at \"#{ctype}\" in \"#{relation_name}\". #{pp}"
+      msg = "#{ee}Invalid field data type at \"#{ctype}\" in \"#{dataset_name}\". #{pp}"
       Zillabyte::Harness::Helper.print_error(msg)
     end
   end
@@ -466,11 +481,11 @@ class Zillabyte::Harness::Helper
   @@_print_check_sink = <<-OUTPUT
 \n\n"Sink" Syntax:
     stream.sink do
-       name "name_of_relation"
+       name "name_of_dataset"
        column "field_1", :type_1
        column "field_2", :type_2 ...
     end
-  - "Sink" relation "name" must be specified as a non-empty STRING with only alphanumeric and underscore characters!
+  - "Sink" dataset "name" must be specified as a non-empty STRING with only alphanumeric and underscore characters!
   - Field names must be non-empty STRINGS with only alphanumeric or underscore characters.
   - Field names cannot be "v[number]", "id", "confidence", "since" or "source" which are reserved Zillabyte names.
   - Field types must be SYMBOLS. The following types are allowed #{ALLOWED_TYPES.join(", ")}.
@@ -478,8 +493,8 @@ OUTPUT
 
   @@_print_check_source = <<-OUTPUT
 \n\n"Source" Syntax:
-  Sourcing from a relation:
-    app.source("SQL query" or [SXP queries])
+  Sourcing from a dataset:
+    app.source("dataset" name)
 
   Custom source:
     app.source do
@@ -495,6 +510,19 @@ OUTPUT
   - The "begin_cycle" and "next_tuple" blocks can be in do...end format or {} format.
       * the "begin_cycle" block is where any setup is done to initialize the content and quantity of tuples emitted by the "next_tuple" block.
       * the "next_tuple" block is where the tuples are actually emitted.
+OUTPUT
+
+  @@_print_check_source_from_csv = <<-OUTPUT
+\n\n"SourceFromCSV" Syntax:
+  Sourcing from a CSV with headers:
+    app.source_from_csv("csv_path")
+
+  Source from a CSV without headers
+    app.source_from_csv("csv_path", headers = ["name", "email"])
+
+  - The "csv_path" is the path to the CSV within your app directory.
+  - The "headers" field is an array of header names for the source to use.
+
 OUTPUT
 
   @@_print_check_filter = <<-OUTPUT

data/ruby/lib/zillabyte/harness/live_delegator.rb

@@ -399,13 +399,40 @@ module Zillabyte
       end
 
       def next_tuple
-        self.instance_eval &@harness._next_tuple
+        self.instance_exec(&@harness._next_tuple)
       end
 
     end
 
+
+
+    class SourceFromCSVController < Storm::Source
+
+      def initialize(harness, progress)
+        @harness = harness
+        @progress = progress
+
+        Storm::Protocol.emits = harness._emits
+        Storm::Protocol.end_cycle_policy = harness._end_cycle_policy
+      end
+
+      def begin_cycle(*args)
+        @harness.begin_cycle
+      end
+      
+      def prepare(*args)
+      end
+
+      def next_tuple
+        @harness.next_tuple(self)
+      end
+
+    end
+
+
     class EachController < Storm::Each
 
+
       def initialize(harness, progress)
         @harness = harness
         @progress = progress

data/ruby/lib/zillabyte/harness/operation_handler.rb

@@ -18,6 +18,8 @@ class Zillabyte::Harness::OperationHandler
     case type
     when "source"
       @_operation = Zillabyte::Harness::Source.new(@_app, *args)
+    when "source_from_csv"
+      @_operation = Zillabyte::Harness::SourceFromCSV.new(@_app, *args)
     when "each"
       @_operation = Zillabyte::Harness::Each.new(@_app, *args)
     when "filter"

data/ruby/lib/zillabyte/harness/source.rb

@@ -71,7 +71,7 @@ class Zillabyte::Harness::Source
   attr_accessor :_app, :_node, :_relation, :_options
 
   class Node < Zillabyte::Harness::CommonNode 
-    attr_accessor :_matches, :_relation, :_end_cycle_policy, :_begin_cycle, :_next_tuple
+    attr_accessor :_filename, :_headers, :_matches, :_relation, :_end_cycle_policy, :_begin_cycle, :_next_tuple
 
     def initialize(v, options = {})
       @_name = options[:name] || "source_"+Zillabyte::Harness::Counter.get()
@@ -87,8 +87,8 @@ class Zillabyte::Harness::Source
       when Array
         @_matches = v
       end
-    end
- 
+    end 
+
     def parallelism(v)
       throw "parallelism cannot be specified for sources"
     end

data/ruby/lib/zillabyte/harness/source_from_csv.rb

@@ -0,0 +1,111 @@
+# OPERATION SourceFromCSV
+# TAGLINE 
+# A source from CSV allows your app to easy source from a CSV on your local machine
+# DESCRIPTION
+#
+# ## Sourcing From A CSV With Headers
+#
+# By default, the SourceFromCSV will parse the first line of the CSV for headers, these headers are the keys of the tuple hash object 
+# emitted by the source.
+#
+# ## Source From A CSV Without Headers 
+#
+# You can also specify the headers as a keyword argument to the SourceFromCSV operation. See the examples below for the syntax.
+#
+# RUBY_SYNTAX
+#
+#
+# # Sourcing from a CSV with Headers
+# # Assume that companies.csv is a CSV with headers ["name", "email"]
+# app.source_from_csv("companies.csv") # emits tuples in form {:name => "company", :email => "example@company.com"}
+#
+# # Sourcing from a CSV without Headers
+#
+# app.source_from_csv("companies.csv", headers=["name", "email"])
+#
+#
+# SYNTAX_NOTES
+#
+# The first argument to the source_from_csv is the relative path of the CSV within the app directory. Absolute paths are 
+# not available when pushing to the Zillabyte Servers.
+#
+#
+#
+# RUBY_EXAMPLE
+#
+# # # Sourcing from a CSV with Headers
+# # Assume that companies.csv is a CSV with headers ["name", "email"]
+# stream = app.source_from_csv("companies.csv") # emits tuples in form {:name => "company", :email => "example@company.com"}
+#
+# stream = stream.each do |tuple|
+#   name = stream["name"]
+#   email = tuple["email"]
+#
+#   # use the name and email here ...
+#
+# end
+#
+class Zillabyte::Harness::SourceFromCSV
+
+  attr_accessor :_app, :_node, :_options
+
+
+  class Node < Zillabyte::Harness::CommonNode
+    attr_accessor :_filename, :_file, :_headers, :_end_cycle_policy, :_harness
+
+    def initialize(v, options = {})
+      @_name = options[:name] || "source_"+Zillabyte::Harness::Counter.get()
+      @_type = 'source'
+      @_end_cycle_policy = :null_emit
+      require 'csv'
+    end
+
+    def begin_cycle
+      @_file = File.open(@_filename)
+      if @_headers.nil?
+        @_headers = CSV.parse_line(@_file.readline)
+      end
+    end
+
+    def next_tuple(harness)
+      begin
+        line = @_file.readline
+        csvline = CSV.parse_line(line)
+        tuple = {}
+        @_headers.each do |header|
+          tuple[header] = csvline.shift
+        end
+        harness.emit tuple
+      rescue EOFError => e
+      end
+    end
+  end
+
+  def initialize(app, *args)
+    @_app = app
+    @_args, @_options = Zillabyte::Harness::Helper.get_vector_and_hashes(args)
+  end
+
+  def build_node(&block)
+    @_node = Node.new(@_args, @_options)
+    @_node._filename = @_args[0]
+
+    if @_args.length > 1
+      case @_args[1]
+      when Array
+        @_node._headers = @_args[1]
+      else
+        throw "Header argument must be an array of filenames"
+      end
+    end
+
+    @_node._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
+  end
+
+      
+  def run_operation
+    c = Zillabyte::Harness::SourceFromCSVController.new(@_node, Zillabyte::Common::Progress.new)
+    c.run(@_app._options)
+  end
+
+end

data/ruby/lib/zillabyte/version.rb

@@ -1,3 +1,3 @@
 module Zillabyte
-  VERSION = "0.9.48" unless defined?(VERSION)
+  VERSION = "0.9.49" unless defined?(VERSION)
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zillabyte
 version: !ruby/object:Gem::Version
-  version: 0.9.48
+  version: 0.9.49
 platform: ruby
 authors:
 - zillabyte
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-22 00:00:00.000000000 Z
+date: 2014-12-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.9.48
+        version: 0.9.49
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.9.48
+        version: 0.9.49
 description: The Official Zillabyte Gem
 email:
 - gem@zillabyte.com
@@ -68,6 +68,7 @@ files:
 - ruby/lib/zillabyte/harness/operation_handler.rb
 - ruby/lib/zillabyte/harness/sink.rb
 - ruby/lib/zillabyte/harness/source.rb
+- ruby/lib/zillabyte/harness/source_from_csv.rb
 - ruby/lib/zillabyte/harness/stream.rb
 - ruby/lib/zillabyte/harness/stream_builder.rb
 - ruby/lib/zillabyte/harness/tuple.rb