zillabyte 0.9.9 → 0.9.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,15 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: 17fe4f4d78abd8577811505fc480db3714eaafa5
4
- data.tar.gz: 46dba9537746b0dd6f7c41825a00b3deb91d269b
5
- SHA512:
6
- metadata.gz: a926a7beeeb77de3ac37838af7fedeac12802b958d02ed453b75b9d806b74966a27831564e752bf4bd12a3c0b7100ef6cabbe67c749473f192558b1809e8267f
7
- data.tar.gz: 07b329196b76990a06315fa7fe36df5fb7a63b2f696d7c1715522eebbcbc808a80c3d04badbcc797ce616e82652f5e4b6ad3536114101bb6a5708a3bce03aec8
2
+ !binary "U0hBMQ==":
3
+ metadata.gz: !binary |-
4
+ NGUxNzIxMjQyMmFmY2Y1YjNhYTg2YWJmNjcwN2U5MTIwYzM3MjJhMw==
5
+ data.tar.gz: !binary |-
6
+ ZTkwYTI4NDcyMTJjNDQxMmM5Y2UzODM5ZmZiYzFmNDAxZDg0NGY4Mg==
7
+ !binary "U0hBNTEy":
8
+ metadata.gz: !binary |-
9
+ YTM3YWE0MWNhNzI5ZDkyY2Y5YTI3ODk1M2MzZmE5MTY0OWE1ZDU5ODgwOWI0
10
+ MDhiMmY5OTNmMWJlNzQwN2RiZGM4YmE0NTEwMzdmNTBjNTRhYWM4MDA3MmYx
11
+ ZmY0ODU4NjBhOTEyNWJlY2E4MzI5ZTI3MzM4YTkxMTllYmY4YjM=
12
+ data.tar.gz: !binary |-
13
+ NDQzMjQ4OTk0YTVkYzVhZGU2NDk5YzM1ZWE5ZWJiN2YxMGZiOWQ4NjRkNDVl
14
+ NTNjMjIwMjhkMTdlYTdkYmY3NDQzMzQwYTczZDI3Njc0YThhNjkzYzE3MTA5
15
+ NzRkZDY0YzY0YjJiNjU5ZjNlYWI3NGUzNDUwYTNiMzMxYWRkYTY=
@@ -31,20 +31,11 @@ class Zillabyte::Harness::App < Zillabyte::Harness::Base
31
31
  end
32
32
 
33
33
 
34
- def sink_to_redshift(*args, &block)
35
- _source_common(Zillabyte::Harness::Stream, *args, &block)
36
- end
37
-
38
-
39
34
  def source_from_kinesis(*args, &block)
40
35
  _source_common(Zillabyte::Harness::Stream, "kinesis_demo_source")
41
36
  end
42
37
 
43
38
 
44
- def sink_to_kinesis(*args, &block)
45
- _source_common(Zillabyte::Harness::Stream, "kinesis_demo_sink")
46
- end
47
-
48
39
 
49
40
 
50
41
  def call_component(*args, &block)
@@ -31,4 +31,6 @@ class Zillabyte::Harness::Component < Zillabyte::Harness::Base
31
31
  .get_output_streams
32
32
  return Zillabyte::Harness::StreamBuilder.new(stream)
33
33
  end
34
+ alias_method :input, :inputs
35
+
34
36
  end
@@ -17,6 +17,8 @@ class Zillabyte::Harness::ComponentInput
17
17
  def field(name, type)
18
18
  @_fields << {name => type}
19
19
  end
20
+ alias_method :column, :field
21
+
20
22
  end
21
23
 
22
24
  def initialize(app, *args)
@@ -21,6 +21,7 @@ class Zillabyte::Harness::ComponentOutput
21
21
  col = {cname => ctype}
22
22
  @_columns << col
23
23
  end
24
+ alias_method :column, :field
24
25
 
25
26
  def scope(v)
26
27
  @_scope = v
@@ -10,5 +10,6 @@ class Zillabyte::Harness::ComponentStream < Zillabyte::Harness::Stream
10
10
  .create_arc_info_from_stream(self)
11
11
  .handle_operation
12
12
  end
13
+ alias_method :output, :outputs
13
14
 
14
15
  end
@@ -54,10 +54,11 @@ class Zillabyte::Harness::Each
54
54
  class Node < Zillabyte::Harness::CommonNode
55
55
  attr_accessor :_output_format, :_execute
56
56
 
57
- def initialize(*args)
58
- @_name = args[0] || "each_"+Zillabyte::Harness::Counter.get()
57
+ def initialize(v, options = {})
58
+ @_name = v[0] || "each_"+Zillabyte::Harness::Counter.get()
59
59
  @_type = "each"
60
60
  @_output_format = :replace
61
+ self._emits = options[:emits]
61
62
  end
62
63
 
63
64
  def output_format(v)
@@ -71,11 +72,11 @@ class Zillabyte::Harness::Each
71
72
 
72
73
  def initialize(app, *args)
73
74
  @_app = app
74
- @_vector_options, @_options = Zillabyte::Harness::Helper.get_vector_and_hashes(args)
75
+ @_args, @_options = Zillabyte::Harness::Helper.get_vector_and_hashes(args)
75
76
  end
76
77
 
77
78
  def build_node(&block)
78
- @_node = Node.new(*@_vector_options)
79
+ @_node = Node.new(@_args, @_options)
79
80
  # Does the block take 0 arguments? If so it's not just an execute block.
80
81
  if(block.arity == 0)
81
82
  @_node.instance_eval(&block)
@@ -87,7 +88,7 @@ class Zillabyte::Harness::Each
87
88
  end
88
89
  # Takes more than 0? Then it takes |tuple| and is an execute block. Give it a generated stream name.
89
90
  else
90
- @_node._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
91
+ @_node._emits ||= ["stream_"+Zillabyte::Harness::Counter.get()]
91
92
  @_node._execute = block
92
93
  end
93
94
 
@@ -39,7 +39,7 @@ class Zillabyte::Harness::Filter
39
39
  class Node < Zillabyte::Harness::CommonNode
40
40
  attr_accessor :_keep
41
41
 
42
- def initialize()
42
+ def initialize(args, options = {})
43
43
  @_name = "filter_"+Zillabyte::Harness::Counter.get()
44
44
  @_type = 'filter'
45
45
  end
@@ -51,11 +51,11 @@ class Zillabyte::Harness::Filter
51
51
 
52
52
  def initialize(app, *args)
53
53
  @_app = app
54
- @_vector_options, @_options = Zillabyte::Harness::Helper.get_vector_and_hashes(args)
54
+ @_args, @_options = Zillabyte::Harness::Helper.get_vector_and_hashes(args)
55
55
  end
56
56
 
57
57
  def build_node(&block)
58
- @_node = Node.new
58
+ @_node = Node.new(@_args, @_options)
59
59
  # Does the block take 0 arguments? If so it's not just an execute block.
60
60
  if(block.arity == 0)
61
61
  @_node.instance_eval(&block)
@@ -408,12 +408,12 @@ class Zillabyte::Harness::Helper
408
408
  Zillabyte::Harness::Helper.print_error(msg)
409
409
  end
410
410
  end
411
- if node._name == previous_node_name
412
- if node._emits.size < 2
413
- msg = "#{ee}The preceding operation does not emit multiple streams. Please make sure that it emits one stream for the loop back and another stream for downstream operations. #{pp}"
414
- Zillabyte::Harness::Helper.print_error(msg)
415
- end
416
- end
411
+ # if node._name == previous_node_name
412
+ # if node._emits.size < 2
413
+ # msg = "#{ee}The preceding operation does not emit multiple streams. Please make sure that it emits one stream for the loop back and another stream for downstream operations. #{pp}"
414
+ # Zillabyte::Harness::Helper.print_error(msg)
415
+ # end
416
+ # end
417
417
  end
418
418
  if !found
419
419
  msg = "#{ee}The specified loop-back node \"#{node_name}\" was not found in the operations preceding it. #{pp}"
@@ -4,11 +4,12 @@ class Zillabyte::Harness::InjectedComponent
4
4
  class Node
5
5
  attr_accessor :_name, :_type, :_id, :_consumes, :_emits, :_output_format
6
6
 
7
- def initialize(input_stream_1)
7
+ def initialize(input_stream_1, options = {})
8
8
  @_name = "component_"+Zillabyte::Harness::Counter.get()
9
9
  @_type = "component"
10
10
  @_consumes = [input_stream_1]
11
11
  @_output_format = :replace
12
+ @_emits ||= options[:emits]
12
13
  end
13
14
 
14
15
  def name(v)
@@ -46,7 +47,7 @@ class Zillabyte::Harness::InjectedComponent
46
47
  end
47
48
 
48
49
  def build_node(&block)
49
- @_node = Node.new @_input_stream_1
50
+ @_node = Node.new @_input_stream_1, @_options
50
51
  @_node.instance_eval(&block) if block_given?
51
52
  @_node._id ||= @_id
52
53
  @_node._emits ||= ["stream_"+Zillabyte::Harness::Counter.get()] # We default to single-stream when called like this..
@@ -1,23 +1,24 @@
1
1
  class Zillabyte::Harness::JvmOperation
2
- attr_accessor :_type, :_node, :_args
2
+ attr_accessor :_type, :_node, :_args, :_options
3
3
 
4
4
  class Node
5
5
  attr_accessor :_name, :_type, :_emits
6
6
 
7
- def initialize(type)
8
- @_name = "#{type}_"+Zillabyte::Harness::Counter.get()
7
+ def initialize(type, options = {})
8
+ @_name = options[:name] || "#{type}_"+Zillabyte::Harness::Counter.get()
9
9
  @_type = type
10
+ @_options = options
10
11
  end
11
12
 
12
13
  end
13
14
 
14
15
  def initialize(type, *args)
15
16
  @_type = type
16
- @_args = args
17
+ @_args, @_options = Zillabyte::Harness::Helper.get_vector_and_hashes(args)
17
18
  end
18
19
 
19
20
  def build_node
20
- @_node = Node.new @_type
21
+ @_node = Node.new @_type, @_options
21
22
  @_node._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
22
23
  @_node
23
24
  end
@@ -80,7 +80,7 @@ class Zillabyte::Harness::OperationHandler
80
80
  output_streams
81
81
  end
82
82
 
83
- def handle_operation
83
+ def handle_operation()
84
84
  @_app._nodes << node
85
85
  if(@_app._options[:command] == :info)
86
86
  node_hash = @_node_hash_elements
@@ -106,4 +106,16 @@ class Zillabyte::Harness::OperationHandler
106
106
  end
107
107
  self
108
108
  end
109
+
110
+
111
+ def self.build(stream, operaiton_name, *args)
112
+ op = Zillabyte::Harness::OperationHandler.new(stream._app, stream.class)
113
+ op.build_jvm_operation(operaiton_name, *args)
114
+ .add_operation_properties_to_info(:name, :type)
115
+ .create_arc_info_from_stream(stream)
116
+ .handle_operation
117
+ .get_output_streams
118
+ end
119
+
120
+
109
121
  end
@@ -61,7 +61,7 @@ class Zillabyte::Harness::Source
61
61
  class Node < Zillabyte::Harness::CommonNode
62
62
  attr_accessor :_matches, :_relation, :_end_cycle_policy, :_begin_cycle, :_next_tuple
63
63
 
64
- def initialize(options = {})
64
+ def initialize(v, options = {})
65
65
  @_name = options[:name] || "source_"+Zillabyte::Harness::Counter.get()
66
66
  @_type = 'source'
67
67
  @_end_cycle_policy = :null_emit
@@ -101,7 +101,7 @@ class Zillabyte::Harness::Source
101
101
  end
102
102
 
103
103
  def build_node(&block)
104
- @_node = Node.new(@_options)
104
+ @_node = Node.new(@_args, @_options)
105
105
  # Are we given a block?
106
106
  if block_given?
107
107
  @_node.instance_eval(&block)
@@ -13,7 +13,6 @@ class Zillabyte::Harness::Stream
13
13
  @_name
14
14
  end
15
15
 
16
-
17
16
  def each(*args, &block)
18
17
  op = Zillabyte::Harness::OperationHandler.new(@_app, self.class)
19
18
  op.build_multilang_operation("each", *args, &block)
@@ -89,7 +88,9 @@ class Zillabyte::Harness::Stream
89
88
  Zillabyte::Harness::Helper.check_loop_back(self, loop_back_node, max_iterations, @_app._nodes)
90
89
  arc_hash = {"name" => (@_options[:name] || self._name), "origin" => self._previous_node_name, "dest" => loop_back_node, "loop_back" => 1}
91
90
  arc_hash["max_iterations"] = max_iterations if max_iterations
92
- Zillabyte::Harness::Helper.write_arc_to_file(arc_hash, @_app._socket)
91
+ if(@_app._options[:command] == :info)
92
+ Zillabyte::Harness::Helper.write_arc_to_file(arc_hash, @_app._socket)
93
+ end
93
94
  end
94
95
 
95
96
  def sink(*args, &block)
@@ -102,6 +103,18 @@ class Zillabyte::Harness::Stream
102
103
  end
103
104
 
104
105
 
106
+ def sink_to_redshift(*args, &block)
107
+ sink(*args, &block)
108
+ end
109
+
110
+
111
+
112
+ def sink_to_kinesis(*args, &block)
113
+ sink(*args, &block)
114
+ end
115
+
116
+
117
+
105
118
 
106
119
  # Unique's a stream. Executed on backend
107
120
  def unique(*args)
@@ -114,6 +127,26 @@ class Zillabyte::Harness::Stream
114
127
  .get_output_streams
115
128
  end
116
129
 
130
+
131
+
132
+ def route_by(*fields)
133
+ options = {}
134
+ options[:fields] = fields
135
+ Zillabyte::Harness::OperationHandler.build(self, "route_by", options)
136
+ end
137
+
138
+ def rate_limit(rate = 1, options = {})
139
+ options[:rate] = rate
140
+ Zillabyte::Harness::OperationHandler.build(self, "rate_limit", options)
141
+ end
142
+
143
+ def fetch_url(*args)
144
+ args, options = Zillabyte::Harness::Helper.get_vector_and_hashes(args)
145
+ self.call_component(:fetch_url, options)
146
+ end
147
+
148
+
149
+
117
150
  # Counts fields. Executed on backend
118
151
  def count(*args)
119
152
  op = Zillabyte::Harness::OperationHandler.new(@_app, self.class)
@@ -159,14 +192,10 @@ class Zillabyte::Harness::Stream
159
192
  end
160
193
 
161
194
  # Renames fields..
162
- def rename(*args)
163
- op = Zillabyte::Harness::OperationHandler.new(@_app, self.class)
164
- op.build_jvm_operation("rename", *args)
165
- .add_operation_properties_to_info(:name, :type)
166
- .add_input_args_to_info_as(:rename, 0)
167
- .create_arc_info_from_stream(self)
168
- .handle_operation
169
- .get_output_streams
195
+ def rename(rename_map = {})
196
+ options = {}
197
+ options[:rename] = rename_map
198
+ Zillabyte::Harness::OperationHandler.build(self, "rename", options)
170
199
  end
171
200
 
172
201
  end
@@ -11,7 +11,9 @@ class Zillabyte::Harness::StreamBuilder < Array
11
11
  end
12
12
  end
13
13
 
14
+
14
15
  # Override these methods explicitly because they are provided by 'Array'
16
+ alias_method :super_each, :each
15
17
  def each(*args, &block)
16
18
  _handle_function(:each, *args, &block)
17
19
  end
@@ -42,7 +44,18 @@ class Zillabyte::Harness::StreamBuilder < Array
42
44
  if self.size == 0
43
45
  raise "The previous operation did not emit a split stream"
44
46
  else
45
- return Zillabyte::Harness::StreamBuilder.new(super)
47
+ if v.is_a?(String)
48
+ # String based
49
+ super_each do |stream|
50
+ if stream._name == v
51
+ return Zillabyte::Harness::StreamBuilder.new(stream)
52
+ end
53
+ end
54
+ raise "Could not find a stream with name '#{v}'"
55
+ else
56
+ # Intger based
57
+ return Zillabyte::Harness::StreamBuilder.new(super)
58
+ end
46
59
  end
47
60
  end
48
61
 
@@ -1,3 +1,3 @@
1
1
  module Zillabyte
2
- VERSION = "0.9.9" unless defined?(VERSION)
2
+ VERSION = "0.9.10" unless defined?(VERSION)
3
3
  end
metadata CHANGED
@@ -1,43 +1,43 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: zillabyte
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.9.9
4
+ version: 0.9.10
5
5
  platform: ruby
6
6
  authors:
7
7
  - zillabyte
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-10-03 00:00:00.000000000 Z
11
+ date: 2014-10-08 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rake
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
- - - ">="
17
+ - - ! '>='
18
18
  - !ruby/object:Gem::Version
19
19
  version: '0'
20
20
  type: :development
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
- - - ">="
24
+ - - ! '>='
25
25
  - !ruby/object:Gem::Version
26
26
  version: '0'
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: zillabyte-cli
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
- - - "~>"
31
+ - - ~>
32
32
  - !ruby/object:Gem::Version
33
- version: 0.9.9
33
+ version: 0.9.10
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
- - - "~>"
38
+ - - ~>
39
39
  - !ruby/object:Gem::Version
40
- version: 0.9.9
40
+ version: 0.9.10
41
41
  description: The Official Zillabyte Gem
42
42
  email:
43
43
  - gem@zillabyte.com
@@ -45,9 +45,7 @@ executables: []
45
45
  extensions: []
46
46
  extra_rdoc_files: []
47
47
  files:
48
- - ruby/lib/zillabyte.rb
49
48
  - ruby/lib/zillabyte/common/progress.rb
50
- - ruby/lib/zillabyte/harness.rb
51
49
  - ruby/lib/zillabyte/harness/app.rb
52
50
  - ruby/lib/zillabyte/harness/base.rb
53
51
  - ruby/lib/zillabyte/harness/clump.rb
@@ -71,7 +69,9 @@ files:
71
69
  - ruby/lib/zillabyte/harness/stream.rb
72
70
  - ruby/lib/zillabyte/harness/stream_builder.rb
73
71
  - ruby/lib/zillabyte/harness/tuple.rb
72
+ - ruby/lib/zillabyte/harness.rb
74
73
  - ruby/lib/zillabyte/version.rb
74
+ - ruby/lib/zillabyte.rb
75
75
  homepage: http://www.zillabyte.com
76
76
  licenses:
77
77
  - MIT
@@ -82,17 +82,17 @@ require_paths:
82
82
  - ruby/lib
83
83
  required_ruby_version: !ruby/object:Gem::Requirement
84
84
  requirements:
85
- - - ">="
85
+ - - ! '>='
86
86
  - !ruby/object:Gem::Version
87
87
  version: '0'
88
88
  required_rubygems_version: !ruby/object:Gem::Requirement
89
89
  requirements:
90
- - - ">="
90
+ - - ! '>='
91
91
  - !ruby/object:Gem::Version
92
92
  version: '0'
93
93
  requirements: []
94
94
  rubyforge_project:
95
- rubygems_version: 2.2.2
95
+ rubygems_version: 2.0.6
96
96
  signing_key:
97
97
  specification_version: 4
98
98
  summary: The Official Zillabyte Gem