streamdal 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: cdf556c16ea4239101cc21f803b78c458c5e2490a1c5df6b2211f5268cc94799
4
+ data.tar.gz: 1bca44db3048358a5bdf978504a8559c1a81ede836a11269c5ab8020698d71b6
5
+ SHA512:
6
+ metadata.gz: 9f0fb7a83fc6e924f520d5402385bd857aff3a27884fd2a1aa9572a002723ad26d3849a2930829e6a05e450547ab1e80293992a693bbeb2bb9bacad8756011f9
7
+ data.tar.gz: 2176e44cd407be37c0137f48c77f7a56550cddacc36a5a8043643049b416138c1b96b35c850c630dfd1f9febdc4e72e9224fb055b8dfd871c54e9bef5fbdbe47
data/lib/audiences.rb ADDED
@@ -0,0 +1,45 @@
1
+ module Audiences
2
+ def aud_to_str(aud)
3
+ "#{aud.service_name}.#{aud.component_name}.#{aud.operation_type}.#{aud.operation_name}"
4
+ end
5
+
6
+ def str_to_aud(str)
7
+ # TODO: move to common package
8
+ parts = str.split(".")
9
+ aud = Streamdal::Protos::Audience.new
10
+ aud.service_name = parts[0]
11
+ aud.component_name = parts[1]
12
+ aud.operation_type = parts[2]
13
+ aud.operation_name = parts[3]
14
+ aud
15
+ end
16
+
17
+ def _seen_audience(aud)
18
+ @audiences.key?(aud_to_str(aud))
19
+ end
20
+
21
+ def _add_audience(aud)
22
+ # Add an audience to the local cache map and send to server
23
+ if _seen_audience(aud)
24
+ return
25
+ end
26
+
27
+ @audiences[aud_to_str(aud)] = aud
28
+
29
+ req = Streamdal::Protos::NewAudienceRequest.new
30
+ req.session_id = @session_id
31
+ req.audience = aud
32
+ @stub.new_audience(req, metadata: _metadata)
33
+ end
34
+
35
+ def _add_audiences
36
+ # This method is used to re-announce audiences after a disconnect
37
+
38
+ @audiences.each do |aud|
39
+ req = Streamdal::Protos::NewAudienceRequest.new
40
+ req.session_id = @session_id
41
+ req.audience = aud
42
+ @stub.new_audience(req, metadata: _metadata)
43
+ end
44
+ end
45
+ end
@@ -0,0 +1,5 @@
1
+ require_relative 'spec_helper'
2
+
3
+ RSpec.describe "Streamdal::Audiences" do
4
+
5
+ end
data/lib/hostfunc.rb ADDED
@@ -0,0 +1,109 @@
1
+ require "steps/sp_steps_kv_pb"
2
+
3
+ module Streamdal
4
+ class HostFunc
5
+
6
+ ##
7
+ # This class holds methods that are called by wasm modules
8
+
9
+ def initialize(kv)
10
+ @kv = kv
11
+ end
12
+
13
+ ##
14
+ # kv_exists is a host function that is used to check if a key exists in the KV store
15
+ def kv_exists(caller, ptr, len)
16
+
17
+ data = caller.export("memory").to_memory.read(ptr, len)
18
+
19
+ # Read request from memory and decode into HttpRequest
20
+ req = Streamdal::Protos::KVStep.decode(data)
21
+
22
+ exists = @kv.exists(req.key)
23
+
24
+ msg = exists ? "Key '#{req.key}' exists" : "Key #{req.key} does not exist"
25
+
26
+ status = exists ? :KV_STATUS_SUCCESS : :KV_STATUS_FAILURE
27
+
28
+ wasm_resp = Streamdal::Protos::KVStepResponse.new
29
+ wasm_resp.status = status
30
+ wasm_resp.message = msg
31
+
32
+ write_to_memory(caller, wasm_resp)
33
+ end
34
+
35
+ ##
36
+ # http_request performs a http request on behalf of a wasm module since WASI cannot talk sockets
37
+ def http_request(caller, ptr, len)
38
+ data = caller.export("memory").to_memory.read(ptr, len)
39
+
40
+ # Read request from memory and decode into HttpRequest
41
+ req = Streamdal::Protos::HttpRequest.decode(data)
42
+
43
+ # Attempt to make HTTP request
44
+ # On error, return a mock 400 response with the error as the body
45
+ begin
46
+ response = _make_http_request(req)
47
+ rescue => e
48
+ wasm_resp = Streamdal::Protos::HttpResponse.new
49
+ wasm_resp.code = 400
50
+ wasm_resp.body = "Unable to execute HTTP request: #{e}"
51
+ return wasm_resp
52
+ end
53
+
54
+ # Successful request, build the proto from the httparty response
55
+ wasm_resp = Streamdal::Protos::HttpResponse.new
56
+ wasm_resp.code = response.code
57
+ wasm_resp.body = response.body
58
+ wasm_resp.headers = Google::Protobuf::Map.new(:string, :string, {})
59
+
60
+ # Headers can have multiple values, but we just want a map[string]string here for simplicity
61
+ # The client can pase by the delimiter ";" if needed.
62
+ response.headers.each do |k, values|
63
+ wasm_resp.headers[k] = values.kind_of?(Array) ? values.join("; ") : values
64
+ end
65
+
66
+ # Write the HttpResponse proto message to WASM memory
67
+ # The .wasm module will read/decode this data internally
68
+ write_to_memory(caller, wasm_resp)
69
+ end
70
+
71
+ private
72
+
73
+ ##
74
+ # Performs an http request
75
+ def _make_http_request(req)
76
+ if req.nil?
77
+ raise "req is required"
78
+ end
79
+
80
+ options = {
81
+ headers: { "Content-Type": "application/json", },
82
+ }
83
+
84
+ req.headers.each { |key, value| options.headers[key] = value }
85
+
86
+ case req.to_h[:method]
87
+ when :HTTP_REQUEST_METHOD_GET
88
+ return HTTParty.get(req.url)
89
+ when :HTTP_REQUEST_METHOD_POST
90
+ options.body = req.body
91
+ return HTTParty.post(req.url, options)
92
+ when :HTTP_REQUEST_METHOD_PUT
93
+ options.body = req.body
94
+ return HTTParty.put(req.url, options)
95
+ when :HTTP_REQUEST_METHOD_DELETE
96
+ return HTTParty.delete(req.url)
97
+ when :HTTP_REQUEST_METHOD_PATCH
98
+ options.body = req.body
99
+ return HTTParty.patch(req.url, options)
100
+ when :HTTP_REQUEST_METHOD_HEAD
101
+ return HTTParty.head(req.url)
102
+ when :HTTP_REQUEST_METHOD_OPTIONS
103
+ return HTTParty.options(req.url)
104
+ else
105
+ raise ArgumentError, "Invalid http request method: #{req.method}"
106
+ end
107
+ end
108
+ end
109
+ end
@@ -0,0 +1,5 @@
1
+ require_relative 'spec_helper'
2
+
3
+ RSpec.describe "Streamdal::HostFunc" do
4
+
5
+ end
data/lib/kv.rb ADDED
@@ -0,0 +1,52 @@
1
+ module Streamdal
2
+ class KeyValue
3
+ def initialize
4
+ @kvs = {}
5
+ @mtx = Mutex.new
6
+ end
7
+
8
+ def set(key, value)
9
+ @mtx.synchronize do
10
+ @kvs[key] = value
11
+ end
12
+ end
13
+
14
+ def get(key)
15
+ @mtx.synchronize do
16
+ @kvs[key]
17
+ end
18
+ end
19
+
20
+ def delete(key)
21
+ @mtx.synchronize do
22
+ @kvs.delete(key)
23
+ end
24
+ end
25
+
26
+ def keys
27
+ @mtx.synchronize do
28
+ @kvs.keys
29
+ end
30
+ end
31
+
32
+ def items
33
+ @mtx.synchronize do
34
+ @kvs.values
35
+ end
36
+ end
37
+
38
+ def exists(key)
39
+ @mtx.synchronize do
40
+ @kvs.key?(key)
41
+ end
42
+ end
43
+
44
+ def purge
45
+ @mtx.synchronize do
46
+ num_keys = @kvs.keys.length
47
+ @kvs = {}
48
+ num_keys
49
+ end
50
+ end
51
+ end
52
+ end
data/lib/kv_spec.rb ADDED
@@ -0,0 +1,54 @@
1
+ require 'rspec'
2
+ require_relative 'spec_helper'
3
+ require_relative 'kv'
4
+
5
+ RSpec.describe 'KeyValue' do
6
+ before(:each) do
7
+ @kv = Streamdal::KeyValue.new
8
+ end
9
+
10
+ it 'stores and retrieves key-value pair' do
11
+ @kv.set('key', 'value')
12
+ expect(@kv.get('key')).to eq('value')
13
+ end
14
+
15
+ it 'deletes a key' do
16
+ expect(@kv.keys.length).to eq(0)
17
+ @kv.set('key1', 'value')
18
+ @kv.set('key2', 'value')
19
+ expect(@kv.keys.length).to eq(2)
20
+ @kv.delete('key1')
21
+ expect(@kv.keys.length).to eq(1)
22
+ end
23
+
24
+ it 'returns array of keys' do
25
+ expect(@kv.keys.length).to eq(0)
26
+ @kv.set('key1', 'value')
27
+ expect(@kv.keys.length).to eq(1)
28
+ expect(@kv.keys[0]).to eq('key1')
29
+ end
30
+
31
+ it 'returns array of item values' do
32
+ expect(@kv.keys.length).to eq(0)
33
+ @kv.set('key1', 'value1')
34
+ expect(@kv.keys.length).to eq(1)
35
+ expect(@kv.items[0]).to eq('value1')
36
+ end
37
+
38
+ it 'returns if a key exists or not' do
39
+ expect(@kv.exists('key1')).to eq(false)
40
+ @kv.set('key1', 'value')
41
+ expect(@kv.exists('key1')).to eq(true)
42
+ end
43
+
44
+ it 'purges all keys' do
45
+ expect(@kv.keys.length).to eq(0)
46
+ @kv.set('key1', 'value')
47
+ @kv.set('key2', 'value')
48
+ expect(@kv.keys.length).to eq(2)
49
+ num_keys = @kv.purge
50
+ expect(@kv.keys.length).to eq(0)
51
+ expect(num_keys).to eq(2)
52
+ end
53
+
54
+ end
data/lib/metrics.rb ADDED
@@ -0,0 +1,265 @@
1
+ module Streamdal
2
+ class Counter
3
+ attr_accessor :last_updated, :name, :aud, :labels
4
+
5
+ def initialize(name, aud, labels = {}, value = 0.0)
6
+ @name = name
7
+ @aud = aud
8
+ @labels = labels
9
+ @value = 0
10
+ @last_updated = Time::now
11
+ @value_mtx = Mutex.new
12
+ end
13
+
14
+ def incr(val)
15
+ @value_mtx.synchronize do
16
+ @value = @value + val
17
+ @last_updated = Time::now
18
+ end
19
+ end
20
+
21
+ def reset
22
+ @value_mtx.synchronize do
23
+ @value = 0.0
24
+ end
25
+ end
26
+
27
+ def val
28
+ @value_mtx.synchronize do
29
+ @value
30
+ end
31
+ end
32
+ end
33
+
34
+ class Metrics
35
+
36
+ COUNTER_CONSUME_BYTES = "counter_consume_bytes"
37
+ COUNTER_CONSUME_PROCESSED = "counter_consume_processed"
38
+ COUNTER_CONSUME_ERRORS = "counter_consume_errors"
39
+ COUNTER_PRODUCE_BYTES = "counter_produce_bytes"
40
+ COUNTER_PRODUCE_PROCESSED = "counter_produce_processed"
41
+ COUNTER_PRODUCE_ERRORS = "counter_produce_errors"
42
+ COUNTER_NOTIFY = "counter_notify"
43
+ COUNTER_DROPPED_TAIL_MESSAGES = "counter_dropped_tail_messages"
44
+ COUNTER_CONSUME_BYTES_RATE = "counter_consume_bytes_rate"
45
+ COUNTER_PRODUCE_BYTES_RATE = "counter_produce_bytes_rate"
46
+ COUNTER_CONSUME_PROCESSED_RATE = "counter_consume_processed_rate"
47
+ COUNTER_PRODUCE_PROCESSED_RATE = "counter_produce_processed_rate"
48
+
49
+ WORKER_POOL_SIZE = 3
50
+ DEFAULT_COUNTER_REAPER_INTERVAL = 10
51
+ DEFAULT_COUNTER_TTL = 10
52
+ DEFAULT_COUNTER_PUBLISH_INTERVAL = 1
53
+
54
+ CounterEntry = Struct.new(:name, :aud, :labels, :value)
55
+
56
+ def initialize(cfg)
57
+ if cfg.nil?
58
+ raise ArgumentError, "cfg is nil"
59
+ end
60
+
61
+ @cfg = cfg
62
+ @log = cfg[:log]
63
+ @counters = {}
64
+ @counters_mtx = Mutex.new
65
+ @exit = false
66
+ @incr_queue = Queue.new
67
+ @publish_queue = Queue.new
68
+ @workers = []
69
+ @stub = Streamdal::Protos::Internal::Stub.new(@cfg[:streamdal_url], :this_channel_is_insecure)
70
+
71
+ _start
72
+ end
73
+
74
+ def shutdown
75
+ # Set exit flag so workers exit
76
+ @exit = true
77
+
78
+ # Let loops exit
79
+ sleep(1)
80
+
81
+ # Exit any remaining threads
82
+ @workers.each do |w|
83
+ if w.running?
84
+ w.exit
85
+ end
86
+ end
87
+ end
88
+
89
+ def self.composite_id(counter_name, labels = {})
90
+ if labels.nil?
91
+ labels = {}
92
+ end
93
+ "#{counter_name}-#{labels.values.join("-")}".freeze
94
+ end
95
+
96
+ def get_counter(ce)
97
+ if ce.nil?
98
+ raise ArgumentError, "ce is nil"
99
+ end
100
+
101
+ k = Metrics::composite_id(ce.name, ce.labels)
102
+
103
+ @counters_mtx.synchronize do
104
+ if @counters.key?(k)
105
+ @counters[k]
106
+ end
107
+ end
108
+
109
+ # No counter exists, create a new one and return it
110
+ new_counter(ce)
111
+ end
112
+
113
+ def new_counter(ce)
114
+ c = Counter.new(ce.name, ce.aud, ce.labels, ce.value)
115
+
116
+ @counters_mtx.synchronize do
117
+ @counters[Metrics::composite_id(ce.name, ce.labels)] = c
118
+ end
119
+
120
+ c
121
+ end
122
+
123
+ def incr(ce)
124
+ c = get_counter(ce)
125
+
126
+ if c.nil?
127
+ new_counter(ce)
128
+ nil
129
+ end
130
+
131
+ @incr_queue.push(ce)
132
+ end
133
+
134
+ def remove_counter(name)
135
+ @counters_mtx.synchronize do
136
+ @counters.delete(name)
137
+ end
138
+ end
139
+
140
+ private
141
+
142
+ def _start
143
+ WORKER_POOL_SIZE.times do |i|
144
+ @workers << Thread.new { _run_incrementer_worker(i) }
145
+ @workers << Thread.new { _run_publisher_worker(i) }
146
+ end
147
+
148
+ @workers << Thread.new { _run_publisher }
149
+ @workers << Thread.new { _run_reaper }
150
+ end
151
+
152
+ def _publish_metrics(ce)
153
+ metric = Streamdal::Protos::Metric.new
154
+ metric.name = ce.name
155
+ metric.labels = Google::Protobuf::Map.new(:string, :string, ce.labels)
156
+ metric.value = ce.value
157
+ metric.audience = ce.aud
158
+
159
+ req = Streamdal::Protos::MetricsRequest.new
160
+ req.metrics = Google::Protobuf::RepeatedField.new(:message, Streamdal::Protos::Metric, [metric])
161
+
162
+ @log.debug("Published metric: #{ce.name} #{ce.labels} #{ce.value}")
163
+ @stub.metrics(req, metadata: _metadata)
164
+ end
165
+
166
+ def _run_publisher
167
+ # Background thread that reads values from counters, adds them to the publish queue, and then
168
+ # resets the counter's value back to zero
169
+ unless @exit
170
+ @log.debug("Starting publisher")
171
+
172
+ # Sleep on startup and then and between each loop run
173
+ sleep(DEFAULT_COUNTER_PUBLISH_INTERVAL)
174
+
175
+ # Get all counters
176
+ # Loop over each counter, get the value,
177
+ # if value > 0, continue
178
+ # if now() - last_updated > 10 seconds, remove counter
179
+ # Grab copy of counters
180
+ @counters_mtx.lock
181
+ new_counters = @counters.dup
182
+ @counters_mtx.unlock
183
+
184
+ new_counters.each do |_, counter|
185
+ if counter.val == 0
186
+ next
187
+ end
188
+
189
+ ce = CounterEntry.new(counter.name, counter.aud, counter.labels, counter.val)
190
+ counter.reset
191
+
192
+ @publish_queue.push(ce)
193
+ end
194
+ end
195
+ end
196
+
197
+ def _run_publisher_worker(worker_id)
198
+ @log.debug("Starting publisher worker '#{worker_id}'")
199
+
200
+ until @exit
201
+ ce = @incr_queue.pop
202
+ if ce.nil?
203
+ next
204
+ end
205
+ begin
206
+ _publish_metrics(ce)
207
+ rescue => e
208
+ @log.error("Failed to publish metrics: #{e}: #{ce.inspect}")
209
+ end
210
+ end
211
+
212
+ @log.debug("Exiting publisher worker '#{worker_id}'")
213
+ end
214
+
215
+ def _run_reaper
216
+ @log.debug("Starting reaper")
217
+
218
+ until @exit
219
+ # Sleep on startup and then and between each loop run
220
+ sleep(DEFAULT_COUNTER_REAPER_INTERVAL)
221
+
222
+ # Get all counters
223
+ # Loop over each counter, get the value,
224
+ # if value > 0, continue
225
+ # if now() - last_updated > 10 seconds, remove counter
226
+ # Grab copy of counters
227
+ @counters_mtx.synchronize do
228
+ @counters.each do |name, counter|
229
+ if counter.val > 0
230
+ next
231
+ end
232
+
233
+ if Time::now - counter.last_updated > DEFAULT_COUNTER_TTL
234
+ @log.debug("Reaping counter '#{name}'")
235
+ @counters.delete(name)
236
+ end
237
+ end
238
+ end
239
+ end
240
+
241
+ @log.debug("Exiting reaper")
242
+ end
243
+
244
+ def _run_incrementer_worker(worker_id)
245
+ @log.debug("Starting incrementer worker '#{worker_id}'")
246
+
247
+ until @exit
248
+ ce = @incr_queue.pop
249
+
250
+ next if ce.nil?
251
+
252
+ c = get_counter(ce)
253
+
254
+ c.incr(ce.value)
255
+ end
256
+
257
+ @log.debug("Exiting incrementer worker '#{worker_id}'")
258
+ end
259
+
260
+ # Returns metadata for gRPC requests to the internal gRPC API
261
+ def _metadata
262
+ { "auth-token" => @cfg[:streamdal_token].to_s }
263
+ end
264
+ end
265
+ end
@@ -0,0 +1,5 @@
1
+ require_relative 'spec_helper'
2
+
3
+ RSpec.describe "Streamdal::Metrics" do
4
+
5
+ end
data/lib/schema.rb ADDED
@@ -0,0 +1,47 @@
1
+ include Streamdal::Protos
2
+
3
+ module Schemas
4
+ def _set_schema(aud, schema)
5
+ s = Streamdal::Protos::Schema.new
6
+ s.json_schema = schema
7
+ @schemas[aud_to_str(aud)] = s
8
+ end
9
+
10
+ def _get_schema(aud)
11
+ if @schemas.key?(aud_to_str(aud))
12
+ return @schemas[aud_to_str(aud)].json_schema
13
+ end
14
+
15
+ ""
16
+ end
17
+
18
+ def _handle_schema(aud, step, wasm_resp)
19
+ # Only handle schema steps
20
+ if step.infer_schema.nil?
21
+ return nil
22
+ end
23
+
24
+ # Only successful schema inferences
25
+ if wasm_resp.exit_code != :WASM_EXIT_CODE_TRUE
26
+ return nil
27
+ end
28
+
29
+ # If existing schema matches, do nothing
30
+ existing_schema = _get_schema(aud)
31
+ if existing_schema == wasm_resp.output_step
32
+ return nil
33
+ end
34
+
35
+ _set_schema(aud, wasm_resp.output_step)
36
+
37
+ req = Streamdal::Protos::SendSchemaRequest.new
38
+ req.audience = aud
39
+ req.schema = Streamdal::Protos::Schema.new
40
+ req.schema.json_schema = wasm_resp.output_step
41
+
42
+ # Run in thread so we don't block on gRPC call
43
+ Thread.new do
44
+ @stub.send_schema(req, metadata: _metadata)
45
+ end
46
+ end
47
+ end
@@ -0,0 +1,59 @@
1
+ require 'rspec'
2
+ require 'sp_wsm_pb'
3
+ require 'steps/sp_steps_inferschema_pb'
4
+ require 'sp_pipeline_pb'
5
+ require 'sp_common_pb'
6
+ require_relative 'streamdal'
7
+ require_relative 'spec_helper'
8
+ require_relative 'schema'
9
+ require_relative 'audiences'
10
+
11
+ module Streamdal
12
+ class TestObj
13
+ include Schemas
14
+ include Audiences
15
+
16
+ @schemas
17
+ @stub
18
+
19
+ attr_accessor :stub
20
+
21
+ def initialize
22
+ @schemas = {}
23
+ @stub = RSpec::Mocks::Double.new("stub", { send_schema: nil })
24
+ end
25
+
26
+ def _metadata
27
+ {}
28
+ end
29
+ end
30
+ end
31
+
32
+ RSpec.describe "Streamdal::Schema" do
33
+ before(:each) do
34
+ @test_obj = Streamdal::TestObj.new
35
+
36
+ public_aud = Streamdal::Audience.new(1, "consume", "kafka")
37
+ @aud = public_aud.to_proto("test-svc")
38
+ expect(@test_obj.aud_to_str(@aud)).to eq("test-svc.kafka.OPERATION_TYPE_CONSUMER.consume")
39
+ end
40
+
41
+ it "should set and get a schema" do
42
+ @test_obj._set_schema(@aud, "{}")
43
+
44
+ got_schema = @test_obj._get_schema(@aud)
45
+ expect(got_schema).to eq("{}")
46
+ end
47
+ it "should handle schema" do
48
+ wasm_resp = Streamdal::Protos::WASMResponse.new
49
+ wasm_resp.exit_code = :WASM_EXIT_CODE_TRUE
50
+ wasm_resp.output_step = "{}"
51
+
52
+ @test_obj._handle_schema(@aud, Streamdal::Protos::PipelineStep.new(infer_schema: Streamdal::Protos::InferSchemaStep.new), wasm_resp)
53
+
54
+ got_schema = @test_obj._get_schema(@aud)
55
+ expect(got_schema).to eq("{}")
56
+ # sleep(1)
57
+ expect(@test_obj.stub).to have_received(:send_schema).with(Streamdal::Protos::SendSchemaRequest.new(audience: @aud, schema: Streamdal::Protos::Schema.new(json_schema: "{}")), metadata: @test_obj._metadata)
58
+ end
59
+ end
@@ -0,0 +1,2 @@
1
+ require 'simplecov'
2
+ SimpleCov.start