streamdal 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/lib/audiences.rb +45 -0
- data/lib/audiences_spec.rb +5 -0
- data/lib/hostfunc.rb +109 -0
- data/lib/hostfunc_spec.rb +5 -0
- data/lib/kv.rb +52 -0
- data/lib/kv_spec.rb +54 -0
- data/lib/metrics.rb +265 -0
- data/lib/metrics_spec.rb +5 -0
- data/lib/schema.rb +47 -0
- data/lib/schema_spec.rb +59 -0
- data/lib/spec_helper.rb +2 -0
- data/lib/streamdal.rb +852 -0
- data/lib/streamdal_spec.rb +5 -0
- data/lib/tail.rb +97 -0
- data/lib/tail_spec.rb +5 -0
- data/lib/validation.rb +88 -0
- data/lib/validation_spec.rb +77 -0
- metadata +59 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: cdf556c16ea4239101cc21f803b78c458c5e2490a1c5df6b2211f5268cc94799
|
4
|
+
data.tar.gz: 1bca44db3048358a5bdf978504a8559c1a81ede836a11269c5ab8020698d71b6
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 9f0fb7a83fc6e924f520d5402385bd857aff3a27884fd2a1aa9572a002723ad26d3849a2930829e6a05e450547ab1e80293992a693bbeb2bb9bacad8756011f9
|
7
|
+
data.tar.gz: 2176e44cd407be37c0137f48c77f7a56550cddacc36a5a8043643049b416138c1b96b35c850c630dfd1f9febdc4e72e9224fb055b8dfd871c54e9bef5fbdbe47
|
data/lib/audiences.rb
ADDED
@@ -0,0 +1,45 @@
|
|
1
|
+
module Audiences
|
2
|
+
def aud_to_str(aud)
|
3
|
+
"#{aud.service_name}.#{aud.component_name}.#{aud.operation_type}.#{aud.operation_name}"
|
4
|
+
end
|
5
|
+
|
6
|
+
def str_to_aud(str)
|
7
|
+
# TODO: move to common package
|
8
|
+
parts = str.split(".")
|
9
|
+
aud = Streamdal::Protos::Audience.new
|
10
|
+
aud.service_name = parts[0]
|
11
|
+
aud.component_name = parts[1]
|
12
|
+
aud.operation_type = parts[2]
|
13
|
+
aud.operation_name = parts[3]
|
14
|
+
aud
|
15
|
+
end
|
16
|
+
|
17
|
+
def _seen_audience(aud)
|
18
|
+
@audiences.key?(aud_to_str(aud))
|
19
|
+
end
|
20
|
+
|
21
|
+
def _add_audience(aud)
|
22
|
+
# Add an audience to the local cache map and send to server
|
23
|
+
if _seen_audience(aud)
|
24
|
+
return
|
25
|
+
end
|
26
|
+
|
27
|
+
@audiences[aud_to_str(aud)] = aud
|
28
|
+
|
29
|
+
req = Streamdal::Protos::NewAudienceRequest.new
|
30
|
+
req.session_id = @session_id
|
31
|
+
req.audience = aud
|
32
|
+
@stub.new_audience(req, metadata: _metadata)
|
33
|
+
end
|
34
|
+
|
35
|
+
def _add_audiences
|
36
|
+
# This method is used to re-announce audiences after a disconnect
|
37
|
+
|
38
|
+
@audiences.each do |aud|
|
39
|
+
req = Streamdal::Protos::NewAudienceRequest.new
|
40
|
+
req.session_id = @session_id
|
41
|
+
req.audience = aud
|
42
|
+
@stub.new_audience(req, metadata: _metadata)
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
data/lib/hostfunc.rb
ADDED
@@ -0,0 +1,109 @@
|
|
1
|
+
require "steps/sp_steps_kv_pb"
|
2
|
+
|
3
|
+
module Streamdal
|
4
|
+
class HostFunc
|
5
|
+
|
6
|
+
##
|
7
|
+
# This class holds methods that are called by wasm modules
|
8
|
+
|
9
|
+
def initialize(kv)
|
10
|
+
@kv = kv
|
11
|
+
end
|
12
|
+
|
13
|
+
##
|
14
|
+
# kv_exists is a host function that is used to check if a key exists in the KV store
|
15
|
+
def kv_exists(caller, ptr, len)
|
16
|
+
|
17
|
+
data = caller.export("memory").to_memory.read(ptr, len)
|
18
|
+
|
19
|
+
# Read request from memory and decode into HttpRequest
|
20
|
+
req = Streamdal::Protos::KVStep.decode(data)
|
21
|
+
|
22
|
+
exists = @kv.exists(req.key)
|
23
|
+
|
24
|
+
msg = exists ? "Key '#{req.key}' exists" : "Key #{req.key} does not exist"
|
25
|
+
|
26
|
+
status = exists ? :KV_STATUS_SUCCESS : :KV_STATUS_FAILURE
|
27
|
+
|
28
|
+
wasm_resp = Streamdal::Protos::KVStepResponse.new
|
29
|
+
wasm_resp.status = status
|
30
|
+
wasm_resp.message = msg
|
31
|
+
|
32
|
+
write_to_memory(caller, wasm_resp)
|
33
|
+
end
|
34
|
+
|
35
|
+
##
|
36
|
+
# http_request performs a http request on behalf of a wasm module since WASI cannot talk sockets
|
37
|
+
def http_request(caller, ptr, len)
|
38
|
+
data = caller.export("memory").to_memory.read(ptr, len)
|
39
|
+
|
40
|
+
# Read request from memory and decode into HttpRequest
|
41
|
+
req = Streamdal::Protos::HttpRequest.decode(data)
|
42
|
+
|
43
|
+
# Attempt to make HTTP request
|
44
|
+
# On error, return a mock 400 response with the error as the body
|
45
|
+
begin
|
46
|
+
response = _make_http_request(req)
|
47
|
+
rescue => e
|
48
|
+
wasm_resp = Streamdal::Protos::HttpResponse.new
|
49
|
+
wasm_resp.code = 400
|
50
|
+
wasm_resp.body = "Unable to execute HTTP request: #{e}"
|
51
|
+
return wasm_resp
|
52
|
+
end
|
53
|
+
|
54
|
+
# Successful request, build the proto from the httparty response
|
55
|
+
wasm_resp = Streamdal::Protos::HttpResponse.new
|
56
|
+
wasm_resp.code = response.code
|
57
|
+
wasm_resp.body = response.body
|
58
|
+
wasm_resp.headers = Google::Protobuf::Map.new(:string, :string, {})
|
59
|
+
|
60
|
+
# Headers can have multiple values, but we just want a map[string]string here for simplicity
|
61
|
+
# The client can pase by the delimiter ";" if needed.
|
62
|
+
response.headers.each do |k, values|
|
63
|
+
wasm_resp.headers[k] = values.kind_of?(Array) ? values.join("; ") : values
|
64
|
+
end
|
65
|
+
|
66
|
+
# Write the HttpResponse proto message to WASM memory
|
67
|
+
# The .wasm module will read/decode this data internally
|
68
|
+
write_to_memory(caller, wasm_resp)
|
69
|
+
end
|
70
|
+
|
71
|
+
private
|
72
|
+
|
73
|
+
##
|
74
|
+
# Performs an http request
|
75
|
+
def _make_http_request(req)
|
76
|
+
if req.nil?
|
77
|
+
raise "req is required"
|
78
|
+
end
|
79
|
+
|
80
|
+
options = {
|
81
|
+
headers: { "Content-Type": "application/json", },
|
82
|
+
}
|
83
|
+
|
84
|
+
req.headers.each { |key, value| options.headers[key] = value }
|
85
|
+
|
86
|
+
case req.to_h[:method]
|
87
|
+
when :HTTP_REQUEST_METHOD_GET
|
88
|
+
return HTTParty.get(req.url)
|
89
|
+
when :HTTP_REQUEST_METHOD_POST
|
90
|
+
options.body = req.body
|
91
|
+
return HTTParty.post(req.url, options)
|
92
|
+
when :HTTP_REQUEST_METHOD_PUT
|
93
|
+
options.body = req.body
|
94
|
+
return HTTParty.put(req.url, options)
|
95
|
+
when :HTTP_REQUEST_METHOD_DELETE
|
96
|
+
return HTTParty.delete(req.url)
|
97
|
+
when :HTTP_REQUEST_METHOD_PATCH
|
98
|
+
options.body = req.body
|
99
|
+
return HTTParty.patch(req.url, options)
|
100
|
+
when :HTTP_REQUEST_METHOD_HEAD
|
101
|
+
return HTTParty.head(req.url)
|
102
|
+
when :HTTP_REQUEST_METHOD_OPTIONS
|
103
|
+
return HTTParty.options(req.url)
|
104
|
+
else
|
105
|
+
raise ArgumentError, "Invalid http request method: #{req.method}"
|
106
|
+
end
|
107
|
+
end
|
108
|
+
end
|
109
|
+
end
|
data/lib/kv.rb
ADDED
@@ -0,0 +1,52 @@
|
|
1
|
+
module Streamdal
|
2
|
+
class KeyValue
|
3
|
+
def initialize
|
4
|
+
@kvs = {}
|
5
|
+
@mtx = Mutex.new
|
6
|
+
end
|
7
|
+
|
8
|
+
def set(key, value)
|
9
|
+
@mtx.synchronize do
|
10
|
+
@kvs[key] = value
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
def get(key)
|
15
|
+
@mtx.synchronize do
|
16
|
+
@kvs[key]
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def delete(key)
|
21
|
+
@mtx.synchronize do
|
22
|
+
@kvs.delete(key)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def keys
|
27
|
+
@mtx.synchronize do
|
28
|
+
@kvs.keys
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def items
|
33
|
+
@mtx.synchronize do
|
34
|
+
@kvs.values
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
def exists(key)
|
39
|
+
@mtx.synchronize do
|
40
|
+
@kvs.key?(key)
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
def purge
|
45
|
+
@mtx.synchronize do
|
46
|
+
num_keys = @kvs.keys.length
|
47
|
+
@kvs = {}
|
48
|
+
num_keys
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
data/lib/kv_spec.rb
ADDED
@@ -0,0 +1,54 @@
|
|
1
|
+
require 'rspec'
|
2
|
+
require_relative 'spec_helper'
|
3
|
+
require_relative 'kv'
|
4
|
+
|
5
|
+
RSpec.describe 'KeyValue' do
|
6
|
+
before(:each) do
|
7
|
+
@kv = Streamdal::KeyValue.new
|
8
|
+
end
|
9
|
+
|
10
|
+
it 'stores and retrieves key-value pair' do
|
11
|
+
@kv.set('key', 'value')
|
12
|
+
expect(@kv.get('key')).to eq('value')
|
13
|
+
end
|
14
|
+
|
15
|
+
it 'deletes a key' do
|
16
|
+
expect(@kv.keys.length).to eq(0)
|
17
|
+
@kv.set('key1', 'value')
|
18
|
+
@kv.set('key2', 'value')
|
19
|
+
expect(@kv.keys.length).to eq(2)
|
20
|
+
@kv.delete('key1')
|
21
|
+
expect(@kv.keys.length).to eq(1)
|
22
|
+
end
|
23
|
+
|
24
|
+
it 'returns array of keys' do
|
25
|
+
expect(@kv.keys.length).to eq(0)
|
26
|
+
@kv.set('key1', 'value')
|
27
|
+
expect(@kv.keys.length).to eq(1)
|
28
|
+
expect(@kv.keys[0]).to eq('key1')
|
29
|
+
end
|
30
|
+
|
31
|
+
it 'returns array of item values' do
|
32
|
+
expect(@kv.keys.length).to eq(0)
|
33
|
+
@kv.set('key1', 'value1')
|
34
|
+
expect(@kv.keys.length).to eq(1)
|
35
|
+
expect(@kv.items[0]).to eq('value1')
|
36
|
+
end
|
37
|
+
|
38
|
+
it 'returns if a key exists or not' do
|
39
|
+
expect(@kv.exists('key1')).to eq(false)
|
40
|
+
@kv.set('key1', 'value')
|
41
|
+
expect(@kv.exists('key1')).to eq(true)
|
42
|
+
end
|
43
|
+
|
44
|
+
it 'purges all keys' do
|
45
|
+
expect(@kv.keys.length).to eq(0)
|
46
|
+
@kv.set('key1', 'value')
|
47
|
+
@kv.set('key2', 'value')
|
48
|
+
expect(@kv.keys.length).to eq(2)
|
49
|
+
num_keys = @kv.purge
|
50
|
+
expect(@kv.keys.length).to eq(0)
|
51
|
+
expect(num_keys).to eq(2)
|
52
|
+
end
|
53
|
+
|
54
|
+
end
|
data/lib/metrics.rb
ADDED
@@ -0,0 +1,265 @@
|
|
1
|
+
module Streamdal
|
2
|
+
class Counter
|
3
|
+
attr_accessor :last_updated, :name, :aud, :labels
|
4
|
+
|
5
|
+
def initialize(name, aud, labels = {}, value = 0.0)
|
6
|
+
@name = name
|
7
|
+
@aud = aud
|
8
|
+
@labels = labels
|
9
|
+
@value = 0
|
10
|
+
@last_updated = Time::now
|
11
|
+
@value_mtx = Mutex.new
|
12
|
+
end
|
13
|
+
|
14
|
+
def incr(val)
|
15
|
+
@value_mtx.synchronize do
|
16
|
+
@value = @value + val
|
17
|
+
@last_updated = Time::now
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def reset
|
22
|
+
@value_mtx.synchronize do
|
23
|
+
@value = 0.0
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def val
|
28
|
+
@value_mtx.synchronize do
|
29
|
+
@value
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
class Metrics
|
35
|
+
|
36
|
+
COUNTER_CONSUME_BYTES = "counter_consume_bytes"
|
37
|
+
COUNTER_CONSUME_PROCESSED = "counter_consume_processed"
|
38
|
+
COUNTER_CONSUME_ERRORS = "counter_consume_errors"
|
39
|
+
COUNTER_PRODUCE_BYTES = "counter_produce_bytes"
|
40
|
+
COUNTER_PRODUCE_PROCESSED = "counter_produce_processed"
|
41
|
+
COUNTER_PRODUCE_ERRORS = "counter_produce_errors"
|
42
|
+
COUNTER_NOTIFY = "counter_notify"
|
43
|
+
COUNTER_DROPPED_TAIL_MESSAGES = "counter_dropped_tail_messages"
|
44
|
+
COUNTER_CONSUME_BYTES_RATE = "counter_consume_bytes_rate"
|
45
|
+
COUNTER_PRODUCE_BYTES_RATE = "counter_produce_bytes_rate"
|
46
|
+
COUNTER_CONSUME_PROCESSED_RATE = "counter_consume_processed_rate"
|
47
|
+
COUNTER_PRODUCE_PROCESSED_RATE = "counter_produce_processed_rate"
|
48
|
+
|
49
|
+
WORKER_POOL_SIZE = 3
|
50
|
+
DEFAULT_COUNTER_REAPER_INTERVAL = 10
|
51
|
+
DEFAULT_COUNTER_TTL = 10
|
52
|
+
DEFAULT_COUNTER_PUBLISH_INTERVAL = 1
|
53
|
+
|
54
|
+
CounterEntry = Struct.new(:name, :aud, :labels, :value)
|
55
|
+
|
56
|
+
def initialize(cfg)
|
57
|
+
if cfg.nil?
|
58
|
+
raise ArgumentError, "cfg is nil"
|
59
|
+
end
|
60
|
+
|
61
|
+
@cfg = cfg
|
62
|
+
@log = cfg[:log]
|
63
|
+
@counters = {}
|
64
|
+
@counters_mtx = Mutex.new
|
65
|
+
@exit = false
|
66
|
+
@incr_queue = Queue.new
|
67
|
+
@publish_queue = Queue.new
|
68
|
+
@workers = []
|
69
|
+
@stub = Streamdal::Protos::Internal::Stub.new(@cfg[:streamdal_url], :this_channel_is_insecure)
|
70
|
+
|
71
|
+
_start
|
72
|
+
end
|
73
|
+
|
74
|
+
def shutdown
|
75
|
+
# Set exit flag so workers exit
|
76
|
+
@exit = true
|
77
|
+
|
78
|
+
# Let loops exit
|
79
|
+
sleep(1)
|
80
|
+
|
81
|
+
# Exit any remaining threads
|
82
|
+
@workers.each do |w|
|
83
|
+
if w.running?
|
84
|
+
w.exit
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
def self.composite_id(counter_name, labels = {})
|
90
|
+
if labels.nil?
|
91
|
+
labels = {}
|
92
|
+
end
|
93
|
+
"#{counter_name}-#{labels.values.join("-")}".freeze
|
94
|
+
end
|
95
|
+
|
96
|
+
def get_counter(ce)
|
97
|
+
if ce.nil?
|
98
|
+
raise ArgumentError, "ce is nil"
|
99
|
+
end
|
100
|
+
|
101
|
+
k = Metrics::composite_id(ce.name, ce.labels)
|
102
|
+
|
103
|
+
@counters_mtx.synchronize do
|
104
|
+
if @counters.key?(k)
|
105
|
+
@counters[k]
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
# No counter exists, create a new one and return it
|
110
|
+
new_counter(ce)
|
111
|
+
end
|
112
|
+
|
113
|
+
def new_counter(ce)
|
114
|
+
c = Counter.new(ce.name, ce.aud, ce.labels, ce.value)
|
115
|
+
|
116
|
+
@counters_mtx.synchronize do
|
117
|
+
@counters[Metrics::composite_id(ce.name, ce.labels)] = c
|
118
|
+
end
|
119
|
+
|
120
|
+
c
|
121
|
+
end
|
122
|
+
|
123
|
+
def incr(ce)
|
124
|
+
c = get_counter(ce)
|
125
|
+
|
126
|
+
if c.nil?
|
127
|
+
new_counter(ce)
|
128
|
+
nil
|
129
|
+
end
|
130
|
+
|
131
|
+
@incr_queue.push(ce)
|
132
|
+
end
|
133
|
+
|
134
|
+
def remove_counter(name)
|
135
|
+
@counters_mtx.synchronize do
|
136
|
+
@counters.delete(name)
|
137
|
+
end
|
138
|
+
end
|
139
|
+
|
140
|
+
private
|
141
|
+
|
142
|
+
def _start
|
143
|
+
WORKER_POOL_SIZE.times do |i|
|
144
|
+
@workers << Thread.new { _run_incrementer_worker(i) }
|
145
|
+
@workers << Thread.new { _run_publisher_worker(i) }
|
146
|
+
end
|
147
|
+
|
148
|
+
@workers << Thread.new { _run_publisher }
|
149
|
+
@workers << Thread.new { _run_reaper }
|
150
|
+
end
|
151
|
+
|
152
|
+
def _publish_metrics(ce)
|
153
|
+
metric = Streamdal::Protos::Metric.new
|
154
|
+
metric.name = ce.name
|
155
|
+
metric.labels = Google::Protobuf::Map.new(:string, :string, ce.labels)
|
156
|
+
metric.value = ce.value
|
157
|
+
metric.audience = ce.aud
|
158
|
+
|
159
|
+
req = Streamdal::Protos::MetricsRequest.new
|
160
|
+
req.metrics = Google::Protobuf::RepeatedField.new(:message, Streamdal::Protos::Metric, [metric])
|
161
|
+
|
162
|
+
@log.debug("Published metric: #{ce.name} #{ce.labels} #{ce.value}")
|
163
|
+
@stub.metrics(req, metadata: _metadata)
|
164
|
+
end
|
165
|
+
|
166
|
+
def _run_publisher
|
167
|
+
# Background thread that reads values from counters, adds them to the publish queue, and then
|
168
|
+
# resets the counter's value back to zero
|
169
|
+
unless @exit
|
170
|
+
@log.debug("Starting publisher")
|
171
|
+
|
172
|
+
# Sleep on startup and then and between each loop run
|
173
|
+
sleep(DEFAULT_COUNTER_PUBLISH_INTERVAL)
|
174
|
+
|
175
|
+
# Get all counters
|
176
|
+
# Loop over each counter, get the value,
|
177
|
+
# if value > 0, continue
|
178
|
+
# if now() - last_updated > 10 seconds, remove counter
|
179
|
+
# Grab copy of counters
|
180
|
+
@counters_mtx.lock
|
181
|
+
new_counters = @counters.dup
|
182
|
+
@counters_mtx.unlock
|
183
|
+
|
184
|
+
new_counters.each do |_, counter|
|
185
|
+
if counter.val == 0
|
186
|
+
next
|
187
|
+
end
|
188
|
+
|
189
|
+
ce = CounterEntry.new(counter.name, counter.aud, counter.labels, counter.val)
|
190
|
+
counter.reset
|
191
|
+
|
192
|
+
@publish_queue.push(ce)
|
193
|
+
end
|
194
|
+
end
|
195
|
+
end
|
196
|
+
|
197
|
+
def _run_publisher_worker(worker_id)
|
198
|
+
@log.debug("Starting publisher worker '#{worker_id}'")
|
199
|
+
|
200
|
+
until @exit
|
201
|
+
ce = @incr_queue.pop
|
202
|
+
if ce.nil?
|
203
|
+
next
|
204
|
+
end
|
205
|
+
begin
|
206
|
+
_publish_metrics(ce)
|
207
|
+
rescue => e
|
208
|
+
@log.error("Failed to publish metrics: #{e}: #{ce.inspect}")
|
209
|
+
end
|
210
|
+
end
|
211
|
+
|
212
|
+
@log.debug("Exiting publisher worker '#{worker_id}'")
|
213
|
+
end
|
214
|
+
|
215
|
+
def _run_reaper
|
216
|
+
@log.debug("Starting reaper")
|
217
|
+
|
218
|
+
until @exit
|
219
|
+
# Sleep on startup and then and between each loop run
|
220
|
+
sleep(DEFAULT_COUNTER_REAPER_INTERVAL)
|
221
|
+
|
222
|
+
# Get all counters
|
223
|
+
# Loop over each counter, get the value,
|
224
|
+
# if value > 0, continue
|
225
|
+
# if now() - last_updated > 10 seconds, remove counter
|
226
|
+
# Grab copy of counters
|
227
|
+
@counters_mtx.synchronize do
|
228
|
+
@counters.each do |name, counter|
|
229
|
+
if counter.val > 0
|
230
|
+
next
|
231
|
+
end
|
232
|
+
|
233
|
+
if Time::now - counter.last_updated > DEFAULT_COUNTER_TTL
|
234
|
+
@log.debug("Reaping counter '#{name}'")
|
235
|
+
@counters.delete(name)
|
236
|
+
end
|
237
|
+
end
|
238
|
+
end
|
239
|
+
end
|
240
|
+
|
241
|
+
@log.debug("Exiting reaper")
|
242
|
+
end
|
243
|
+
|
244
|
+
def _run_incrementer_worker(worker_id)
|
245
|
+
@log.debug("Starting incrementer worker '#{worker_id}'")
|
246
|
+
|
247
|
+
until @exit
|
248
|
+
ce = @incr_queue.pop
|
249
|
+
|
250
|
+
next if ce.nil?
|
251
|
+
|
252
|
+
c = get_counter(ce)
|
253
|
+
|
254
|
+
c.incr(ce.value)
|
255
|
+
end
|
256
|
+
|
257
|
+
@log.debug("Exiting incrementer worker '#{worker_id}'")
|
258
|
+
end
|
259
|
+
|
260
|
+
# Returns metadata for gRPC requests to the internal gRPC API
|
261
|
+
def _metadata
|
262
|
+
{ "auth-token" => @cfg[:streamdal_token].to_s }
|
263
|
+
end
|
264
|
+
end
|
265
|
+
end
|
data/lib/metrics_spec.rb
ADDED
data/lib/schema.rb
ADDED
@@ -0,0 +1,47 @@
|
|
1
|
+
include Streamdal::Protos
|
2
|
+
|
3
|
+
module Schemas
|
4
|
+
def _set_schema(aud, schema)
|
5
|
+
s = Streamdal::Protos::Schema.new
|
6
|
+
s.json_schema = schema
|
7
|
+
@schemas[aud_to_str(aud)] = s
|
8
|
+
end
|
9
|
+
|
10
|
+
def _get_schema(aud)
|
11
|
+
if @schemas.key?(aud_to_str(aud))
|
12
|
+
return @schemas[aud_to_str(aud)].json_schema
|
13
|
+
end
|
14
|
+
|
15
|
+
""
|
16
|
+
end
|
17
|
+
|
18
|
+
def _handle_schema(aud, step, wasm_resp)
|
19
|
+
# Only handle schema steps
|
20
|
+
if step.infer_schema.nil?
|
21
|
+
return nil
|
22
|
+
end
|
23
|
+
|
24
|
+
# Only successful schema inferences
|
25
|
+
if wasm_resp.exit_code != :WASM_EXIT_CODE_TRUE
|
26
|
+
return nil
|
27
|
+
end
|
28
|
+
|
29
|
+
# If existing schema matches, do nothing
|
30
|
+
existing_schema = _get_schema(aud)
|
31
|
+
if existing_schema == wasm_resp.output_step
|
32
|
+
return nil
|
33
|
+
end
|
34
|
+
|
35
|
+
_set_schema(aud, wasm_resp.output_step)
|
36
|
+
|
37
|
+
req = Streamdal::Protos::SendSchemaRequest.new
|
38
|
+
req.audience = aud
|
39
|
+
req.schema = Streamdal::Protos::Schema.new
|
40
|
+
req.schema.json_schema = wasm_resp.output_step
|
41
|
+
|
42
|
+
# Run in thread so we don't block on gRPC call
|
43
|
+
Thread.new do
|
44
|
+
@stub.send_schema(req, metadata: _metadata)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
data/lib/schema_spec.rb
ADDED
@@ -0,0 +1,59 @@
|
|
1
|
+
require 'rspec'
|
2
|
+
require 'sp_wsm_pb'
|
3
|
+
require 'steps/sp_steps_inferschema_pb'
|
4
|
+
require 'sp_pipeline_pb'
|
5
|
+
require 'sp_common_pb'
|
6
|
+
require_relative 'streamdal'
|
7
|
+
require_relative 'spec_helper'
|
8
|
+
require_relative 'schema'
|
9
|
+
require_relative 'audiences'
|
10
|
+
|
11
|
+
module Streamdal
|
12
|
+
class TestObj
|
13
|
+
include Schemas
|
14
|
+
include Audiences
|
15
|
+
|
16
|
+
@schemas
|
17
|
+
@stub
|
18
|
+
|
19
|
+
attr_accessor :stub
|
20
|
+
|
21
|
+
def initialize
|
22
|
+
@schemas = {}
|
23
|
+
@stub = RSpec::Mocks::Double.new("stub", { send_schema: nil })
|
24
|
+
end
|
25
|
+
|
26
|
+
def _metadata
|
27
|
+
{}
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
RSpec.describe "Streamdal::Schema" do
|
33
|
+
before(:each) do
|
34
|
+
@test_obj = Streamdal::TestObj.new
|
35
|
+
|
36
|
+
public_aud = Streamdal::Audience.new(1, "consume", "kafka")
|
37
|
+
@aud = public_aud.to_proto("test-svc")
|
38
|
+
expect(@test_obj.aud_to_str(@aud)).to eq("test-svc.kafka.OPERATION_TYPE_CONSUMER.consume")
|
39
|
+
end
|
40
|
+
|
41
|
+
it "should set and get a schema" do
|
42
|
+
@test_obj._set_schema(@aud, "{}")
|
43
|
+
|
44
|
+
got_schema = @test_obj._get_schema(@aud)
|
45
|
+
expect(got_schema).to eq("{}")
|
46
|
+
end
|
47
|
+
it "should handle schema" do
|
48
|
+
wasm_resp = Streamdal::Protos::WASMResponse.new
|
49
|
+
wasm_resp.exit_code = :WASM_EXIT_CODE_TRUE
|
50
|
+
wasm_resp.output_step = "{}"
|
51
|
+
|
52
|
+
@test_obj._handle_schema(@aud, Streamdal::Protos::PipelineStep.new(infer_schema: Streamdal::Protos::InferSchemaStep.new), wasm_resp)
|
53
|
+
|
54
|
+
got_schema = @test_obj._get_schema(@aud)
|
55
|
+
expect(got_schema).to eq("{}")
|
56
|
+
# sleep(1)
|
57
|
+
expect(@test_obj.stub).to have_received(:send_schema).with(Streamdal::Protos::SendSchemaRequest.new(audience: @aud, schema: Streamdal::Protos::Schema.new(json_schema: "{}")), metadata: @test_obj._metadata)
|
58
|
+
end
|
59
|
+
end
|
data/lib/spec_helper.rb
ADDED