streamdal 0.0.1 → 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/audiences.rb +5 -5
- data/lib/hostfunc.rb +53 -21
- data/lib/kv.rb +3 -1
- data/lib/metrics.rb +52 -65
- data/lib/schema.rb +8 -14
- data/lib/streamdal.rb +51 -55
- data/lib/tail.rb +22 -28
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ca014e8334d38a589d1cd3aab58362ac3095fc077a93c076301f2e1552f9b375
|
4
|
+
data.tar.gz: 77110e0cc07e851e7b21ef831bed28495353bd70fce865159c41b18d4a40f7d0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 227c07d36fd54e8bf35a79b434417f59c80e45c72678ca82ada629b8268b744b614616d28b196e48b85c93f7a031330ebc8884a8ca826553eb647953069d7890
|
7
|
+
data.tar.gz: 9c83c8388b8e871d60b1b670b6f919b6a05878c731b6b8978545f9b782885c195cb30d02f1ec160cba58974ad53f99d9506410301185c5e036128eca4c56bb3b
|
data/lib/audiences.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Audiences
|
2
4
|
def aud_to_str(aud)
|
3
5
|
"#{aud.service_name}.#{aud.component_name}.#{aud.operation_type}.#{aud.operation_name}"
|
@@ -5,7 +7,7 @@ module Audiences
|
|
5
7
|
|
6
8
|
def str_to_aud(str)
|
7
9
|
# TODO: move to common package
|
8
|
-
parts = str.split(
|
10
|
+
parts = str.split('.')
|
9
11
|
aud = Streamdal::Protos::Audience.new
|
10
12
|
aud.service_name = parts[0]
|
11
13
|
aud.component_name = parts[1]
|
@@ -20,9 +22,7 @@ module Audiences
|
|
20
22
|
|
21
23
|
def _add_audience(aud)
|
22
24
|
# Add an audience to the local cache map and send to server
|
23
|
-
if _seen_audience(aud)
|
24
|
-
return
|
25
|
-
end
|
25
|
+
return if _seen_audience(aud)
|
26
26
|
|
27
27
|
@audiences[aud_to_str(aud)] = aud
|
28
28
|
|
@@ -42,4 +42,4 @@ module Audiences
|
|
42
42
|
@stub.new_audience(req, metadata: _metadata)
|
43
43
|
end
|
44
44
|
end
|
45
|
-
end
|
45
|
+
end
|
data/lib/hostfunc.rb
CHANGED
@@ -1,4 +1,6 @@
|
|
1
|
-
require
|
1
|
+
require 'steps/sp_steps_kv_pb'
|
2
|
+
require 'sp_wsm_pb'
|
3
|
+
require 'steps/sp_steps_httprequest_pb'
|
2
4
|
|
3
5
|
module Streamdal
|
4
6
|
class HostFunc
|
@@ -14,7 +16,7 @@ module Streamdal
|
|
14
16
|
# kv_exists is a host function that is used to check if a key exists in the KV store
|
15
17
|
def kv_exists(caller, ptr, len)
|
16
18
|
|
17
|
-
data = caller.export(
|
19
|
+
data = caller.export('memory').to_memory.read(ptr, len)
|
18
20
|
|
19
21
|
# Read request from memory and decode into HttpRequest
|
20
22
|
req = Streamdal::Protos::KVStep.decode(data)
|
@@ -35,32 +37,40 @@ module Streamdal
|
|
35
37
|
##
|
36
38
|
# http_request performs a http request on behalf of a wasm module since WASI cannot talk sockets
|
37
39
|
def http_request(caller, ptr, len)
|
38
|
-
data = caller.export(
|
40
|
+
data = caller.export('memory').to_memory.read(ptr, len)
|
39
41
|
|
40
42
|
# Read request from memory and decode into HttpRequest
|
41
|
-
req = Streamdal::Protos::
|
43
|
+
req = Streamdal::Protos::WASMRequest.decode(data)
|
44
|
+
|
45
|
+
begin
|
46
|
+
req_body = self._get_request_body_for_mode(req)
|
47
|
+
rescue => e
|
48
|
+
return self._http_request_response(caller, 400, e.to_s, {})
|
49
|
+
end
|
42
50
|
|
43
51
|
# Attempt to make HTTP request
|
44
52
|
# On error, return a mock 400 response with the error as the body
|
45
53
|
begin
|
46
|
-
response = _make_http_request(req)
|
54
|
+
response = _make_http_request(req.step.http_request.request, req_body)
|
47
55
|
rescue => e
|
48
|
-
|
49
|
-
wasm_resp.code = 400
|
50
|
-
wasm_resp.body = "Unable to execute HTTP request: #{e}"
|
51
|
-
return wasm_resp
|
56
|
+
return self._http_request_response(caller, 400, "Unable to execute HTTP request: #{e}", {})
|
52
57
|
end
|
53
58
|
|
54
|
-
|
59
|
+
self._http_request_response(caller, response.code, response.body, response.headers)
|
60
|
+
end
|
61
|
+
|
62
|
+
private
|
63
|
+
|
64
|
+
def _http_request_response(caller, code, body, headers)
|
55
65
|
wasm_resp = Streamdal::Protos::HttpResponse.new
|
56
|
-
wasm_resp.code =
|
57
|
-
wasm_resp.body =
|
66
|
+
wasm_resp.code = code
|
67
|
+
wasm_resp.body = body
|
58
68
|
wasm_resp.headers = Google::Protobuf::Map.new(:string, :string, {})
|
59
69
|
|
60
70
|
# Headers can have multiple values, but we just want a map[string]string here for simplicity
|
61
71
|
# The client can pase by the delimiter ";" if needed.
|
62
|
-
|
63
|
-
wasm_resp.headers[k] = values.
|
72
|
+
headers.each do |k, values|
|
73
|
+
wasm_resp.headers[k] = values.is_a?(Array) ? values.join('; ') : values
|
64
74
|
end
|
65
75
|
|
66
76
|
# Write the HttpResponse proto message to WASM memory
|
@@ -68,17 +78,39 @@ module Streamdal
|
|
68
78
|
write_to_memory(caller, wasm_resp)
|
69
79
|
end
|
70
80
|
|
71
|
-
|
81
|
+
def _get_request_body_for_mode(req)
|
82
|
+
http_req = req.step.http_request.request
|
83
|
+
|
84
|
+
case http_req.body_mode
|
85
|
+
when :HTTP_REQUEST_BODY_MODE_INTER_STEP_RESULT
|
86
|
+
raise 'Inter step result is empty' if req.inter_step_result.nil?
|
87
|
+
|
88
|
+
detective_res = req.inter_step_result.detective_result
|
89
|
+
|
90
|
+
raise 'Detective result is empty' if detective_res.nil?
|
91
|
+
|
92
|
+
# Wipe values to prevent PII from being leaked
|
93
|
+
detective_res.matches.each { |step_res|
|
94
|
+
step_res.value = ''
|
95
|
+
}
|
96
|
+
|
97
|
+
req.inter_step_result.to_json
|
98
|
+
when :HTTP_REQUEST_BODY_MODE_STATIC
|
99
|
+
http_req.body
|
100
|
+
else
|
101
|
+
raise 'invalid http request body mode'
|
102
|
+
end
|
103
|
+
end
|
72
104
|
|
73
105
|
##
|
74
106
|
# Performs an http request
|
75
|
-
def _make_http_request(req)
|
107
|
+
def _make_http_request(req, body)
|
76
108
|
if req.nil?
|
77
|
-
raise
|
109
|
+
raise 'req is required'
|
78
110
|
end
|
79
111
|
|
80
112
|
options = {
|
81
|
-
headers: { "Content-Type":
|
113
|
+
headers: { "Content-Type": 'application/json', },
|
82
114
|
}
|
83
115
|
|
84
116
|
req.headers.each { |key, value| options.headers[key] = value }
|
@@ -87,15 +119,15 @@ module Streamdal
|
|
87
119
|
when :HTTP_REQUEST_METHOD_GET
|
88
120
|
return HTTParty.get(req.url)
|
89
121
|
when :HTTP_REQUEST_METHOD_POST
|
90
|
-
options.body =
|
122
|
+
options.body = body
|
91
123
|
return HTTParty.post(req.url, options)
|
92
124
|
when :HTTP_REQUEST_METHOD_PUT
|
93
|
-
options.body =
|
125
|
+
options.body = body
|
94
126
|
return HTTParty.put(req.url, options)
|
95
127
|
when :HTTP_REQUEST_METHOD_DELETE
|
96
128
|
return HTTParty.delete(req.url)
|
97
129
|
when :HTTP_REQUEST_METHOD_PATCH
|
98
|
-
options.body =
|
130
|
+
options.body = body
|
99
131
|
return HTTParty.patch(req.url, options)
|
100
132
|
when :HTTP_REQUEST_METHOD_HEAD
|
101
133
|
return HTTParty.head(req.url)
|
data/lib/kv.rb
CHANGED
data/lib/metrics.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Streamdal
|
2
4
|
class Counter
|
3
5
|
attr_accessor :last_updated, :name, :aud, :labels
|
@@ -6,15 +8,15 @@ module Streamdal
|
|
6
8
|
@name = name
|
7
9
|
@aud = aud
|
8
10
|
@labels = labels
|
9
|
-
@value =
|
10
|
-
@last_updated = Time
|
11
|
+
@value = value
|
12
|
+
@last_updated = Time.now
|
11
13
|
@value_mtx = Mutex.new
|
12
14
|
end
|
13
15
|
|
14
16
|
def incr(val)
|
15
17
|
@value_mtx.synchronize do
|
16
|
-
@value
|
17
|
-
@last_updated = Time
|
18
|
+
@value += val
|
19
|
+
@last_updated = Time.now
|
18
20
|
end
|
19
21
|
end
|
20
22
|
|
@@ -33,18 +35,18 @@ module Streamdal
|
|
33
35
|
|
34
36
|
class Metrics
|
35
37
|
|
36
|
-
COUNTER_CONSUME_BYTES =
|
37
|
-
COUNTER_CONSUME_PROCESSED =
|
38
|
-
COUNTER_CONSUME_ERRORS =
|
39
|
-
COUNTER_PRODUCE_BYTES =
|
40
|
-
COUNTER_PRODUCE_PROCESSED =
|
41
|
-
COUNTER_PRODUCE_ERRORS =
|
42
|
-
COUNTER_NOTIFY =
|
43
|
-
COUNTER_DROPPED_TAIL_MESSAGES =
|
44
|
-
COUNTER_CONSUME_BYTES_RATE =
|
45
|
-
COUNTER_PRODUCE_BYTES_RATE =
|
46
|
-
COUNTER_CONSUME_PROCESSED_RATE =
|
47
|
-
COUNTER_PRODUCE_PROCESSED_RATE =
|
38
|
+
COUNTER_CONSUME_BYTES = 'counter_consume_bytes'
|
39
|
+
COUNTER_CONSUME_PROCESSED = 'counter_consume_processed'
|
40
|
+
COUNTER_CONSUME_ERRORS = 'counter_consume_errors'
|
41
|
+
COUNTER_PRODUCE_BYTES = 'counter_produce_bytes'
|
42
|
+
COUNTER_PRODUCE_PROCESSED = 'counter_produce_processed'
|
43
|
+
COUNTER_PRODUCE_ERRORS = 'counter_produce_errors'
|
44
|
+
COUNTER_NOTIFY = 'counter_notify'
|
45
|
+
COUNTER_DROPPED_TAIL_MESSAGES = 'counter_dropped_tail_messages'
|
46
|
+
COUNTER_CONSUME_BYTES_RATE = 'counter_consume_bytes_rate'
|
47
|
+
COUNTER_PRODUCE_BYTES_RATE = 'counter_produce_bytes_rate'
|
48
|
+
COUNTER_CONSUME_PROCESSED_RATE = 'counter_consume_processed_rate'
|
49
|
+
COUNTER_PRODUCE_PROCESSED_RATE = 'counter_produce_processed_rate'
|
48
50
|
|
49
51
|
WORKER_POOL_SIZE = 3
|
50
52
|
DEFAULT_COUNTER_REAPER_INTERVAL = 10
|
@@ -54,9 +56,7 @@ module Streamdal
|
|
54
56
|
CounterEntry = Struct.new(:name, :aud, :labels, :value)
|
55
57
|
|
56
58
|
def initialize(cfg)
|
57
|
-
if cfg.nil?
|
58
|
-
raise ArgumentError, "cfg is nil"
|
59
|
-
end
|
59
|
+
raise ArgumentError, 'cfg is nil' if cfg.nil?
|
60
60
|
|
61
61
|
@cfg = cfg
|
62
62
|
@log = cfg[:log]
|
@@ -80,30 +80,22 @@ module Streamdal
|
|
80
80
|
|
81
81
|
# Exit any remaining threads
|
82
82
|
@workers.each do |w|
|
83
|
-
if w.running?
|
84
|
-
w.exit
|
85
|
-
end
|
83
|
+
w.exit if w.running?
|
86
84
|
end
|
87
85
|
end
|
88
86
|
|
89
87
|
def self.composite_id(counter_name, labels = {})
|
90
|
-
if labels.nil?
|
91
|
-
|
92
|
-
end
|
93
|
-
"#{counter_name}-#{labels.values.join("-")}".freeze
|
88
|
+
labels = {} if labels.nil?
|
89
|
+
"#{counter_name}-#{labels.values.join('-')}"
|
94
90
|
end
|
95
91
|
|
96
92
|
def get_counter(ce)
|
97
|
-
if ce.nil?
|
98
|
-
raise ArgumentError, "ce is nil"
|
99
|
-
end
|
93
|
+
raise ArgumentError, 'ce is nil' if ce.nil?
|
100
94
|
|
101
|
-
k = Metrics
|
95
|
+
k = Metrics.composite_id(ce.name, ce.labels)
|
102
96
|
|
103
97
|
@counters_mtx.synchronize do
|
104
|
-
if @counters.key?(k)
|
105
|
-
@counters[k]
|
106
|
-
end
|
98
|
+
@counters[k] if @counters.key?(k)
|
107
99
|
end
|
108
100
|
|
109
101
|
# No counter exists, create a new one and return it
|
@@ -114,7 +106,7 @@ module Streamdal
|
|
114
106
|
c = Counter.new(ce.name, ce.aud, ce.labels, ce.value)
|
115
107
|
|
116
108
|
@counters_mtx.synchronize do
|
117
|
-
@counters[Metrics
|
109
|
+
@counters[Metrics.composite_id(ce.name, ce.labels)] = c
|
118
110
|
end
|
119
111
|
|
120
112
|
c
|
@@ -166,31 +158,29 @@ module Streamdal
|
|
166
158
|
def _run_publisher
|
167
159
|
# Background thread that reads values from counters, adds them to the publish queue, and then
|
168
160
|
# resets the counter's value back to zero
|
169
|
-
|
170
|
-
@log.debug("Starting publisher")
|
161
|
+
return if @exit
|
171
162
|
|
172
|
-
|
173
|
-
sleep(DEFAULT_COUNTER_PUBLISH_INTERVAL)
|
163
|
+
@log.debug('Starting publisher')
|
174
164
|
|
175
|
-
|
176
|
-
|
177
|
-
# if value > 0, continue
|
178
|
-
# if now() - last_updated > 10 seconds, remove counter
|
179
|
-
# Grab copy of counters
|
180
|
-
@counters_mtx.lock
|
181
|
-
new_counters = @counters.dup
|
182
|
-
@counters_mtx.unlock
|
165
|
+
# Sleep on startup and then and between each loop run
|
166
|
+
sleep(DEFAULT_COUNTER_PUBLISH_INTERVAL)
|
183
167
|
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
168
|
+
# Get all counters
|
169
|
+
# Loop over each counter, get the value,
|
170
|
+
# if value > 0, continue
|
171
|
+
# if now() - last_updated > 10 seconds, remove counter
|
172
|
+
# Grab copy of counters
|
173
|
+
@counters_mtx.lock
|
174
|
+
new_counters = @counters.dup
|
175
|
+
@counters_mtx.unlock
|
188
176
|
|
189
|
-
|
190
|
-
|
177
|
+
new_counters.each_value do |counter|
|
178
|
+
next if counter.val.zero?
|
191
179
|
|
192
|
-
|
193
|
-
|
180
|
+
ce = CounterEntry.new(counter.name, counter.aud, counter.labels, counter.val)
|
181
|
+
counter.reset
|
182
|
+
|
183
|
+
@publish_queue.push(ce)
|
194
184
|
end
|
195
185
|
end
|
196
186
|
|
@@ -199,9 +189,8 @@ module Streamdal
|
|
199
189
|
|
200
190
|
until @exit
|
201
191
|
ce = @incr_queue.pop
|
202
|
-
if ce.nil?
|
203
|
-
|
204
|
-
end
|
192
|
+
next if ce.nil?
|
193
|
+
|
205
194
|
begin
|
206
195
|
_publish_metrics(ce)
|
207
196
|
rescue => e
|
@@ -213,7 +202,7 @@ module Streamdal
|
|
213
202
|
end
|
214
203
|
|
215
204
|
def _run_reaper
|
216
|
-
@log.debug(
|
205
|
+
@log.debug('Starting reaper')
|
217
206
|
|
218
207
|
until @exit
|
219
208
|
# Sleep on startup and then and between each loop run
|
@@ -226,11 +215,9 @@ module Streamdal
|
|
226
215
|
# Grab copy of counters
|
227
216
|
@counters_mtx.synchronize do
|
228
217
|
@counters.each do |name, counter|
|
229
|
-
if counter.val
|
230
|
-
next
|
231
|
-
end
|
218
|
+
next if counter.val.positive?
|
232
219
|
|
233
|
-
if Time
|
220
|
+
if Time.now - counter.last_updated > DEFAULT_COUNTER_TTL
|
234
221
|
@log.debug("Reaping counter '#{name}'")
|
235
222
|
@counters.delete(name)
|
236
223
|
end
|
@@ -238,7 +225,7 @@ module Streamdal
|
|
238
225
|
end
|
239
226
|
end
|
240
227
|
|
241
|
-
@log.debug(
|
228
|
+
@log.debug('Exiting reaper')
|
242
229
|
end
|
243
230
|
|
244
231
|
def _run_incrementer_worker(worker_id)
|
@@ -259,7 +246,7 @@ module Streamdal
|
|
259
246
|
|
260
247
|
# Returns metadata for gRPC requests to the internal gRPC API
|
261
248
|
def _metadata
|
262
|
-
{
|
249
|
+
{ 'auth-token' => @cfg[:streamdal_token].to_s }
|
263
250
|
end
|
264
251
|
end
|
265
|
-
end
|
252
|
+
end
|
data/lib/schema.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
include Streamdal::Protos
|
2
4
|
|
3
5
|
module Schemas
|
@@ -8,29 +10,21 @@ module Schemas
|
|
8
10
|
end
|
9
11
|
|
10
12
|
def _get_schema(aud)
|
11
|
-
if @schemas.key?(aud_to_str(aud))
|
12
|
-
return @schemas[aud_to_str(aud)].json_schema
|
13
|
-
end
|
13
|
+
return @schemas[aud_to_str(aud)].json_schema if @schemas.key?(aud_to_str(aud))
|
14
14
|
|
15
|
-
|
15
|
+
''
|
16
16
|
end
|
17
17
|
|
18
18
|
def _handle_schema(aud, step, wasm_resp)
|
19
19
|
# Only handle schema steps
|
20
|
-
if step.infer_schema.nil?
|
21
|
-
return nil
|
22
|
-
end
|
20
|
+
return nil if step.infer_schema.nil?
|
23
21
|
|
24
22
|
# Only successful schema inferences
|
25
|
-
if wasm_resp.exit_code != :WASM_EXIT_CODE_TRUE
|
26
|
-
return nil
|
27
|
-
end
|
23
|
+
return nil if wasm_resp.exit_code != :WASM_EXIT_CODE_TRUE
|
28
24
|
|
29
25
|
# If existing schema matches, do nothing
|
30
26
|
existing_schema = _get_schema(aud)
|
31
|
-
if existing_schema == wasm_resp.output_step
|
32
|
-
return nil
|
33
|
-
end
|
27
|
+
return nil if existing_schema == wasm_resp.output_step
|
34
28
|
|
35
29
|
_set_schema(aud, wasm_resp.output_step)
|
36
30
|
|
@@ -44,4 +38,4 @@ module Schemas
|
|
44
38
|
@stub.send_schema(req, metadata: _metadata)
|
45
39
|
end
|
46
40
|
end
|
47
|
-
end
|
41
|
+
end
|
data/lib/streamdal.rb
CHANGED
@@ -8,11 +8,11 @@ require 'sp_sdk_pb'
|
|
8
8
|
require 'sp_common_pb'
|
9
9
|
require 'sp_info_pb'
|
10
10
|
require 'sp_internal_pb'
|
11
|
-
require
|
11
|
+
require 'sp_internal_services_pb'
|
12
12
|
require 'sp_pipeline_pb'
|
13
|
-
require
|
14
|
-
require
|
15
|
-
require
|
13
|
+
require 'sp_wsm_pb'
|
14
|
+
require 'steps/sp_steps_httprequest_pb'
|
15
|
+
require 'steps/sp_steps_kv_pb'
|
16
16
|
require 'timeout'
|
17
17
|
require 'google/protobuf'
|
18
18
|
require_relative 'audiences'
|
@@ -124,12 +124,12 @@ module Streamdal
|
|
124
124
|
end
|
125
125
|
|
126
126
|
def process(data, audience)
|
127
|
-
if data.
|
128
|
-
raise
|
127
|
+
if data.empty?
|
128
|
+
raise 'data is required'
|
129
129
|
end
|
130
130
|
|
131
131
|
if audience.nil?
|
132
|
-
raise
|
132
|
+
raise 'audience is required'
|
133
133
|
end
|
134
134
|
|
135
135
|
resp = Streamdal::Protos::SDKResponse.new
|
@@ -144,8 +144,8 @@ module Streamdal
|
|
144
144
|
"operation_type": aud.operation_type,
|
145
145
|
"operation": aud.operation_name,
|
146
146
|
"component": aud.component_name,
|
147
|
-
"pipeline_name":
|
148
|
-
"pipeline_id":
|
147
|
+
"pipeline_name": '',
|
148
|
+
"pipeline_id": '',
|
149
149
|
}
|
150
150
|
|
151
151
|
# TODO: metrics
|
@@ -166,7 +166,7 @@ module Streamdal
|
|
166
166
|
if payload_size > MAX_PAYLOAD_SIZE
|
167
167
|
# TODO: add metrics
|
168
168
|
resp.status = :EXEC_STATUS_ERROR
|
169
|
-
resp.error =
|
169
|
+
resp.error = 'payload size exceeds maximum allowed size'
|
170
170
|
resp
|
171
171
|
end
|
172
172
|
|
@@ -174,8 +174,8 @@ module Streamdal
|
|
174
174
|
original_data = data
|
175
175
|
|
176
176
|
pipelines = _get_pipelines(aud)
|
177
|
-
if pipelines.
|
178
|
-
_send_tail(aud,
|
177
|
+
if pipelines.empty?
|
178
|
+
_send_tail(aud, '', original_data, original_data)
|
179
179
|
return resp
|
180
180
|
end
|
181
181
|
|
@@ -219,7 +219,7 @@ module Streamdal
|
|
219
219
|
@log.debug "Running step '#{step.name}' in dry-run mode"
|
220
220
|
end
|
221
221
|
|
222
|
-
if wasm_resp.output_payload.length
|
222
|
+
if wasm_resp.output_payload.length.positive?
|
223
223
|
resp.data = wasm_resp.output_payload
|
224
224
|
end
|
225
225
|
|
@@ -296,10 +296,10 @@ module Streamdal
|
|
296
296
|
end # pipelines.each
|
297
297
|
end # timeout
|
298
298
|
|
299
|
-
_send_tail(aud,
|
299
|
+
_send_tail(aud, '', original_data, resp.data)
|
300
300
|
|
301
301
|
if @cfg[:dry_run]
|
302
|
-
@log.debug
|
302
|
+
@log.debug 'Dry-run, setting response data to original data'
|
303
303
|
resp.data = original_data
|
304
304
|
end
|
305
305
|
|
@@ -310,19 +310,19 @@ module Streamdal
|
|
310
310
|
|
311
311
|
def _validate_cfg(cfg)
|
312
312
|
if cfg[:streamdal_url].nil? || cfg[:streamdal_url].empty?
|
313
|
-
raise
|
313
|
+
raise 'streamdal_url is required'
|
314
314
|
end
|
315
315
|
|
316
316
|
if cfg[:streamdal_token].nil? || cfg[:streamdal_token].empty?
|
317
|
-
raise
|
317
|
+
raise 'streamdal_token is required'
|
318
318
|
end
|
319
319
|
|
320
320
|
if cfg[:service_name].nil? || cfg[:streamdal_token].empty?
|
321
|
-
raise
|
321
|
+
raise 'service_name is required'
|
322
322
|
end
|
323
323
|
|
324
324
|
if cfg[:log].nil? || cfg[:streamdal_token].empty?
|
325
|
-
logger = Logger.new(
|
325
|
+
logger = Logger.new($stdout)
|
326
326
|
logger.level = Logger::ERROR
|
327
327
|
cfg[:log] = logger
|
328
328
|
end
|
@@ -338,13 +338,13 @@ module Streamdal
|
|
338
338
|
|
339
339
|
def _handle_command(cmd)
|
340
340
|
case cmd.command.to_s
|
341
|
-
when
|
341
|
+
when 'kv'
|
342
342
|
_handle_kv(cmd)
|
343
|
-
when
|
343
|
+
when 'tail'
|
344
344
|
_handle_tail_request(cmd)
|
345
|
-
when
|
345
|
+
when 'set_pipelines'
|
346
346
|
_set_pipelines(cmd)
|
347
|
-
when
|
347
|
+
when 'keep_alive'
|
348
348
|
# Do nothing
|
349
349
|
else
|
350
350
|
@log.error "unknown command type #{cmd.command}"
|
@@ -378,13 +378,11 @@ module Streamdal
|
|
378
378
|
end
|
379
379
|
|
380
380
|
def _set_pipelines(cmd)
|
381
|
-
if cmd.nil?
|
382
|
-
raise "cmd is required"
|
383
|
-
end
|
381
|
+
raise 'cmd is required' if cmd.nil?
|
384
382
|
|
385
383
|
cmd.set_pipelines.pipelines.each_with_index { |p, pIdx|
|
386
384
|
p.steps.each_with_index { |step, idx|
|
387
|
-
if step._wasm_bytes ==
|
385
|
+
if step._wasm_bytes == ''
|
388
386
|
if cmd.set_pipelines.wasm_modules.has_key?(step._wasm_id)
|
389
387
|
step._wasm_bytes = cmd.set_pipelines.wasm_modules[step._wasm_id].bytes
|
390
388
|
cmd.set_pipelines.pipelines[pIdx].steps[idx] = step
|
@@ -424,11 +422,11 @@ module Streamdal
|
|
424
422
|
mod = Wasmtime::Module.new(engine, step._wasm_bytes)
|
425
423
|
linker = Wasmtime::Linker.new(engine, wasi: true)
|
426
424
|
|
427
|
-
linker.func_new(
|
425
|
+
linker.func_new('env', 'httpRequest', %i[i32 i32], [:i64]) do |caller, ptr, len|
|
428
426
|
@hostfunc.http_request(caller, ptr, len)
|
429
427
|
end
|
430
428
|
|
431
|
-
linker.func_new(
|
429
|
+
linker.func_new('env', 'kvExists', %i[i32 i32], [:i64]) do |caller, ptr, len|
|
432
430
|
@hostfunc.kv_exists(caller, ptr, len)
|
433
431
|
end
|
434
432
|
|
@@ -442,8 +440,6 @@ module Streamdal
|
|
442
440
|
|
443
441
|
instance = linker.instantiate(store, mod)
|
444
442
|
|
445
|
-
# TODO: host funcs
|
446
|
-
|
447
443
|
# Store in cache
|
448
444
|
func = WasmFunction.new
|
449
445
|
func.instance = instance
|
@@ -455,11 +451,11 @@ module Streamdal
|
|
455
451
|
|
456
452
|
def _call_wasm(step, data, isr)
|
457
453
|
if step.nil?
|
458
|
-
raise
|
454
|
+
raise 'step is required'
|
459
455
|
end
|
460
456
|
|
461
457
|
if data.nil?
|
462
|
-
raise
|
458
|
+
raise 'data is required'
|
463
459
|
end
|
464
460
|
|
465
461
|
if isr.nil?
|
@@ -480,7 +476,7 @@ module Streamdal
|
|
480
476
|
resp = Streamdal::Protos::WASMResponse.new
|
481
477
|
resp.exit_code = :WASM_EXIT_CODE_ERROR
|
482
478
|
resp.exit_msg = "Failed to execute WASM: #{e}"
|
483
|
-
resp.output_payload =
|
479
|
+
resp.output_payload = ''
|
484
480
|
return resp
|
485
481
|
end
|
486
482
|
end
|
@@ -500,9 +496,9 @@ module Streamdal
|
|
500
496
|
|
501
497
|
ci = Streamdal::Protos::ClientInfo.new
|
502
498
|
ci.client_type = :CLIENT_TYPE_SDK
|
503
|
-
ci.library_name =
|
504
|
-
ci.library_version =
|
505
|
-
ci.language =
|
499
|
+
ci.library_name = 'ruby-sdk'
|
500
|
+
ci.library_version = '0.0.1'
|
501
|
+
ci.language = 'ruby'
|
506
502
|
ci.arch = arch
|
507
503
|
ci.os = os
|
508
504
|
|
@@ -511,11 +507,11 @@ module Streamdal
|
|
511
507
|
|
512
508
|
# Returns metadata for gRPC requests to the internal gRPC API
|
513
509
|
def _metadata
|
514
|
-
{
|
510
|
+
{ 'auth-token' => @cfg[:streamdal_token].to_s }
|
515
511
|
end
|
516
512
|
|
517
513
|
def _register
|
518
|
-
@log.info(
|
514
|
+
@log.info('register started')
|
519
515
|
|
520
516
|
# Register with Streamdal External gRPC API
|
521
517
|
resps = @stub.register(_gen_register_request, metadata: _metadata)
|
@@ -527,7 +523,7 @@ module Streamdal
|
|
527
523
|
_handle_command(r)
|
528
524
|
end
|
529
525
|
|
530
|
-
@log.info(
|
526
|
+
@log.info('register exited')
|
531
527
|
end
|
532
528
|
|
533
529
|
def _exec_wasm(req)
|
@@ -535,14 +531,14 @@ module Streamdal
|
|
535
531
|
|
536
532
|
# Empty out _wasm_bytes, we don't need it anymore
|
537
533
|
# TODO: does this actually update the original object?
|
538
|
-
req.step._wasm_bytes =
|
534
|
+
req.step._wasm_bytes = ''
|
539
535
|
|
540
536
|
data = req.to_proto
|
541
537
|
|
542
|
-
memory = wasm_func.instance.export(
|
543
|
-
alloc = wasm_func.instance.export(
|
544
|
-
dealloc = wasm_func.instance.export(
|
545
|
-
f = wasm_func.instance.export(
|
538
|
+
memory = wasm_func.instance.export('memory').to_memory
|
539
|
+
alloc = wasm_func.instance.export('alloc').to_func
|
540
|
+
dealloc = wasm_func.instance.export('dealloc').to_func
|
541
|
+
f = wasm_func.instance.export('f').to_func
|
546
542
|
|
547
543
|
start_ptr = alloc.call(data.length)
|
548
544
|
|
@@ -581,7 +577,7 @@ module Streamdal
|
|
581
577
|
req.session_id = @session_id
|
582
578
|
req.audiences = Google::Protobuf::RepeatedField.new(:message, Streamdal::Protos::Audience, [])
|
583
579
|
|
584
|
-
@audiences.
|
580
|
+
@audiences.each_value do |aud|
|
585
581
|
req.audiences.push(aud)
|
586
582
|
end
|
587
583
|
|
@@ -625,7 +621,7 @@ module Streamdal
|
|
625
621
|
|
626
622
|
def _send_tail(aud, pipeline_id, original_data, new_data)
|
627
623
|
tails = _get_active_tails_for_audience(aud)
|
628
|
-
if tails.
|
624
|
+
if tails.empty?
|
629
625
|
return nil
|
630
626
|
end
|
631
627
|
|
@@ -652,7 +648,7 @@ module Streamdal
|
|
652
648
|
return nil
|
653
649
|
end
|
654
650
|
|
655
|
-
@log.debug
|
651
|
+
@log.debug 'Notifying'
|
656
652
|
|
657
653
|
if @cfg[:dry_run]
|
658
654
|
return nil
|
@@ -739,7 +735,7 @@ module Streamdal
|
|
739
735
|
# Remove from active tails
|
740
736
|
@tails[key].delete(cmd.tail.request.id)
|
741
737
|
|
742
|
-
if @tails[key].
|
738
|
+
if @tails[key].empty?
|
743
739
|
@tails.delete(key)
|
744
740
|
end
|
745
741
|
end
|
@@ -747,7 +743,7 @@ module Streamdal
|
|
747
743
|
if @paused_tails.key?(key) && @paused_tails[key].key?(cmd.tail.request.id)
|
748
744
|
@paused_tails[key].delete(cmd.tail.request.id)
|
749
745
|
|
750
|
-
if @paused_tails[key].
|
746
|
+
if @paused_tails[key].empty?
|
751
747
|
@paused_tails.delete(key)
|
752
748
|
end
|
753
749
|
end
|
@@ -766,7 +762,7 @@ module Streamdal
|
|
766
762
|
t.stop_tail
|
767
763
|
tails[aud].delete(tail.request.id)
|
768
764
|
|
769
|
-
if tails[aud].
|
765
|
+
if tails[aud].empty?
|
770
766
|
tails.delete(aud)
|
771
767
|
end
|
772
768
|
end
|
@@ -805,7 +801,7 @@ module Streamdal
|
|
805
801
|
|
806
802
|
@tails[key].delete(tail_id)
|
807
803
|
|
808
|
-
if @tails[key].
|
804
|
+
if @tails[key].empty?
|
809
805
|
@tails.delete(key)
|
810
806
|
end
|
811
807
|
|
@@ -821,7 +817,7 @@ module Streamdal
|
|
821
817
|
|
822
818
|
@paused_tails[key].delete(tail_id)
|
823
819
|
|
824
|
-
if @paused_tails[key].
|
820
|
+
if @paused_tails[key].empty?
|
825
821
|
@paused_tails.delete(key)
|
826
822
|
end
|
827
823
|
|
@@ -832,8 +828,8 @@ module Streamdal
|
|
832
828
|
# Called by host functions to write memory to wasm instance so that
|
833
829
|
# the wasm module can read the result of a host function call
|
834
830
|
def write_to_memory(caller, res)
|
835
|
-
alloc = caller.export(
|
836
|
-
memory = caller.export(
|
831
|
+
alloc = caller.export('alloc').to_func
|
832
|
+
memory = caller.export('memory').to_memory
|
837
833
|
|
838
834
|
# Serialize protobuf message
|
839
835
|
resp = res.to_proto
|
data/lib/tail.rb
CHANGED
@@ -1,5 +1,4 @@
|
|
1
|
-
|
2
|
-
require "bozos_buckets"
|
1
|
+
require 'bozos_buckets'
|
3
2
|
|
4
3
|
NUM_TAIL_WORKERS = 2
|
5
4
|
MIN_TAIL_RESPONSE_INTERVAL_MS = 100
|
@@ -15,18 +14,19 @@ module Streamdal
|
|
15
14
|
@logger = log
|
16
15
|
@metrics = metrics
|
17
16
|
@active = active
|
18
|
-
@last_msg = Time
|
17
|
+
@last_msg = Time.at(0)
|
19
18
|
@queue = Queue.new
|
20
19
|
@workers = []
|
21
20
|
|
22
21
|
# Only use rate limiting if sample_options is set
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
22
|
+
return if request.sample_options.nil?
|
23
|
+
|
24
|
+
@limiter = BozosBuckets::Bucket.new(
|
25
|
+
initial_token_count: request.sample_options.sample_rate,
|
26
|
+
refill_rate: request.sample_options.sample_interval_seconds,
|
27
|
+
max_token_count: request.sample_options.sample_rate
|
28
|
+
)
|
29
|
+
|
30
30
|
end
|
31
31
|
|
32
32
|
def start_tail_workers
|
@@ -43,11 +43,8 @@ module Streamdal
|
|
43
43
|
sleep(1)
|
44
44
|
|
45
45
|
@workers.each do |worker|
|
46
|
-
if worker.alive?
|
47
|
-
worker.exit
|
48
|
-
end
|
46
|
+
worker.exit if worker.alive?
|
49
47
|
end
|
50
|
-
|
51
48
|
end
|
52
49
|
|
53
50
|
def start_tail_worker(worker_id)
|
@@ -63,35 +60,32 @@ module Streamdal
|
|
63
60
|
next
|
64
61
|
end
|
65
62
|
|
66
|
-
if Time
|
63
|
+
if Time.now - @last_msg < MIN_TAIL_RESPONSE_INTERVAL_MS
|
67
64
|
sleep(MIN_TAIL_RESPONSE_INTERVAL_MS)
|
68
65
|
@metrics.incr(Metrics::CounterEntry.new(COUNTER_DROPPED_TAIL_MESSAGES, nil, {}, 1))
|
69
66
|
@logger.debug("Dropped tail message for '#{@request.id}' due to rate limiting")
|
70
67
|
next
|
71
68
|
end
|
72
69
|
|
73
|
-
|
74
|
-
|
75
|
-
|
70
|
+
next if stub.nil?
|
71
|
+
|
72
|
+
tail_response = @queue.pop(false)
|
73
|
+
@logger.debug("Sending tail request for '#{tail_response.tail_request_id}'")
|
76
74
|
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
end
|
75
|
+
begin
|
76
|
+
stub.send_tail([tail_response], metadata: { 'auth-token' => @auth_token })
|
77
|
+
rescue Error => e
|
78
|
+
@logger.error("Error sending tail request: #{e}")
|
82
79
|
end
|
83
80
|
end
|
84
81
|
|
85
82
|
@logger.debug "Tail worker #{worker_id} exited"
|
86
|
-
|
87
83
|
end
|
88
84
|
|
89
85
|
def should_send
|
90
|
-
if @limiter.nil?
|
91
|
-
true
|
92
|
-
end
|
86
|
+
true if @limiter.nil?
|
93
87
|
|
94
88
|
@limiter.use_tokens(1)
|
95
89
|
end
|
96
90
|
end
|
97
|
-
end
|
91
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: streamdal
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Mark Gregan
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2024-
|
11
|
+
date: 2024-06-19 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description:
|
14
14
|
email: mark@streamdal.com
|
@@ -45,7 +45,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
45
45
|
requirements:
|
46
46
|
- - '='
|
47
47
|
- !ruby/object:Gem::Version
|
48
|
-
version: 0.0.
|
48
|
+
version: 0.0.2
|
49
49
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
50
50
|
requirements:
|
51
51
|
- - ">="
|