streamdal 0.0.1 → 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: cdf556c16ea4239101cc21f803b78c458c5e2490a1c5df6b2211f5268cc94799
4
- data.tar.gz: 1bca44db3048358a5bdf978504a8559c1a81ede836a11269c5ab8020698d71b6
3
+ metadata.gz: ca014e8334d38a589d1cd3aab58362ac3095fc077a93c076301f2e1552f9b375
4
+ data.tar.gz: 77110e0cc07e851e7b21ef831bed28495353bd70fce865159c41b18d4a40f7d0
5
5
  SHA512:
6
- metadata.gz: 9f0fb7a83fc6e924f520d5402385bd857aff3a27884fd2a1aa9572a002723ad26d3849a2930829e6a05e450547ab1e80293992a693bbeb2bb9bacad8756011f9
7
- data.tar.gz: 2176e44cd407be37c0137f48c77f7a56550cddacc36a5a8043643049b416138c1b96b35c850c630dfd1f9febdc4e72e9224fb055b8dfd871c54e9bef5fbdbe47
6
+ metadata.gz: 227c07d36fd54e8bf35a79b434417f59c80e45c72678ca82ada629b8268b744b614616d28b196e48b85c93f7a031330ebc8884a8ca826553eb647953069d7890
7
+ data.tar.gz: 9c83c8388b8e871d60b1b670b6f919b6a05878c731b6b8978545f9b782885c195cb30d02f1ec160cba58974ad53f99d9506410301185c5e036128eca4c56bb3b
data/lib/audiences.rb CHANGED
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module Audiences
2
4
  def aud_to_str(aud)
3
5
  "#{aud.service_name}.#{aud.component_name}.#{aud.operation_type}.#{aud.operation_name}"
@@ -5,7 +7,7 @@ module Audiences
5
7
 
6
8
  def str_to_aud(str)
7
9
  # TODO: move to common package
8
- parts = str.split(".")
10
+ parts = str.split('.')
9
11
  aud = Streamdal::Protos::Audience.new
10
12
  aud.service_name = parts[0]
11
13
  aud.component_name = parts[1]
@@ -20,9 +22,7 @@ module Audiences
20
22
 
21
23
  def _add_audience(aud)
22
24
  # Add an audience to the local cache map and send to server
23
- if _seen_audience(aud)
24
- return
25
- end
25
+ return if _seen_audience(aud)
26
26
 
27
27
  @audiences[aud_to_str(aud)] = aud
28
28
 
@@ -42,4 +42,4 @@ module Audiences
42
42
  @stub.new_audience(req, metadata: _metadata)
43
43
  end
44
44
  end
45
- end
45
+ end
data/lib/hostfunc.rb CHANGED
@@ -1,4 +1,6 @@
1
- require "steps/sp_steps_kv_pb"
1
+ require 'steps/sp_steps_kv_pb'
2
+ require 'sp_wsm_pb'
3
+ require 'steps/sp_steps_httprequest_pb'
2
4
 
3
5
  module Streamdal
4
6
  class HostFunc
@@ -14,7 +16,7 @@ module Streamdal
14
16
  # kv_exists is a host function that is used to check if a key exists in the KV store
15
17
  def kv_exists(caller, ptr, len)
16
18
 
17
- data = caller.export("memory").to_memory.read(ptr, len)
19
+ data = caller.export('memory').to_memory.read(ptr, len)
18
20
 
19
21
  # Read request from memory and decode into HttpRequest
20
22
  req = Streamdal::Protos::KVStep.decode(data)
@@ -35,32 +37,40 @@ module Streamdal
35
37
  ##
36
38
  # http_request performs a http request on behalf of a wasm module since WASI cannot talk sockets
37
39
  def http_request(caller, ptr, len)
38
- data = caller.export("memory").to_memory.read(ptr, len)
40
+ data = caller.export('memory').to_memory.read(ptr, len)
39
41
 
40
42
  # Read request from memory and decode into HttpRequest
41
- req = Streamdal::Protos::HttpRequest.decode(data)
43
+ req = Streamdal::Protos::WASMRequest.decode(data)
44
+
45
+ begin
46
+ req_body = self._get_request_body_for_mode(req)
47
+ rescue => e
48
+ return self._http_request_response(caller, 400, e.to_s, {})
49
+ end
42
50
 
43
51
  # Attempt to make HTTP request
44
52
  # On error, return a mock 400 response with the error as the body
45
53
  begin
46
- response = _make_http_request(req)
54
+ response = _make_http_request(req.step.http_request.request, req_body)
47
55
  rescue => e
48
- wasm_resp = Streamdal::Protos::HttpResponse.new
49
- wasm_resp.code = 400
50
- wasm_resp.body = "Unable to execute HTTP request: #{e}"
51
- return wasm_resp
56
+ return self._http_request_response(caller, 400, "Unable to execute HTTP request: #{e}", {})
52
57
  end
53
58
 
54
- # Successful request, build the proto from the httparty response
59
+ self._http_request_response(caller, response.code, response.body, response.headers)
60
+ end
61
+
62
+ private
63
+
64
+ def _http_request_response(caller, code, body, headers)
55
65
  wasm_resp = Streamdal::Protos::HttpResponse.new
56
- wasm_resp.code = response.code
57
- wasm_resp.body = response.body
66
+ wasm_resp.code = code
67
+ wasm_resp.body = body
58
68
  wasm_resp.headers = Google::Protobuf::Map.new(:string, :string, {})
59
69
 
60
70
  # Headers can have multiple values, but we just want a map[string]string here for simplicity
61
71
  # The client can pase by the delimiter ";" if needed.
62
- response.headers.each do |k, values|
63
- wasm_resp.headers[k] = values.kind_of?(Array) ? values.join("; ") : values
72
+ headers.each do |k, values|
73
+ wasm_resp.headers[k] = values.is_a?(Array) ? values.join('; ') : values
64
74
  end
65
75
 
66
76
  # Write the HttpResponse proto message to WASM memory
@@ -68,17 +78,39 @@ module Streamdal
68
78
  write_to_memory(caller, wasm_resp)
69
79
  end
70
80
 
71
- private
81
+ def _get_request_body_for_mode(req)
82
+ http_req = req.step.http_request.request
83
+
84
+ case http_req.body_mode
85
+ when :HTTP_REQUEST_BODY_MODE_INTER_STEP_RESULT
86
+ raise 'Inter step result is empty' if req.inter_step_result.nil?
87
+
88
+ detective_res = req.inter_step_result.detective_result
89
+
90
+ raise 'Detective result is empty' if detective_res.nil?
91
+
92
+ # Wipe values to prevent PII from being leaked
93
+ detective_res.matches.each { |step_res|
94
+ step_res.value = ''
95
+ }
96
+
97
+ req.inter_step_result.to_json
98
+ when :HTTP_REQUEST_BODY_MODE_STATIC
99
+ http_req.body
100
+ else
101
+ raise 'invalid http request body mode'
102
+ end
103
+ end
72
104
 
73
105
  ##
74
106
  # Performs an http request
75
- def _make_http_request(req)
107
+ def _make_http_request(req, body)
76
108
  if req.nil?
77
- raise "req is required"
109
+ raise 'req is required'
78
110
  end
79
111
 
80
112
  options = {
81
- headers: { "Content-Type": "application/json", },
113
+ headers: { "Content-Type": 'application/json', },
82
114
  }
83
115
 
84
116
  req.headers.each { |key, value| options.headers[key] = value }
@@ -87,15 +119,15 @@ module Streamdal
87
119
  when :HTTP_REQUEST_METHOD_GET
88
120
  return HTTParty.get(req.url)
89
121
  when :HTTP_REQUEST_METHOD_POST
90
- options.body = req.body
122
+ options.body = body
91
123
  return HTTParty.post(req.url, options)
92
124
  when :HTTP_REQUEST_METHOD_PUT
93
- options.body = req.body
125
+ options.body = body
94
126
  return HTTParty.put(req.url, options)
95
127
  when :HTTP_REQUEST_METHOD_DELETE
96
128
  return HTTParty.delete(req.url)
97
129
  when :HTTP_REQUEST_METHOD_PATCH
98
- options.body = req.body
130
+ options.body = body
99
131
  return HTTParty.patch(req.url, options)
100
132
  when :HTTP_REQUEST_METHOD_HEAD
101
133
  return HTTParty.head(req.url)
data/lib/kv.rb CHANGED
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module Streamdal
2
4
  class KeyValue
3
5
  def initialize
@@ -49,4 +51,4 @@ module Streamdal
49
51
  end
50
52
  end
51
53
  end
52
- end
54
+ end
data/lib/metrics.rb CHANGED
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module Streamdal
2
4
  class Counter
3
5
  attr_accessor :last_updated, :name, :aud, :labels
@@ -6,15 +8,15 @@ module Streamdal
6
8
  @name = name
7
9
  @aud = aud
8
10
  @labels = labels
9
- @value = 0
10
- @last_updated = Time::now
11
+ @value = value
12
+ @last_updated = Time.now
11
13
  @value_mtx = Mutex.new
12
14
  end
13
15
 
14
16
  def incr(val)
15
17
  @value_mtx.synchronize do
16
- @value = @value + val
17
- @last_updated = Time::now
18
+ @value += val
19
+ @last_updated = Time.now
18
20
  end
19
21
  end
20
22
 
@@ -33,18 +35,18 @@ module Streamdal
33
35
 
34
36
  class Metrics
35
37
 
36
- COUNTER_CONSUME_BYTES = "counter_consume_bytes"
37
- COUNTER_CONSUME_PROCESSED = "counter_consume_processed"
38
- COUNTER_CONSUME_ERRORS = "counter_consume_errors"
39
- COUNTER_PRODUCE_BYTES = "counter_produce_bytes"
40
- COUNTER_PRODUCE_PROCESSED = "counter_produce_processed"
41
- COUNTER_PRODUCE_ERRORS = "counter_produce_errors"
42
- COUNTER_NOTIFY = "counter_notify"
43
- COUNTER_DROPPED_TAIL_MESSAGES = "counter_dropped_tail_messages"
44
- COUNTER_CONSUME_BYTES_RATE = "counter_consume_bytes_rate"
45
- COUNTER_PRODUCE_BYTES_RATE = "counter_produce_bytes_rate"
46
- COUNTER_CONSUME_PROCESSED_RATE = "counter_consume_processed_rate"
47
- COUNTER_PRODUCE_PROCESSED_RATE = "counter_produce_processed_rate"
38
+ COUNTER_CONSUME_BYTES = 'counter_consume_bytes'
39
+ COUNTER_CONSUME_PROCESSED = 'counter_consume_processed'
40
+ COUNTER_CONSUME_ERRORS = 'counter_consume_errors'
41
+ COUNTER_PRODUCE_BYTES = 'counter_produce_bytes'
42
+ COUNTER_PRODUCE_PROCESSED = 'counter_produce_processed'
43
+ COUNTER_PRODUCE_ERRORS = 'counter_produce_errors'
44
+ COUNTER_NOTIFY = 'counter_notify'
45
+ COUNTER_DROPPED_TAIL_MESSAGES = 'counter_dropped_tail_messages'
46
+ COUNTER_CONSUME_BYTES_RATE = 'counter_consume_bytes_rate'
47
+ COUNTER_PRODUCE_BYTES_RATE = 'counter_produce_bytes_rate'
48
+ COUNTER_CONSUME_PROCESSED_RATE = 'counter_consume_processed_rate'
49
+ COUNTER_PRODUCE_PROCESSED_RATE = 'counter_produce_processed_rate'
48
50
 
49
51
  WORKER_POOL_SIZE = 3
50
52
  DEFAULT_COUNTER_REAPER_INTERVAL = 10
@@ -54,9 +56,7 @@ module Streamdal
54
56
  CounterEntry = Struct.new(:name, :aud, :labels, :value)
55
57
 
56
58
  def initialize(cfg)
57
- if cfg.nil?
58
- raise ArgumentError, "cfg is nil"
59
- end
59
+ raise ArgumentError, 'cfg is nil' if cfg.nil?
60
60
 
61
61
  @cfg = cfg
62
62
  @log = cfg[:log]
@@ -80,30 +80,22 @@ module Streamdal
80
80
 
81
81
  # Exit any remaining threads
82
82
  @workers.each do |w|
83
- if w.running?
84
- w.exit
85
- end
83
+ w.exit if w.running?
86
84
  end
87
85
  end
88
86
 
89
87
  def self.composite_id(counter_name, labels = {})
90
- if labels.nil?
91
- labels = {}
92
- end
93
- "#{counter_name}-#{labels.values.join("-")}".freeze
88
+ labels = {} if labels.nil?
89
+ "#{counter_name}-#{labels.values.join('-')}"
94
90
  end
95
91
 
96
92
  def get_counter(ce)
97
- if ce.nil?
98
- raise ArgumentError, "ce is nil"
99
- end
93
+ raise ArgumentError, 'ce is nil' if ce.nil?
100
94
 
101
- k = Metrics::composite_id(ce.name, ce.labels)
95
+ k = Metrics.composite_id(ce.name, ce.labels)
102
96
 
103
97
  @counters_mtx.synchronize do
104
- if @counters.key?(k)
105
- @counters[k]
106
- end
98
+ @counters[k] if @counters.key?(k)
107
99
  end
108
100
 
109
101
  # No counter exists, create a new one and return it
@@ -114,7 +106,7 @@ module Streamdal
114
106
  c = Counter.new(ce.name, ce.aud, ce.labels, ce.value)
115
107
 
116
108
  @counters_mtx.synchronize do
117
- @counters[Metrics::composite_id(ce.name, ce.labels)] = c
109
+ @counters[Metrics.composite_id(ce.name, ce.labels)] = c
118
110
  end
119
111
 
120
112
  c
@@ -166,31 +158,29 @@ module Streamdal
166
158
  def _run_publisher
167
159
  # Background thread that reads values from counters, adds them to the publish queue, and then
168
160
  # resets the counter's value back to zero
169
- unless @exit
170
- @log.debug("Starting publisher")
161
+ return if @exit
171
162
 
172
- # Sleep on startup and then and between each loop run
173
- sleep(DEFAULT_COUNTER_PUBLISH_INTERVAL)
163
+ @log.debug('Starting publisher')
174
164
 
175
- # Get all counters
176
- # Loop over each counter, get the value,
177
- # if value > 0, continue
178
- # if now() - last_updated > 10 seconds, remove counter
179
- # Grab copy of counters
180
- @counters_mtx.lock
181
- new_counters = @counters.dup
182
- @counters_mtx.unlock
165
+ # Sleep on startup and then and between each loop run
166
+ sleep(DEFAULT_COUNTER_PUBLISH_INTERVAL)
183
167
 
184
- new_counters.each do |_, counter|
185
- if counter.val == 0
186
- next
187
- end
168
+ # Get all counters
169
+ # Loop over each counter, get the value,
170
+ # if value > 0, continue
171
+ # if now() - last_updated > 10 seconds, remove counter
172
+ # Grab copy of counters
173
+ @counters_mtx.lock
174
+ new_counters = @counters.dup
175
+ @counters_mtx.unlock
188
176
 
189
- ce = CounterEntry.new(counter.name, counter.aud, counter.labels, counter.val)
190
- counter.reset
177
+ new_counters.each_value do |counter|
178
+ next if counter.val.zero?
191
179
 
192
- @publish_queue.push(ce)
193
- end
180
+ ce = CounterEntry.new(counter.name, counter.aud, counter.labels, counter.val)
181
+ counter.reset
182
+
183
+ @publish_queue.push(ce)
194
184
  end
195
185
  end
196
186
 
@@ -199,9 +189,8 @@ module Streamdal
199
189
 
200
190
  until @exit
201
191
  ce = @incr_queue.pop
202
- if ce.nil?
203
- next
204
- end
192
+ next if ce.nil?
193
+
205
194
  begin
206
195
  _publish_metrics(ce)
207
196
  rescue => e
@@ -213,7 +202,7 @@ module Streamdal
213
202
  end
214
203
 
215
204
  def _run_reaper
216
- @log.debug("Starting reaper")
205
+ @log.debug('Starting reaper')
217
206
 
218
207
  until @exit
219
208
  # Sleep on startup and then and between each loop run
@@ -226,11 +215,9 @@ module Streamdal
226
215
  # Grab copy of counters
227
216
  @counters_mtx.synchronize do
228
217
  @counters.each do |name, counter|
229
- if counter.val > 0
230
- next
231
- end
218
+ next if counter.val.positive?
232
219
 
233
- if Time::now - counter.last_updated > DEFAULT_COUNTER_TTL
220
+ if Time.now - counter.last_updated > DEFAULT_COUNTER_TTL
234
221
  @log.debug("Reaping counter '#{name}'")
235
222
  @counters.delete(name)
236
223
  end
@@ -238,7 +225,7 @@ module Streamdal
238
225
  end
239
226
  end
240
227
 
241
- @log.debug("Exiting reaper")
228
+ @log.debug('Exiting reaper')
242
229
  end
243
230
 
244
231
  def _run_incrementer_worker(worker_id)
@@ -259,7 +246,7 @@ module Streamdal
259
246
 
260
247
  # Returns metadata for gRPC requests to the internal gRPC API
261
248
  def _metadata
262
- { "auth-token" => @cfg[:streamdal_token].to_s }
249
+ { 'auth-token' => @cfg[:streamdal_token].to_s }
263
250
  end
264
251
  end
265
- end
252
+ end
data/lib/schema.rb CHANGED
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  include Streamdal::Protos
2
4
 
3
5
  module Schemas
@@ -8,29 +10,21 @@ module Schemas
8
10
  end
9
11
 
10
12
  def _get_schema(aud)
11
- if @schemas.key?(aud_to_str(aud))
12
- return @schemas[aud_to_str(aud)].json_schema
13
- end
13
+ return @schemas[aud_to_str(aud)].json_schema if @schemas.key?(aud_to_str(aud))
14
14
 
15
- ""
15
+ ''
16
16
  end
17
17
 
18
18
  def _handle_schema(aud, step, wasm_resp)
19
19
  # Only handle schema steps
20
- if step.infer_schema.nil?
21
- return nil
22
- end
20
+ return nil if step.infer_schema.nil?
23
21
 
24
22
  # Only successful schema inferences
25
- if wasm_resp.exit_code != :WASM_EXIT_CODE_TRUE
26
- return nil
27
- end
23
+ return nil if wasm_resp.exit_code != :WASM_EXIT_CODE_TRUE
28
24
 
29
25
  # If existing schema matches, do nothing
30
26
  existing_schema = _get_schema(aud)
31
- if existing_schema == wasm_resp.output_step
32
- return nil
33
- end
27
+ return nil if existing_schema == wasm_resp.output_step
34
28
 
35
29
  _set_schema(aud, wasm_resp.output_step)
36
30
 
@@ -44,4 +38,4 @@ module Schemas
44
38
  @stub.send_schema(req, metadata: _metadata)
45
39
  end
46
40
  end
47
- end
41
+ end
data/lib/streamdal.rb CHANGED
@@ -8,11 +8,11 @@ require 'sp_sdk_pb'
8
8
  require 'sp_common_pb'
9
9
  require 'sp_info_pb'
10
10
  require 'sp_internal_pb'
11
- require "sp_internal_services_pb"
11
+ require 'sp_internal_services_pb'
12
12
  require 'sp_pipeline_pb'
13
- require "sp_wsm_pb"
14
- require "steps/sp_steps_httprequest_pb"
15
- require "steps/sp_steps_kv_pb"
13
+ require 'sp_wsm_pb'
14
+ require 'steps/sp_steps_httprequest_pb'
15
+ require 'steps/sp_steps_kv_pb'
16
16
  require 'timeout'
17
17
  require 'google/protobuf'
18
18
  require_relative 'audiences'
@@ -124,12 +124,12 @@ module Streamdal
124
124
  end
125
125
 
126
126
  def process(data, audience)
127
- if data.length == 0
128
- raise "data is required"
127
+ if data.empty?
128
+ raise 'data is required'
129
129
  end
130
130
 
131
131
  if audience.nil?
132
- raise "audience is required"
132
+ raise 'audience is required'
133
133
  end
134
134
 
135
135
  resp = Streamdal::Protos::SDKResponse.new
@@ -144,8 +144,8 @@ module Streamdal
144
144
  "operation_type": aud.operation_type,
145
145
  "operation": aud.operation_name,
146
146
  "component": aud.component_name,
147
- "pipeline_name": "",
148
- "pipeline_id": "",
147
+ "pipeline_name": '',
148
+ "pipeline_id": '',
149
149
  }
150
150
 
151
151
  # TODO: metrics
@@ -166,7 +166,7 @@ module Streamdal
166
166
  if payload_size > MAX_PAYLOAD_SIZE
167
167
  # TODO: add metrics
168
168
  resp.status = :EXEC_STATUS_ERROR
169
- resp.error = "payload size exceeds maximum allowed size"
169
+ resp.error = 'payload size exceeds maximum allowed size'
170
170
  resp
171
171
  end
172
172
 
@@ -174,8 +174,8 @@ module Streamdal
174
174
  original_data = data
175
175
 
176
176
  pipelines = _get_pipelines(aud)
177
- if pipelines.length == 0
178
- _send_tail(aud, "", original_data, original_data)
177
+ if pipelines.empty?
178
+ _send_tail(aud, '', original_data, original_data)
179
179
  return resp
180
180
  end
181
181
 
@@ -219,7 +219,7 @@ module Streamdal
219
219
  @log.debug "Running step '#{step.name}' in dry-run mode"
220
220
  end
221
221
 
222
- if wasm_resp.output_payload.length > 0
222
+ if wasm_resp.output_payload.length.positive?
223
223
  resp.data = wasm_resp.output_payload
224
224
  end
225
225
 
@@ -296,10 +296,10 @@ module Streamdal
296
296
  end # pipelines.each
297
297
  end # timeout
298
298
 
299
- _send_tail(aud, "", original_data, resp.data)
299
+ _send_tail(aud, '', original_data, resp.data)
300
300
 
301
301
  if @cfg[:dry_run]
302
- @log.debug "Dry-run, setting response data to original data"
302
+ @log.debug 'Dry-run, setting response data to original data'
303
303
  resp.data = original_data
304
304
  end
305
305
 
@@ -310,19 +310,19 @@ module Streamdal
310
310
 
311
311
  def _validate_cfg(cfg)
312
312
  if cfg[:streamdal_url].nil? || cfg[:streamdal_url].empty?
313
- raise "streamdal_url is required"
313
+ raise 'streamdal_url is required'
314
314
  end
315
315
 
316
316
  if cfg[:streamdal_token].nil? || cfg[:streamdal_token].empty?
317
- raise "streamdal_token is required"
317
+ raise 'streamdal_token is required'
318
318
  end
319
319
 
320
320
  if cfg[:service_name].nil? || cfg[:streamdal_token].empty?
321
- raise "service_name is required"
321
+ raise 'service_name is required'
322
322
  end
323
323
 
324
324
  if cfg[:log].nil? || cfg[:streamdal_token].empty?
325
- logger = Logger.new(STDOUT)
325
+ logger = Logger.new($stdout)
326
326
  logger.level = Logger::ERROR
327
327
  cfg[:log] = logger
328
328
  end
@@ -338,13 +338,13 @@ module Streamdal
338
338
 
339
339
  def _handle_command(cmd)
340
340
  case cmd.command.to_s
341
- when "kv"
341
+ when 'kv'
342
342
  _handle_kv(cmd)
343
- when "tail"
343
+ when 'tail'
344
344
  _handle_tail_request(cmd)
345
- when "set_pipelines"
345
+ when 'set_pipelines'
346
346
  _set_pipelines(cmd)
347
- when "keep_alive"
347
+ when 'keep_alive'
348
348
  # Do nothing
349
349
  else
350
350
  @log.error "unknown command type #{cmd.command}"
@@ -378,13 +378,11 @@ module Streamdal
378
378
  end
379
379
 
380
380
  def _set_pipelines(cmd)
381
- if cmd.nil?
382
- raise "cmd is required"
383
- end
381
+ raise 'cmd is required' if cmd.nil?
384
382
 
385
383
  cmd.set_pipelines.pipelines.each_with_index { |p, pIdx|
386
384
  p.steps.each_with_index { |step, idx|
387
- if step._wasm_bytes == ""
385
+ if step._wasm_bytes == ''
388
386
  if cmd.set_pipelines.wasm_modules.has_key?(step._wasm_id)
389
387
  step._wasm_bytes = cmd.set_pipelines.wasm_modules[step._wasm_id].bytes
390
388
  cmd.set_pipelines.pipelines[pIdx].steps[idx] = step
@@ -424,11 +422,11 @@ module Streamdal
424
422
  mod = Wasmtime::Module.new(engine, step._wasm_bytes)
425
423
  linker = Wasmtime::Linker.new(engine, wasi: true)
426
424
 
427
- linker.func_new("env", "httpRequest", [:i32, :i32], [:i64]) do |caller, ptr, len|
425
+ linker.func_new('env', 'httpRequest', %i[i32 i32], [:i64]) do |caller, ptr, len|
428
426
  @hostfunc.http_request(caller, ptr, len)
429
427
  end
430
428
 
431
- linker.func_new("env", "kvExists", [:i32, :i32], [:i64]) do |caller, ptr, len|
429
+ linker.func_new('env', 'kvExists', %i[i32 i32], [:i64]) do |caller, ptr, len|
432
430
  @hostfunc.kv_exists(caller, ptr, len)
433
431
  end
434
432
 
@@ -442,8 +440,6 @@ module Streamdal
442
440
 
443
441
  instance = linker.instantiate(store, mod)
444
442
 
445
- # TODO: host funcs
446
-
447
443
  # Store in cache
448
444
  func = WasmFunction.new
449
445
  func.instance = instance
@@ -455,11 +451,11 @@ module Streamdal
455
451
 
456
452
  def _call_wasm(step, data, isr)
457
453
  if step.nil?
458
- raise "step is required"
454
+ raise 'step is required'
459
455
  end
460
456
 
461
457
  if data.nil?
462
- raise "data is required"
458
+ raise 'data is required'
463
459
  end
464
460
 
465
461
  if isr.nil?
@@ -480,7 +476,7 @@ module Streamdal
480
476
  resp = Streamdal::Protos::WASMResponse.new
481
477
  resp.exit_code = :WASM_EXIT_CODE_ERROR
482
478
  resp.exit_msg = "Failed to execute WASM: #{e}"
483
- resp.output_payload = ""
479
+ resp.output_payload = ''
484
480
  return resp
485
481
  end
486
482
  end
@@ -500,9 +496,9 @@ module Streamdal
500
496
 
501
497
  ci = Streamdal::Protos::ClientInfo.new
502
498
  ci.client_type = :CLIENT_TYPE_SDK
503
- ci.library_name = "ruby-sdk"
504
- ci.library_version = "0.0.1"
505
- ci.language = "ruby"
499
+ ci.library_name = 'ruby-sdk'
500
+ ci.library_version = '0.0.1'
501
+ ci.language = 'ruby'
506
502
  ci.arch = arch
507
503
  ci.os = os
508
504
 
@@ -511,11 +507,11 @@ module Streamdal
511
507
 
512
508
  # Returns metadata for gRPC requests to the internal gRPC API
513
509
  def _metadata
514
- { "auth-token" => @cfg[:streamdal_token].to_s }
510
+ { 'auth-token' => @cfg[:streamdal_token].to_s }
515
511
  end
516
512
 
517
513
  def _register
518
- @log.info("register started")
514
+ @log.info('register started')
519
515
 
520
516
  # Register with Streamdal External gRPC API
521
517
  resps = @stub.register(_gen_register_request, metadata: _metadata)
@@ -527,7 +523,7 @@ module Streamdal
527
523
  _handle_command(r)
528
524
  end
529
525
 
530
- @log.info("register exited")
526
+ @log.info('register exited')
531
527
  end
532
528
 
533
529
  def _exec_wasm(req)
@@ -535,14 +531,14 @@ module Streamdal
535
531
 
536
532
  # Empty out _wasm_bytes, we don't need it anymore
537
533
  # TODO: does this actually update the original object?
538
- req.step._wasm_bytes = ""
534
+ req.step._wasm_bytes = ''
539
535
 
540
536
  data = req.to_proto
541
537
 
542
- memory = wasm_func.instance.export("memory").to_memory
543
- alloc = wasm_func.instance.export("alloc").to_func
544
- dealloc = wasm_func.instance.export("dealloc").to_func
545
- f = wasm_func.instance.export("f").to_func
538
+ memory = wasm_func.instance.export('memory').to_memory
539
+ alloc = wasm_func.instance.export('alloc').to_func
540
+ dealloc = wasm_func.instance.export('dealloc').to_func
541
+ f = wasm_func.instance.export('f').to_func
546
542
 
547
543
  start_ptr = alloc.call(data.length)
548
544
 
@@ -581,7 +577,7 @@ module Streamdal
581
577
  req.session_id = @session_id
582
578
  req.audiences = Google::Protobuf::RepeatedField.new(:message, Streamdal::Protos::Audience, [])
583
579
 
584
- @audiences.each do |_, aud|
580
+ @audiences.each_value do |aud|
585
581
  req.audiences.push(aud)
586
582
  end
587
583
 
@@ -625,7 +621,7 @@ module Streamdal
625
621
 
626
622
  def _send_tail(aud, pipeline_id, original_data, new_data)
627
623
  tails = _get_active_tails_for_audience(aud)
628
- if tails.length == 0
624
+ if tails.empty?
629
625
  return nil
630
626
  end
631
627
 
@@ -652,7 +648,7 @@ module Streamdal
652
648
  return nil
653
649
  end
654
650
 
655
- @log.debug "Notifying"
651
+ @log.debug 'Notifying'
656
652
 
657
653
  if @cfg[:dry_run]
658
654
  return nil
@@ -739,7 +735,7 @@ module Streamdal
739
735
  # Remove from active tails
740
736
  @tails[key].delete(cmd.tail.request.id)
741
737
 
742
- if @tails[key].length == 0
738
+ if @tails[key].empty?
743
739
  @tails.delete(key)
744
740
  end
745
741
  end
@@ -747,7 +743,7 @@ module Streamdal
747
743
  if @paused_tails.key?(key) && @paused_tails[key].key?(cmd.tail.request.id)
748
744
  @paused_tails[key].delete(cmd.tail.request.id)
749
745
 
750
- if @paused_tails[key].length == 0
746
+ if @paused_tails[key].empty?
751
747
  @paused_tails.delete(key)
752
748
  end
753
749
  end
@@ -766,7 +762,7 @@ module Streamdal
766
762
  t.stop_tail
767
763
  tails[aud].delete(tail.request.id)
768
764
 
769
- if tails[aud].length == 0
765
+ if tails[aud].empty?
770
766
  tails.delete(aud)
771
767
  end
772
768
  end
@@ -805,7 +801,7 @@ module Streamdal
805
801
 
806
802
  @tails[key].delete(tail_id)
807
803
 
808
- if @tails[key].length == 0
804
+ if @tails[key].empty?
809
805
  @tails.delete(key)
810
806
  end
811
807
 
@@ -821,7 +817,7 @@ module Streamdal
821
817
 
822
818
  @paused_tails[key].delete(tail_id)
823
819
 
824
- if @paused_tails[key].length == 0
820
+ if @paused_tails[key].empty?
825
821
  @paused_tails.delete(key)
826
822
  end
827
823
 
@@ -832,8 +828,8 @@ module Streamdal
832
828
  # Called by host functions to write memory to wasm instance so that
833
829
  # the wasm module can read the result of a host function call
834
830
  def write_to_memory(caller, res)
835
- alloc = caller.export("alloc").to_func
836
- memory = caller.export("memory").to_memory
831
+ alloc = caller.export('alloc').to_func
832
+ memory = caller.export('memory').to_memory
837
833
 
838
834
  # Serialize protobuf message
839
835
  resp = res.to_proto
data/lib/tail.rb CHANGED
@@ -1,5 +1,4 @@
1
- # TODO: implement token bucket limiter
2
- require "bozos_buckets"
1
+ require 'bozos_buckets'
3
2
 
4
3
  NUM_TAIL_WORKERS = 2
5
4
  MIN_TAIL_RESPONSE_INTERVAL_MS = 100
@@ -15,18 +14,19 @@ module Streamdal
15
14
  @logger = log
16
15
  @metrics = metrics
17
16
  @active = active
18
- @last_msg = Time::at(0)
17
+ @last_msg = Time.at(0)
19
18
  @queue = Queue.new
20
19
  @workers = []
21
20
 
22
21
  # Only use rate limiting if sample_options is set
23
- unless request.sample_options.nil?
24
- @limiter = BozosBuckets::Bucket.new(
25
- initial_token_count: request.sample_options.sample_rate,
26
- refill_rate: request.sample_options.sample_interval_seconds,
27
- max_token_count: request.sample_options.sample_rate
28
- )
29
- end
22
+ return if request.sample_options.nil?
23
+
24
+ @limiter = BozosBuckets::Bucket.new(
25
+ initial_token_count: request.sample_options.sample_rate,
26
+ refill_rate: request.sample_options.sample_interval_seconds,
27
+ max_token_count: request.sample_options.sample_rate
28
+ )
29
+
30
30
  end
31
31
 
32
32
  def start_tail_workers
@@ -43,11 +43,8 @@ module Streamdal
43
43
  sleep(1)
44
44
 
45
45
  @workers.each do |worker|
46
- if worker.alive?
47
- worker.exit
48
- end
46
+ worker.exit if worker.alive?
49
47
  end
50
-
51
48
  end
52
49
 
53
50
  def start_tail_worker(worker_id)
@@ -63,35 +60,32 @@ module Streamdal
63
60
  next
64
61
  end
65
62
 
66
- if Time::now - @last_msg < MIN_TAIL_RESPONSE_INTERVAL_MS
63
+ if Time.now - @last_msg < MIN_TAIL_RESPONSE_INTERVAL_MS
67
64
  sleep(MIN_TAIL_RESPONSE_INTERVAL_MS)
68
65
  @metrics.incr(Metrics::CounterEntry.new(COUNTER_DROPPED_TAIL_MESSAGES, nil, {}, 1))
69
66
  @logger.debug("Dropped tail message for '#{@request.id}' due to rate limiting")
70
67
  next
71
68
  end
72
69
 
73
- unless stub.nil?
74
- tail_response = @queue.pop(non_block = false)
75
- @logger.debug("Sending tail request for '#{tail_response.tail_request_id}'")
70
+ next if stub.nil?
71
+
72
+ tail_response = @queue.pop(false)
73
+ @logger.debug("Sending tail request for '#{tail_response.tail_request_id}'")
76
74
 
77
- begin
78
- stub.send_tail([tail_response], metadata: { "auth-token" => @auth_token })
79
- rescue => e
80
- @logger.error("Error sending tail request: #{e}")
81
- end
75
+ begin
76
+ stub.send_tail([tail_response], metadata: { 'auth-token' => @auth_token })
77
+ rescue Error => e
78
+ @logger.error("Error sending tail request: #{e}")
82
79
  end
83
80
  end
84
81
 
85
82
  @logger.debug "Tail worker #{worker_id} exited"
86
-
87
83
  end
88
84
 
89
85
  def should_send
90
- if @limiter.nil?
91
- true
92
- end
86
+ true if @limiter.nil?
93
87
 
94
88
  @limiter.use_tokens(1)
95
89
  end
96
90
  end
97
- end
91
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: streamdal
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.1
4
+ version: 0.0.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Mark Gregan
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2024-05-03 00:00:00.000000000 Z
11
+ date: 2024-06-19 00:00:00.000000000 Z
12
12
  dependencies: []
13
13
  description:
14
14
  email: mark@streamdal.com
@@ -45,7 +45,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
45
45
  requirements:
46
46
  - - '='
47
47
  - !ruby/object:Gem::Version
48
- version: 0.0.1
48
+ version: 0.0.2
49
49
  required_rubygems_version: !ruby/object:Gem::Requirement
50
50
  requirements:
51
51
  - - ">="