streamdal 0.0.2 → 0.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/hostfunc.rb +34 -11
- data/lib/spec_helper.rb +11 -1
- data/lib/streamdal.rb +43 -123
- data/lib/wasm_spec.rb +259 -0
- metadata +4 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: bd49e6aba633028e6bd0af296c6ed7a69ca00f2b4bc9c26646d40bdd1e19d313
|
4
|
+
data.tar.gz: 745eb525e4290b1d23bda53000683e9a8761fe070a4b50bb011605f2f03d64a5
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c65cd7b9a1342e31092bbb183e55537b26cb1be2ca34609d5df44ae763ec6bdcf514febb0d85f95a02107ec5d6b65f66b02956ef2b4516cce178e4185af6712e
|
7
|
+
data.tar.gz: 22319f8bffbafea0aa41748f7a745f44698c6061589ebe1ba8ee2621127e0f4fb3b4d5990d476e172d3e54f45c66f88310c7d295becab73ae0cc0172a79bc432
|
data/lib/hostfunc.rb
CHANGED
@@ -43,9 +43,9 @@ module Streamdal
|
|
43
43
|
req = Streamdal::Protos::WASMRequest.decode(data)
|
44
44
|
|
45
45
|
begin
|
46
|
-
req_body =
|
46
|
+
req_body = _get_request_body_for_mode(req)
|
47
47
|
rescue => e
|
48
|
-
return
|
48
|
+
return _http_request_response(caller, 400, e.to_s, {})
|
49
49
|
end
|
50
50
|
|
51
51
|
# Attempt to make HTTP request
|
@@ -53,10 +53,17 @@ module Streamdal
|
|
53
53
|
begin
|
54
54
|
response = _make_http_request(req.step.http_request.request, req_body)
|
55
55
|
rescue => e
|
56
|
-
return
|
56
|
+
return _http_request_response(caller, 400, "Unable to execute HTTP request: #{e}", {})
|
57
57
|
end
|
58
58
|
|
59
|
-
|
59
|
+
# Convert body to utf8
|
60
|
+
out = encode(response.body)
|
61
|
+
|
62
|
+
_http_request_response(caller, response.code, out, response.headers)
|
63
|
+
end
|
64
|
+
|
65
|
+
def encode(str)
|
66
|
+
str.force_encoding('ascii-8bit').encode('utf-8', invalid: :replace, undef: :replace, replace: '?')
|
60
67
|
end
|
61
68
|
|
62
69
|
private
|
@@ -95,22 +102,18 @@ module Streamdal
|
|
95
102
|
}
|
96
103
|
|
97
104
|
req.inter_step_result.to_json
|
98
|
-
when :HTTP_REQUEST_BODY_MODE_STATIC
|
99
|
-
http_req.body
|
100
105
|
else
|
101
|
-
|
106
|
+
http_req.body
|
102
107
|
end
|
103
108
|
end
|
104
109
|
|
105
110
|
##
|
106
111
|
# Performs an http request
|
107
112
|
def _make_http_request(req, body)
|
108
|
-
if req.nil?
|
109
|
-
raise 'req is required'
|
110
|
-
end
|
113
|
+
raise 'req is required' if req.nil?
|
111
114
|
|
112
115
|
options = {
|
113
|
-
headers: { "Content-Type": 'application/json'
|
116
|
+
headers: { "Content-Type": 'application/json' }
|
114
117
|
}
|
115
118
|
|
116
119
|
req.headers.each { |key, value| options.headers[key] = value }
|
@@ -137,5 +140,25 @@ module Streamdal
|
|
137
140
|
raise ArgumentError, "Invalid http request method: #{req.method}"
|
138
141
|
end
|
139
142
|
end
|
143
|
+
|
144
|
+
# Called by host functions to write memory to wasm instance so that
|
145
|
+
# the wasm module can read the result of a host function call
|
146
|
+
def write_to_memory(caller, res)
|
147
|
+
alloc = caller.export('alloc').to_func
|
148
|
+
memory = caller.export('memory').to_memory
|
149
|
+
|
150
|
+
# Serialize protobuf message
|
151
|
+
resp = res.to_proto
|
152
|
+
|
153
|
+
# Allocate memory for response
|
154
|
+
resp_ptr = alloc.call(resp.length)
|
155
|
+
|
156
|
+
# Write response to memory
|
157
|
+
memory.write(resp_ptr, resp)
|
158
|
+
|
159
|
+
# return 64bit integer where first 32 bits is the pointer, and the last 32 is the length
|
160
|
+
resp_ptr << 32 | resp.length
|
161
|
+
end
|
162
|
+
|
140
163
|
end
|
141
164
|
end
|
data/lib/spec_helper.rb
CHANGED
@@ -1,2 +1,12 @@
|
|
1
1
|
require 'simplecov'
|
2
|
-
SimpleCov.start
|
2
|
+
SimpleCov.start
|
3
|
+
|
4
|
+
# This environment variable exists so that we can run WASM tests
|
5
|
+
# via the CI process for libs/wasm* and not just through the CI
|
6
|
+
# of the ruby-sdk
|
7
|
+
WASM_DIR = ENV['WASM_DIR'] || File.join(File.dirname(__FILE__), '..', 'test-assets', 'wasm')
|
8
|
+
|
9
|
+
# Load the WebAssembly module
|
10
|
+
def load_wasm(name)
|
11
|
+
File.read(File.join(WASM_DIR, name))
|
12
|
+
end
|
data/lib/streamdal.rb
CHANGED
@@ -31,7 +31,6 @@ DEFAULT_HEARTBEAT_INTERVAL = 1 # 1 second
|
|
31
31
|
MAX_PAYLOAD_SIZE = 1024 * 1024 # 1 megabyte
|
32
32
|
|
33
33
|
module Streamdal
|
34
|
-
|
35
34
|
OPERATION_TYPE_PRODUCER = 2
|
36
35
|
OPERATION_TYPE_CONSUMER = 1
|
37
36
|
CLIENT_TYPE_SDK = 1
|
@@ -39,7 +38,6 @@ module Streamdal
|
|
39
38
|
|
40
39
|
# Data class to hold instantiated wasm functions
|
41
40
|
class WasmFunction
|
42
|
-
|
43
41
|
##
|
44
42
|
# Instance of an initialized wasm module and associated memory store
|
45
43
|
|
@@ -66,7 +64,6 @@ module Streamdal
|
|
66
64
|
end
|
67
65
|
end
|
68
66
|
|
69
|
-
|
70
67
|
class Client
|
71
68
|
|
72
69
|
##
|
@@ -117,20 +114,14 @@ module Streamdal
|
|
117
114
|
|
118
115
|
# Exit any remaining threads
|
119
116
|
@workers.each do |w|
|
120
|
-
if w.running?
|
121
|
-
w.exit
|
122
|
-
end
|
117
|
+
w.exit if w.running?
|
123
118
|
end
|
124
119
|
end
|
125
120
|
|
126
121
|
def process(data, audience)
|
127
|
-
if data.empty?
|
128
|
-
raise 'data is required'
|
129
|
-
end
|
122
|
+
raise 'data is required' if data.empty?
|
130
123
|
|
131
|
-
if audience.nil?
|
132
|
-
raise 'audience is required'
|
133
|
-
end
|
124
|
+
raise 'audience is required' if audience.nil?
|
134
125
|
|
135
126
|
resp = Streamdal::Protos::SDKResponse.new
|
136
127
|
resp.status = :EXEC_STATUS_TRUE
|
@@ -215,13 +206,9 @@ module Streamdal
|
|
215
206
|
break
|
216
207
|
end
|
217
208
|
|
218
|
-
if @cfg[:dry_run]
|
219
|
-
@log.debug "Running step '#{step.name}' in dry-run mode"
|
220
|
-
end
|
209
|
+
@log.debug "Running step '#{step.name}' in dry-run mode" if @cfg[:dry_run]
|
221
210
|
|
222
|
-
if wasm_resp.output_payload.length.positive?
|
223
|
-
resp.data = wasm_resp.output_payload
|
224
|
-
end
|
211
|
+
resp.data = wasm_resp.output_payload if wasm_resp.output_payload.length.positive?
|
225
212
|
|
226
213
|
_handle_schema(aud, step, wasm_resp)
|
227
214
|
|
@@ -309,17 +296,11 @@ module Streamdal
|
|
309
296
|
private
|
310
297
|
|
311
298
|
def _validate_cfg(cfg)
|
312
|
-
if cfg[:streamdal_url].nil? || cfg[:streamdal_url].empty?
|
313
|
-
raise 'streamdal_url is required'
|
314
|
-
end
|
299
|
+
raise 'streamdal_url is required' if cfg[:streamdal_url].nil? || cfg[:streamdal_url].empty?
|
315
300
|
|
316
|
-
if cfg[:streamdal_token].nil? || cfg[:streamdal_token].empty?
|
317
|
-
raise 'streamdal_token is required'
|
318
|
-
end
|
301
|
+
raise 'streamdal_token is required' if cfg[:streamdal_token].nil? || cfg[:streamdal_token].empty?
|
319
302
|
|
320
|
-
if cfg[:service_name].nil? || cfg[:streamdal_token].empty?
|
321
|
-
raise 'service_name is required'
|
322
|
-
end
|
303
|
+
raise 'service_name is required' if cfg[:service_name].nil? || cfg[:streamdal_token].empty?
|
323
304
|
|
324
305
|
if cfg[:log].nil? || cfg[:streamdal_token].empty?
|
325
306
|
logger = Logger.new($stdout)
|
@@ -327,13 +308,9 @@ module Streamdal
|
|
327
308
|
cfg[:log] = logger
|
328
309
|
end
|
329
310
|
|
330
|
-
if cfg[:pipeline_timeout].nil?
|
331
|
-
cfg[:pipeline_timeout] = DEFAULT_PIPELINE_TIMEOUT
|
332
|
-
end
|
311
|
+
cfg[:pipeline_timeout] = DEFAULT_PIPELINE_TIMEOUT if cfg[:pipeline_timeout].nil?
|
333
312
|
|
334
|
-
if cfg[:step_timeout].nil?
|
335
|
-
cfg[:step_timeout] = DEFAULT_STEP_TIMEOUT
|
336
|
-
end
|
313
|
+
cfg[:step_timeout] = DEFAULT_STEP_TIMEOUT if cfg[:step_timeout].nil?
|
337
314
|
end
|
338
315
|
|
339
316
|
def _handle_command(cmd)
|
@@ -414,9 +391,7 @@ module Streamdal
|
|
414
391
|
def _get_function(step)
|
415
392
|
# We cache functions so we can eliminate the wasm bytes from steps to save on memory
|
416
393
|
# And also to avoid re-initializing the same function multiple times
|
417
|
-
if @functions.key?(step._wasm_id)
|
418
|
-
return @functions[step._wasm_id]
|
419
|
-
end
|
394
|
+
return @functions[step._wasm_id] if @functions.key?(step._wasm_id)
|
420
395
|
|
421
396
|
engine = Wasmtime::Engine.new
|
422
397
|
mod = Wasmtime::Module.new(engine, step._wasm_bytes)
|
@@ -450,17 +425,11 @@ module Streamdal
|
|
450
425
|
end
|
451
426
|
|
452
427
|
def _call_wasm(step, data, isr)
|
453
|
-
if step.nil?
|
454
|
-
raise 'step is required'
|
455
|
-
end
|
428
|
+
raise 'step is required' if step.nil?
|
456
429
|
|
457
|
-
if data.nil?
|
458
|
-
raise 'data is required'
|
459
|
-
end
|
430
|
+
raise 'data is required' if data.nil?
|
460
431
|
|
461
|
-
if isr.nil?
|
462
|
-
isr = Streamdal::Protos::InterStepResult.new
|
463
|
-
end
|
432
|
+
isr = Streamdal::Protos::InterStepResult.new if isr.nil?
|
464
433
|
|
465
434
|
req = Streamdal::Protos::WASMRequest.new
|
466
435
|
req.step = step.clone
|
@@ -477,7 +446,7 @@ module Streamdal
|
|
477
446
|
resp.exit_code = :WASM_EXIT_CODE_ERROR
|
478
447
|
resp.exit_msg = "Failed to execute WASM: #{e}"
|
479
448
|
resp.output_payload = ''
|
480
|
-
|
449
|
+
resp
|
481
450
|
end
|
482
451
|
end
|
483
452
|
|
@@ -516,9 +485,7 @@ module Streamdal
|
|
516
485
|
# Register with Streamdal External gRPC API
|
517
486
|
resps = @stub.register(_gen_register_request, metadata: _metadata)
|
518
487
|
resps.each do |r|
|
519
|
-
if @exit
|
520
|
-
break
|
521
|
-
end
|
488
|
+
break if @exit
|
522
489
|
|
523
490
|
_handle_command(r)
|
524
491
|
end
|
@@ -564,9 +531,7 @@ module Streamdal
|
|
564
531
|
|
565
532
|
_add_audience(aud)
|
566
533
|
|
567
|
-
if @pipelines.key?(aud_str)
|
568
|
-
return @pipelines[aud_str]
|
569
|
-
end
|
534
|
+
return @pipelines[aud_str] if @pipelines.key?(aud_str)
|
570
535
|
|
571
536
|
[]
|
572
537
|
end
|
@@ -612,18 +577,14 @@ module Streamdal
|
|
612
577
|
|
613
578
|
def _get_active_tails_for_audience(aud)
|
614
579
|
aud_str = aud_to_str(aud)
|
615
|
-
if @tails.key?(aud_str)
|
616
|
-
return @tails[aud_str].values
|
617
|
-
end
|
580
|
+
return @tails[aud_str].values if @tails.key?(aud_str)
|
618
581
|
|
619
582
|
[]
|
620
583
|
end
|
621
584
|
|
622
585
|
def _send_tail(aud, pipeline_id, original_data, new_data)
|
623
586
|
tails = _get_active_tails_for_audience(aud)
|
624
|
-
if tails.empty?
|
625
|
-
return nil
|
626
|
-
end
|
587
|
+
return nil if tails.empty?
|
627
588
|
|
628
589
|
tails.each do |tail|
|
629
590
|
req = Streamdal::Protos::TailResponse.new
|
@@ -640,19 +601,13 @@ module Streamdal
|
|
640
601
|
end
|
641
602
|
|
642
603
|
def _notify_condition(pipeline, step, aud, cond, data, cond_type)
|
643
|
-
if cond.nil?
|
644
|
-
return nil
|
645
|
-
end
|
604
|
+
return nil if cond.nil?
|
646
605
|
|
647
|
-
if cond.notification.nil?
|
648
|
-
return nil
|
649
|
-
end
|
606
|
+
return nil if cond.notification.nil?
|
650
607
|
|
651
608
|
@log.debug 'Notifying'
|
652
609
|
|
653
|
-
if @cfg[:dry_run]
|
654
|
-
return nil
|
655
|
-
end
|
610
|
+
return nil if @cfg[:dry_run]
|
656
611
|
|
657
612
|
@metrics.incr(CounterEntry.new(Metrics::COUNTER_NOTIFY, aud, {
|
658
613
|
"service": @cfg[:service_name],
|
@@ -702,15 +657,12 @@ module Streamdal
|
|
702
657
|
t.start_tail_workers
|
703
658
|
|
704
659
|
_set_active_tail(t)
|
705
|
-
|
706
660
|
end
|
707
661
|
|
708
662
|
def _set_active_tail(tail)
|
709
663
|
key = aud_to_str(tail.request.audience)
|
710
664
|
|
711
|
-
unless @tails.key?(key)
|
712
|
-
@tails[key] = {}
|
713
|
-
end
|
665
|
+
@tails[key] = {} unless @tails.key?(key)
|
714
666
|
|
715
667
|
@tails[key][tail.request.id] = tail
|
716
668
|
end
|
@@ -718,9 +670,7 @@ module Streamdal
|
|
718
670
|
def _set_paused_tail(tail)
|
719
671
|
key = aud_to_str(tail.request.aud)
|
720
672
|
|
721
|
-
unless @paused_tails.key?(key)
|
722
|
-
@paused_tails[key] = {}
|
723
|
-
end
|
673
|
+
@paused_tails[key] = {} unless @paused_tails.key?(key)
|
724
674
|
|
725
675
|
@paused_tails[key][tail.request.id] = tail
|
726
676
|
end
|
@@ -735,18 +685,14 @@ module Streamdal
|
|
735
685
|
# Remove from active tails
|
736
686
|
@tails[key].delete(cmd.tail.request.id)
|
737
687
|
|
738
|
-
if @tails[key].empty?
|
739
|
-
@tails.delete(key)
|
740
|
-
end
|
688
|
+
@tails.delete(key) if @tails[key].empty?
|
741
689
|
end
|
742
690
|
|
743
|
-
|
744
|
-
@paused_tails[key].delete(cmd.tail.request.id)
|
691
|
+
return unless @paused_tails.key?(key) && @paused_tails[key].key?(cmd.tail.request.id)
|
745
692
|
|
746
|
-
|
747
|
-
|
748
|
-
|
749
|
-
end
|
693
|
+
@paused_tails[key].delete(cmd.tail.request.id)
|
694
|
+
|
695
|
+
@paused_tails.delete(key) if @paused_tails[key].empty?
|
750
696
|
end
|
751
697
|
|
752
698
|
def _stop_all_tails
|
@@ -762,9 +708,7 @@ module Streamdal
|
|
762
708
|
t.stop_tail
|
763
709
|
tails[aud].delete(tail.request.id)
|
764
710
|
|
765
|
-
if tails[aud].empty?
|
766
|
-
tails.delete(aud)
|
767
|
-
end
|
711
|
+
tails.delete(aud) if tails[aud].empty?
|
768
712
|
end
|
769
713
|
end
|
770
714
|
end
|
@@ -795,54 +739,30 @@ module Streamdal
|
|
795
739
|
def _remove_active_tail(aud, tail_id)
|
796
740
|
key = aud_to_str(aud)
|
797
741
|
|
798
|
-
|
799
|
-
t = @tails[key][tail_id]
|
800
|
-
t.stop_tail
|
742
|
+
return unless @tails.key?(key) && @tails[key].key?(tail_id)
|
801
743
|
|
802
|
-
|
744
|
+
t = @tails[key][tail_id]
|
745
|
+
t.stop_tail
|
803
746
|
|
804
|
-
|
805
|
-
@tails.delete(key)
|
806
|
-
end
|
747
|
+
@tails[key].delete(tail_id)
|
807
748
|
|
808
|
-
|
809
|
-
|
749
|
+
@tails.delete(key) if @tails[key].empty?
|
750
|
+
|
751
|
+
t
|
810
752
|
end
|
811
753
|
|
812
754
|
def _remove_paused_tail(aud, tail_id)
|
813
755
|
key = aud_to_str(aud)
|
814
756
|
|
815
|
-
|
816
|
-
t = @paused_tails[key][tail_id]
|
757
|
+
return unless @paused_tails.key?(key) && @paused_tails[key].key?(tail_id)
|
817
758
|
|
818
|
-
|
759
|
+
t = @paused_tails[key][tail_id]
|
819
760
|
|
820
|
-
|
821
|
-
@paused_tails.delete(key)
|
822
|
-
end
|
761
|
+
@paused_tails[key].delete(tail_id)
|
823
762
|
|
824
|
-
|
825
|
-
end
|
826
|
-
end
|
827
|
-
|
828
|
-
# Called by host functions to write memory to wasm instance so that
|
829
|
-
# the wasm module can read the result of a host function call
|
830
|
-
def write_to_memory(caller, res)
|
831
|
-
alloc = caller.export('alloc').to_func
|
832
|
-
memory = caller.export('memory').to_memory
|
833
|
-
|
834
|
-
# Serialize protobuf message
|
835
|
-
resp = res.to_proto
|
763
|
+
@paused_tails.delete(key) if @paused_tails[key].empty?
|
836
764
|
|
837
|
-
|
838
|
-
resp_ptr = alloc.call(resp.length)
|
839
|
-
|
840
|
-
# Write response to memory
|
841
|
-
memory.write(resp_ptr, resp)
|
842
|
-
|
843
|
-
# return 64bit integer where first 32 bits is the pointer, and the last 32 is the length
|
844
|
-
resp_ptr << 32 | resp.length
|
765
|
+
t
|
845
766
|
end
|
846
|
-
|
847
767
|
end
|
848
|
-
end
|
768
|
+
end
|
data/lib/wasm_spec.rb
ADDED
@@ -0,0 +1,259 @@
|
|
1
|
+
require_relative 'spec_helper'
|
2
|
+
require 'steps/sp_steps_httprequest_pb'
|
3
|
+
require 'steps/sp_steps_detective_pb'
|
4
|
+
require 'steps/sp_steps_transform_pb'
|
5
|
+
require_relative 'streamdal'
|
6
|
+
|
7
|
+
class TestClient < Streamdal::Client
|
8
|
+
|
9
|
+
attr_accessor :kv
|
10
|
+
|
11
|
+
# Ignore rubocop warning
|
12
|
+
# rubocop:disable Lint/MissingSuper
|
13
|
+
def initialize
|
14
|
+
@cfg = {
|
15
|
+
step_timeout: 2
|
16
|
+
}
|
17
|
+
@functions = {}
|
18
|
+
|
19
|
+
logger = Logger.new($stdout)
|
20
|
+
logger.level = Logger::ERROR
|
21
|
+
|
22
|
+
@log = logger
|
23
|
+
@kv = Streamdal::KeyValue.new
|
24
|
+
@hostfunc = Streamdal::HostFunc.new(@kv)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
RSpec.describe 'WASM' do
|
29
|
+
let(:client) { TestClient.new }
|
30
|
+
|
31
|
+
context '_call_wasm' do
|
32
|
+
it 'raises an error if step is nil' do
|
33
|
+
expect { client.send(:_call_wasm, nil, nil, nil) }.to raise_error('step is required')
|
34
|
+
end
|
35
|
+
|
36
|
+
it 'raises an error if data is nil' do
|
37
|
+
step = Streamdal::Protos::HttpRequestStep.new
|
38
|
+
expect { client.send(:_call_wasm, step, nil, nil) }.to raise_error('data is required')
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
context 'detective.wasm' do
|
43
|
+
before(:each) do
|
44
|
+
wasm_bytes = load_wasm('detective.wasm')
|
45
|
+
|
46
|
+
@step = Streamdal::Protos::PipelineStep.new
|
47
|
+
@step.name = 'detective'
|
48
|
+
@step._wasm_bytes = wasm_bytes.b
|
49
|
+
@step._wasm_id = SecureRandom.uuid
|
50
|
+
@step._wasm_function = 'f'
|
51
|
+
@step.detective = Streamdal::Protos::DetectiveStep.new
|
52
|
+
@step.detective.path = 'object.field'
|
53
|
+
@step.detective.args = Google::Protobuf::RepeatedField.new(:string, ['streamdal'])
|
54
|
+
@step.detective.negate = false
|
55
|
+
@step.detective.type = :DETECTIVE_TYPE_STRING_CONTAINS_ANY
|
56
|
+
end
|
57
|
+
|
58
|
+
it 'detects email in JSON payload' do
|
59
|
+
data = '{"object":{"field":"streamdal@gmail.com"}}'
|
60
|
+
|
61
|
+
res = client.send(:_call_wasm, @step, data, nil)
|
62
|
+
|
63
|
+
expect(res).not_to be_nil
|
64
|
+
expect(res.exit_code).to eq(:WASM_EXIT_CODE_TRUE)
|
65
|
+
expect(res.output_payload).to eq(data)
|
66
|
+
|
67
|
+
end
|
68
|
+
|
69
|
+
it 'does not detect email in JSON payload' do
|
70
|
+
data = '{"object":{"field":"mark@gmail.com"}}'
|
71
|
+
|
72
|
+
res = client.send(:_call_wasm, @step, data, nil)
|
73
|
+
expect(res).not_to be_nil
|
74
|
+
expect(res.exit_code).to eq(:WASM_EXIT_CODE_FALSE)
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
context 'httprequest.wasm' do
|
79
|
+
before(:each) do
|
80
|
+
wasm_bytes = load_wasm('httprequest.wasm')
|
81
|
+
|
82
|
+
http_req_step = Streamdal::Protos::HttpRequestStep.new
|
83
|
+
http_req_step.request = Streamdal::Protos::HttpRequest.new
|
84
|
+
http_req_step.request.url = 'https://www.google.com/404_me'
|
85
|
+
http_req_step.request.method = :HTTP_REQUEST_METHOD_GET
|
86
|
+
http_req_step.request.body = ''
|
87
|
+
|
88
|
+
@step = Streamdal::Protos::PipelineStep.new
|
89
|
+
@step.name = 'http request'
|
90
|
+
@step._wasm_bytes = wasm_bytes.b
|
91
|
+
@step._wasm_id = SecureRandom.uuid
|
92
|
+
@step._wasm_function = 'f'
|
93
|
+
@step.http_request = http_req_step
|
94
|
+
end
|
95
|
+
|
96
|
+
it 'returns false on 404' do
|
97
|
+
res = client.send(:_call_wasm, @step, '', nil)
|
98
|
+
|
99
|
+
expect(res).not_to be_nil
|
100
|
+
expect(res.exit_msg).to eq('Request returned non-200 response code: 404')
|
101
|
+
expect(res.exit_code).to eq(:WASM_EXIT_CODE_FALSE)
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
context 'inferschema.wasm' do
|
106
|
+
before(:each) do
|
107
|
+
wasm_bytes = load_wasm('inferschema.wasm')
|
108
|
+
|
109
|
+
@step = Streamdal::Protos::PipelineStep.new
|
110
|
+
@step.name = 'schema inference'
|
111
|
+
@step._wasm_bytes = wasm_bytes.b
|
112
|
+
@step._wasm_id = SecureRandom.uuid
|
113
|
+
@step._wasm_function = 'f'
|
114
|
+
@step.infer_schema = Streamdal::Protos::InferSchemaStep.new
|
115
|
+
end
|
116
|
+
|
117
|
+
it 'infers schema' do
|
118
|
+
payload = '{"object": {"payload": "test"}}'
|
119
|
+
res = client.send(:_call_wasm, @step, payload, nil)
|
120
|
+
|
121
|
+
expected_schema = '{"$schema":"http://json-schema.org/draft-07/schema#","properties":{"object":{"properties":{"payload":{"type":"string"}},"required":["payload"],"type":"object"}},"required":["object"],"type":"object"}'
|
122
|
+
|
123
|
+
expect(res).not_to be_nil
|
124
|
+
expect(res.exit_msg).to eq('inferred fresh schema')
|
125
|
+
expect(res.exit_code).to eq(:WASM_EXIT_CODE_TRUE)
|
126
|
+
expect(res.output_payload).to eq(payload)
|
127
|
+
expect(res.output_step).to eq(expected_schema)
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
131
|
+
context 'transform.wasm' do
|
132
|
+
before(:each) do
|
133
|
+
wasm_bytes = load_wasm('transform.wasm')
|
134
|
+
|
135
|
+
@step = Streamdal::Protos::PipelineStep.new
|
136
|
+
@step.name = 'transform'
|
137
|
+
@step._wasm_bytes = wasm_bytes.b
|
138
|
+
@step._wasm_id = SecureRandom.uuid
|
139
|
+
@step._wasm_function = 'f'
|
140
|
+
end
|
141
|
+
|
142
|
+
it 'deletes a field from a payload' do
|
143
|
+
@step.transform = Streamdal::Protos::TransformStep.new
|
144
|
+
@step.transform.type = :TRANSFORM_TYPE_DELETE_FIELD
|
145
|
+
@step.transform.delete_field_options = Streamdal::Protos::TransformDeleteFieldOptions.new
|
146
|
+
@step.transform.delete_field_options.paths = Google::Protobuf::RepeatedField.new(:string, ['object.another'])
|
147
|
+
|
148
|
+
payload = '{"object": {"payload": "old val", "another": "field"}}'
|
149
|
+
res = client.send(:_call_wasm, @step, payload, nil)
|
150
|
+
|
151
|
+
expect(res).not_to be_nil
|
152
|
+
expect(res.exit_msg).to eq('Successfully transformed payload')
|
153
|
+
expect(res.exit_code).to eq(:WASM_EXIT_CODE_TRUE)
|
154
|
+
expect(res.output_payload).to eq('{"object": {"payload": "old val"}}')
|
155
|
+
end
|
156
|
+
|
157
|
+
it 'replaces a fields value with a new one' do
|
158
|
+
@step.transform = Streamdal::Protos::TransformStep.new
|
159
|
+
@step.transform.type = :TRANSFORM_TYPE_REPLACE_VALUE
|
160
|
+
@step.transform.replace_value_options = Streamdal::Protos::TransformReplaceValueOptions.new
|
161
|
+
@step.transform.replace_value_options.path = 'object.payload'
|
162
|
+
@step.transform.replace_value_options.value = '"new val"'
|
163
|
+
|
164
|
+
payload = '{"object": {"payload": "old val"}}'
|
165
|
+
res = client.send(:_call_wasm, @step, payload, nil)
|
166
|
+
|
167
|
+
expect(res).not_to be_nil
|
168
|
+
expect(res.exit_msg).to eq('Successfully transformed payload')
|
169
|
+
expect(res.exit_code).to eq(:WASM_EXIT_CODE_TRUE)
|
170
|
+
expect(res.output_payload).to eq('{"object": {"payload": "new val"}}')
|
171
|
+
end
|
172
|
+
|
173
|
+
it 'truncates the value of a field' do
|
174
|
+
@step.transform = Streamdal::Protos::TransformStep.new
|
175
|
+
@step.transform.type = :TRANSFORM_TYPE_TRUNCATE_VALUE
|
176
|
+
@step.transform.truncate_options = Streamdal::Protos::TransformTruncateOptions.new
|
177
|
+
@step.transform.truncate_options.type = :TRANSFORM_TRUNCATE_TYPE_LENGTH
|
178
|
+
@step.transform.truncate_options.path = 'object.payload'
|
179
|
+
@step.transform.truncate_options.value = 3
|
180
|
+
|
181
|
+
payload = '{"object": {"payload": "old val"}}'
|
182
|
+
res = client.send(:_call_wasm, @step, payload, nil)
|
183
|
+
|
184
|
+
expect(res).not_to be_nil
|
185
|
+
expect(res.exit_msg).to eq('Successfully transformed payload')
|
186
|
+
expect(res.exit_code).to eq(:WASM_EXIT_CODE_TRUE)
|
187
|
+
expect(res.output_payload).to eq('{"object": {"payload": "old"}}')
|
188
|
+
end
|
189
|
+
|
190
|
+
it 'performs dynamic transformation' do
|
191
|
+
# TODO: add this test
|
192
|
+
end
|
193
|
+
end
|
194
|
+
|
195
|
+
context 'validjson.wasm' do
|
196
|
+
before(:each) do
|
197
|
+
wasm_bytes = load_wasm('validjson.wasm')
|
198
|
+
|
199
|
+
@step = Streamdal::Protos::PipelineStep.new
|
200
|
+
@step.name = 'validate json'
|
201
|
+
@step._wasm_bytes = wasm_bytes.b
|
202
|
+
@step._wasm_id = SecureRandom.uuid
|
203
|
+
@step._wasm_function = 'f'
|
204
|
+
@step.valid_json = Streamdal::Protos::ValidJSONStep.new
|
205
|
+
|
206
|
+
end
|
207
|
+
|
208
|
+
it 'validates a valid JSON payload' do
|
209
|
+
|
210
|
+
payload = '{"object": {"payload": "test"}}'
|
211
|
+
res = client.send(:_call_wasm, @step, payload, nil)
|
212
|
+
|
213
|
+
expect(res).not_to be_nil
|
214
|
+
expect(res.exit_code).to eq(:WASM_EXIT_CODE_TRUE)
|
215
|
+
expect(res.output_payload).to eq(payload)
|
216
|
+
end
|
217
|
+
|
218
|
+
it 'returns false on invalid JSON payload' do
|
219
|
+
payload = '{"object": {"payload": "test"'
|
220
|
+
res = client.send(:_call_wasm, @step, payload, nil)
|
221
|
+
|
222
|
+
expect(res).not_to be_nil
|
223
|
+
expect(res.exit_code).to eq(:WASM_EXIT_CODE_FALSE)
|
224
|
+
end
|
225
|
+
end
|
226
|
+
|
227
|
+
context 'kv.wasm' do
|
228
|
+
before(:each) do
|
229
|
+
wasm_bytes = load_wasm('kv.wasm')
|
230
|
+
|
231
|
+
client.kv.set('test', 'test')
|
232
|
+
|
233
|
+
@step = Streamdal::Protos::PipelineStep.new
|
234
|
+
@step.name = 'kv exists'
|
235
|
+
@step._wasm_bytes = wasm_bytes.b
|
236
|
+
@step._wasm_id = SecureRandom.uuid
|
237
|
+
@step._wasm_function = 'f'
|
238
|
+
@step.kv = Streamdal::Protos::KVStep.new
|
239
|
+
@step.kv.key = 'test'
|
240
|
+
@step.kv.mode = :KV_MODE_STATIC
|
241
|
+
@step.kv.action = :KV_ACTION_EXISTS
|
242
|
+
end
|
243
|
+
|
244
|
+
it 'returns true if a key exists' do
|
245
|
+
res = client.send(:_call_wasm, @step, '', nil)
|
246
|
+
|
247
|
+
expect(res).not_to be_nil
|
248
|
+
expect(res.exit_code).to eq(:WASM_EXIT_CODE_TRUE)
|
249
|
+
end
|
250
|
+
|
251
|
+
it 'returns false when a key doesnt exist' do
|
252
|
+
@step.kv.key = 'not_exists'
|
253
|
+
res = client.send(:_call_wasm, @step, '', nil)
|
254
|
+
|
255
|
+
expect(res).not_to be_nil
|
256
|
+
expect(res.exit_code).to eq(:WASM_EXIT_CODE_FALSE)
|
257
|
+
end
|
258
|
+
end
|
259
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: streamdal
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Mark Gregan
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2024-06-
|
11
|
+
date: 2024-06-20 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description:
|
14
14
|
email: mark@streamdal.com
|
@@ -33,6 +33,7 @@ files:
|
|
33
33
|
- lib/tail_spec.rb
|
34
34
|
- lib/validation.rb
|
35
35
|
- lib/validation_spec.rb
|
36
|
+
- lib/wasm_spec.rb
|
36
37
|
homepage: https://docs.streamdal.com
|
37
38
|
licenses:
|
38
39
|
- Apache-2.0
|
@@ -45,7 +46,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
45
46
|
requirements:
|
46
47
|
- - '='
|
47
48
|
- !ruby/object:Gem::Version
|
48
|
-
version: 0.0.
|
49
|
+
version: 0.0.4
|
49
50
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
50
51
|
requirements:
|
51
52
|
- - ">="
|