streamdal 0.0.2 → 0.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: ca014e8334d38a589d1cd3aab58362ac3095fc077a93c076301f2e1552f9b375
4
- data.tar.gz: 77110e0cc07e851e7b21ef831bed28495353bd70fce865159c41b18d4a40f7d0
3
+ metadata.gz: bd49e6aba633028e6bd0af296c6ed7a69ca00f2b4bc9c26646d40bdd1e19d313
4
+ data.tar.gz: 745eb525e4290b1d23bda53000683e9a8761fe070a4b50bb011605f2f03d64a5
5
5
  SHA512:
6
- metadata.gz: 227c07d36fd54e8bf35a79b434417f59c80e45c72678ca82ada629b8268b744b614616d28b196e48b85c93f7a031330ebc8884a8ca826553eb647953069d7890
7
- data.tar.gz: 9c83c8388b8e871d60b1b670b6f919b6a05878c731b6b8978545f9b782885c195cb30d02f1ec160cba58974ad53f99d9506410301185c5e036128eca4c56bb3b
6
+ metadata.gz: c65cd7b9a1342e31092bbb183e55537b26cb1be2ca34609d5df44ae763ec6bdcf514febb0d85f95a02107ec5d6b65f66b02956ef2b4516cce178e4185af6712e
7
+ data.tar.gz: 22319f8bffbafea0aa41748f7a745f44698c6061589ebe1ba8ee2621127e0f4fb3b4d5990d476e172d3e54f45c66f88310c7d295becab73ae0cc0172a79bc432
data/lib/hostfunc.rb CHANGED
@@ -43,9 +43,9 @@ module Streamdal
43
43
  req = Streamdal::Protos::WASMRequest.decode(data)
44
44
 
45
45
  begin
46
- req_body = self._get_request_body_for_mode(req)
46
+ req_body = _get_request_body_for_mode(req)
47
47
  rescue => e
48
- return self._http_request_response(caller, 400, e.to_s, {})
48
+ return _http_request_response(caller, 400, e.to_s, {})
49
49
  end
50
50
 
51
51
  # Attempt to make HTTP request
@@ -53,10 +53,17 @@ module Streamdal
53
53
  begin
54
54
  response = _make_http_request(req.step.http_request.request, req_body)
55
55
  rescue => e
56
- return self._http_request_response(caller, 400, "Unable to execute HTTP request: #{e}", {})
56
+ return _http_request_response(caller, 400, "Unable to execute HTTP request: #{e}", {})
57
57
  end
58
58
 
59
- self._http_request_response(caller, response.code, response.body, response.headers)
59
+ # Convert body to utf8
60
+ out = encode(response.body)
61
+
62
+ _http_request_response(caller, response.code, out, response.headers)
63
+ end
64
+
65
+ def encode(str)
66
+ str.force_encoding('ascii-8bit').encode('utf-8', invalid: :replace, undef: :replace, replace: '?')
60
67
  end
61
68
 
62
69
  private
@@ -95,22 +102,18 @@ module Streamdal
95
102
  }
96
103
 
97
104
  req.inter_step_result.to_json
98
- when :HTTP_REQUEST_BODY_MODE_STATIC
99
- http_req.body
100
105
  else
101
- raise 'invalid http request body mode'
106
+ http_req.body
102
107
  end
103
108
  end
104
109
 
105
110
  ##
106
111
  # Performs an http request
107
112
  def _make_http_request(req, body)
108
- if req.nil?
109
- raise 'req is required'
110
- end
113
+ raise 'req is required' if req.nil?
111
114
 
112
115
  options = {
113
- headers: { "Content-Type": 'application/json', },
116
+ headers: { "Content-Type": 'application/json' }
114
117
  }
115
118
 
116
119
  req.headers.each { |key, value| options.headers[key] = value }
@@ -137,5 +140,25 @@ module Streamdal
137
140
  raise ArgumentError, "Invalid http request method: #{req.method}"
138
141
  end
139
142
  end
143
+
144
+ # Called by host functions to write memory to wasm instance so that
145
+ # the wasm module can read the result of a host function call
146
+ def write_to_memory(caller, res)
147
+ alloc = caller.export('alloc').to_func
148
+ memory = caller.export('memory').to_memory
149
+
150
+ # Serialize protobuf message
151
+ resp = res.to_proto
152
+
153
+ # Allocate memory for response
154
+ resp_ptr = alloc.call(resp.length)
155
+
156
+ # Write response to memory
157
+ memory.write(resp_ptr, resp)
158
+
159
+ # return 64bit integer where first 32 bits is the pointer, and the last 32 is the length
160
+ resp_ptr << 32 | resp.length
161
+ end
162
+
140
163
  end
141
164
  end
data/lib/spec_helper.rb CHANGED
@@ -1,2 +1,12 @@
1
1
  require 'simplecov'
2
- SimpleCov.start
2
+ SimpleCov.start
3
+
4
+ # This environment variable exists so that we can run WASM tests
5
+ # via the CI process for libs/wasm* and not just through the CI
6
+ # of the ruby-sdk
7
+ WASM_DIR = ENV['WASM_DIR'] || File.join(File.dirname(__FILE__), '..', 'test-assets', 'wasm')
8
+
9
+ # Load the WebAssembly module
10
+ def load_wasm(name)
11
+ File.read(File.join(WASM_DIR, name))
12
+ end
data/lib/streamdal.rb CHANGED
@@ -31,7 +31,6 @@ DEFAULT_HEARTBEAT_INTERVAL = 1 # 1 second
31
31
  MAX_PAYLOAD_SIZE = 1024 * 1024 # 1 megabyte
32
32
 
33
33
  module Streamdal
34
-
35
34
  OPERATION_TYPE_PRODUCER = 2
36
35
  OPERATION_TYPE_CONSUMER = 1
37
36
  CLIENT_TYPE_SDK = 1
@@ -39,7 +38,6 @@ module Streamdal
39
38
 
40
39
  # Data class to hold instantiated wasm functions
41
40
  class WasmFunction
42
-
43
41
  ##
44
42
  # Instance of an initialized wasm module and associated memory store
45
43
 
@@ -66,7 +64,6 @@ module Streamdal
66
64
  end
67
65
  end
68
66
 
69
-
70
67
  class Client
71
68
 
72
69
  ##
@@ -117,20 +114,14 @@ module Streamdal
117
114
 
118
115
  # Exit any remaining threads
119
116
  @workers.each do |w|
120
- if w.running?
121
- w.exit
122
- end
117
+ w.exit if w.running?
123
118
  end
124
119
  end
125
120
 
126
121
  def process(data, audience)
127
- if data.empty?
128
- raise 'data is required'
129
- end
122
+ raise 'data is required' if data.empty?
130
123
 
131
- if audience.nil?
132
- raise 'audience is required'
133
- end
124
+ raise 'audience is required' if audience.nil?
134
125
 
135
126
  resp = Streamdal::Protos::SDKResponse.new
136
127
  resp.status = :EXEC_STATUS_TRUE
@@ -215,13 +206,9 @@ module Streamdal
215
206
  break
216
207
  end
217
208
 
218
- if @cfg[:dry_run]
219
- @log.debug "Running step '#{step.name}' in dry-run mode"
220
- end
209
+ @log.debug "Running step '#{step.name}' in dry-run mode" if @cfg[:dry_run]
221
210
 
222
- if wasm_resp.output_payload.length.positive?
223
- resp.data = wasm_resp.output_payload
224
- end
211
+ resp.data = wasm_resp.output_payload if wasm_resp.output_payload.length.positive?
225
212
 
226
213
  _handle_schema(aud, step, wasm_resp)
227
214
 
@@ -309,17 +296,11 @@ module Streamdal
309
296
  private
310
297
 
311
298
  def _validate_cfg(cfg)
312
- if cfg[:streamdal_url].nil? || cfg[:streamdal_url].empty?
313
- raise 'streamdal_url is required'
314
- end
299
+ raise 'streamdal_url is required' if cfg[:streamdal_url].nil? || cfg[:streamdal_url].empty?
315
300
 
316
- if cfg[:streamdal_token].nil? || cfg[:streamdal_token].empty?
317
- raise 'streamdal_token is required'
318
- end
301
+ raise 'streamdal_token is required' if cfg[:streamdal_token].nil? || cfg[:streamdal_token].empty?
319
302
 
320
- if cfg[:service_name].nil? || cfg[:streamdal_token].empty?
321
- raise 'service_name is required'
322
- end
303
+ raise 'service_name is required' if cfg[:service_name].nil? || cfg[:streamdal_token].empty?
323
304
 
324
305
  if cfg[:log].nil? || cfg[:streamdal_token].empty?
325
306
  logger = Logger.new($stdout)
@@ -327,13 +308,9 @@ module Streamdal
327
308
  cfg[:log] = logger
328
309
  end
329
310
 
330
- if cfg[:pipeline_timeout].nil?
331
- cfg[:pipeline_timeout] = DEFAULT_PIPELINE_TIMEOUT
332
- end
311
+ cfg[:pipeline_timeout] = DEFAULT_PIPELINE_TIMEOUT if cfg[:pipeline_timeout].nil?
333
312
 
334
- if cfg[:step_timeout].nil?
335
- cfg[:step_timeout] = DEFAULT_STEP_TIMEOUT
336
- end
313
+ cfg[:step_timeout] = DEFAULT_STEP_TIMEOUT if cfg[:step_timeout].nil?
337
314
  end
338
315
 
339
316
  def _handle_command(cmd)
@@ -414,9 +391,7 @@ module Streamdal
414
391
  def _get_function(step)
415
392
  # We cache functions so we can eliminate the wasm bytes from steps to save on memory
416
393
  # And also to avoid re-initializing the same function multiple times
417
- if @functions.key?(step._wasm_id)
418
- return @functions[step._wasm_id]
419
- end
394
+ return @functions[step._wasm_id] if @functions.key?(step._wasm_id)
420
395
 
421
396
  engine = Wasmtime::Engine.new
422
397
  mod = Wasmtime::Module.new(engine, step._wasm_bytes)
@@ -450,17 +425,11 @@ module Streamdal
450
425
  end
451
426
 
452
427
  def _call_wasm(step, data, isr)
453
- if step.nil?
454
- raise 'step is required'
455
- end
428
+ raise 'step is required' if step.nil?
456
429
 
457
- if data.nil?
458
- raise 'data is required'
459
- end
430
+ raise 'data is required' if data.nil?
460
431
 
461
- if isr.nil?
462
- isr = Streamdal::Protos::InterStepResult.new
463
- end
432
+ isr = Streamdal::Protos::InterStepResult.new if isr.nil?
464
433
 
465
434
  req = Streamdal::Protos::WASMRequest.new
466
435
  req.step = step.clone
@@ -477,7 +446,7 @@ module Streamdal
477
446
  resp.exit_code = :WASM_EXIT_CODE_ERROR
478
447
  resp.exit_msg = "Failed to execute WASM: #{e}"
479
448
  resp.output_payload = ''
480
- return resp
449
+ resp
481
450
  end
482
451
  end
483
452
 
@@ -516,9 +485,7 @@ module Streamdal
516
485
  # Register with Streamdal External gRPC API
517
486
  resps = @stub.register(_gen_register_request, metadata: _metadata)
518
487
  resps.each do |r|
519
- if @exit
520
- break
521
- end
488
+ break if @exit
522
489
 
523
490
  _handle_command(r)
524
491
  end
@@ -564,9 +531,7 @@ module Streamdal
564
531
 
565
532
  _add_audience(aud)
566
533
 
567
- if @pipelines.key?(aud_str)
568
- return @pipelines[aud_str]
569
- end
534
+ return @pipelines[aud_str] if @pipelines.key?(aud_str)
570
535
 
571
536
  []
572
537
  end
@@ -612,18 +577,14 @@ module Streamdal
612
577
 
613
578
  def _get_active_tails_for_audience(aud)
614
579
  aud_str = aud_to_str(aud)
615
- if @tails.key?(aud_str)
616
- return @tails[aud_str].values
617
- end
580
+ return @tails[aud_str].values if @tails.key?(aud_str)
618
581
 
619
582
  []
620
583
  end
621
584
 
622
585
  def _send_tail(aud, pipeline_id, original_data, new_data)
623
586
  tails = _get_active_tails_for_audience(aud)
624
- if tails.empty?
625
- return nil
626
- end
587
+ return nil if tails.empty?
627
588
 
628
589
  tails.each do |tail|
629
590
  req = Streamdal::Protos::TailResponse.new
@@ -640,19 +601,13 @@ module Streamdal
640
601
  end
641
602
 
642
603
  def _notify_condition(pipeline, step, aud, cond, data, cond_type)
643
- if cond.nil?
644
- return nil
645
- end
604
+ return nil if cond.nil?
646
605
 
647
- if cond.notification.nil?
648
- return nil
649
- end
606
+ return nil if cond.notification.nil?
650
607
 
651
608
  @log.debug 'Notifying'
652
609
 
653
- if @cfg[:dry_run]
654
- return nil
655
- end
610
+ return nil if @cfg[:dry_run]
656
611
 
657
612
  @metrics.incr(CounterEntry.new(Metrics::COUNTER_NOTIFY, aud, {
658
613
  "service": @cfg[:service_name],
@@ -702,15 +657,12 @@ module Streamdal
702
657
  t.start_tail_workers
703
658
 
704
659
  _set_active_tail(t)
705
-
706
660
  end
707
661
 
708
662
  def _set_active_tail(tail)
709
663
  key = aud_to_str(tail.request.audience)
710
664
 
711
- unless @tails.key?(key)
712
- @tails[key] = {}
713
- end
665
+ @tails[key] = {} unless @tails.key?(key)
714
666
 
715
667
  @tails[key][tail.request.id] = tail
716
668
  end
@@ -718,9 +670,7 @@ module Streamdal
718
670
  def _set_paused_tail(tail)
719
671
  key = aud_to_str(tail.request.aud)
720
672
 
721
- unless @paused_tails.key?(key)
722
- @paused_tails[key] = {}
723
- end
673
+ @paused_tails[key] = {} unless @paused_tails.key?(key)
724
674
 
725
675
  @paused_tails[key][tail.request.id] = tail
726
676
  end
@@ -735,18 +685,14 @@ module Streamdal
735
685
  # Remove from active tails
736
686
  @tails[key].delete(cmd.tail.request.id)
737
687
 
738
- if @tails[key].empty?
739
- @tails.delete(key)
740
- end
688
+ @tails.delete(key) if @tails[key].empty?
741
689
  end
742
690
 
743
- if @paused_tails.key?(key) && @paused_tails[key].key?(cmd.tail.request.id)
744
- @paused_tails[key].delete(cmd.tail.request.id)
691
+ return unless @paused_tails.key?(key) && @paused_tails[key].key?(cmd.tail.request.id)
745
692
 
746
- if @paused_tails[key].empty?
747
- @paused_tails.delete(key)
748
- end
749
- end
693
+ @paused_tails[key].delete(cmd.tail.request.id)
694
+
695
+ @paused_tails.delete(key) if @paused_tails[key].empty?
750
696
  end
751
697
 
752
698
  def _stop_all_tails
@@ -762,9 +708,7 @@ module Streamdal
762
708
  t.stop_tail
763
709
  tails[aud].delete(tail.request.id)
764
710
 
765
- if tails[aud].empty?
766
- tails.delete(aud)
767
- end
711
+ tails.delete(aud) if tails[aud].empty?
768
712
  end
769
713
  end
770
714
  end
@@ -795,54 +739,30 @@ module Streamdal
795
739
  def _remove_active_tail(aud, tail_id)
796
740
  key = aud_to_str(aud)
797
741
 
798
- if @tails.key?(key) && @tails[key].key?(tail_id)
799
- t = @tails[key][tail_id]
800
- t.stop_tail
742
+ return unless @tails.key?(key) && @tails[key].key?(tail_id)
801
743
 
802
- @tails[key].delete(tail_id)
744
+ t = @tails[key][tail_id]
745
+ t.stop_tail
803
746
 
804
- if @tails[key].empty?
805
- @tails.delete(key)
806
- end
747
+ @tails[key].delete(tail_id)
807
748
 
808
- t
809
- end
749
+ @tails.delete(key) if @tails[key].empty?
750
+
751
+ t
810
752
  end
811
753
 
812
754
  def _remove_paused_tail(aud, tail_id)
813
755
  key = aud_to_str(aud)
814
756
 
815
- if @paused_tails.key?(key) && @paused_tails[key].key?(tail_id)
816
- t = @paused_tails[key][tail_id]
757
+ return unless @paused_tails.key?(key) && @paused_tails[key].key?(tail_id)
817
758
 
818
- @paused_tails[key].delete(tail_id)
759
+ t = @paused_tails[key][tail_id]
819
760
 
820
- if @paused_tails[key].empty?
821
- @paused_tails.delete(key)
822
- end
761
+ @paused_tails[key].delete(tail_id)
823
762
 
824
- t
825
- end
826
- end
827
-
828
- # Called by host functions to write memory to wasm instance so that
829
- # the wasm module can read the result of a host function call
830
- def write_to_memory(caller, res)
831
- alloc = caller.export('alloc').to_func
832
- memory = caller.export('memory').to_memory
833
-
834
- # Serialize protobuf message
835
- resp = res.to_proto
763
+ @paused_tails.delete(key) if @paused_tails[key].empty?
836
764
 
837
- # Allocate memory for response
838
- resp_ptr = alloc.call(resp.length)
839
-
840
- # Write response to memory
841
- memory.write(resp_ptr, resp)
842
-
843
- # return 64bit integer where first 32 bits is the pointer, and the last 32 is the length
844
- resp_ptr << 32 | resp.length
765
+ t
845
766
  end
846
-
847
767
  end
848
- end
768
+ end
data/lib/wasm_spec.rb ADDED
@@ -0,0 +1,259 @@
1
+ require_relative 'spec_helper'
2
+ require 'steps/sp_steps_httprequest_pb'
3
+ require 'steps/sp_steps_detective_pb'
4
+ require 'steps/sp_steps_transform_pb'
5
+ require_relative 'streamdal'
6
+
7
+ class TestClient < Streamdal::Client
8
+
9
+ attr_accessor :kv
10
+
11
+ # Ignore rubocop warning
12
+ # rubocop:disable Lint/MissingSuper
13
+ def initialize
14
+ @cfg = {
15
+ step_timeout: 2
16
+ }
17
+ @functions = {}
18
+
19
+ logger = Logger.new($stdout)
20
+ logger.level = Logger::ERROR
21
+
22
+ @log = logger
23
+ @kv = Streamdal::KeyValue.new
24
+ @hostfunc = Streamdal::HostFunc.new(@kv)
25
+ end
26
+ end
27
+
28
+ RSpec.describe 'WASM' do
29
+ let(:client) { TestClient.new }
30
+
31
+ context '_call_wasm' do
32
+ it 'raises an error if step is nil' do
33
+ expect { client.send(:_call_wasm, nil, nil, nil) }.to raise_error('step is required')
34
+ end
35
+
36
+ it 'raises an error if data is nil' do
37
+ step = Streamdal::Protos::HttpRequestStep.new
38
+ expect { client.send(:_call_wasm, step, nil, nil) }.to raise_error('data is required')
39
+ end
40
+ end
41
+
42
+ context 'detective.wasm' do
43
+ before(:each) do
44
+ wasm_bytes = load_wasm('detective.wasm')
45
+
46
+ @step = Streamdal::Protos::PipelineStep.new
47
+ @step.name = 'detective'
48
+ @step._wasm_bytes = wasm_bytes.b
49
+ @step._wasm_id = SecureRandom.uuid
50
+ @step._wasm_function = 'f'
51
+ @step.detective = Streamdal::Protos::DetectiveStep.new
52
+ @step.detective.path = 'object.field'
53
+ @step.detective.args = Google::Protobuf::RepeatedField.new(:string, ['streamdal'])
54
+ @step.detective.negate = false
55
+ @step.detective.type = :DETECTIVE_TYPE_STRING_CONTAINS_ANY
56
+ end
57
+
58
+ it 'detects email in JSON payload' do
59
+ data = '{"object":{"field":"streamdal@gmail.com"}}'
60
+
61
+ res = client.send(:_call_wasm, @step, data, nil)
62
+
63
+ expect(res).not_to be_nil
64
+ expect(res.exit_code).to eq(:WASM_EXIT_CODE_TRUE)
65
+ expect(res.output_payload).to eq(data)
66
+
67
+ end
68
+
69
+ it 'does not detect email in JSON payload' do
70
+ data = '{"object":{"field":"mark@gmail.com"}}'
71
+
72
+ res = client.send(:_call_wasm, @step, data, nil)
73
+ expect(res).not_to be_nil
74
+ expect(res.exit_code).to eq(:WASM_EXIT_CODE_FALSE)
75
+ end
76
+ end
77
+
78
+ context 'httprequest.wasm' do
79
+ before(:each) do
80
+ wasm_bytes = load_wasm('httprequest.wasm')
81
+
82
+ http_req_step = Streamdal::Protos::HttpRequestStep.new
83
+ http_req_step.request = Streamdal::Protos::HttpRequest.new
84
+ http_req_step.request.url = 'https://www.google.com/404_me'
85
+ http_req_step.request.method = :HTTP_REQUEST_METHOD_GET
86
+ http_req_step.request.body = ''
87
+
88
+ @step = Streamdal::Protos::PipelineStep.new
89
+ @step.name = 'http request'
90
+ @step._wasm_bytes = wasm_bytes.b
91
+ @step._wasm_id = SecureRandom.uuid
92
+ @step._wasm_function = 'f'
93
+ @step.http_request = http_req_step
94
+ end
95
+
96
+ it 'returns false on 404' do
97
+ res = client.send(:_call_wasm, @step, '', nil)
98
+
99
+ expect(res).not_to be_nil
100
+ expect(res.exit_msg).to eq('Request returned non-200 response code: 404')
101
+ expect(res.exit_code).to eq(:WASM_EXIT_CODE_FALSE)
102
+ end
103
+ end
104
+
105
+ context 'inferschema.wasm' do
106
+ before(:each) do
107
+ wasm_bytes = load_wasm('inferschema.wasm')
108
+
109
+ @step = Streamdal::Protos::PipelineStep.new
110
+ @step.name = 'schema inference'
111
+ @step._wasm_bytes = wasm_bytes.b
112
+ @step._wasm_id = SecureRandom.uuid
113
+ @step._wasm_function = 'f'
114
+ @step.infer_schema = Streamdal::Protos::InferSchemaStep.new
115
+ end
116
+
117
+ it 'infers schema' do
118
+ payload = '{"object": {"payload": "test"}}'
119
+ res = client.send(:_call_wasm, @step, payload, nil)
120
+
121
+ expected_schema = '{"$schema":"http://json-schema.org/draft-07/schema#","properties":{"object":{"properties":{"payload":{"type":"string"}},"required":["payload"],"type":"object"}},"required":["object"],"type":"object"}'
122
+
123
+ expect(res).not_to be_nil
124
+ expect(res.exit_msg).to eq('inferred fresh schema')
125
+ expect(res.exit_code).to eq(:WASM_EXIT_CODE_TRUE)
126
+ expect(res.output_payload).to eq(payload)
127
+ expect(res.output_step).to eq(expected_schema)
128
+ end
129
+ end
130
+
131
+ context 'transform.wasm' do
132
+ before(:each) do
133
+ wasm_bytes = load_wasm('transform.wasm')
134
+
135
+ @step = Streamdal::Protos::PipelineStep.new
136
+ @step.name = 'transform'
137
+ @step._wasm_bytes = wasm_bytes.b
138
+ @step._wasm_id = SecureRandom.uuid
139
+ @step._wasm_function = 'f'
140
+ end
141
+
142
+ it 'deletes a field from a payload' do
143
+ @step.transform = Streamdal::Protos::TransformStep.new
144
+ @step.transform.type = :TRANSFORM_TYPE_DELETE_FIELD
145
+ @step.transform.delete_field_options = Streamdal::Protos::TransformDeleteFieldOptions.new
146
+ @step.transform.delete_field_options.paths = Google::Protobuf::RepeatedField.new(:string, ['object.another'])
147
+
148
+ payload = '{"object": {"payload": "old val", "another": "field"}}'
149
+ res = client.send(:_call_wasm, @step, payload, nil)
150
+
151
+ expect(res).not_to be_nil
152
+ expect(res.exit_msg).to eq('Successfully transformed payload')
153
+ expect(res.exit_code).to eq(:WASM_EXIT_CODE_TRUE)
154
+ expect(res.output_payload).to eq('{"object": {"payload": "old val"}}')
155
+ end
156
+
157
+ it 'replaces a fields value with a new one' do
158
+ @step.transform = Streamdal::Protos::TransformStep.new
159
+ @step.transform.type = :TRANSFORM_TYPE_REPLACE_VALUE
160
+ @step.transform.replace_value_options = Streamdal::Protos::TransformReplaceValueOptions.new
161
+ @step.transform.replace_value_options.path = 'object.payload'
162
+ @step.transform.replace_value_options.value = '"new val"'
163
+
164
+ payload = '{"object": {"payload": "old val"}}'
165
+ res = client.send(:_call_wasm, @step, payload, nil)
166
+
167
+ expect(res).not_to be_nil
168
+ expect(res.exit_msg).to eq('Successfully transformed payload')
169
+ expect(res.exit_code).to eq(:WASM_EXIT_CODE_TRUE)
170
+ expect(res.output_payload).to eq('{"object": {"payload": "new val"}}')
171
+ end
172
+
173
+ it 'truncates the value of a field' do
174
+ @step.transform = Streamdal::Protos::TransformStep.new
175
+ @step.transform.type = :TRANSFORM_TYPE_TRUNCATE_VALUE
176
+ @step.transform.truncate_options = Streamdal::Protos::TransformTruncateOptions.new
177
+ @step.transform.truncate_options.type = :TRANSFORM_TRUNCATE_TYPE_LENGTH
178
+ @step.transform.truncate_options.path = 'object.payload'
179
+ @step.transform.truncate_options.value = 3
180
+
181
+ payload = '{"object": {"payload": "old val"}}'
182
+ res = client.send(:_call_wasm, @step, payload, nil)
183
+
184
+ expect(res).not_to be_nil
185
+ expect(res.exit_msg).to eq('Successfully transformed payload')
186
+ expect(res.exit_code).to eq(:WASM_EXIT_CODE_TRUE)
187
+ expect(res.output_payload).to eq('{"object": {"payload": "old"}}')
188
+ end
189
+
190
+ it 'performs dynamic transformation' do
191
+ # TODO: add this test
192
+ end
193
+ end
194
+
195
+ context 'validjson.wasm' do
196
+ before(:each) do
197
+ wasm_bytes = load_wasm('validjson.wasm')
198
+
199
+ @step = Streamdal::Protos::PipelineStep.new
200
+ @step.name = 'validate json'
201
+ @step._wasm_bytes = wasm_bytes.b
202
+ @step._wasm_id = SecureRandom.uuid
203
+ @step._wasm_function = 'f'
204
+ @step.valid_json = Streamdal::Protos::ValidJSONStep.new
205
+
206
+ end
207
+
208
+ it 'validates a valid JSON payload' do
209
+
210
+ payload = '{"object": {"payload": "test"}}'
211
+ res = client.send(:_call_wasm, @step, payload, nil)
212
+
213
+ expect(res).not_to be_nil
214
+ expect(res.exit_code).to eq(:WASM_EXIT_CODE_TRUE)
215
+ expect(res.output_payload).to eq(payload)
216
+ end
217
+
218
+ it 'returns false on invalid JSON payload' do
219
+ payload = '{"object": {"payload": "test"'
220
+ res = client.send(:_call_wasm, @step, payload, nil)
221
+
222
+ expect(res).not_to be_nil
223
+ expect(res.exit_code).to eq(:WASM_EXIT_CODE_FALSE)
224
+ end
225
+ end
226
+
227
+ context 'kv.wasm' do
228
+ before(:each) do
229
+ wasm_bytes = load_wasm('kv.wasm')
230
+
231
+ client.kv.set('test', 'test')
232
+
233
+ @step = Streamdal::Protos::PipelineStep.new
234
+ @step.name = 'kv exists'
235
+ @step._wasm_bytes = wasm_bytes.b
236
+ @step._wasm_id = SecureRandom.uuid
237
+ @step._wasm_function = 'f'
238
+ @step.kv = Streamdal::Protos::KVStep.new
239
+ @step.kv.key = 'test'
240
+ @step.kv.mode = :KV_MODE_STATIC
241
+ @step.kv.action = :KV_ACTION_EXISTS
242
+ end
243
+
244
+ it 'returns true if a key exists' do
245
+ res = client.send(:_call_wasm, @step, '', nil)
246
+
247
+ expect(res).not_to be_nil
248
+ expect(res.exit_code).to eq(:WASM_EXIT_CODE_TRUE)
249
+ end
250
+
251
+ it 'returns false when a key doesnt exist' do
252
+ @step.kv.key = 'not_exists'
253
+ res = client.send(:_call_wasm, @step, '', nil)
254
+
255
+ expect(res).not_to be_nil
256
+ expect(res.exit_code).to eq(:WASM_EXIT_CODE_FALSE)
257
+ end
258
+ end
259
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: streamdal
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.2
4
+ version: 0.0.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Mark Gregan
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2024-06-19 00:00:00.000000000 Z
11
+ date: 2024-06-20 00:00:00.000000000 Z
12
12
  dependencies: []
13
13
  description:
14
14
  email: mark@streamdal.com
@@ -33,6 +33,7 @@ files:
33
33
  - lib/tail_spec.rb
34
34
  - lib/validation.rb
35
35
  - lib/validation_spec.rb
36
+ - lib/wasm_spec.rb
36
37
  homepage: https://docs.streamdal.com
37
38
  licenses:
38
39
  - Apache-2.0
@@ -45,7 +46,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
45
46
  requirements:
46
47
  - - '='
47
48
  - !ruby/object:Gem::Version
48
- version: 0.0.2
49
+ version: 0.0.4
49
50
  required_rubygems_version: !ruby/object:Gem::Requirement
50
51
  requirements:
51
52
  - - ">="