logstash-filter-fix_protocol 0.1.3 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bbf914880612f207ca4535d38a6adbba00d75713
-  data.tar.gz: 1de673ca2eb056627156f0984df6a7a4ac41ec9f
+  metadata.gz: 9c738b5a16d38b8c62fb2c1f03a6656af33e9ca8
+  data.tar.gz: 54367647f6258f5c17ad2739868ca855f5432439
 SHA512:
-  metadata.gz: 79be0dc6622bd4bca51a524443b40414b07c581cca300c38104010c794a114175b1274af517e90bb024b6ac73fba1f725b6ad4ee719318ec241db651c1c2122b
-  data.tar.gz: b5ac803915f4886257ed3849f02887521ccbb5b39f5bd3123edebfcbcd96f96727481a3ad7abee64532fb35359a1ce00fcad4eba8a90ec5ab10dda3ac3c77002
+  metadata.gz: 86c6277a75cce71efa8f413f887e30669bc4954aeba5acd9ce52d831272ab68215d103ea69e0ec8b950c8bf2198a099f8edbe30efeea18c283769b68c4b66015
+  data.tar.gz: c24afff68ef9a9bcf109a796aba773b6c17a5c0dfb4fcf3fd91d5928c0428e5d8e2d332795815d523db697d415e61059888bf6f7a75ce7959242708213d386cb

data/lib/logstash/filters/fix_message.rb

@@ -1,17 +1,16 @@
 require 'quickfix'
-require 'active_support/core_ext'
 
 module LogStash
   module Filters
     class FixMessage < quickfix.Message
-      attr_reader :type, :msg_string, :session_dictionary, :data_dictionary, :all_dictionaries
+      attr_reader :type, :msg_string, :session_dictionary, :data_dictionary, :all_dictionaries, :unknown_fields
 
       def initialize(msg_string, data_dictionary, session_dictionary)
         @session_dictionary = session_dictionary
         @data_dictionary = data_dictionary
         @msg_string = msg_string
         @type = quickfix.MessageUtils.getMessageType(msg_string)
-
+        @unknown_fields = []
         @all_dictionaries = [@data_dictionary, @session_dictionary]
 
         super(msg_string, data_dictionary, false)
@@ -44,6 +43,14 @@ module LogStash
           value = dd.get_field_name(tag)
           return value if value.present?
         end
+
+        tag = to_string(tag)
+        @unknown_fields << tag
+        tag
+      end
+
+      def to_string(tag)
+        tag.to_s.force_encoding("UTF-8")
       end
 
       def field_map_to_hash(field_map, msg_type = nil)
@@ -56,17 +63,21 @@ module LogStash
           tag   = field.get_tag
           value = field.get_value
 
-          # IF GROUP
           if msg_type.present? and @data_dictionary.is_group(msg_type, tag)
             groups = []
 
             for i in 1..value.to_i
-              group_map = field_map.get_group(i, tag)
-              groups << field_map_to_hash(group_map, msg_type)
+              begin
+                group_map  = field_map.get_group(i, tag)
+                group_hash = field_map_to_hash(group_map, msg_type)
+              rescue Java::Quickfix::FieldNotFound
+                group_hash = {to_string(tag) => i}
+                self.unknown_fields << to_string(tag)
+              end
+              groups << group_hash
             end
 
             value = groups
-          # IF FIELD
           elsif @data_dictionary.is_field(tag)
             value =
               case field_type(tag)

data/lib/logstash/filters/fix_protocol.rb

@@ -3,6 +3,7 @@ require "logstash/filters/base"
 require "logstash/namespace"
 require "logstash/filters/data_dictionary"
 require "logstash/filters/fix_message"
+require 'active_support/core_ext'
 
 module LogStash
   module  Filters
@@ -48,18 +49,15 @@ module LogStash
           if fix_message
             fix_hash = fix_message.to_hash
 
+            # TODO: Unknown tags are only set after calling #to_hash
+            # this creates an implicit timing issue
+            if fix_message.unknown_fields.any?
+              event["unknown_fields"] = fix_message.unknown_fields
+              event["tags"] = ["_fix_field_not_found"]
+            end
+
             fix_hash.each do |key, value|
-              begin
-                event[key] = value
-              rescue NoMethodError => e
-                puts "********"
-                puts "WARNING: Could not correctly parse #{event["fix_message"]}"
-                puts JSON.pretty_generate(fix_hash)
-                puts "Message: #{e.message}"
-                puts "********"
-              ensure
-                next
-              end
+              event[key] = value
             end
           end
         end

data/logstash-filter-fix_protocol.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |s|
   s.name          = "logstash-filter-fix_protocol"
-  s.version       = "0.1.3"
+  s.version       = "0.2.0"
   s.authors       = ["Connamara Systems"]
   s.email         = ["info@connamara.com"]
 

data/spec/filters/fix_message_spec.rb

@@ -9,6 +9,9 @@ describe LF::FixMessage do
   let(:message) { LF::FixMessage.new(message_str, data_dictionary, session_dictionary) }
   let(:message2) { LF::FixMessage.new(another_str, data_dictionary, session_dictionary) }
 
+  let(:fix_4)  { {data_dictionary: "FIX42.xml", session_dictionary: nil} }
+  let(:fix_5)  { {data_dictionary: "FIX50SP1.xml", session_dictionary: "FIXT11.xml"} }
+
   describe '#to_hash' do
     it 'converts the FIX message string to a hash in human readable format' do
       expect(message.to_hash).to eq({
@@ -59,10 +62,41 @@ describe LF::FixMessage do
     end
   end
 
+  describe '#field_map_to_hash' do
+    context 'invalid message - field name not found' do
+      let(:fix_string) { "8=FIX.4.2
9=240
35=8
34=6
49=DUMMY_INC
52=20150826-23:10:17.744
56=ANOTHER_INC
57=Firm_B
1=Inst_B
6=0
11=best_buy
14=5
17=ITRZ1201508261_24
20=0
22=8
31=1010
32=5
37=ITRZ1201508261_12
38=5
39=2
7012=227
40=2
41=best_buy
44=1011
54=1
55=ITRZ1
60=20150826-23:10:15.547
150=2
151=0
10=227
" }
+
+      it 'adds an error object and uses the data dictionary index number as the key' do
+        data_dictionary = LF::DataDictionary.new(load_fixture(fix_5[:data_dictionary]))
+        sess_dictionary = LF::DataDictionary.new(load_fixture(fix_5[:session_dictionary]))
+
+        fix_message = LF::FixMessage.new(fix_string, data_dictionary, sess_dictionary)
+        hash = fix_message.to_hash
+        # Side Note: field 20, LastShares, was changed between FIX 4 / 5
+        expect(hash["20"]).to eq("0")
+        expect(hash["7012"]).to eq("227")
+        expect(fix_message.unknown_fields.include?("20")).to be true
+        expect(fix_message.unknown_fields.include?("7012")).to be true
+      end
+    end
+
+    context 'invalid message - group field name not found (Java::Quickfix::FieldNotFound)' do
+      let(:fix_string) { "8=FIX.4.2
35=D
34=2
49=TW
52=<TIME>
56=ISLD
11=ID
21=1
40=1
54=1
38=200.00
55=INTC
386=3
336=PRE-OPEN
336=AFTER-HOURS
60=<TIME>
" }
+
+      it 'adds an error object and uses the data dictionary index number as the key' do
+        data_dictionary = LF::DataDictionary.new(load_fixture(fix_5[:data_dictionary]))
+        sess_dictionary = LF::DataDictionary.new(load_fixture(fix_5[:session_dictionary]))
+
+        fix_message = LF::FixMessage.new(fix_string, data_dictionary, sess_dictionary)
+        hash = fix_message.to_hash
+        expect(fix_message.unknown_fields.include?("386")).to be true
+        expect(hash["NoTradingSessions"]).to eq([{"TradingSessionID"=>"PRE-OPEN"}, {"TradingSessionID"=>"AFTER-HOURS"}, {"386"=>3}])
+      end
+    end
+  end
+
   context 'message types' do
-    let(:fix_4)  { {data_dictionary: "FIX42.xml", session_dictionary: nil} }
-    let(:fix_5)  { {data_dictionary: "FIX50SP1.xml", session_dictionary: "FIXT11.xml"} }
-    # data is from: http://fixparser.targetcompid.com/
+    # most data is from: http://fixparser.targetcompid.com/
     context 'heartbeats' do
       it 'can parse these' do
         [fix_4, fix_5].each do |version|
@@ -88,8 +122,7 @@ describe LF::FixMessage do
             expect(["Logon", "LOGON"].include?(hash["MsgType"])).to be true
 
             expect(hash["BeginString"]).to be_a String
-            # NOTE: This field was changed between FIX 4 / 5
-            # expect([String, Fixnum].include?(hash["HeartBtInt"].class)).to be true
+            expect([String, Fixnum].include?(hash["HeartBtInt"].class)).to be true
 
             expect(["BANZAI", "EXEC"].include?(hash["TargetCompID"])).to be true
             expect(["BANZAI", "EXEC"].include?(hash["SenderCompID"])).to be true

data/spec/filters/fix_protocol_spec.rb

@@ -36,14 +36,31 @@ describe LF::FixProtocol do
     end
   end
 
+  context 'an incoming execution report' do
+    config fix_4_configuration
+
+    execution = "8=FIXT.1.1\x0135=8\x0149=ITG\x0156=SILO\x01315=8\x016=100.25\x01410=50.25\x01424=23.45\x01411=Y\x0143=N\x0140=1\x015=N\x01"
+
+    sample(execution) do
+      filtered_event = subject
+      insist { filtered_event["BeginString"] } == "FIXT.1.1"
+      insist { filtered_event["MsgType"] } == "ExecutionReport"
+      insist { filtered_event["SenderCompID"] } == "ITG"
+      insist { filtered_event["AvgPx"] } == 100.25
+      insist { filtered_event["OrdType"] } == "MARKET"
+      insist { filtered_event["UnderlyingPutOrCall"] } == 8
+    end
+  end
+
   context 'invalid message - Java::Quickfix::InvalidMessage' do
     config fix_4_configuration
 
     invalid_msg = "8=invalid_stuff"
 
     sample(invalid_msg) do
-      insist { subject["tags"] } == ["_fix_parse_failure"]
-      insist { subject["message"] } == invalid_msg
+      filtered_event = subject
+      insist { filtered_event["tags"] } == ["_fix_parse_failure"]
+      insist { filtered_event["message"] } == invalid_msg
     end
   end
 
@@ -53,41 +70,48 @@ describe LF::FixProtocol do
     invalid_msg = "8=FIX.4.0
9=81
35=D
34=3
49garbled=TW
52=<TIME>
56=ISLD
11=ID
21=3
40=1
54=1
55=INTC
10=0
"
 
     sample(invalid_msg) do
-      insist { subject["tags"] } == ["_fix_parse_failure"]
-      insist { subject["message"] } == invalid_msg
+      filtered_event = subject
+      insist { filtered_event["tags"] } == ["_fix_parse_failure"]
+      insist { filtered_event["message"] } == invalid_msg
     end
   end
 
-  context 'an incoming execution report' do
+  context 'invalid message - group field not found - Java::Quickfix::FieldNotFound' do
     config fix_4_configuration
 
-    execution = "8=FIXT.1.1\x0135=8\x0149=ITG\x0156=SILO\x01315=8\x016=100.25\x01410=50.25\x01424=23.45\x01411=Y\x0143=N\x0140=1\x015=N\x01"
+    invalid_msg = "8=FIX.4.2
35=D
34=2
49=TW
52=<TIME>
56=ISLD
11=ID
21=1
40=1
54=1
38=200.00
55=INTC
386=3
336=PRE-OPEN
336=AFTER-HOURS
60=<TIME>
"
 
-    sample(execution) do
+    sample(invalid_msg) do
       filtered_event = subject
-      insist { filtered_event["BeginString"] } == "FIXT.1.1"
-      insist { filtered_event["MsgType"] } == "ExecutionReport"
-      insist { filtered_event["SenderCompID"] } == "ITG"
-      insist { filtered_event["AvgPx"] } == 100.25
-      insist { filtered_event["OrdType"] } == "MARKET"
-      insist { filtered_event["UnderlyingPutOrCall"] } == 8
+      # adds a tag to warn that fields weren't found in the DD
+      insist { filtered_event["tags"] } == ["_fix_field_not_found"]
+      insist { filtered_event["unknown_fields"] } == ["386"]
+      # correctly parses as much as it can
+      insist { filtered_event["OrderQty"]} == 200.0
+      insist { filtered_event["OrdType"]} == "MARKET"
+      insist { filtered_event["Side"]} == "BUY"
+      insist { filtered_event["Symbol"]} == "INTC"
+      insist { filtered_event["NoTradingSessions"]} == [{"TradingSessionID"=>"PRE-OPEN"}, {"TradingSessionID"=>"AFTER-HOURS"}, {"386"=>3}] # here's the rescued group field
     end
   end
 
-  context 'it removes unparseable key-value pairs' do
+  context 'invalid message - field name not found' do
     config fix_5_configuration
 
     execution = "8=FIX.4.2\x019=240\x0135=8\x0134=6\x0149=DUMMY_INC\x0152=20150826-23:10:17.744\x0156=ANOTHER_INC\x0157=Firm_B\x011=Inst_B\x016=0\x0111=151012569\x0117=ITRZ1201508261_24\x0120=0\x0122=8\x0131=1010\x0132=5\x0137=ITRZ1201508261_12\x0138=5\x0139=2\x0140=2\x0141=best_buy\x0144=1011\x0154=1\x0155=ITRZ1\x0160=20150826-23:10:15.547\x01150=2\x01151=0\x0110=227\x01"
 
     sample(execution) do
-      expect { subject }.to output.to_stdout
       filtered_event = subject
+      # adds a tag to warn that fields weren't found in the DD
+      insist { filtered_event["tags"] } == ["_fix_field_not_found"]
+      insist { filtered_event["unknown_fields"] } == ["20"]
+      # correctly parses as much as it can
       insist { filtered_event["BeginString"] } == "FIX.4.2"
       insist { filtered_event["MsgType"] } == "ExecutionReport"
       insist { filtered_event["SenderCompID"] } == "DUMMY_INC"
       insist { filtered_event["AvgPx"] } == 0.0
       insist { filtered_event["OrdType"] } == "LIMIT"
-      insist { filtered_event["LeavesQty"] } == 0.0 # this should fail if parsing gets rescued, but doesnt finish setting on the event object
+      insist { filtered_event["LeavesQty"] } == 0.0
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-fix_protocol
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.2.0
 platform: ruby
 authors:
 - Connamara Systems
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-03-02 00:00:00.000000000 Z
+date: 2016-03-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement