logstash-filter-fix_protocol 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CODE_OF_CONDUCT.md +13 -0
- data/CONTRIBUTING.md +34 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +13 -0
- data/QUICKFIX_LICENSE.txt +46 -0
- data/README.md +151 -0
- data/lib/logstash/filters/data_dictionary.rb +16 -0
- data/lib/logstash/filters/fix_message.rb +91 -0
- data/lib/logstash/filters/fix_protocol.rb +61 -0
- data/logstash-filter-fix_message_filter.gemspec +34 -0
- data/spec/filters/fix_message_spec.rb +212 -0
- data/spec/filters/fix_protocol_spec.rb +71 -0
- data/spec/fixtures/FIX42.xml +2670 -0
- data/spec/fixtures/FIX50SP1.xml +9419 -0
- data/spec/fixtures/FIXT11.xml +383 -0
- data/spec/fixtures/fix-input.log +11 -0
- data/spec/fixtures/message_types/execution_report.txt +6 -0
- data/spec/fixtures/message_types/heartbeat.txt +2 -0
- data/spec/fixtures/message_types/logon.txt +2 -0
- data/spec/fixtures/message_types/market_data_snapshot.txt +2 -0
- data/spec/fixtures/message_types/new_order_single.txt +4 -0
- data/spec/fixtures/message_types/order_cancel_request.txt +3 -0
- data/spec/fixtures/message_types/reject.txt +2 -0
- data/spec/spec_helper.rb +14 -0
- data/spec/support/data_dictionary_helper.rb +45 -0
- data/spec/support/fix_configuration.rb +33 -0
- data/spec/support/fixture_helper.rb +10 -0
- data/spec/support/logstash_helper.rb +3 -0
- metadata +224 -0
@@ -0,0 +1,212 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe LF::FixMessage do
|
4
|
+
let(:message_str) { "8=FIXT.1.1\x0135=8\x0149=ITG\x0156=SILO\x01315=8\x016=100.25\x01410=50.25\x01424=23.45\x01411=Y\x0143=N\x0140=1\x015=N\x01" }
|
5
|
+
let(:another_str) { "8=FIXT.1.1\x0135=B\x0149=ITG\x0156=SILO\x01148=Market Bulls Have Short Sellers on the Run\x0133=2\x0158=The bears have been cowed by the bulls.\x0158=Buy buy buy\x01354=0\x0143=N\x0140=1\x015=N\x01" }
|
6
|
+
|
7
|
+
let(:data_dictionary) { LF::DataDictionary.new(load_fixture("FIX50SP1.xml")) }
|
8
|
+
let(:session_dictionary) { LF::DataDictionary.new(load_fixture("FIXT11.xml")) }
|
9
|
+
let(:message) { LF::FixMessage.new(message_str, data_dictionary, session_dictionary) }
|
10
|
+
let(:message2) { LF::FixMessage.new(another_str, data_dictionary, session_dictionary) }
|
11
|
+
|
12
|
+
describe '#to_hash' do
|
13
|
+
it 'converts the FIX message string to a hash in human readable format' do
|
14
|
+
expect(message.to_hash).to eq({
|
15
|
+
"BeginString"=>"FIXT.1.1",
|
16
|
+
"MsgType"=>"ExecutionReport",
|
17
|
+
"PossDupFlag"=>false,
|
18
|
+
"SenderCompID"=>"ITG",
|
19
|
+
"TargetCompID"=>"SILO",
|
20
|
+
"AdvTransType"=>"NEW",
|
21
|
+
"AvgPx"=>100.25,
|
22
|
+
"OrdType"=>"MARKET",
|
23
|
+
"UnderlyingPutOrCall"=>8,
|
24
|
+
"WtAverageLiquidity"=>"50.25",
|
25
|
+
"ExchangeForPhysical"=>true,
|
26
|
+
"DayOrderQty"=>23.45
|
27
|
+
})
|
28
|
+
|
29
|
+
expect(message2.to_hash).to eq({
|
30
|
+
"BeginString"=>"FIXT.1.1",
|
31
|
+
"MsgType"=>"News",
|
32
|
+
"PossDupFlag"=>false,
|
33
|
+
"SenderCompID"=>"ITG",
|
34
|
+
"TargetCompID"=>"SILO",
|
35
|
+
"AdvTransType"=>"NEW",
|
36
|
+
"NoLinesOfText"=>[
|
37
|
+
{
|
38
|
+
"Text"=>"The bears have been cowed by the bulls."
|
39
|
+
}, {
|
40
|
+
"Text"=>"Buy buy buy", "EncodedTextLen"=>"0"
|
41
|
+
}
|
42
|
+
],
|
43
|
+
"OrdType"=>"MARKET",
|
44
|
+
"Headline"=>"Market Bulls Have Short Sellers on the Run"
|
45
|
+
})
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
def should_parse_fix_messages(file_path, dictionary = "FIX42.xml", session_dictionary = nil)
|
50
|
+
data_dictionary = LF::DataDictionary.new(load_fixture(dictionary))
|
51
|
+
session_dictionary = session_dictionary.present? ? LF::DataDictionary.new(load_fixture(session_dictionary)) : data_dictionary
|
52
|
+
|
53
|
+
File.open(load_fixture(file_path), encoding: 'UTF-8') do |file|
|
54
|
+
file.each_entry do |line|
|
55
|
+
line.chomp! # remove new line character
|
56
|
+
message = LF::FixMessage.new(line, data_dictionary, session_dictionary)
|
57
|
+
yield(message.to_hash)
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
context 'message types' do
|
63
|
+
let(:fix_4) { {data_dictionary: "FIX42.xml", session_dictionary: nil} }
|
64
|
+
let(:fix_5) { {data_dictionary: "FIX50SP1.xml", session_dictionary: "FIXT11.xml"} }
|
65
|
+
# data is from: http://fixparser.targetcompid.com/
|
66
|
+
context 'heartbeats' do
|
67
|
+
it 'can parse these' do
|
68
|
+
[fix_4, fix_5].each do |version|
|
69
|
+
should_parse_fix_messages('message_types/heartbeat.txt', version[:data_dictionary], version[:session_dictionary]) do |hash|
|
70
|
+
expect(["Heartbeat", "HEARTBEAT"].include?(hash["MsgType"])).to be true
|
71
|
+
|
72
|
+
expect(hash["BeginString"]).to be_a String
|
73
|
+
expect(hash["SendingTime"]).to be_a String
|
74
|
+
expect(hash["CheckSum"]).to be_a String
|
75
|
+
expect([String, Fixnum].include?(hash["BodyLength"].class)).to be true
|
76
|
+
expect([String, Fixnum].include?(hash["MsgSeqNum"].class)).to be true
|
77
|
+
expect(["BANZAI", "EXEC"].include?(hash["TargetCompID"])).to be true
|
78
|
+
expect(["BANZAI", "EXEC"].include?(hash["SenderCompID"])).to be true
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
context 'logon' do
|
85
|
+
it 'can parse these' do
|
86
|
+
[fix_4, fix_5].each do |version|
|
87
|
+
should_parse_fix_messages('message_types/logon.txt', version[:data_dictionary], version[:session_dictionary]) do |hash|
|
88
|
+
expect(["Logon", "LOGON"].include?(hash["MsgType"])).to be true
|
89
|
+
|
90
|
+
expect(hash["BeginString"]).to be_a String
|
91
|
+
# NOTE: This field was changed between FIX 4 / 5
|
92
|
+
# expect([String, Fixnum].include?(hash["HeartBtInt"].class)).to be true
|
93
|
+
|
94
|
+
expect(["BANZAI", "EXEC"].include?(hash["TargetCompID"])).to be true
|
95
|
+
expect(["BANZAI", "EXEC"].include?(hash["SenderCompID"])).to be true
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
context 'execution_report' do
|
102
|
+
it 'can parse these' do
|
103
|
+
[fix_4, fix_5].each do |version|
|
104
|
+
should_parse_fix_messages('message_types/execution_report.txt', version[:data_dictionary], version[:session_dictionary]) do |hash|
|
105
|
+
expect(["ExecutionReport"].include?(hash["MsgType"])).to be true
|
106
|
+
|
107
|
+
expect(hash["ClOrdID"]).to be_a String
|
108
|
+
expect(hash["Symbol"]).to be_a String
|
109
|
+
|
110
|
+
expect(["NEW", "FILLED"].include?(hash["OrdStatus"])).to be true
|
111
|
+
expect(["BUY", "SELL"].include?(hash["Side"])).to be true
|
112
|
+
|
113
|
+
expect([String, Float].include?(hash["LastPx"].class)).to be true
|
114
|
+
# NOTE: This field was changed between FIX 4 / 5
|
115
|
+
# expect([String, Float].include?(hash["LastShares"].class)).to be true
|
116
|
+
expect([String, Float].include?(hash["OrderQty"].class)).to be true
|
117
|
+
|
118
|
+
expect(hash["TargetSubID"]).to be_a(String) if hash["TargetSubID"].present?
|
119
|
+
|
120
|
+
expect(["BANZAI", "EXEC", "ANOTHER_INC"].include?(hash["TargetCompID"])).to be true
|
121
|
+
expect(["BANZAI", "EXEC", "DUMMY_INC"].include?(hash["SenderCompID"])).to be true
|
122
|
+
end
|
123
|
+
end
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
127
|
+
context 'new order single' do
|
128
|
+
it 'can parse these' do
|
129
|
+
[fix_4, fix_5].each do |version|
|
130
|
+
should_parse_fix_messages('message_types/new_order_single.txt', version[:data_dictionary], version[:session_dictionary]) do |hash|
|
131
|
+
expect(hash["MsgType"]).to eq "NewOrderSingle"
|
132
|
+
|
133
|
+
expect(hash["ClOrdID"]).to be_a String
|
134
|
+
expect(hash["Symbol"]).to be_a String
|
135
|
+
expect(hash["TimeInForce"]).to be_a String
|
136
|
+
expect(hash["OrderQty"]).to be_a Float
|
137
|
+
|
138
|
+
expect(["MARKET", "LIMIT"].include?(hash["OrdType"])).to be true
|
139
|
+
expect(["BUY", "SELL"].include?(hash["Side"])).to be true
|
140
|
+
|
141
|
+
expect(["BANZAI", "EXEC", "DUMMY_INC"].include?(hash["TargetCompID"])).to be true
|
142
|
+
expect(["BANZAI", "EXEC", "ANOTHER_INC"].include?(hash["SenderCompID"])).to be true
|
143
|
+
end
|
144
|
+
end
|
145
|
+
end
|
146
|
+
end
|
147
|
+
|
148
|
+
context 'order cancel request' do
|
149
|
+
it 'can parse these' do
|
150
|
+
[fix_4, fix_5].each do |version|
|
151
|
+
should_parse_fix_messages('message_types/order_cancel_request.txt', version[:data_dictionary], version[:session_dictionary]) do |hash|
|
152
|
+
expect(hash["MsgType"]).to eq "OrderCancelRequest"
|
153
|
+
|
154
|
+
expect(hash["ClOrdID"]).to be_a String
|
155
|
+
expect(hash["OrigClOrdID"]).to be_a String
|
156
|
+
expect(hash["Symbol"]).to be_a String
|
157
|
+
|
158
|
+
expect(hash["OrderQty"]).to be_a(Float) if hash["OrderQty"].present?
|
159
|
+
expect(["FUTURE"].include?(hash["SecurityType"])).to be(true) if hash["SecurityType"].present?
|
160
|
+
|
161
|
+
expect(["BUY", "SELL"].include?(hash["Side"])).to be true
|
162
|
+
|
163
|
+
expect(["BANZAI", "EXEC", "DUMMY_INC"].include?(hash["TargetCompID"])).to be true
|
164
|
+
expect(["BANZAI", "EXEC", "ANOTHER_INC"].include?(hash["SenderCompID"])).to be true
|
165
|
+
end
|
166
|
+
end
|
167
|
+
end
|
168
|
+
end
|
169
|
+
|
170
|
+
context 'rejects' do
|
171
|
+
it 'can parse these' do
|
172
|
+
[fix_4, fix_5].each do |version|
|
173
|
+
should_parse_fix_messages('message_types/reject.txt', version[:data_dictionary], version[:session_dictionary]) do |hash|
|
174
|
+
expect(["Reject", "REJECT"].include?(hash["MsgType"])).to be true
|
175
|
+
expect(hash["Text"]).to eq "Unsupported message type"
|
176
|
+
|
177
|
+
expect(["BANZAI", "EXEC"].include?(hash["TargetCompID"])).to be true
|
178
|
+
expect(["BANZAI", "EXEC"].include?(hash["SenderCompID"])).to be true
|
179
|
+
end
|
180
|
+
end
|
181
|
+
end
|
182
|
+
end
|
183
|
+
|
184
|
+
context 'market data snapshots' do
|
185
|
+
it 'can parse these' do
|
186
|
+
[fix_4, fix_5].each do |version|
|
187
|
+
should_parse_fix_messages('message_types/market_data_snapshot.txt', version[:data_dictionary], version[:session_dictionary]) do |hash|
|
188
|
+
expect(["MarketDataSnapshotFullRefresh", "REJECT"].include?(hash["MsgType"])).to be true
|
189
|
+
|
190
|
+
expect(hash["NoMDEntries"]).to be_a(Array)
|
191
|
+
expect(hash["NoMDEntries"].first).to be_a(Hash)
|
192
|
+
expect(hash["NoMDEntries"].first["MDEntryPx"]).to be_a(Float)
|
193
|
+
expect(hash["NoMDEntries"].first["MDEntrySize"]).to be_a(Float) if hash["NoMDEntries"].first["MDEntrySize"].present?
|
194
|
+
|
195
|
+
expect(["ANOTHER_INC"].include?(hash["TargetCompID"])).to be true
|
196
|
+
expect(["DUMMY_INC"].include?(hash["SenderCompID"])).to be true
|
197
|
+
end
|
198
|
+
end
|
199
|
+
end
|
200
|
+
end
|
201
|
+
|
202
|
+
context 'human-readable group enums' do
|
203
|
+
it 'can parse these' do
|
204
|
+
[fix_4, fix_5].each do |version|
|
205
|
+
should_parse_fix_messages('message_types/market_data_snapshot.txt', version[:data_dictionary], version[:session_dictionary]) do |hash|
|
206
|
+
expect(["BID", "OFFER", "INDEX_VALUE"].include?(hash["NoMDEntries"].first["MDEntryType"])).to be true
|
207
|
+
end
|
208
|
+
end
|
209
|
+
end
|
210
|
+
end
|
211
|
+
end
|
212
|
+
end
|
@@ -0,0 +1,71 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe LF::FixProtocol do
|
4
|
+
let(:fix_5_config) do
|
5
|
+
{
|
6
|
+
"message" => "fix_message",
|
7
|
+
"session_dictionary_path" => load_fixture("FIXT11.xml"),
|
8
|
+
"data_dictionary_path" => load_fixture("FIX50SP1.xml")
|
9
|
+
}
|
10
|
+
end
|
11
|
+
|
12
|
+
let(:fix_4_config) do
|
13
|
+
{
|
14
|
+
"message" => "fix_message",
|
15
|
+
"data_dictionary_path" => load_fixture("FIX42.xml")
|
16
|
+
}
|
17
|
+
end
|
18
|
+
|
19
|
+
describe 'config' do
|
20
|
+
context 'fix 4 configuration' do
|
21
|
+
let(:filter) { LF::FixProtocol.new(fix_4_config) }
|
22
|
+
|
23
|
+
it 'reuses the data dictionary as the session dictionary' do
|
24
|
+
expect(filter.data_dictionary).to be_a(LF::DataDictionary)
|
25
|
+
expect(filter.session_dictionary == filter.data_dictionary).to be true
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
context 'fix 5 configuration' do
|
30
|
+
let(:filter) { LF::FixProtocol.new(fix_5_config) }
|
31
|
+
|
32
|
+
it 'instantiates a new data dictionary for a session dictionary' do
|
33
|
+
expect(filter.data_dictionary).to be_a(LF::DataDictionary)
|
34
|
+
expect(filter.session_dictionary == filter.data_dictionary).to be false
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
context 'an incoming execution report' do
|
40
|
+
config fix_4_configuration
|
41
|
+
|
42
|
+
execution = "8=FIXT.1.1\x0135=8\x0149=ITG\x0156=SILO\x01315=8\x016=100.25\x01410=50.25\x01424=23.45\x01411=Y\x0143=N\x0140=1\x015=N\x01"
|
43
|
+
|
44
|
+
sample("fix_message" => execution) do
|
45
|
+
filtered_event = subject
|
46
|
+
insist { filtered_event["BeginString"] } == "FIXT.1.1"
|
47
|
+
insist { filtered_event["MsgType"] } == "ExecutionReport"
|
48
|
+
insist { filtered_event["SenderCompID"] } == "ITG"
|
49
|
+
insist { filtered_event["AvgPx"] } == 100.25
|
50
|
+
insist { filtered_event["OrdType"] } == "MARKET"
|
51
|
+
insist { filtered_event["UnderlyingPutOrCall"] } == 8
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
context 'it removes unparseable key-value pairs' do
|
56
|
+
config fix_5_configuration
|
57
|
+
|
58
|
+
execution = "8=FIX.4.2\x019=240\x0135=8\x0134=6\x0149=DUMMY_INC\x0152=20150826-23:10:17.744\x0156=ANOTHER_INC\x0157=Firm_B\x011=Inst_B\x016=0\x0111=151012569\x0117=ITRZ1201508261_24\x0120=0\x0122=8\x0131=1010\x0132=5\x0137=ITRZ1201508261_12\x0138=5\x0139=2\x0140=2\x0141=best_buy\x0144=1011\x0154=1\x0155=ITRZ1\x0160=20150826-23:10:15.547\x01150=2\x01151=0\x0110=227\x01"
|
59
|
+
|
60
|
+
sample("fix_message" => execution) do
|
61
|
+
expect { subject }.to output.to_stdout
|
62
|
+
filtered_event = subject
|
63
|
+
insist { filtered_event["BeginString"] } == "FIX.4.2"
|
64
|
+
insist { filtered_event["MsgType"] } == "ExecutionReport"
|
65
|
+
insist { filtered_event["SenderCompID"] } == "DUMMY_INC"
|
66
|
+
insist { filtered_event["AvgPx"] } == 0.0
|
67
|
+
insist { filtered_event["OrdType"] } == "LIMIT"
|
68
|
+
insist { filtered_event["LeavesQty"] } == 0.0 # this should fail if parsing gets rescued, but doesnt finish setting on the event object
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|