avro-salsify-fork 1.9.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,551 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing, software
12
+ # distributed under the License is distributed on an "AS IS" BASIS,
13
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ # See the License for the specific language governing permissions and
15
+ # limitations under the License.
16
+
17
+ require "net/http"
18
+
19
+ module Avro::IPC
20
+
21
+ class AvroRemoteError < Avro::AvroError; end
22
+
23
+ HANDSHAKE_REQUEST_SCHEMA = Avro::Schema.parse <<-JSON
24
+ {
25
+ "type": "record",
26
+ "name": "HandshakeRequest", "namespace":"org.apache.avro.ipc",
27
+ "fields": [
28
+ {"name": "clientHash",
29
+ "type": {"type": "fixed", "name": "MD5", "size": 16}},
30
+ {"name": "clientProtocol", "type": ["null", "string"]},
31
+ {"name": "serverHash", "type": "MD5"},
32
+ {"name": "meta", "type": ["null", {"type": "map", "values": "bytes"}]}
33
+ ]
34
+ }
35
+ JSON
36
+
37
+ HANDSHAKE_RESPONSE_SCHEMA = Avro::Schema.parse <<-JSON
38
+ {
39
+ "type": "record",
40
+ "name": "HandshakeResponse", "namespace": "org.apache.avro.ipc",
41
+ "fields": [
42
+ {"name": "match",
43
+ "type": {"type": "enum", "name": "HandshakeMatch",
44
+ "symbols": ["BOTH", "CLIENT", "NONE"]}},
45
+ {"name": "serverProtocol", "type": ["null", "string"]},
46
+ {"name": "serverHash",
47
+ "type": ["null", {"type": "fixed", "name": "MD5", "size": 16}]},
48
+ {"name": "meta",
49
+ "type": ["null", {"type": "map", "values": "bytes"}]}
50
+ ]
51
+ }
52
+ JSON
53
+
54
+ HANDSHAKE_REQUESTOR_WRITER = Avro::IO::DatumWriter.new(HANDSHAKE_REQUEST_SCHEMA)
55
+ HANDSHAKE_REQUESTOR_READER = Avro::IO::DatumReader.new(HANDSHAKE_RESPONSE_SCHEMA)
56
+ HANDSHAKE_RESPONDER_WRITER = Avro::IO::DatumWriter.new(HANDSHAKE_RESPONSE_SCHEMA)
57
+ HANDSHAKE_RESPONDER_READER = Avro::IO::DatumReader.new(HANDSHAKE_REQUEST_SCHEMA)
58
+
59
+ META_SCHEMA = Avro::Schema.parse('{"type": "map", "values": "bytes"}')
60
+ META_WRITER = Avro::IO::DatumWriter.new(META_SCHEMA)
61
+ META_READER = Avro::IO::DatumReader.new(META_SCHEMA)
62
+
63
+ SYSTEM_ERROR_SCHEMA = Avro::Schema.parse('["string"]')
64
+
65
+ # protocol cache
66
+ REMOTE_HASHES = {}
67
+ REMOTE_PROTOCOLS = {}
68
+
69
+ BUFFER_HEADER_LENGTH = 4
70
+ BUFFER_SIZE = 8192
71
+
72
+ # Raised when an error message is sent by an Avro requestor or responder.
73
+ class AvroRemoteException < Avro::AvroError; end
74
+
75
+ class ConnectionClosedException < Avro::AvroError; end
76
+
77
+ class Requestor
78
+ """Base class for the client side of a protocol interaction."""
79
+ attr_reader :local_protocol, :transport
80
+ attr_accessor :remote_protocol, :remote_hash, :send_protocol
81
+
82
+ def initialize(local_protocol, transport)
83
+ @local_protocol = local_protocol
84
+ @transport = transport
85
+ @remote_protocol = nil
86
+ @remote_hash = nil
87
+ @send_protocol = nil
88
+ end
89
+
90
+ def remote_protocol=(new_remote_protocol)
91
+ @remote_protocol = new_remote_protocol
92
+ REMOTE_PROTOCOLS[transport.remote_name] = remote_protocol
93
+ end
94
+
95
+ def remote_hash=(new_remote_hash)
96
+ @remote_hash = new_remote_hash
97
+ REMOTE_HASHES[transport.remote_name] = remote_hash
98
+ end
99
+
100
+ def request(message_name, request_datum)
101
+ # Writes a request message and reads a response or error message.
102
+ # build handshake and call request
103
+ buffer_writer = StringIO.new(''.force_encoding('BINARY'))
104
+ buffer_encoder = Avro::IO::BinaryEncoder.new(buffer_writer)
105
+ write_handshake_request(buffer_encoder)
106
+ write_call_request(message_name, request_datum, buffer_encoder)
107
+
108
+ # send the handshake and call request; block until call response
109
+ call_request = buffer_writer.string
110
+ call_response = transport.transceive(call_request)
111
+
112
+ # process the handshake and call response
113
+ buffer_decoder = Avro::IO::BinaryDecoder.new(StringIO.new(call_response))
114
+ if read_handshake_response(buffer_decoder)
115
+ read_call_response(message_name, buffer_decoder)
116
+ else
117
+ request(message_name, request_datum)
118
+ end
119
+ end
120
+
121
+ def write_handshake_request(encoder)
122
+ local_hash = local_protocol.md5
123
+ remote_name = transport.remote_name
124
+ remote_hash = REMOTE_HASHES[remote_name]
125
+ unless remote_hash
126
+ remote_hash = local_hash
127
+ self.remote_protocol = local_protocol
128
+ end
129
+ request_datum = {
130
+ 'clientHash' => local_hash,
131
+ 'serverHash' => remote_hash
132
+ }
133
+ if send_protocol
134
+ request_datum['clientProtocol'] = local_protocol.to_s
135
+ end
136
+ HANDSHAKE_REQUESTOR_WRITER.write(request_datum, encoder)
137
+ end
138
+
139
+ def write_call_request(message_name, request_datum, encoder)
140
+ # The format of a call request is:
141
+ # * request metadata, a map with values of type bytes
142
+ # * the message name, an Avro string, followed by
143
+ # * the message parameters. Parameters are serialized according to
144
+ # the message's request declaration.
145
+
146
+ # TODO request metadata (not yet implemented)
147
+ request_metadata = {}
148
+ META_WRITER.write(request_metadata, encoder)
149
+
150
+ message = local_protocol.messages[message_name]
151
+ unless message
152
+ raise AvroError, "Unknown message: #{message_name}"
153
+ end
154
+ encoder.write_string(message.name)
155
+
156
+ write_request(message.request, request_datum, encoder)
157
+ end
158
+
159
+ def write_request(request_schema, request_datum, encoder)
160
+ datum_writer = Avro::IO::DatumWriter.new(request_schema)
161
+ datum_writer.write(request_datum, encoder)
162
+ end
163
+
164
+ def read_handshake_response(decoder)
165
+ handshake_response = HANDSHAKE_REQUESTOR_READER.read(decoder)
166
+ we_have_matching_schema = false
167
+
168
+ case handshake_response['match']
169
+ when 'BOTH'
170
+ self.send_protocol = false
171
+ we_have_matching_schema = true
172
+ when 'CLIENT'
173
+ raise AvroError.new('Handshake failure. match == CLIENT') if send_protocol
174
+ self.remote_protocol = Avro::Protocol.parse(handshake_response['serverProtocol'])
175
+ self.remote_hash = handshake_response['serverHash']
176
+ self.send_protocol = false
177
+ we_have_matching_schema = true
178
+ when 'NONE'
179
+ raise AvroError.new('Handshake failure. match == NONE') if send_protocol
180
+ self.remote_protocol = Avro::Protocol.parse(handshake_response['serverProtocol'])
181
+ self.remote_hash = handshake_response['serverHash']
182
+ self.send_protocol = true
183
+ else
184
+ raise AvroError.new("Unexpected match: #{match}")
185
+ end
186
+
187
+ return we_have_matching_schema
188
+ end
189
+
190
+ def read_call_response(message_name, decoder)
191
+ # The format of a call response is:
192
+ # * response metadata, a map with values of type bytes
193
+ # * a one-byte error flag boolean, followed by either:
194
+ # * if the error flag is false,
195
+ # the message response, serialized per the message's response schema.
196
+ # * if the error flag is true,
197
+ # the error, serialized per the message's error union schema.
198
+ response_metadata = META_READER.read(decoder)
199
+
200
+ # remote response schema
201
+ remote_message_schema = remote_protocol.messages[message_name]
202
+ raise AvroError.new("Unknown remote message: #{message_name}") unless remote_message_schema
203
+
204
+ # local response schema
205
+ local_message_schema = local_protocol.messages[message_name]
206
+ unless local_message_schema
207
+ raise AvroError.new("Unknown local message: #{message_name}")
208
+ end
209
+
210
+ # error flag
211
+ if !decoder.read_boolean
212
+ writers_schema = remote_message_schema.response
213
+ readers_schema = local_message_schema.response
214
+ read_response(writers_schema, readers_schema, decoder)
215
+ else
216
+ writers_schema = remote_message_schema.errors || SYSTEM_ERROR_SCHEMA
217
+ readers_schema = local_message_schema.errors || SYSTEM_ERROR_SCHEMA
218
+ raise read_error(writers_schema, readers_schema, decoder)
219
+ end
220
+ end
221
+
222
+ def read_response(writers_schema, readers_schema, decoder)
223
+ datum_reader = Avro::IO::DatumReader.new(writers_schema, readers_schema)
224
+ datum_reader.read(decoder)
225
+ end
226
+
227
+ def read_error(writers_schema, readers_schema, decoder)
228
+ datum_reader = Avro::IO::DatumReader.new(writers_schema, readers_schema)
229
+ AvroRemoteError.new(datum_reader.read(decoder))
230
+ end
231
+ end
232
+
233
+ # Base class for the server side of a protocol interaction.
234
+ class Responder
235
+ attr_reader :local_protocol, :local_hash, :protocol_cache
236
+ def initialize(local_protocol)
237
+ @local_protocol = local_protocol
238
+ @local_hash = self.local_protocol.md5
239
+ @protocol_cache = {}
240
+ protocol_cache[local_hash] = local_protocol
241
+ end
242
+
243
+ # Called by a server to deserialize a request, compute and serialize
244
+ # a response or error. Compare to 'handle()' in Thrift.
245
+ def respond(call_request, transport=nil)
246
+ buffer_decoder = Avro::IO::BinaryDecoder.new(StringIO.new(call_request))
247
+ buffer_writer = StringIO.new(''.force_encoding('BINARY'))
248
+ buffer_encoder = Avro::IO::BinaryEncoder.new(buffer_writer)
249
+ error = nil
250
+ response_metadata = {}
251
+
252
+ begin
253
+ remote_protocol = process_handshake(buffer_decoder, buffer_encoder, transport)
254
+ # handshake failure
255
+ unless remote_protocol
256
+ return buffer_writer.string
257
+ end
258
+
259
+ # read request using remote protocol
260
+ request_metadata = META_READER.read(buffer_decoder)
261
+ remote_message_name = buffer_decoder.read_string
262
+
263
+ # get remote and local request schemas so we can do
264
+ # schema resolution (one fine day)
265
+ remote_message = remote_protocol.messages[remote_message_name]
266
+ unless remote_message
267
+ raise AvroError.new("Unknown remote message: #{remote_message_name}")
268
+ end
269
+ local_message = local_protocol.messages[remote_message_name]
270
+ unless local_message
271
+ raise AvroError.new("Unknown local message: #{remote_message_name}")
272
+ end
273
+ writers_schema = remote_message.request
274
+ readers_schema = local_message.request
275
+ request = read_request(writers_schema, readers_schema, buffer_decoder)
276
+ # perform server logic
277
+ begin
278
+ response = call(local_message, request)
279
+ rescue AvroRemoteError => e
280
+ error = e
281
+ rescue Exception => e
282
+ error = AvroRemoteError.new(e.to_s)
283
+ end
284
+
285
+ # write response using local protocol
286
+ META_WRITER.write(response_metadata, buffer_encoder)
287
+ buffer_encoder.write_boolean(!!error)
288
+ if error.nil?
289
+ writers_schema = local_message.response
290
+ write_response(writers_schema, response, buffer_encoder)
291
+ else
292
+ writers_schema = local_message.errors || SYSTEM_ERROR_SCHEMA
293
+ write_error(writers_schema, error, buffer_encoder)
294
+ end
295
+ rescue Avro::AvroError => e
296
+ error = AvroRemoteException.new(e.to_s)
297
+ # TODO does the stuff written here ever get used?
298
+ buffer_encoder = Avro::IO::BinaryEncoder.new(StringIO.new)
299
+ META_WRITER.write(response_metadata, buffer_encoder)
300
+ buffer_encoder.write_boolean(true)
301
+ self.write_error(SYSTEM_ERROR_SCHEMA, error, buffer_encoder)
302
+ end
303
+ buffer_writer.string
304
+ end
305
+
306
+ def process_handshake(decoder, encoder, connection=nil)
307
+ if connection && connection.is_connected?
308
+ return connection.protocol
309
+ end
310
+ handshake_request = HANDSHAKE_RESPONDER_READER.read(decoder)
311
+ handshake_response = {}
312
+
313
+ # determine the remote protocol
314
+ client_hash = handshake_request['clientHash']
315
+ client_protocol = handshake_request['clientProtocol']
316
+ remote_protocol = protocol_cache[client_hash]
317
+
318
+ if !remote_protocol && client_protocol
319
+ remote_protocol = Avro::Protocol.parse(client_protocol)
320
+ protocol_cache[client_hash] = remote_protocol
321
+ end
322
+
323
+ # evaluate remote's guess of the local protocol
324
+ server_hash = handshake_request['serverHash']
325
+ if local_hash == server_hash
326
+ if !remote_protocol
327
+ handshake_response['match'] = 'NONE'
328
+ else
329
+ handshake_response['match'] = 'BOTH'
330
+ end
331
+ else
332
+ if !remote_protocol
333
+ handshake_response['match'] = 'NONE'
334
+ else
335
+ handshake_response['match'] = 'CLIENT'
336
+ end
337
+ end
338
+
339
+ if handshake_response['match'] != 'BOTH'
340
+ handshake_response['serverProtocol'] = local_protocol.to_s
341
+ handshake_response['serverHash'] = local_hash
342
+ end
343
+
344
+ HANDSHAKE_RESPONDER_WRITER.write(handshake_response, encoder)
345
+
346
+ if connection && handshake_response['match'] != 'NONE'
347
+ connection.protocol = remote_protocol
348
+ end
349
+
350
+ remote_protocol
351
+ end
352
+
353
+ def call(local_message, request)
354
+ # Actual work done by server: cf. handler in thrift.
355
+ raise NotImplementedError
356
+ end
357
+
358
+ def read_request(writers_schema, readers_schema, decoder)
359
+ datum_reader = Avro::IO::DatumReader.new(writers_schema, readers_schema)
360
+ datum_reader.read(decoder)
361
+ end
362
+
363
+ def write_response(writers_schema, response_datum, encoder)
364
+ datum_writer = Avro::IO::DatumWriter.new(writers_schema)
365
+ datum_writer.write(response_datum, encoder)
366
+ end
367
+
368
+ def write_error(writers_schema, error_exception, encoder)
369
+ datum_writer = Avro::IO::DatumWriter.new(writers_schema)
370
+ datum_writer.write(error_exception.to_s, encoder)
371
+ end
372
+ end
373
+
374
+ class SocketTransport
375
+ # A simple socket-based Transport implementation.
376
+
377
+ attr_reader :sock, :remote_name
378
+ attr_accessor :protocol
379
+
380
+ def initialize(sock)
381
+ @sock = sock
382
+ @protocol = nil
383
+ end
384
+
385
+ def is_connected?()
386
+ !!@protocol
387
+ end
388
+
389
+ def transceive(request)
390
+ write_framed_message(request)
391
+ read_framed_message
392
+ end
393
+
394
+ def read_framed_message
395
+ message = []
396
+ loop do
397
+ buffer = StringIO.new(''.force_encoding('BINARY'))
398
+ buffer_length = read_buffer_length
399
+ if buffer_length == 0
400
+ return message.join
401
+ end
402
+ while buffer.tell < buffer_length
403
+ chunk = sock.read(buffer_length - buffer.tell)
404
+ if chunk == ''
405
+ raise ConnectionClosedException.new("Socket read 0 bytes.")
406
+ end
407
+ buffer.write(chunk)
408
+ end
409
+ message << buffer.string
410
+ end
411
+ end
412
+
413
+ def write_framed_message(message)
414
+ message_length = message.bytesize
415
+ total_bytes_sent = 0
416
+ while message_length - total_bytes_sent > 0
417
+ if message_length - total_bytes_sent > BUFFER_SIZE
418
+ buffer_length = BUFFER_SIZE
419
+ else
420
+ buffer_length = message_length - total_bytes_sent
421
+ end
422
+ write_buffer(message[total_bytes_sent,buffer_length])
423
+ total_bytes_sent += buffer_length
424
+ end
425
+ # A message is always terminated by a zero-length buffer.
426
+ write_buffer_length(0)
427
+ end
428
+
429
+ def write_buffer(chunk)
430
+ buffer_length = chunk.bytesize
431
+ write_buffer_length(buffer_length)
432
+ total_bytes_sent = 0
433
+ while total_bytes_sent < buffer_length
434
+ bytes_sent = self.sock.write(chunk[total_bytes_sent..-1])
435
+ if bytes_sent == 0
436
+ raise ConnectionClosedException.new("Socket sent 0 bytes.")
437
+ end
438
+ total_bytes_sent += bytes_sent
439
+ end
440
+ end
441
+
442
+ def write_buffer_length(n)
443
+ bytes_sent = sock.write([n].pack('N'))
444
+ if bytes_sent == 0
445
+ raise ConnectionClosedException.new("socket sent 0 bytes")
446
+ end
447
+ end
448
+
449
+ def read_buffer_length
450
+ read = sock.read(BUFFER_HEADER_LENGTH)
451
+ if read == '' || read == nil
452
+ raise ConnectionClosedException.new("Socket read 0 bytes.")
453
+ end
454
+ read.unpack('N')[0]
455
+ end
456
+
457
+ def close
458
+ sock.close
459
+ end
460
+ end
461
+
462
+ class ConnectionClosedError < StandardError; end
463
+
464
+ class FramedWriter
465
+ attr_reader :writer
466
+ def initialize(writer)
467
+ @writer = writer
468
+ end
469
+
470
+ def write_framed_message(message)
471
+ message_size = message.bytesize
472
+ total_bytes_sent = 0
473
+ while message_size - total_bytes_sent > 0
474
+ if message_size - total_bytes_sent > BUFFER_SIZE
475
+ buffer_size = BUFFER_SIZE
476
+ else
477
+ buffer_size = message_size - total_bytes_sent
478
+ end
479
+ write_buffer(message[total_bytes_sent, buffer_size])
480
+ total_bytes_sent += buffer_size
481
+ end
482
+ write_buffer_size(0)
483
+ end
484
+
485
+ def to_s; writer.string; end
486
+
487
+ private
488
+ def write_buffer(chunk)
489
+ buffer_size = chunk.bytesize
490
+ write_buffer_size(buffer_size)
491
+ writer << chunk
492
+ end
493
+
494
+ def write_buffer_size(n)
495
+ writer.write([n].pack('N'))
496
+ end
497
+ end
498
+
499
+ class FramedReader
500
+ attr_reader :reader
501
+
502
+ def initialize(reader)
503
+ @reader = reader
504
+ end
505
+
506
+ def read_framed_message
507
+ message = []
508
+ loop do
509
+ buffer = ''.force_encoding('BINARY')
510
+ buffer_size = read_buffer_size
511
+
512
+ return message.join if buffer_size == 0
513
+
514
+ while buffer.bytesize < buffer_size
515
+ chunk = reader.read(buffer_size - buffer.bytesize)
516
+ chunk_error?(chunk)
517
+ buffer << chunk
518
+ end
519
+ message << buffer
520
+ end
521
+ end
522
+
523
+ private
524
+ def read_buffer_size
525
+ header = reader.read(BUFFER_HEADER_LENGTH)
526
+ chunk_error?(header)
527
+ header.unpack('N')[0]
528
+ end
529
+
530
+ def chunk_error?(chunk)
531
+ raise ConnectionClosedError.new("Reader read 0 bytes") if chunk == ''
532
+ end
533
+ end
534
+
535
+ # Only works for clients. Sigh.
536
+ class HTTPTransceiver
537
+ attr_reader :remote_name, :host, :port
538
+ def initialize(host, port)
539
+ @host, @port = host, port
540
+ @remote_name = "#{host}:#{port}"
541
+ @conn = Net::HTTP.start host, port
542
+ end
543
+
544
+ def transceive(message)
545
+ writer = FramedWriter.new(StringIO.new(''.force_encoding('BINARY')))
546
+ writer.write_framed_message(message)
547
+ resp = @conn.post('/', writer.to_s, {'Content-Type' => 'avro/binary'})
548
+ FramedReader.new(StringIO.new(resp.body)).read_framed_message
549
+ end
550
+ end
551
+ end