amq-protocol 2.4.0 → 2.5.1

data/codegen/codegen.py

@@ -1,151 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-# Documentation for Mako templates:
-# http://www.makotemplates.org/docs/syntax.html
-
-import os, sys, re
-
-sys.path.append(os.path.join("codegen", "rabbitmq-codegen"))
-
-from amqp_codegen import *
-try:
-    from mako.template import Template
-except ImportError:
-    print("Mako isn't installed. Please install mako via pip or similar.")
-    sys.exit(1)
-
-# main class
-class AmqpSpecObject(AmqpSpec):
-    IGNORED_CLASSES = ["access"]
-    IGNORED_FIELDS = {
-        'ticket': 0,
-        'capabilities': '',
-        'insist' : 0,
-        'out_of_band': '',
-        'known_hosts': '',
-    }
-
-    def __init__(self, path):
-        AmqpSpec.__init__(self, path)
-
-        def extend_field(field):
-            field.ruby_name = re.sub("[- ]", "_", field.name)
-            field.type = self.resolveDomain(field.domain)
-            field.ignored = bool(field.name in self.__class__.IGNORED_FIELDS) # I. e. deprecated
-
-        for klass in self.classes:
-            klass.ignored = bool(klass.name in self.__class__.IGNORED_CLASSES)
-
-            for field in klass.fields:
-                extend_field(field)
-
-            for method in klass.methods:
-                for field in method.arguments:
-                    extend_field(field)
-
-        self.classes = filter(lambda klass: not klass.ignored, self.classes)
-
-original_init = AmqpEntity.__init__
-def new_init(self, arg):
-    original_init(self, arg)
-    constant_name = ""
-    for chunk in self.name.split("-"):
-        constant_name += chunk.capitalize()
-    self.constant_name = constant_name
-AmqpEntity.__init__ = new_init
-
-# method.accepted_by("server")
-# method.accepted_by("client", "server")
-accepted_by_update = json.loads(open("codegen/amqp_0.9.1_changes.json").read())
-
-def accepted_by(self, *receivers):
-    def get_accepted_by(self):
-        try:
-            return accepted_by_update[self.klass.name][self.name]
-        except KeyError:
-            return ["server", "client"]
-
-    actual_receivers = get_accepted_by(self)
-    return all(map(lambda receiver: receiver in actual_receivers, receivers))
-
-AmqpMethod.accepted_by = accepted_by
-
-def convert_value_to_ruby(value):
-    values = {None: "nil", False: "false", True: "true", "": "EMPTY_STRING"}
-
-    try:
-        return values[value]
-    except:
-        return value.__repr__()
-
-def convert_to_ruby(field):
-    name = re.sub("-", "_", field.name) # TODO: use ruby_name
-    if name == "ticket":
-        return "%s = %s" % (name, field.defaultvalue) # we want to keep it as an int, not as a boolean
-    else:
-        return "%s = %s" % (name, convert_value_to_ruby(field.defaultvalue))
-
-def not_ignored_args(self):
-    if self.hasContent:
-        return ["payload", "user_headers"] + map(lambda argument: argument.ruby_name, filter(lambda argument: not argument.ignored, self.arguments)) + ["frame_size"]
-    else:
-        return map(lambda argument: argument.ruby_name, filter(lambda argument: not argument.ignored, self.arguments))
-
-AmqpMethod.not_ignored_args = not_ignored_args
-
-def ignored_args(self):
-    return filter(lambda argument: argument.ignored, self.arguments)
-
-AmqpMethod.ignored_args = ignored_args
-
-# helpers
-def to_ruby_name(name):
-    return re.sub("[- ]", "_", name)
-
-def to_ruby_class_name(name):
-    parts = re.split("[- ]", name)
-    ruby_class_name = ""
-    for part in parts:
-        ruby_class_name = ruby_class_name + part[0].upper() + part[1:].lower()
-    return ruby_class_name
-
-def params(self):
-    buffer = []
-    for f in self.arguments:
-        buffer.append(convert_to_ruby(f))
-    if self.hasContent:
-        buffer.append("user_headers = nil")
-        buffer.append("payload = \"\"")
-        buffer.append("frame_size = nil")
-    return buffer
-
-AmqpMethod.params = params
-
-def args(self):
-    return map(lambda item: item.split(" ")[0], self.params())
-
-AmqpMethod.args = args
-
-def binary(self):
-    method_id = self.klass.index << 16 | self.index
-    return "0x%08X # %i, %i, %i" % (method_id, self.klass.index, self.index, method_id)
-
-AmqpMethod.binary = binary
-
-# helpers
-def render(path, **context):
-    file = open(path)
-    template = Template(file.read())
-    return template.render(**context)
-
-def generateMain(type):
-    def main(json_spec_path):
-        spec = AmqpSpecObject(json_spec_path)
-        spec.type = type
-        print(render("codegen/protocol.rb.pytemplate", spec = spec))
-
-    return main
-
-if __name__ == "__main__":
-    do_main_dict({"client": generateMain("client")})

data/codegen/codegen_helpers.py

@@ -1,162 +0,0 @@
-# -*- coding: utf-8 -*-
-
-from __future__ import print_function
-
-def genSingleEncode(spec, cValue, unresolved_domain):
-    buffer = []
-    type = spec.resolveDomain(unresolved_domain)
-    if type == 'shortstr':
-        buffer.append("buffer << %s.to_s.bytesize.chr" % (cValue,))
-        buffer.append("buffer << %s.to_s" % (cValue,))
-    elif type == 'longstr':
-        buffer.append("buffer << [%s.to_s.bytesize].pack(PACK_UINT32)" % (cValue,))
-        buffer.append("buffer << %s.to_s" % (cValue,))
-    elif type == 'octet':
-        buffer.append("buffer << [%s].pack(PACK_CHAR)" % (cValue,))
-    elif type == 'short':
-        buffer.append("buffer << [%s].pack(PACK_UINT16)" % (cValue,))
-    elif type == 'long':
-        buffer.append("buffer << [%s].pack(PACK_UINT32)" % (cValue,))
-    elif type == 'longlong':
-        buffer.append("buffer << AMQ::Pack.pack_uint64_big_endian(%s)" % (cValue,))
-    elif type == 'timestamp':
-        buffer.append("buffer << AMQ::Pack.pack_uint64_big_endian(%s)" % (cValue,))
-    elif type == 'bit':
-        raise "Can't encode bit in genSingleEncode"
-    elif type == 'table':
-        buffer.append("buffer << AMQ::Protocol::Table.encode(%s)" % (cValue,))
-    else:
-        raise "Illegal domain in genSingleEncode: {0}".format(type)
-
-    return buffer
-
-def genSingleDecode(spec, field):
-    cLvalue = field.ruby_name
-    unresolved_domain = field.domain
-
-    if cLvalue == "known_hosts":
-        import sys
-        print(field, field.ignored, file = sys.stderr)
-
-    type = spec.resolveDomain(unresolved_domain)
-    buffer = []
-    if type == 'shortstr':
-        buffer.append("length = data[offset, 1].unpack(PACK_CHAR).first")
-        buffer.append("offset += 1")
-        buffer.append("%s = data[offset, length]" % (cLvalue,))
-        buffer.append("offset += length")
-    elif type == 'longstr':
-        buffer.append("length = data[offset, 4].unpack(PACK_UINT32).first")
-        buffer.append("offset += 4")
-        buffer.append("%s = data[offset, length]" % (cLvalue,))
-        buffer.append("offset += length")
-    elif type == 'octet':
-        buffer.append("%s = data[offset, 1].unpack(PACK_CHAR).first" % (cLvalue,))
-        buffer.append("offset += 1")
-    elif type == 'short':
-        buffer.append("%s = data[offset, 2].unpack(PACK_UINT16).first" % (cLvalue,))
-        buffer.append("offset += 2")
-    elif type == 'long':
-        buffer.append("%s = data[offset, 4].unpack(PACK_UINT32).first" % (cLvalue,))
-        buffer.append("offset += 4")
-    elif type == 'longlong':
-        buffer.append("%s = AMQ::Pack.unpack_uint64_big_endian(data[offset, 8]).first" % (cLvalue,))
-        buffer.append("offset += 8")
-    elif type == 'timestamp':
-        buffer.append("%s = data[offset, 8].unpack(PACK_UINT64_BE).first" % (cLvalue,))
-        buffer.append("offset += 8")
-    elif type == 'bit':
-        raise "Can't decode bit in genSingleDecode"
-    elif type == 'table':
-        buffer.append("table_length = Table.length(data[offset, 4])")
-        buffer.append("%s = Table.decode(data[offset, table_length + 4])" % (cLvalue,))
-        buffer.append("offset += table_length + 4")
-    else:
-        raise StandardError("Illegal domain '{0}' in genSingleDecode".format(type))
-
-    return buffer
-
-
-
-def genSingleSimpleDecode(spec, field):
-    cLvalue = field.ruby_name
-    unresolved_domain = field.domain
-
-    if cLvalue == "known_hosts":
-        import sys
-        print >> sys.stderr, field, field.ignored
-
-    type = spec.resolveDomain(unresolved_domain)
-    buffer = []
-    if type == 'shortstr':
-        buffer.append("data.to_s")
-    elif type == 'longstr':
-        buffer.append("data.to_s")
-    elif type == 'octet':
-        buffer.append("data.unpack(PACK_INT8).first")
-    elif type == 'short':
-        buffer.append("data.unpack(PACK_UINT16).first")
-    elif type == 'long':
-        buffer.append("data.unpack(PACK_UINT32).first")
-    elif type == 'longlong':
-        buffer.append("AMQ::Pack.unpack_uint64_big_endian(data).first")
-    elif type == 'timestamp':
-        buffer.append("Time.at(data.unpack(PACK_UINT64_BE).last)")
-    elif type == 'bit':
-        raise "Can't decode bit in genSingleDecode"
-    elif type == 'table':
-        buffer.append("Table.decode(data)")
-    else:
-        raise StandardError("Illegal domain '" + type + "' in genSingleSimpleDecode")
-
-    return buffer
-
-
-def genEncodeMethodDefinition(spec, m):
-    def finishBits():
-        if bit_index is not None:
-            buffer.append("buffer << [bit_buffer].pack(PACK_CHAR)")
-
-    bit_index = None
-    buffer = []
-
-    for f in m.arguments:
-        if spec.resolveDomain(f.domain) == 'bit':
-            if bit_index is None:
-                bit_index = 0
-                buffer.append("bit_buffer = 0")
-            if bit_index >= 8:
-                finishBits()
-                buffer.append("bit_buffer = 0")
-                bit_index = 0
-            buffer.append("bit_buffer = bit_buffer | (1 << %d) if %s" % (bit_index, f.ruby_name))
-            bit_index = bit_index + 1
-        else:
-            finishBits()
-            bit_index = None
-            buffer += genSingleEncode(spec, f.ruby_name, f.domain)
-
-    finishBits()
-    return buffer
-
-def genDecodeMethodDefinition(spec, m):
-    buffer = []
-    bitindex = None
-    for f in m.arguments:
-        if spec.resolveDomain(f.domain) == 'bit':
-            if bitindex is None:
-                bitindex = 0
-            if bitindex >= 8:
-                bitindex = 0
-            if bitindex == 0:
-                buffer.append("bit_buffer = data[offset, 1].unpack(PACK_CHAR).first")
-                buffer.append("offset += 1")
-                buffer.append("%s = (bit_buffer & (1 << %d)) != 0" % (f.ruby_name, bitindex))
-                #### TODO: ADD bitindex TO THE buffer
-            else:
-                buffer.append("%s = (bit_buffer & (1 << %d)) != 0" % (f.ruby_name, bitindex))
-            bitindex = bitindex + 1
-        else:
-            bitindex = None
-            buffer += genSingleDecode(spec, f)
-    return buffer

data/codegen/protocol.rb.pytemplate

@@ -1,320 +0,0 @@
-# encoding: utf-8
-# encoding: binary
-# frozen_string_literal: true
-
-# THIS IS AN AUTOGENERATED FILE, DO NOT MODIFY
-# IT DIRECTLY ! FOR CHANGES, PLEASE UPDATE FILES
-# IN THE ./codegen DIRECTORY OF THE AMQ-PROTOCOL REPOSITORY.<% import codegen_helpers as helpers %><% import re, os, codegen %>
-
-require "amq/pack"
-
-require "amq/protocol/table"
-require "amq/protocol/frame"
-
-require "amq/protocol/constants"
-require "amq/protocol/exceptions"
-
-module AMQ
-  module Protocol
-    PROTOCOL_VERSION = "${spec.major}.${spec.minor}.${spec.revision}".freeze
-    PREAMBLE         = "${'AMQP\\x00\\x%02x\\x%02x\\x%02x' % (spec.major, spec.minor, spec.revision)}".freeze
-    DEFAULT_PORT     = ${spec.port}
-
-    # @return [Array] Collection of subclasses of AMQ::Protocol::Class.
-    def self.classes
-      Protocol::Class.classes
-    end
-
-    # @return [Array] Collection of subclasses of AMQ::Protocol::Method.
-    def self.methods
-      Protocol::Method.methods
-    end
-
-    % for tuple in spec.constants:
-    % if tuple[2] == "soft-error" or tuple[2] == "hard-error":
-    class ${codegen.to_ruby_class_name(tuple[0])} < ${codegen.to_ruby_class_name(tuple[2])}
-      VALUE = ${tuple[1]}
-    end
-
-    % endif
-    % endfor
-
-    class Class
-      @classes = Array.new
-
-      def self.method_id
-        @method_id
-      end
-
-      def self.name
-        @name
-      end
-
-      def self.inherited(base)
-        if self == Protocol::Class
-          @classes << base
-        end
-      end
-
-      def self.classes
-        @classes
-      end
-    end
-
-    class Method
-      @methods = Array.new
-      def self.method_id
-        @method_id
-      end
-
-      def self.name
-        @name
-      end
-
-      def self.index
-        @index
-      end
-
-      def self.inherited(base)
-        if self == Protocol::Method
-          @methods << base
-        end
-      end
-
-      def self.methods
-        @methods
-      end
-
-      def self.split_headers(user_headers)
-        properties, headers = {}, {}
-        user_headers.each do |key, value|
-          # key MUST be a symbol since symbols are not garbage-collected
-          if Basic::PROPERTIES.include?(key)
-            properties[key] = value
-          else
-            headers[key] = value
-          end
-        end
-
-        return [properties, headers]
-      end
-
-      def self.encode_body(body, channel, frame_size)
-        return [] if body.empty?
-
-        # 8 = 1 + 2 + 4 + 1
-        # 1 byte of frame type
-        # 2 bytes of channel number
-        # 4 bytes of frame payload length
-        # 1 byte of payload trailer FRAME_END byte
-        limit        = frame_size - 8
-        return [BodyFrame.new(body, channel)] if body.bytesize < limit
-
-        # Otherwise String#slice on 1.9 will operate with code points,
-        # and we need bytes. MK.
-        body.force_encoding("ASCII-8BIT") if RUBY_VERSION.to_f >= 1.9
-
-        array = Array.new
-        while body && !body.empty?
-          payload, body = body[0, limit], body[limit, body.length - limit]
-          array << BodyFrame.new(payload, channel)
-        end
-
-        array
-      end
-
-      def self.instantiate(*args, &block)
-        self.new(*args, &block)
-      end
-    end
-
-    % for klass in spec.classes :
-    class ${klass.constant_name} < Protocol::Class
-      @name = "${klass.name}"
-      @method_id = ${klass.index}
-
-      % if klass.fields: ## only the Basic class has fields (refered as properties in the JSON)
-      PROPERTIES = [
-      % for field in klass.fields:
-        :${field.ruby_name}, # ${spec.resolveDomain(field.domain)}
-        % endfor
-      ]
-
-      % for f in klass.fields:
-      # <% i = klass.fields.index(f) %>1 << ${15 - i}
-      def self.encode_${f.ruby_name}(value)
-        buffer = +''
-        % for line in helpers.genSingleEncode(spec, "value", f.domain):
-        ${line}
-        % endfor
-        [${i}, ${"0x%04x" % ( 1 << (15-i),)}, buffer]
-      end
-
-      % endfor
-
-      % endif
-
-      % if klass.name == "basic" :
-      def self.encode_properties(body_size, properties)
-        pieces, flags = [], 0
-
-        properties.reject {|key, value| value.nil?}.each do |key, value|
-          i, f, result = self.__send__(:"encode_#{key}", value)
-          flags |= f
-          pieces[i] = result
-        end
-
-        # result = [${klass.index}, 0, body_size, flags].pack('n2Qn')
-        result = [${klass.index}, 0].pack(PACK_UINT16_X2)
-        result += AMQ::Pack.pack_uint64_big_endian(body_size)
-        result += [flags].pack(PACK_UINT16)
-        pieces_joined = pieces.join(EMPTY_STRING)
-        result.force_encoding(pieces_joined.encoding) + pieces_joined
-      end
-
-      # THIS DECODES ONLY FLAGS
-      DECODE_PROPERTIES = {
-        % for f in klass.fields:
-        ${"0x%04x" % ( 1 << (15 - klass.fields.index(f)),)} => :${f.ruby_name},
-        % endfor
-      }
-
-      DECODE_PROPERTIES_TYPE = {
-        % for f in klass.fields:
-        ${"0x%04x" % ( 1 << (15 - klass.fields.index(f)),)} => :${spec.resolveDomain(f.domain)},
-        % endfor
-      }
-
-      # Hash doesn't give any guarantees on keys order, we will do it in a
-      # straightforward way
-      DECODE_PROPERTIES_KEYS = [
-        % for f in klass.fields:
-        ${"0x%04x" % ( 1 << (15 - klass.fields.index(f)),)},
-        % endfor
-      ]
-
-      def self.decode_properties(data)
-        offset, data_length, properties = 0, data.bytesize, {}
-
-        compressed_index = data[offset, 2].unpack(PACK_UINT16)[0]
-        offset += 2
-        while data_length > offset
-          DECODE_PROPERTIES_KEYS.each do |key|
-            next unless compressed_index >= key
-            compressed_index -= key
-            name = DECODE_PROPERTIES[key] || raise(RuntimeError.new("No property found for index #{index.inspect}!"))
-            case DECODE_PROPERTIES_TYPE[key]
-            when :shortstr
-              size = data[offset, 1].unpack(PACK_CHAR)[0]
-              offset += 1
-              result = data[offset, size]
-            when :octet
-              size = 1
-              result = data[offset, size].unpack(PACK_CHAR).first
-            when :timestamp
-              size = 8
-              result = Time.at(data[offset, size].unpack(PACK_UINT64_BE).last)
-            when :table
-              size = 4 + data[offset, 4].unpack(PACK_UINT32)[0]
-              result = Table.decode(data[offset, size])
-            end
-            properties[name] = result
-            offset += size
-          end
-        end
-
-        properties
-      end
-      % endif
-
-      % for method in klass.methods:
-      class ${method.constant_name} < Protocol::Method
-        @name = "${klass.name}.${method.name}"
-        @method_id = ${method.index}
-        @index = ${method.binary()}
-        @packed_indexes = [${klass.index}, ${method.index}].pack(PACK_UINT16_X2).freeze
-
-        % if (spec.type == "client" and method.accepted_by("client")) or (spec.type == "server" and method.accepted_by("server") or spec.type == "all"):
-        # @return
-        def self.decode(data)
-          offset = offset = 0 # self-assigning offset to eliminate "assigned but unused variable" warning even if offset is not used in this method
-          % for line in helpers.genDecodeMethodDefinition(spec, method):
-          ${line}
-          % endfor
-          % if (method.klass.name == "connection" or method.klass.name == "channel") and method.name == "close":
-          self.new(${', '.join([f.ruby_name for f in method.arguments])})
-          % else:
-          self.new(${', '.join([f.ruby_name for f in method.arguments])})
-          % endif
-        end
-
-        % if len(method.arguments) > 0:
-        attr_reader ${', '.join([":" + f.ruby_name for f in method.arguments])}
-        % endif
-        def initialize(${', '.join([f.ruby_name for f in method.arguments])})
-          % for f in method.arguments:
-          @${f.ruby_name} = ${f.ruby_name}
-          % endfor
-        end
-        % endif
-
-        def self.has_content?
-          % if method.hasContent:
-          true
-          % else:
-          false
-          % endif
-        end
-
-        % if (spec.type == "client" and method.accepted_by("server")) or (spec.type == "server" and method.accepted_by("client")) or spec.type == "all":
-        # @return
-        # ${method.params()}
-        % if klass.name == "connection":
-        def self.encode(${(", ").join(method.not_ignored_args())})
-        % else:
-        def self.encode(${(", ").join(["channel"] + method.not_ignored_args())})
-        % endif
-          % for argument in method.ignored_args():
-          ${codegen.convert_to_ruby(argument)}
-          % endfor
-          % if klass.name == "connection":
-          channel = 0
-          % endif
-          buffer = @packed_indexes.dup
-          % for line in helpers.genEncodeMethodDefinition(spec, method):
-          ${line}
-          % endfor
-          % if "payload" in method.args() or "user_headers" in method.args():
-          frames = [MethodFrame.new(buffer, channel)]
-          % if "user_headers" in method.args():
-          properties, _headers = self.split_headers(user_headers)
-          if properties.nil? or properties.empty?
-            raise RuntimeError.new("Properties can not be empty!")
-          end
-          properties_payload = Basic.encode_properties(payload.bytesize, properties)
-          frames << HeaderFrame.new(properties_payload, channel)
-          % endif
-          % if "payload" in method.args():
-          frames += self.encode_body(payload, channel, frame_size)
-          frames
-          % endif
-          % else:
-          MethodFrame.new(buffer, channel)
-          % endif
-        end
-        % endif
-
-      end
-
-      % endfor
-    end
-
-    % endfor
-
-    METHODS = begin
-      Method.methods.inject(Hash.new) do |hash, klass|
-        hash.merge!(klass.index => klass)
-      end
-    end
-  end
-end

data/generate.rb

@@ -1,24 +0,0 @@
-#!/usr/bin/env ruby
-# encoding: utf-8
-
-# rabbitmq-codegen is Python 3 compatible and so is
-# the code in this repo but Mako still fails with 3.6 as of May 2017 :( MK.
-python = ENV.fetch("PYTHON", "python2")
-
-def sh(*args)
-  system(*args)
-end
-
-extensions = []
-
-spec = "codegen/rabbitmq-codegen/amqp-rabbitmq-0.9.1.json"
-unless File.exist?(spec)
-  sh "git submodule update --init"
-end
-
-path = "lib/amq/protocol/client.rb"
-puts "Running '#{python} ./codegen/codegen.py client #{spec} #{extensions.join(' ')} #{path}'"
-sh "#{python} ./codegen/codegen.py client #{spec} #{extensions.join(' ')} #{path}"
-if File.file?(path)
-  sh "ruby -c #{path}"
-end

data/profiling/README.md

@@ -1,9 +0,0 @@
-# Profiling Scripts
-
-This directory contains profiling scripts. Currently they use [stackprof](https://github.com/tmm1/stackprof) which
-requires Ruby 2.1+ (preview2 or later).
-
-## Running the Profiler
-
-    ruby profiling/stackprof/body_framing_with_2k_payload.rb
-    stackprof profiling/dumps/body_framing_with_2k_payload.dump --text

data/profiling/stackprof/body_framing_with_2k_payload.rb

@@ -1,33 +0,0 @@
-#!/usr/bin/env ruby
-# encoding: utf-8
-
-$LOAD_PATH << File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "lib"))
-
-require "amq/protocol/client"
-
-FRAME_SIZE = 128 * 1024
-
-puts
-puts "-" * 80
-puts "Profiling on #{RUBY_DESCRIPTION}"
-
-n = 250_000
-
-# warm up the JIT, etc
-puts "Doing a warmup run..."
-15_000.times { AMQ::Protocol::Method.encode_body("ab" * 1024, 1, FRAME_SIZE) }
-
-require 'stackprof'
-
-# preallocate
-ary = Array.new(n) { "ab" * 1024 }
-
-puts "Doing main run..."
-result = StackProf.run(mode: :wall) do
-  n.times { |i| AMQ::Protocol::Method.encode_body(ary[i], 1, FRAME_SIZE) }
-end
-
-File.open('./profiling/dumps/body_framing_with_2k_payload.dump', "w+") do |f|
-  f.write Marshal.dump(result)
-end
-