icss 0.0.2

data/lib/icss/code_asset.rb

@@ -0,0 +1,16 @@
+module Icss
+  class CodeAsset
+    include Receiver
+    include Receiver::ActsAsHash
+
+    rcvr_accessor :name,      String
+    rcvr_accessor :location,  String
+
+    def to_hash()
+      { :name => name, :location => location}
+    end
+
+    def to_json() to_hash.to_json ; end
+
+  end
+end

data/lib/icss/core_ext.rb

@@ -0,0 +1,4 @@
+require 'gorillib/string/inflections'
+require 'gorillib/string/constantize'
+require 'gorillib/hash/keys'
+require 'gorillib/metaprogramming/class_attribute'

data/lib/icss/data_asset.rb

@@ -0,0 +1,22 @@
+module Icss
+  class DataAsset
+    include Receiver
+    include Receiver::ActsAsHash
+
+    rcvr_accessor :name,      String
+    rcvr_accessor :location,  String
+    #overriding ruby's deprecated but still present type attr on objects
+    attr_accessor :type
+    rcvr_accessor :type,      String
+    rcvr_accessor :doc,       String
+
+    def named? nm
+      name == nm
+    end
+
+    def to_hash()
+      { :name => name, :location => location, :type => type, :doc => doc }
+    end
+    def to_json() to_hash.to_json ; end
+  end
+end

data/lib/icss/message.rb

@@ -0,0 +1,72 @@
+module Icss
+
+  #
+  # Describes an Avro Message
+  #
+  # A message has attributes:
+  #
+  # * doc:        an optional description of the message,
+  # * request:    a list of named, typed parameter schemas (this has the same form as the fields of a record declaration);
+  # * response:   a valid schema for the response
+  # * errors:     an optional union of error schemas.
+  #
+  # A request parameter list is processed equivalently to an anonymous
+  # record. Since record field lists may vary between reader and writer, request
+  # parameters may also differ between the caller and responder, and such
+  # differences are resolved in the same manner as record field differences.
+  #
+  class Message
+    include Receiver
+    include Receiver::ActsAsHash
+    rcvr_accessor :name,     String
+    rcvr_accessor :doc,      String
+
+    #we're starting to attach a lot of pork to this lib...
+    rcvr_accessor :initial_free_qty,      Integer
+    rcvr_accessor :price_per_k_in_cents,  Integer
+
+    rcvr_accessor :request,  Array, :of => Icss::RecordField, :default => []
+    rcvr_accessor :response, Icss::TypeFactory
+    rcvr_accessor :errors,   Icss::UnionType, :default => []
+    attr_accessor :protocol
+    # this is defined in sample_message_call.rb -- since we don't do referenced types yet
+    # rcvr_accessor :samples, Array, :of => Icss::SampleMessageCall, :default => []
+
+    after_receive do |hsh|
+      # track recursion of type references
+      @response_is_reference = true if hsh['response'].is_a?(String) || hsh['response'].is_a?(Symbol)
+      # tie each sample back to this, its parent message
+      (self.samples ||= []).each{|sample| sample.message = self }
+    end
+
+    def path
+      File.join(protocol.path, name)
+    end
+
+    def first_sample_request_param
+      req = samples.first.request.first rescue nil
+      req || {}
+    end
+
+    #
+    # Conversion
+    #
+    def to_hash()
+      {
+        :doc      => doc,
+        :request  => summary_of_request_attr,
+        :response => summary_of_response_attr,
+        :errors   => errors,
+      }.reject{|k,v| v.nil? }
+    end
+    def to_json(*args) to_hash.to_json(*args) ; end
+
+  private
+    def summary_of_response_attr
+      case when response.blank? then response when @response_is_reference then response.name else response.to_hash end
+    end
+    def summary_of_request_attr
+      request.map(&:to_hash)
+    end
+  end
+end

data/lib/icss/old.rb

@@ -0,0 +1,96 @@
+module Icss
+  Protocol.class_eval do
+
+    #
+    # Returns the asset hash with the passed in asset_name
+    #
+    def asset_for_name nm
+      warn "asset_for_name obsolete, please rewrite"
+      data_assets.find{|asset| asset.named?(nm) }
+    end
+
+    #
+    # Returns the full type for the passed in asset hash
+    #
+    def type_for_asset asset
+      warn "type_for_asset obsolete, please rewrite"
+      # types.find{|type| type.name == asset['type']}
+      raise 'use asset.type_obj'
+    end
+
+    #
+    # Given an asset name, returns the the record it points to
+    #
+    def type_for_asset_name asset_name
+      warn "type_for_asset_name obsolete, please rewrite"
+      asset = asset_for_name(asset_name)
+      type_for_asset(asset)
+    end
+
+    #
+    # Given an asset name, return the name of the type it points to
+    #
+    def type_name_for_asset_name asset_name
+      warn "type_name_for_asset_name obsolete, please rewrite"
+      asset = asset_for_name(asset_name)
+      asset['type']
+    end
+
+    #
+    # Return the avro record with the given name
+    #
+    def type_for_name type_name
+      warn "type_for_name obsolete, please rewrite"
+      # types.find{|record| record.name == type_name}
+      Icss::Type.find(type_name)
+    end
+
+    #
+    # Fetch the avro fields for a named data asset
+    #
+    def fields_for_asset asset_name
+      warn "fields_for_asset obsolete, please rewrite"
+      asset      = asset_for_name(asset_name)
+      asset_type = type_for_asset(asset)
+      asset_type.fields
+    end
+
+    #
+    # Fetch the locations on disk of each data asset. WARNING: Assuming relative paths
+    #
+    def location_for_asset asset_name
+      warn "location_for_asset obsolete, please rewrite"
+      asset = asset_for_name(asset_name)
+      File.join(dirname, asset['location'])
+    end
+
+    #
+    # Return the index of the named field for the named type. Returns (nil) if field
+    # does not exist
+    #
+    def index_of_fieldname asset_name, field_name
+      warn "index_of_fieldname obsolete, please rewrite"
+      type = type_for_asset_name(asset_name)
+      type.index_of_fieldname(field_name)
+    end
+
+    AVRO_PIG_MAPPING = {
+      'string' => 'chararray',
+      'int'    => 'int',
+      'long'   => 'long',
+      'float'  => 'float',
+      'double' => 'double',
+      'bytes'  => 'bytearray',
+      'fixed'  => 'bytearray'
+    }
+
+    #
+    # Add pig fields to a passed in array of avro fields
+    #
+    def augment_with_pig_fields fields
+      warn "augment_with_pig_fields obsolete, please rewrite"
+      fields.map{|field| field.body['pig_type'] = AVRO_PIG_MAPPING[field.type]; field }
+    end
+  end
+
+end

data/lib/icss/protocol.rb

@@ -0,0 +1,138 @@
+module Icss
+
+  #
+  # Describes an Avro Protocol Declaration
+  #
+  # Avro protocols describe RPC interfaces. The Protocol class will receive an
+  # Avro JSON
+  #
+  # A Protocol has the following attributes:
+  #
+  # * protocol, a string, the name of the protocol (required). +name+ is
+  #   provided as an alias for +protocol+.
+  #
+  # * namespace, a string that qualifies the name (optional).
+  #
+  # * doc, a string describing this protocol (optional).
+  #
+  # * types, an optional list of definitions of named types (records, enums,
+  #   fixed and errors). An error definition is just like a record definition
+  #   except it uses "error" instead of "record". Note that forward references
+  #   to named types are not permitted.
+  #
+  # * messages, an optional JSON object whose keys are message names and whose
+  #   values are objects whose attributes are described below. No two messages
+  #   may have the same name.
+  #
+  # The name and namespace qualification rules defined for schema objects apply
+  # to protocols as well: see the documentation for Icss::Type.
+  #
+  # For example, one may define a simple HelloWorld protocol with:
+  #
+  #     {
+  #       "namespace":    "com.acme",
+  #       "protocol":     "HelloWorld",
+  #       "doc":          "Protocol Greetings",
+  #       "types": [
+  #         { "name":     "Greeting",
+  #           "type":     "record",
+  #           "fields":   [ {"name": "message", "type": "string"} ]},
+  #         { "name":     "Curse",
+  #           "type":     "error",
+  #           "fields":   [ {"name": "message", "type": "string"} ]}
+  #       ],
+  #       "messages": {
+  #         "hello": {
+  #           "doc":      "Say hello.",
+  #           "request":  [{"name": "greeting", "type": "Greeting" }],
+  #           "response": "Greeting",
+  #           "errors":   ["Curse"]
+  #         }
+  #       }
+  #     }
+  #
+  class Protocol
+    include Receiver
+    include Receiver::ActsAsHash
+    include Receiver::ActsAsLoadable
+    include Icss::Validations
+
+    rcvr_accessor :protocol,    String, :required => true
+    alias_method  :name, :protocol
+    rcvr_accessor :namespace,   String # must be *dotted* ("foo.bar"), not slashed ("foo/bar")
+    rcvr_accessor :doc,         String
+    #
+    rcvr_accessor :types,       Array, :of => Icss::TypeFactory, :default => []
+    rcvr_accessor :messages,    Hash,  :of => Icss::Message,     :default => {}
+    # extensions to avro
+    rcvr_accessor :data_assets, Array, :of => Icss::DataAsset,   :default => []
+    rcvr_accessor :code_assets, Array, :of => Icss::CodeAsset,   :default => []
+    rcvr_accessor :targets,     Hash,  :of => Icss::TargetListFactory, :default => {}, :merge_as => :hash_of_arrays
+    rcvr_accessor :under_consideration, Boolean
+    rcvr_accessor :update_frequency, String # must be of the form daily, weekly, monthly, quarterly, never
+
+    # attr_accessor :body
+    after_receive do |hsh|
+      # Set each message's protocol to self, and if the name wasn't given, set
+      # it using the message's hash key.
+      self.messages.each{|msg_name, msg| msg.protocol = self; msg.name ||= msg_name }
+      # Set all the type's parent to self (for namespace resolution)
+      self.types.each{|type| type.parent  = self }
+      validate_name
+      validate_namespace
+    end
+
+    # String: namespace.name
+    def fullname
+      [namespace, name].compact.join(".")
+    end
+
+    def path
+      fullname.gsub('.', '/')
+    end
+
+    def find_message nm
+      return if messages.blank?
+      nm = nm.to_s.gsub("/", ".").split(".").last
+      messages[nm]
+    end
+
+    def receive_protocol nm
+      namespace_and_name = nm.to_s.gsub("/", ".").split(".")
+      self.protocol  = namespace_and_name.pop
+      self.namespace = namespace_and_name.join('.')
+    end
+
+    def receive_targets hsh
+      self.targets = hsh.inject({}) do |target_obj_hsh, (target_name, target_info_list)|
+        target_obj_hsh[target_name] = TargetListFactory.receive(target_info_list, target_name) # returns an arry of targets
+        target_obj_hsh
+      end
+    end
+
+    def to_hash()
+      {
+        :namespace   => @namespace, # use accessor so unset namespace isn't given
+        :protocol    => protocol,
+        :doc         => doc,
+        :under_consideration => under_consideration,
+        :update_frequency    => update_frequency,
+        :types       => (types       && types.map(&:to_hash)),
+        :messages    => messages.inject({}){|h,(k,v)| h[k] = v.to_hash; h },
+        :data_assets => data_assets.map(&:to_hash),
+        :code_assets => code_assets.map(&:to_hash),
+        :targets     => targets_to_hash,
+      }.reject{|k,v| v.nil? }.tree_merge! self.message_samples_hash
+    end
+
+    def targets_to_hash
+      return unless targets
+      targets.inject({}) do |hsh,(k,targs)|
+        hsh[k] = targs.map{|t| t.respond_to?(:to_hash) ? t.to_hash : t } ; hsh
+      end
+    end
+
+    # This will cause funny errors when it is an element of something that's to_json'ed
+    def to_json(*args) to_hash.to_json(*args) ; end
+  end
+end

data/lib/icss/protocol_set.rb

@@ -0,0 +1,48 @@
+# you must require 'icss/protocol_set' explicitly to use it.
+
+module Icss
+  #
+  # Holds a set of icss protocols, with helper methods to load them from a
+  # directory tree, to merge in a protocol set yaml file, and so forth
+  #
+  class ProtocolSet
+    attr_accessor :protocols
+
+    def initialize
+      self.protocols = {}
+    end
+
+
+    def register pr
+      if protocols[pr.fullname]
+        protocols[pr.fullname].tree_merge!(pr)
+      else
+        protocols[pr.fullname] = pr
+      end
+    end
+
+    # load and register the protocols in the given filenames
+    def load_protocols *icss_filenames
+      icss_filenames = icss_filenames.flatten.compact
+      icss_filenames.each do |icss_filename|
+        register Icss::Protocol.receive_from_file(icss_filename)
+      end
+    end
+
+    # load and register the set of protocols in the given yaml file.
+    # it must be a hash with element 'protocol_set', holding an array of
+    # protocol hashes.
+    #
+    #   protocol_set:
+    #     - namespace: "..."
+    #       protocol:  "..."
+    def load_protocol_set protocol_set_filename
+      protocol_set_hsh = YAML.load(File.open(protocol_set_filename))
+      protocol_set_hsh['protocol_set'].each do |hsh|
+        register Icss::Protocol.receive(hsh)
+      end
+    end
+  end
+end
+
+

data/lib/icss/sample_message_call.rb

@@ -0,0 +1,140 @@
+module Icss
+  #
+  # Holds a sample call for a message and its expected response
+  #
+  # You may define the request parameters using an array of parameters
+  # or with the corresponding URL it would render to.
+  #
+  # This file also decorates Icss::Message and Icss::Protocol with helper methods for sample calls.
+  #
+  class SampleMessageCall
+    include Receiver
+    rcvr_accessor :name,              String
+    rcvr_accessor :doc,               String
+    rcvr_accessor :request,           Array, :default => []
+    rcvr_accessor :response,          Object  # a hash suitable for populating the message's @response@ type
+    attr_accessor :raw_response               # the raw http response from fetching
+    rcvr_accessor :error,             String
+    rcvr_accessor :url,               String
+    attr_accessor :message
+
+    # The URL implied by the given hostname and the sample request parameters.
+    #
+    # @param [String] hostname  The hostname or hostname:port to include in the URL
+    # @param [Hash]   extra_query_params A hash of extra params to in
+    #
+    # The URI expects string values in the hash used to build the query -- if
+    # calling #to_s on a field won't do what you want, clobber the value beforehand.
+    #
+    def full_url hostname, extra_query_params={}
+      host, port = hostname.split(':', 2)
+      u = Addressable::URI.new(:host => host, :port => port, :path => self.path, :scheme => 'http')
+      u.query_values = query_hash(extra_query_params)
+      u
+    end
+
+    def query_hash extra_query_params={}
+      hsh = (@url.present? ? @url.query_values : request.first.to_hash) rescue {}
+      hsh = hsh.merge extra_query_params
+      hsh.each{|k,v| hsh[k] = v.to_s }
+      hsh
+    end
+
+    def path
+      ((@url && @url.path).present? ? @url.path : "/#{message.path}" )
+    end
+
+    # @param [String, Addressable::URI]
+    #   the URL can be fully-qualified (htttp://api.infochimps.com/this/that?the=other) or relative (this/that?the=other)
+    #   and the path must match that of the message.
+    #
+    def url= new_url
+      if new_url.is_a?(String)
+        unless new_url.include?('?') then warn "sample request url should have a '?' introducing its query parameters: {#{new_url}}" ; end
+        new_url = Addressable::URI.parse(new_url)
+      end
+      @url = new_url
+    end
+
+    # Whips up the class implied by the ICSS type of this message's response,
+    # and populates it using the response hash.
+    def response_obj
+      return if response.blank?
+      klass = message.response.ruby_klass
+      klass.receive(response.compact_blank!)
+    end
+
+    # retrieve the response from the given host, storing it in response.  this
+    # catches all server errors and constructs a dummy response hash if the call
+    # fails.
+    def fetch_response! hostname="", extra_query_params={}
+      self.raw_response = fetch_raw_response( full_url(hostname, extra_query_params) )
+      begin
+        resp_hsh = JSON.load(raw_response.body)
+      rescue StandardError => e
+        warn ["  error parsing response: #{e}"].join("\n")
+        self.response = nil
+        self.error    = "JsonParseError"
+        return
+      end
+      if raw_response.code == 200
+        self.response = resp_hsh
+        self.error    = nil
+      else
+        self.response = nil
+        self.error    = resp_hsh["error"]
+      end
+    end
+
+  private
+
+    def fetch_raw_response full_url
+      RestClient.get(full_url.to_s) do |response, request, result|
+        response
+      end
+    end
+  end
+end
+
+class Icss::Message
+  rcvr_accessor :samples, Array, :of => Icss::SampleMessageCall, :default => []
+end
+
+class Icss::Protocol
+
+  #
+  # a hash for dumping to file:
+  # @example: from the whole thing, would dump only this:
+  #
+  #     namespace: util.time
+  #     protocol: chronic
+  #     messages:
+  #       parse:
+  #         samples:
+  #           - url:            "?now=5%3A06%3A07%202010-08-08&time_str=Yesterday"
+  #             response:       { "time": "2010-08-07 05:06:07 UTC", "epoch_seconds": 1281225967 }
+  #
+  def message_samples_hash
+    hsh = { "namespace" => namespace, "protocol" => protocol, "messages" => {} }
+    messages.each do |msg_name, msg|
+      hsh["messages"][msg_name] = { "samples" => [] }
+      msg.samples.each do |sample_req|
+        sample_hsh = {
+          "name"     => sample_req.name,
+          "doc"      => sample_req.doc,
+        }
+        if sample_req.response.present?
+        then sample_hsh['response'] =  sample_req.response
+        else sample_hsh['error']    =  sample_req.error
+        end
+        if sample_req.url.present?
+        then sample_hsh['url']     = sample_req.url.to_s
+        else sample_hsh['request'] = sample_req.request
+        end
+        hsh["messages"][msg_name]["samples"] << sample_hsh.compact_blank!
+      end
+    end
+    return hsh
+  end
+
+end