icss-activesupport-4 0.4.0

data/lib/icss.rb

@@ -0,0 +1,53 @@
+require 'gorillib/object/blank'
+require 'gorillib/object/try_dup'
+require 'gorillib/string/inflections'
+require 'gorillib/string/constantize'
+require 'gorillib/array/compact_blank'
+require 'gorillib/array/extract_options'
+require 'gorillib/hashlike'
+require 'gorillib/hashlike/tree_merge'
+require 'gorillib/hash/compact'
+require 'gorillib/hash/keys'
+require 'gorillib/hash/tree_merge'
+require 'gorillib/metaprogramming/class_attribute'
+require 'gorillib/serialization'
+require 'gorillib/hashlike'
+
+#
+require 'configliere'
+Settings.define :catalog_root, :type => :filename, :default => (defined?(Rails) ? (Rails.root+'catalog') : File.expand_path('../../ics/infochimps_catalog', __FILE__))
+#
+require 'icss/error'
+require 'icss/receiver_model/acts_as_hash'
+require 'icss/receiver_model/acts_as_loadable'
+require 'icss/receiver_model/acts_as_catalog'
+require 'icss/receiver_model/active_model_shim'
+require 'icss/receiver_model/validations'
+require 'icss/receiver_model/acts_as_tuple'
+#
+require 'icss/type'                   #
+require 'icss/type/simple_types'      # Boolean, Integer, ...
+require 'icss/type/named_type'        # class methods for a named type: .metamodel .doc, .fullname, &c
+require 'icss/type/record_type'       # class methods for a record model: .field, .receive,
+require 'icss/type/record_model'      # instance methods for a record model
+
+
+#
+require 'icss/type/type_factory'      #
+require 'icss/type/structured_schema' # generate type from array, hash, &c schema
+require 'icss/type/union_schema'      # factory for instances based on type
+require 'icss/type/record_schema'
+require 'icss/type/record_field'
+#
+require 'icss/receiver_model'
+require 'icss/serialization'
+#
+require 'icss/message'
+require 'icss/protocol/data_asset'
+require 'icss/protocol/code_asset'
+require 'icss/protocol/target'
+require 'icss/protocol/license'
+require 'icss/protocol/source'
+require 'icss/protocol'
+require 'icss/message/message_sample'
+

data/lib/icss/core_types.rb

@@ -0,0 +1,20 @@
+# p [__FILE__, "core types from ", Settings.catalog_root]
+
+module Icss
+  module Encyclopedic ; module Wikipedia ; module Dbpedia ; end ; end ; end
+  module Web ; module An ; end ; end
+  module Web ; module Analytics ; end ; end
+  module Engineering ; module Chemical; module Msds ; end ; end ; end
+  module Government ; module Public ; module Acs ; end ; end ; end
+  module Language ; module Corpora ; module WordFreq ; end ; end ; end
+  module Sports ; module Stats ; module Baseball ; end ; module Vargatron ; end ; end ; end
+  module Social  ; module Network ; module Tw ; end ; module Qwerly ; end ; end ;  end
+  module Soc  ; module Net ; module Tw ; end ; end ;  end
+  module Meta ; module Req ; class Geolocator ; end  ; end ; end
+  module St ; class Url < String ; end ; end
+  module Geo ; module Location ; end ; end
+end
+
+unless defined?(Icss::Geo::Place)
+  warn "Could not load core type 'place'. Make sure the catalog (#{Settings.catalog_root}) is where you expect it to be."
+end

data/lib/icss/error.rb

@@ -0,0 +1,4 @@
+module Icss
+  class NotFoundError      < ::NameError     ; end unless defined?(NotFoundError)
+  class FactoryTypeMissing < ::ArgumentError ; end unless defined?(FactoryTypeMissing)
+end

data/lib/icss/init.rb

@@ -0,0 +1,3 @@
+require 'icss'
+require 'icss/view_helper'
+

data/lib/icss/message.rb

@@ -0,0 +1,133 @@
+module Icss
+  module Meta
+    #
+    # Describes an Avro Message
+    #
+    # A message has attributes:
+    #
+    # * doc:        an optional description of the message,
+    # * request:    a list of named, typed parameter schemas (this has the same form as the fields of a record declaration);
+    # * response:   a valid schema for the response
+    # * errors:     an optional union of error schemas.
+    #
+    # A request parameter list is processed equivalently to an anonymous
+    # record. Since record field lists may vary between reader and writer, request
+    # parameters may also differ between the caller and responder, and such
+    # differences are resolved in the same manner as record field differences.
+    #
+    class Message
+      include ::Icss::ReceiverModel
+
+      field :name,     String
+      alias_method :basename,  :name
+      alias_method :basename=, :name=
+      field :doc,      String
+
+      field :request_decorators, Hash, :default => {:anchors => []}
+
+      field :request,  Array, :items => Icss::Meta::RecordField, :default => []
+      field :response, Icss::Meta::TypeFactory
+      field :errors,   Object # FIXME: Icss::Meta::UnionType, :default => []
+      # this is defined in sample_message_call.rb -- since we don't do referenced types yet
+
+      attr_accessor :protocol
+
+      #we're starting to attach a lot of pork to this lib...
+      field :initial_free_qty,      Integer
+      field :price_per_k_in_cents,  Integer
+
+      after_receive(:are_my_types_references) do |hsh|
+        # track recursion of type references
+        @response_referenceness = ! hsh[:response].respond_to?(:each_pair)
+      end
+
+      after_receive(:parent_my_samples) do |hsh|
+        # # tie each sample back to this, its parent message
+        (self.samples||=[]).each{|samp| samp.message = self }
+      end
+
+      def fullname
+        "#{protocol.fullname}.#{basename}"
+      end
+      def path
+        fullname.gsub(%r{\.},'/')
+      end
+
+      # the type of the message's params (by convention, its first request field)
+      def params_type
+        request.first ? request.first.type : {}
+      end
+
+      def first_sample_request_param
+        req = samples.first.request.first rescue nil
+        req || {}
+      end
+
+      # ----------------------------------------
+      # GEO
+      #
+
+      def is_a_geo?
+        geolocators.present?
+      end
+
+      rcvr_alias(:is_geo, :is_geo)
+      def receive_is_geo(val)
+        return unless val
+        unless defined?(Icss::Meta::Req::Geolocator) then
+          warn "View helpers can\'t help with geolocators: Icss::Meta::Req::Geolocator type is missing. Is the catalog loaded properly?"
+          return
+        end
+        self.request_decorators = {
+          :anchors => [
+            Icss::Meta::Req::PointWithRadiusGeolocator,
+            Icss::Meta::Req::AddressTextGeolocator,
+            Icss::Meta::Req::TileXYZoomGeolocator,
+            Icss::Meta::Req::BoundingBoxGeolocator,
+            Icss::Meta::Req::IpAddressGeolocator
+          ],
+        }
+      end
+
+      def geolocators
+        request_decorators[:anchors]
+      end
+
+      #
+      # Conversion
+      #
+      def to_hash()
+        {
+          :request  => summary_of_request_attr,
+          :response => summary_of_response_attr,
+          :doc      => doc,
+          :errors   => (errors.blank? ? nil : errors),
+          :samples  => samples.map(&:to_hash).map(&:compact_blank),
+          :initial_free_qty => initial_free_qty,
+          :price_per_k_in_cents => price_per_k_in_cents,
+        }.compact
+      end
+      def to_json(*args) to_hash.to_json(*args) ; end
+
+    private
+      def summary_of_response_attr
+        case
+        when response.blank?        then response
+        when @response_referenceness then response.fullname
+        else response.to_schema.compact_blank
+        end
+      end
+      def summary_of_request_attr
+        request.map do |req|
+          case
+          when req.blank?        then req
+          # Is there a case where this needs to be a string and not a hash?
+          # when req.is_reference? then req.type.fullname
+          else                        req.to_schema.compact_blank
+          end
+        end
+      end
+    end
+  end
+
+end

data/lib/icss/message/message_sample.rb

@@ -0,0 +1,144 @@
+module Icss
+  module Meta
+    #
+    # Holds a sample call for a message and its expected response
+    #
+    # You may define the request parameters using an array of parameters
+    # or with the corresponding URL it would render to.
+    #
+    # This file also decorates Icss::Meta::Message and Icss::Meta::Protocol with helper methods for sample calls.
+    #
+    class MessageSample
+      include ReceiverModel
+      field         :name,         String
+      field         :doc,          String
+      field         :request,      Array, :default => []
+      field         :response_hsh, Hash  # a hash suitable for populating the message's @response@ type
+      field         :response,     Hash
+      attr_accessor :raw_response         # the raw http response from fetching
+      field         :error,        String
+      field         :url,          String
+      attr_accessor :message
+
+      # Whips up the class implied by the ICSS type of this message's response,
+      # and populates it using the response hash.
+      def response_obj
+        return if response.blank?
+        message.response.receive(response)
+      end
+
+      # The URL implied by the given hostname and the sample request parameters.
+      #
+      # @param [String] hostname  The hostname or hostname:port to include in the URL
+      # @param [Hash]   extra_query_params A hash of extra params to in
+      #
+      # The URI expects string values in the hash used to build the query -- if
+      # calling #to_s on a field won't do what you want, clobber the value beforehand.
+      #
+      def full_url hostname, extra_query_params={}
+        host, port = hostname.split(':', 2)
+        u = Addressable::URI.new(:host => host, :port => port, :path => self.path, :scheme => 'http')
+        u.query_values = query_hash(extra_query_params)
+        u
+      end
+
+      def query_hash extra_query_params={}
+        hsh = (@url.present? ? @url.query_values : request.first.to_hash) rescue {}
+        hsh = hsh.merge extra_query_params
+        hsh.each{|k,v| hsh[k] = v.to_s }
+        hsh
+      end
+
+      def path
+        ((@url && @url.path).present? ? @url.path : "/#{message.path}" )
+      end
+
+      # @param [String, Addressable::URI]
+      #   the URL can be fully-qualified (htttp://api.infochimps.com/this/that?the=other) or relative (this/that?the=other)
+      #   and the path must match that of the message.
+      #
+      def url= new_url
+        if new_url.is_a?(String)
+          unless new_url.include?('?') then warn "sample request url should have a '?' introducing its query parameters: {#{new_url}}" ; end
+          new_url = Addressable::URI.parse(new_url)
+        end
+        @url = new_url
+      end
+
+      # retrieve the response from the given host, storing it in response.  this
+      # catches all server errors and constructs a dummy response hash if the call
+      # fails.
+      def fetch_response! hostname="", extra_query_params={}
+        self.raw_response = fetch_raw_response( full_url(hostname, extra_query_params) )
+        begin
+          resp_hsh = JSON.load(raw_response.body)
+        rescue StandardError => e
+          warn ["  error parsing response: #{e}"].join("\n")
+          self.response = nil
+          self.error    = "JsonParseError"
+          return
+        end
+        if raw_response.code == 200
+          self.response = resp_hsh
+          self.error    = nil
+        else
+          self.response = nil
+          self.error    = resp_hsh["error"]
+        end
+      end
+
+      protected
+
+      def fetch_raw_response full_url
+        RestClient.get(full_url.to_s) do |response, request, result|
+          response
+        end
+      end
+    end
+  end
+
+end
+
+class Icss::Meta::Message
+  field :samples, Array, :items => Icss::Meta::MessageSample, :default => []
+end
+
+class Icss::Meta::Protocol
+
+  #
+  # a hash for dumping to file:
+  # @example: from the whole thing, would dump only this:
+  #
+  #     namespace: util.time
+  #     protocol: chronic
+  #     messages:
+  #       parse:
+  #         samples:
+  #           - url:            "?now=5%3A06%3A07%202010-08-08&time_str=Yesterday"
+  #             response:       { "time": "2010-08-07 05:06:07 UTC", "epoch_seconds": 1281225967 }
+  #
+  def message_samples_hash
+    hsh = { :namespace => namespace, :protocol => protocol, :messages => {} }
+    messages.each do |msg_name, msg|
+      hsh[:messages][msg_name] = { :samples => [] }
+      msg.samples.each do |sample_req|
+        sample_hsh = {
+          :name     => sample_req.name,
+          :doc      => sample_req.doc,
+        }
+        if sample_req.response.present?
+        then sample_hsh[:response] =  sample_req.response
+        else sample_hsh[:error]    =  sample_req.error
+        end
+        if sample_req.url.present?
+        then sample_hsh[:url]      = sample_req.url.to_s
+        else sample_hsh[:request]  = sample_req.request
+        end
+        sample_hsh.compact_blank!
+        hsh[:messages][msg_name][:samples] << sample_hsh
+      end
+    end
+    return hsh
+  end
+
+end

data/lib/icss/protocol.rb

@@ -0,0 +1,199 @@
+module Icss
+  module Meta
+
+    # predefine so we can use below
+
+    class Message ; end
+
+    #
+    # Describes an Avro Protocol Declaration
+    #
+    # Avro protocols describe RPC interfaces. The Protocol class will receive an
+    # Avro JSON
+    #
+    # A Protocol has the following attributes:
+    #
+    # * protocol, a string, the name of the protocol (required). +name+ is
+    #   provided as an alias for +protocol+.
+    #
+    # * namespace, a string that qualifies the name (optional).
+    #
+    # * doc, a string describing this protocol (optional).
+    #
+    # * types, an optional list of definitions of named types (records, enums,
+    #   fixed and errors). An error definition is just like a record definition
+    #   except it uses "error" instead of "record". Note that forward references
+    #   to named types are not permitted.
+    #
+    # * messages, an optional JSON object whose keys are message names and whose
+    #   values are objects whose attributes are described below. No two messages
+    #   may have the same name.
+    #
+    # The name and namespace qualification rules defined for schema objects apply
+    # to protocols as well: see the documentation for Icss::Meta::Type.
+    #
+    # For example, one may define a simple HelloWorld protocol with:
+    #
+    #     {
+    #       "namespace":    "com.acme",
+    #       "protocol":     "HelloWorld",
+    #       "doc":          "Protocol Greetings",
+    #       "types": [
+    #         { "name":     "Greeting",
+    #           "type":     "record",
+    #           "fields":   [ {"name": "message", "type": "string"} ]},
+    #         { "name":     "Curse",
+    #           "type":     "error",
+    #           "fields":   [ {"name": "message", "type": "string"} ]}
+    #       ],
+    #       "messages": {
+    #         "hello": {
+    #           "doc":      "Say hello.",
+    #           "request":  [{"name": "greeting", "type": "Greeting" }],
+    #           "response": "Greeting",
+    #           "errors":   ["Curse"]
+    #         }
+    #       }
+    #     }
+    #
+    class Protocol
+      include Icss::ReceiverModel
+      include Icss::ReceiverModel::ActsAsCatalog
+
+      field :protocol,    String, :required => true, :validates => { :format => { :with => /\A[A-Za-z_]\w*\Z/, :message => "must start with [A-Za-z_] and contain only [A-Za-z0-9_]." } }
+      alias_method  :basename, :protocol
+      field :namespace,   String, :required => true, :validates => { :format => { :with => /\A([A-Za-z_]\w*\.?)+\Z/, :message => "Segments that start with [A-Za-z_] and contain only [A-Za-z0-9_], joined by '.'dots" } }
+      field :title,       String
+      field :doc,         String
+      #
+      field :types,       Array, :items => Icss::Meta::TypeFactory, :default => []
+      field :_doc_hints,  Hash,  :default => {}
+
+      field :messages,    Hash,  :values => Icss::Meta::Message,     :default => {}
+      field :data_assets, Array, :items  => Icss::Meta::DataAsset,   :default => []
+      field :code_assets, Array, :items  => Icss::Meta::CodeAsset,   :default => []
+      field :targets,     Hash,  :values => Icss::TargetListFactory, :default => {}, :merge_as => :hash_of_arrays
+      
+      field :tags,        Array, :items => String, :default => []
+      field :categories,  Array, :items => String, :default => []
+      field :license_id,  String
+      field :credits,     Hash,  :values=> String, :default => {} # hash of source_ids
+      
+      def license
+        Icss::Meta::License.find(license_id) unless license_id.blank?
+      end  
+      
+      def sources
+        @sources ||= credits.inject(Hash.new){|hash, credit| hash[credit[0].to_sym] = Icss::Meta::Source.find(credit[1]); hash }
+      end
+
+      field :under_consideration, Boolean
+      field :update_frequency, String, :validates => { :format => { :with => /daily|weekly|monthly|quarterly|never/ }, :allow_blank => true }
+      rcvr_remaining :_extra_params
+
+      after_receive(:parent_my_messages) do |hsh|
+        # Set each message's protocol to self, and if the basename wasn't given, set
+        # it using the message's hash key.
+        self.messages.each{|msg_name, msg| msg.protocol = self; msg.basename ||= msg_name }
+      end
+      after_receive(:warn_if_invalid) do |hsh|
+        warn errors.inspect unless valid?
+        warn "Extra params #{_extra_params.keys.inspect} given to #{self.fullname}" if _extra_params.present?
+      end
+      after_receive(:declare_core_types) do |hsh|
+        self.types.each{|type| type.respond_to?(:_schema) && (type._schema.is_core = (self.fullname  == "icss.core.typedefs")) }
+      end
+      after_receive(:fix_legacy_catalog_info) do
+        if targets[:catalog].present?
+          catalog = targets[:catalog].first
+          if self.title.blank? then self.title = catalog.title ; end
+          if self.tags.blank? then self.tags = catalog.tags ; end
+          if self.doc.blank? then self.doc = catalog.description; end
+        end
+      end
+
+      # String: namespace.basename
+      def fullname
+        [namespace, basename].compact.join(".")
+      end
+
+      # a / separated version of the fullname, with no / at start
+      def path
+        fullname.gsub('.', '/')
+      end
+
+      def find_message nm
+        return if messages.blank?
+        nm = nm.to_s.gsub("/", ".").split(".").last
+        messages[nm]
+      end
+
+      def receive_types(types)
+        # this is a horrible, horrible kludge so that types with simple names ('bob') can become
+        # properly namespaced ('foo.bar.bob') even when they haven't met their parents (and even 
+        # where the calls to receive types are nested/recursive)
+        Icss::Meta::TypeFactory.with_namespace(namespace) do
+          super(types)
+        end
+      end
+
+      def receive_messages(types)
+        # this is a horrible, horrible kludge so that messages with simple names ('do_bob') can become
+        # properly namespaced ('foo.bar.do_bob') even when they haven't met their parents (and even 
+        # where the calls to receive_messages are nested/recursive)
+        Icss::Meta::TypeFactory.with_namespace(namespace) do
+          super(types)
+        end
+      end
+
+      def receive_protocol(nm)
+        name_segs = nm.to_s.gsub("/", ".").split(".")
+        self.protocol  = name_segs.pop
+        self.namespace = name_segs.join('.') if name_segs.present?
+      end
+
+      def receive_targets(tgts)
+        return unless tgts.present?
+        self.targets ||= {}
+        tgts.symbolize_keys!.each do |target_name, target_info_list|
+          targets[target_name] = TargetListFactory.receive(target_info_list, target_name) # array of targets
+        end
+        targets
+      end
+      
+      def self.catalog_sections
+        ['core', 'datasets', 'old']
+      end
+
+      def to_hash()
+        {
+          :namespace   => @namespace, # use accessor so unset namespace isn't given
+          :protocol    => protocol,
+          :license_id  => license_id,
+          :credits     => credits,
+          :tags        => tags,
+          :categories  => categories,
+          :doc         => doc,
+          :types       => (types       && types.map(&:to_schema)),
+          :messages    => messages.inject({}){|h,(k,v)| h[k.to_sym] = v.to_hash; h },
+          :data_assets => data_assets.map(&:to_hash).map(&:compact_blank),
+          :code_assets => code_assets.map(&:to_hash).map(&:compact_blank),
+          :update_frequency    => update_frequency,
+          :under_consideration => under_consideration,
+          :targets     => targets_to_hash,
+        }.reject{|k,v| v.nil? }
+      end
+
+      def targets_to_hash
+        return unless targets
+        targets.inject({}) do |hsh,(k,targs)|
+          hsh[k] = targs.map(&:to_hash).map(&:compact_blank) ; hsh
+        end
+      end
+
+      # This will cause funny errors when it is an element of something that's to_json'ed
+      def to_json(*args) to_hash.to_json(*args) ; end      
+    end
+  end
+
+end