omf_oml 1.0.0 → 1.1.0

data/lib/omf_oml/endpoint.rb

@@ -30,6 +30,7 @@ module OMF::OML
 
     def initialize(port = 5000, host = "0.0.0.0")
       require 'socket'
+      debug "OML client listening on #{port}"
       @serv = TCPServer.new(host, port)
       @running = false
       @on_new_stream_procs = {}
@@ -108,7 +109,7 @@ module OMF::OML
     private
     def parse_header(socket, &reportStreamProc)
       while (l = socket.gets.strip)
-        #puts "H>> '#{l}'"        
+        #puts "H>> '#{l}'"
         return if l.length == 0
 
         key, *value = l.split(':')
@@ -126,29 +127,64 @@ module OMF::OML
       els = desc.split(' ')
       #puts "ELS: #{els.inspect}"
       index = els.shift.to_i - 1
+      if (index < 0)
+        # meta data - ignore
+        return
+      end
+      if @streams[index]
+        warn "Schema '#{index}' already defined"
+        return
+      end
       sname = els.shift
       schema_desc = els.collect do |el|
         name, type = el.split(':')
+        if (type == 'blob'); type = 'blob64' end # base64 encode blob
         {:name => name.to_sym, :type => type.to_sym}
       end
       schema_desc.insert(0, {:name => :oml_ts, :type => :double})
       schema_desc.insert(1, {:name => :sender_id, :type => :string})
-      schema_desc.insert(2, {:name => :oml_seq_no, :type => :integer})            
+      schema_desc.insert(2, {:name => :oml_seq_no, :type => :integer})
       schema = OMF::OML::OmlSchema.create(schema_desc)
+      debug "New schema '#{sname}(#{index})' with schema '#{schema}'"
       @streams[index] = tuple = OmlTuple.new(sname, schema)
       @endpoint.report_new_stream(sname, tuple)
     end
 
+    def parse_meta_row(row)
+      #puts "META>> #{row.inspect}"
+      unless row.length == 5
+        warn "Received mis-formatted META tuple - #{row}"
+        return
+      end
+      key = "#{row[2]}/#{row[3]}".downcase
+      value = row[4]
+      debug "META: '#{key}' - '#{value}'"
+      case key
+      when './schema'
+        parse_schema(value)
+      else
+        warn "Unknwon META type '#{key}'"
+      end
+    end
+
     def parse_rows(socket)
       sender_id = @header['sender-id'] || 'unknown'
       while (l = socket.gets)
         return if l.length == 0
 
         els = l.strip.split("\t")
-        #puts "R>> '#{els.inspect}'"
-        index = els.delete_at(1).to_i - 1
+        #puts "R(#{sender_id})>> '#{els.inspect}'"
+        index = els.delete_at(1).to_i
+        if (index == 0)
+          parse_meta_row(els)
+          next
+        end
+        unless stream = @streams[index - 1]
+          warn "Receiving tuples for unknown schema index '#{index}'"
+          next
+        end
         els.insert(1, sender_id)
-        row = @streams[index].parse_tuple(els)
+        row = @streams[index - 1].parse_tuple(els)
       end
     end
 
@@ -160,13 +196,15 @@ end
 if $0 == __FILE__
 
   require 'omf_oml/table'
-  ep = OMF::OML::OmlEndpoint.new(3000)
-  toml = OMF::OML::OmlTable.new('oml', [[:x], [:y]], :max_size => 20)
-  ep.on_new_stream() do |s|
-    puts "New stream: #{s}"
-    s.on_new_vector() do |v|
-      puts "New vector: #{v.select(:oml_ts, :value).join('|')}"
-      toml.add_row(v.select(:oml_ts, :value))
+  require 'omf_base/lobject'
+  OMF::Base::Loggable.init_log 'endpoint'
+
+  ep = OMF::OML::OmlEndpoint.new(3003)
+  ep.on_new_stream() do |name, stream|
+    puts "New stream: #{name}-#{stream}"
+    table = stream.create_table(name + '_tbl', :max_size => 5)
+    table.on_content_changed do |action, change|
+      puts "TTTT > #{action} - #{change}"
     end
   end
   ep.run(false)

data/lib/omf_oml/oml_tuple.rb

@@ -9,15 +9,15 @@ require 'omf_oml/schema'
 require 'omf_oml/tuple'
 
 module OMF::OML
-  
+
   # This class represents a single vector from an OML measurement stream.
   # It provides various methods to access the vectors elements.
   #
-  # NOTE: Do not store the vector itself, but make a copy as the instance may be 
+  # NOTE: Do not store the vector itself, but make a copy as the instance may be
   # reused over various rows by the sender.
   #
   class OmlTuple < Tuple
-    
+
     # Return the elements of the vector as an array
     # def to_a(include_index_ts = false)
       # res = []
@@ -31,39 +31,39 @@ module OMF::OML
       # end
       # res
     # end
-    
+
     def ts
       @raw[0].to_f
     end
-    
+
     def seq_no
       @raw[1].to_i
     end
-        
+
     # Note: This method assumes that the first two elements in each OML tuple,
     # 'oml_ts' and 'oml_seq_no' are not defined in the associated schema.
     #
-    def process_schema(schema)
-      i = 0
-      @vprocs = {}
-      schema.each_column do |col| #
-        name = col[:name] || raise("Ill-formed schema '#{schema}'")
-        type = col[:type] || raise("Ill-formed schema '#{schema}'")
-        j = i + 2; # need to create a locally scoped variable for the following lambdas
-        @vprocs[name] = @vprocs[i] = case type      
-          when :string
-            lambda do |r| r[j] end
-          when :float
-            lambda do |r| r[j].to_f end
-          when :integer
-            lambda do |r| r[j].to_i end
-          else raise "Unrecognized OML type '#{type}' (#{col.inspect})"
-        end
-        i += 1
-      end
-      @vprocs[:oml_ts] = lambda do |r| r[0].to_f end
-      @vprocs[:oml_seq_no] = lambda do |r| r[1].to_i end
-    end
-    
+    # def process_schema(schema)
+      # i = 0
+      # @vprocs = {}
+      # schema.each_column do |col| #
+        # name = col[:name] || raise("Ill-formed schema '#{schema}'")
+        # type = col[:type] || raise("Ill-formed schema '#{schema}'")
+        # j = i + 2; # need to create a locally scoped variable for the following lambdas
+        # @vprocs[name] = @vprocs[i] = case type
+          # when :string
+            # lambda do |r| r[j] end
+          # when :float
+            # lambda do |r| r[j].to_f end
+          # when :integer
+            # lambda do |r| r[j].to_i end
+          # else raise "Unrecognized OML type '#{type}' (#{col.inspect})"
+        # end
+        # i += 1
+      # end
+      # @vprocs[:oml_ts] = lambda do |r| r[0].to_f end
+      # @vprocs[:oml_seq_no] = lambda do |r| r[1].to_i end
+    # end
+
   end # OmlTuple
 end # OMF::OML

data/lib/omf_oml/schema.rb

@@ -8,21 +8,21 @@ require 'omf_base/lobject'
 require 'omf_oml'
 
 module OMF::OML
-  
+
   # This class represents the schema of an OML measurement stream.
   #
   class OmlSchema < OMF::Base::LObject
-    
+
     CLASS2TYPE = {
       TrueClass => 'boolean',
       FalseClass => 'boolean',
       String => 'string',
-      Symbol => 'string',            
+      Symbol => 'string',
       Fixnum => 'decimal',
       Float => 'double',
       Time => 'dateTime'
     }
-    
+
     # Map various type definitions (all lower case) into a single one
     ANY2TYPE = {
       'integer' => :integer,
@@ -43,9 +43,12 @@ module OMF::OML
       'date' => :date,
       'dateTime'.downcase => :dateTime, # should be 'datetime' but we downcase the string for comparison
       'timestamp' => :dateTime, # Postgreql specific, not sure if this works as it.
-      'key' => :key,      
+      'key' => :key,
+      'bool' => :bool,
+      'blob' => :blob,
+      'blob64' => :blob64 # base64 encoded blob
     }
-    
+
     OML_INTERNALS = [
       :oml_sender,
       :oml_sender_id,
@@ -60,7 +63,7 @@ module OMF::OML
       end
       return self.new(schema_description)
     end
-    
+
     # Return the col name at a specific index
     #
     def name_at(index)
@@ -69,17 +72,25 @@ module OMF::OML
       end
       d[:name]
     end
-    
+
     # Return the col index for column named +name+
     #
     def index_for_col(name)
+      if name.is_a? Integer
+        # seems they already know hte index
+        index = name
+        if (index >= @schema.length)
+          raise "Index '#{index}' points past schema - #{schema.length}"
+        end
+        return index
+      end
       name = name.to_sym
       @schema.each_with_index do |col, i|
-        return i if col[:name] == name 
+        return i if col[:name] == name
       end
-      raise "Unknonw column '#{name}'"
+      raise "Unknown column '#{name}'"
     end
-    
+
     # Return the column names as an array
     #
     def names
@@ -91,25 +102,25 @@ module OMF::OML
     def type_at(index)
       @schema[index][:type]
     end
-    
+
     def columns
       @schema
     end
-    
+
     def insert_column_at(index, col)
-      @schema.insert(index, _create_col_descr(col))       
+      @schema.insert(index, _create_col_descr(col))
     end
-    
+
     def replace_column_at(index, col)
       @schema[index] = _create_col_descr(col)
     end
 
     def each_column(&block)
-      @schema.each do |c| 
-        block.call(c) 
+      @schema.each do |c|
+        block.call(c)
       end
     end
-    
+
     # Translate a record described in a hash of 'col_name => value'
     # to a row array. Note: Will suppress column '__id__'
     #
@@ -127,7 +138,7 @@ module OMF::OML
         end
         unless (v = hrow[cname]) ||  (v = hrow[cname.to_sym])
           next nil if set_nil_when_missing
-          raise "Missing record element '#{cname}' in record '#{hrow}'"
+          raise "Missing record element '#{cname}' in record '#{hrow}' - schema: #{@schema}"
         end
         call_type_conversion ? cdescr[:type_conversion].call(v) : v
       end
@@ -135,7 +146,7 @@ module OMF::OML
       #puts "#{r.inspect} -- #{@schema.map {|c| c[:name]}.inspect}"
       r
     end
-    
+
     # Cast each element in 'row' into its proper type according to this schema
     #
     def cast_row(raw_row, ignore_first_column = false)
@@ -150,15 +161,25 @@ module OMF::OML
       end
       row
     end
-    
+
+    # Cast a named column from a 'raw' row into its proper type according to this schema
+    #
+    def cast_col(col_name, raw_row)
+      index = index_for_col(col_name)
+      unless raw_value = raw_row[index]
+        raise "Row doesn't include a value at '#{index}' (#{raw_row.inspect}-#{describe}) "
+      end
+      value = @schema[index][:type_conversion].call(raw_value)
+    end
+
     def describe
       @schema.map {|c| {:name => c[:name], :type => c[:type], :title => c[:title] }}
     end
-    
+
     def to_json(*opt)
       describe.to_json(*opt)
     end
-    
+
     def clone(exclude_oml_internals = false)
       c = self.dup
       schema = @schema.clone
@@ -170,13 +191,13 @@ module OMF::OML
       c.instance_variable_set('@schema', schema)
       c
     end
-    
+
     def to_s
       "OmlSchema: #{@schema.map {|c| "#{c[:name]}:#{c[:type]}"}.join(', ')}"
     end
-    
+
     protected
-    
+
     # schema_description - Array containing [name, type*] for every column in table
     #   TODO: define format of TYPE
     #
@@ -187,9 +208,9 @@ module OMF::OML
         insert_column_at(i, cdesc)
       end
       debug "schema: '#{describe.inspect}'"
-      
+
     end
-    
+
     # Create a column descriptor from whatever is given by user
     #
     def _create_col_descr(col)
@@ -199,12 +220,12 @@ module OMF::OML
       if col.kind_of? Array
         # should be [name, type]
         if col.length == 1
-          col = {:name => col[0].to_sym, 
-                  :type => :string, 
+          col = {:name => col[0].to_sym,
+                  :type => :string,
                   :title => col[0].to_s.split('_').collect {|s| s.capitalize}.join(' ')}
         elsif col.length == 2
           col = {:name => col[0].to_sym,
-                  :type => col[1].to_sym, 
+                  :type => col[1].to_sym,
                   :title => col[0].to_s.split('_').collect {|s| s.capitalize}.join(' ')}
         elsif col.length == 3
           col = {:name => col[0].to_sym, :type => col[1].to_sym, :title => col[2]}
@@ -217,7 +238,7 @@ module OMF::OML
           col[:title] = col[:name].to_s.split('_').collect {|s| s.capitalize}.join(' ')
         end
       end
-      
+
       # should normalize type
       if type = col[:type]
         tn = type.to_s.split('(')[0] # take care of types like varargs(..)
@@ -226,28 +247,33 @@ module OMF::OML
           type = :string
         end
       else
-        warn "Missing type definition in '#{col[:name]}', default to 'string' (#{col.inspect})"          
+        warn "Missing type definition in '#{col[:name]}', default to 'string' (#{col.inspect})"
         type = :string
       end
       col[:type] = type
-      
+
       col[:type_conversion] = case type
         when :string
           lambda do |r| r end
         when :key
           lambda do |r| r end
-        when :integer 
+        when :integer
           lambda do |r| r.to_i end
-        when :float 
+        when :float
           lambda do |r| r.to_f end
-        when :date 
+        when :bool
+          lambda do |r| (r.downcase.start_with? 't') ? true : false end
+        when :date
           lambda do |r| Date.parse(r) end
         when :dateTime
           lambda do |r| Time.parse(r) end
+        when :blob64
+          require "base64"
+          lambda do |r| Base64.decode64(r) end
         else raise "Unrecognized Schema type '#{type}'"
       end
       col
     end
   end # OmlSchema
-  
+
 end

data/lib/omf_oml/sql_row.rb

@@ -33,7 +33,7 @@ module OMF::OML
     def initialize(sql_table_name, schema, query, opts = {})
       @sname = sql_table_name
       @schema = schema
-      raise "Expected OmlSchema but got '#{schema.class}" unless schema.is_a? OmlSchema
+      raise "Expected OmlSchema but got '#{schema.class}'" unless schema.is_a? OmlSchema
       @query = query
 
       unless @offset = opts[:offset]

data/lib/omf_oml/sql_source.rb

@@ -26,7 +26,8 @@ module OMF::OML
     # for all adaptors
     #
     FALLBACK_MAPPING = {
-      'UNSIGNED INTEGER' => :integer
+      'UNSIGNED INTEGER' => :integer,
+      'UNSIGNED BIGINT' => :integer
     }
 
     # db_opts - Options used to create a Sequel adapter
@@ -62,12 +63,20 @@ module OMF::OML
     # table_name - Name of table in the SQL database
     # opts -
     #   :name - name used for returned OML Table [table_name]
+    #   :schema - Schema to use instead of default table schema
+    #   :query - Query to use instead of default one
     #   All other options defined for OmlSqlRow#new
     #
     def create_table(table_name, opts = {})
       tn = opts.delete(:name) || table_name
-      schema = _schema_for_table(table_name)
-      r = OmlSqlRow.new(table_name, schema, _def_query_for_table(table_name), opts)
+      schema = opts.delete(:schema) || _schema_for_table(table_name)
+      if q = opts.delete(:query)
+        query = (q.is_a? String) ? @db[q] : q
+      else
+        query = _def_query_for_table(table_name)
+      end
+      r = OmlSqlRow.new(table_name, schema, query, opts)
+      opts[:schema] = schema
       r.to_table(tn, opts)
     end
 
@@ -180,7 +189,8 @@ module OMF::OML
       unless table =  @tables[table_name] # check if already reported before
         debug "Found table: #{table_name}"
         schema = _schema_for_table(table_name)
-        table = @tables[table_name] = OmlSqlRow.new(table_name, schema, @db, @row_opts)
+        query = _def_query_for_table(table_name)
+        table = @tables[table_name] = OmlSqlRow.new(table_name, schema, query, @row_opts)
         #table = @tables[table_name] = OmlSqlRow.new(table_name, @db.schema(table_name), @db_opts, self, @row_opts)
         @on_new_stream_procs.each_value do |proc|
           proc.call(table)
@@ -195,6 +205,11 @@ module OMF::OML
           unless type = cd[:type] || FALLBACK_MAPPING[cd[:db_type]]
             warn "Can't find ruby type for database type '#{cd[:db_type]}'"
           end
+          if col_name == :oml_sender_id
+            # see _def_query_for_table(table_name) which replaces sender_id by sender name
+            col_name = :oml_sender
+            type = 'string'
+          end
           {:name => col_name, :type => type}
         end
         #puts "SCHEMA_DESCR>>>> #{schema_descr}"

data/lib/omf_oml/table.rb

@@ -93,10 +93,10 @@ module OMF::OML
     # parameters, an 'action' and the content changed. The 'action' is either
     # ':added', or ':removed' and the content is an array of rows.
     #
-    def on_content_changed(key, offset = -1, &proc)
+    def on_content_changed(key = nil, offset = -1, &proc)
       #puts ">>>>>>> #{offset}"
       if proc
-        @on_content_changed[key] = proc
+        @on_content_changed[key || proc] = proc
         if offset >= 0
           #with_offset = proc.arity == 2
           proc.call(:added, @rows[offset .. -1])

data/lib/omf_oml/tuple.rb

@@ -7,6 +7,7 @@
 require 'omf_base/lobject'
 require 'omf_oml'
 require 'omf_oml/schema'
+require 'omf_oml/table'
 
 module OMF::OML
 
@@ -25,17 +26,11 @@ module OMF::OML
     # by it's name, or its col index
     #
     def [](name_or_index)
-      @vprocs[name_or_index].call(@raw)
+      @schema.cast_col(name_or_index, @raw)
     end
 
     # Return the elements of the tuple as an array
     def to_a()
-      # res = []
-      # r = @raw
-      # @schema.each do |col|
-        # res << @vprocs[col[:name]].call(r)
-      # end
-      # res
       @schema.cast_row(@raw)
     end
 
@@ -44,28 +39,38 @@ module OMF::OML
     #
     def select(*col_names)
       r = @raw
-      col_names.collect do |n|
-        p = @vprocs[n]
-        p ? p.call(r) : nil
+      col_names.map do |n|
+        self[n]
       end
     end
 
+    # Return a table (more precisely an OmlTable instance) fed from
+    # the content of this tuple stream.
+    #
+    # table_name - Name of table
+    # opts -
+    #   All options defined for OmlTable#create
+    #
+    def create_table(table_name, opts = {})
+      table = OmlTable.create(table_name, @schema, opts)
+      id = -1
+      on_new_tuple(table) do |t|
+        row = @schema.cast_row(@raw, true)
+        #puts "ROW>> #{row.inspect}"
+        table.add_row(row)
+      end
+      table
+    end
+
     attr_reader :schema
     attr_reader :stream_name
 
     def initialize(name, schema)
-      # if schema.kind_of? Array
-        # schema = OmlSchema.new(schema)
-      # end
       @stream_name = name
       @schema = OmlSchema.create(schema)
-
       @raw = []
-#      puts "SCHEMA: #{schema.inspect}"
       @on_new_tuple_procs = {}
-      
       super name
-      process_schema(@schema)
     end
 
 
@@ -82,7 +87,7 @@ module OMF::OML
       end
     end
 
-    # Register a proc to be called when a new tuple arrived 
+    # Register a proc to be called when a new tuple arrived
     #
     def on_new_tuple(key = :_, &proc)
       if proc
@@ -94,22 +99,6 @@ module OMF::OML
 
 
     protected
-    def process_schema(schema)
-      i = 0
-      @vprocs = {}
-      schema.each_column do |col| #
-        name = col[:name] || raise("Ill-formed schema '#{schema}'")
-        type = col[:type] || raise("Ill-formed schema '#{schema}'")
-        @vprocs[name] = @vprocs[i] = case type
-          when :string
-            lambda do |r| r[i] end
-          when :float 
-            lambda do |r| r[i].to_f end
-          else raise "Unrecognized Schema type '#{type}'"
-        end
-        i += 1
-      end
-    end
 
   end # Tuple
 end # OMF::OML

data/lib/omf_oml/version.rb

@@ -6,6 +6,6 @@
 #-------------------------------------------------------------------------------
 module OMF
   module OML
-    VERSION = '1.0.0'
+    VERSION = '1.1.0'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: omf_oml
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-11-05 00:00:00.000000000 Z
+date: 2013-12-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: omf_base
@@ -90,9 +90,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: omf_oml
-rubygems_version: 1.8.25
+rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
 summary: Glue between OMF and OML.
 test_files:
 - test/data/brooklynDemo.sq3
+has_rdoc: