omf_web 0.9.1 → 0.9.3

data/lib/omf-oml/sql_row.rb

@@ -1,302 +0,0 @@
-
-require 'omf-oml/tuple'
-
-module OMF::OML
-        
-
-  
-  # Read the content of a table and feeds it out as a tuple store.
-  # After creation of the object. The actual tuple feed is started
-  # with a call to +run+.
-  #
-  class OmlSqlRow < OmlTuple
-    
-    # *opts:  
-    #   - offset: Ignore first +offset+ rows. If negative or zero serve +offset+ rows initially
-    #   - limit: Number of rows to fetch each time [1000]
-    #   - check_interval: Interval in seconds when to check for new data. If 0, only run once.
-    #
-    def initialize(table_name, db_file, source, opts = {})
-      @sname = table_name
-      @db_file = db_file
-      @source = source
-      
-      unless @offset = opts[:offset]
-        @offset = 0
-      end
-      @limit = opts[:limit]
-      @limit = 1000 unless @limit
-      
-      @check_interval = opts[:check_interval]
-      @check_interval = 0 unless @check_interval
-      
-
-      @on_new_vector_proc = {}
-
-      schema = find_schema
-      super table_name, schema 
-    end
-    
-    
-    # Return a specific element of the vector identified either
-    # by it's name, or its col index
-    #
-    def [](name_or_index)
-      @vprocs[name_or_index].call(@raw)
-    end
-    
-    # Return the elements of the vector as an array
-    def to_a(include_oml_internals = false)
-      include_oml_internals ? @row.dup : @row[4 .. -1]
-    end
-    
-    # Return an array including the values for the names elements
-    # given as parameters.
-    #
-    def select(*col_names)
-      r = @row
-      col_names.collect do |n|
-        p = @vprocs[n]
-        #puts "#{n}::#{p}"
-        p ? p.call(r) : nil
-      end
-    end
-        
-    def ts
-      self[:oml_ts_server]
-    end
-    
-    def seq_no
-      self[:oml_seq]
-    end    
-    
-    # Register a proc to be called when a new tuple arrived
-    # on this stream.
-    #
-    def on_new_tuple(key = :_, &proc)
-      if proc
-        @on_new_vector_proc[key] = proc
-      else
-        @on_new_vector_proc.delete key
-      end
-      run() unless @on_new_vector_proc.empty?
-    end
-
-    # Create and return an +OmlTable+ which captures this tuple stream
-    #
-    # The argument to this method are either a list of columns to 
-    # to capture in the table, or an array of column names and
-    # an option hash  or just 
-    # the option hash to be provided to the +OmlTable+ constructor.
-    #
-    # If a block is provided, any arriving tuple is executed by the block
-    # which is expected to return an array which is added to the table
-    # or nil in which case nothing is added. If a selector array is given the 
-    # block is called with an array of values in the order of the columns
-    # listed in the selector. Otherwise, the block is called directly 
-    # with the tuple.
-    #
-    # opts:
-    #   :schema - use this schema instead for the table
-    #   :name   - name to use for table
-    #   ....    - remaining options to be passed to table constructur
-    #
-    def capture_in_table(*args, &block)
-      if args.length == 1
-        if args[0].kind_of?(Array)
-          select = args[0]
-        elsif args[0].kind_of?(Hash)
-          opts = args[0]
-        end
-      elsif args.length == 2 && args[1].kind_of?(Hash)
-        select = args[0]
-        opts = args[1]
-      else
-        opts = {}
-        select = args
-      end
-      
-      if (tschema = opts.delete(:schema))
-        # unless tschema[0].kind_of? Hash
-          # tschema = tschema.collect do |cname| {:name => cname} end
-        # end 
-      else
-        tschema = select.collect do |cname| {:name => cname} end
-      end
-      tname = opts.delete(:name) || stream_name
-      t = OMF::OML::OmlTable.new(tname, tschema, opts)
-      if block
-        self.on_new_tuple() do |v|
-          #puts "New vector(#{tname}): #{v.schema.inspect} ---- #{v.select(*select).size} <#{v.select(*select).join('|')}>"
-          if select
-            row = block.call(v.select(*select))
-          else
-            row = block.call(v)
-          end             
-          if row
-            raise "Expected kind of Array, but got '#{row.inspect}'" unless row.kind_of?(Array)
-            t.add_row(row)
-          end  
-        end
-      else
-        self.on_new_tuple() do |v|
-          #puts "New vector(#{tname}): #{v.select(*select).join('|')}"
-          t.add_row(v.select(*select))   
-        end
-      end
-      t
-    end
-    
-    def to_table(name = nil, opts = {})
-      unless name
-        name = @sname
-      end
-      t = OMF::OML::OmlTable.new(name, self.schema)
-      include_oml_internals = opts[:include_oml_internals] || true
-      self.on_new_tuple() do |v|
-        r = v.to_a(include_oml_internals)
-        t.add_row(r)   
-      end
-      t
-    end
-
-
-    protected
-        
-    def find_schema()
-      stmt = _statement
-      cnames = stmt.columns
-      ctypes = stmt.types
-      schema = []
-      #schema << {:name => :oml_sender, :type => 'STRING'}
-      cnames.size.times do |i|
-        name = cnames[i].to_sym
-        schema << {:name => name, :type => ctypes[i]}
-      end
-      # Rename first col
-      first = schema[0]
-      raise "BUG: Should be 'name'" if first[:name] != :name
-      first[:name] = :oml_sender
-      
-      OmlSchema.new(schema)
-    end
-    
-    # override
-    def process_schema(schema)
-      i = 0
-      @vprocs = {}
-      schema.each_column do |col|
-        name = col[:name]
-        j = i + 0
-        l = @vprocs[name] = lambda do |r| r[j] end
-        @vprocs[i - 4] = l if i > 4
-        i += 1
-      end
-    end
-    
-    def run(in_thread = true)
-      return if @running
-      if in_thread
-        if @db
-          # force opening of database in new thread
-          begin
-            @db.close
-          rescue Exception
-            # ALERT: issues with finalising statments, don't know how to deal with it
-          end
-          @db = nil
-          @stmt = nil
-        end
-        Thread.new do
-          begin
-            _run
-          rescue Exception => ex
-            error "Exception in OmlSqlRow: #{ex}"
-            debug "Exception in OmlSqlRow: #{ex.backtrace.join("\n\t")}"
-          end
-        end
-      else
-        _run
-      end
-    end
-    
-    private
-    
-    def _run
-      if @check_interval <= 0
-        _run_once
-      else
-        @running = true
-        while (@running)
-          begin 
-            unless _run_once
-              # All rows read, wait a bit for news to show up
-              sleep @check_interval
-            end
-          rescue Exception => ex
-            warn ex
-            debug "\t", ex.backtrace.join("\n\t")
-          end
-        end 
-      end
-    end
-      
-    # Run a query on database an serve all rows found one at a time.
-    # Return true if there might be more rows in the database
-    def _run_once
-      row_cnt = 0
-      _statement.execute(@limit, @offset).each do |r|
-        @row = r
-        @on_new_vector_proc.each_value do |proc|
-          proc.call(self)
-        end
-        row_cnt += 1
-      end
-      @offset += row_cnt
-      debug "Read #{row_cnt}/#{@offset} rows from '#{@sname}'"
-      row_cnt >= @limit # there could be more to read     
-    end
-    
-    def _statement
-      unless @stmt
-        db = @db = SQLite3::Database.new(@db_file)
-        @db.type_translation = true   
-        table_name = t = @sname  
-        if @offset < 0
-          cnt = db.execute("select count(*) from #{table_name};")[0][0].to_i
-          #debug "CNT: #{cnt}.#{cnt.class} offset: #{@offset}"
-          @offset = cnt + @offset # @offset was negative here
-          debug("Initial offset #{@offset} in '#{table_name}' with #{cnt} rows")
-          @offset = 0 if @offset < 0
-        end
-        #@stmt = db.prepare("SELECT * FROM #{table_name} LIMIT ? OFFSET ?;")
-        @stmt = db.prepare("SELECT _senders.name, #{t}.* FROM #{t} JOIN _senders WHERE #{t}.oml_sender_id = _senders.id LIMIT ? OFFSET ?;")
-      end
-      @stmt
-    end
-  end # OmlSqlRow
-
-
-end
-
-if $0 == __FILE__
-
-  require 'omf-oml/table'
-  ep = OMF::OML::OmlSqlSource.new('brooklynDemo.sq3')
-  ep.on_new_stream() do |s|
-    puts ">>>>>>>>>>>> New stream #{s.stream_name}: #{s.names.join(', ')}"
-    case s.stream_name
-    when 'wimaxmonitor_wimaxstatus'
-      select = [:oml_ts_server, :sender_hostname, :frequency, :signal, :rssi, :cinr, :avg_tx_pw]
-    when 'GPSlogger_gps_data'
-      select = [:oml_ts_server, :oml_sender_id, :lat, :lon]
-    end
-
-    s.on_new_vector() do |v|
-      puts "New vector(#{s.stream_name}): #{v.select(*select).join('|')}"      
-    end
-  end
-  ep.run()
-
-end
-

data/lib/omf-oml/sql_source.rb

@@ -1,131 +0,0 @@
-
-require 'sqlite3'
-
-require 'omf_common/lobject'
-require 'omf-oml/endpoint'
-require 'omf-oml/tuple'
-require 'omf-oml/sql_row'
-
-module OMF::OML
-
-  # This class fetches the content of an sqlite3 database and serves it as multiple
-  # OML streams.
-  #
-  # After creating the object, the @run@ method needs to be called to
-  # start producing the streams.
-  #
-  class OmlSqlSource < OMF::Common::LObject
-
-    # +opts+ - passed on as +opts+ to the OmlSqlRow constructor
-    #
-    def initialize(db_file, opts = {})
-      raise "Can't find database '#{db_file}'" unless File.readable?(db_file)
-      @db_file = db_file
-      @running = false
-      @on_new_stream_procs = {}
-      @tables = {}
-      @table_opts = opts
-    end
-
-    # Register a proc to be called when a new stream was
-    # discovered on this endpoint.
-    #
-    def on_new_stream(key = :_, &proc)
-      if proc
-        @on_new_stream_procs[key] = proc
-      else
-        @on_new_stream_procs.delete key
-      end
-    end
-
-
-    # def report_new_stream(stream)
-      # @on_new_stream_procs.each_value do |proc|
-        # proc.call(stream)
-      # end
-    # end
-
-    # Start checking the database for tables and create a new stream
-    # by calling the internal +report_new_table+ method.
-    # If +check_every+ > 0 continue checking every +check_every+ seconds
-    # for new tables in the database, otherwise it's only checked once
-    #
-    #
-    def run(check_every = -1)
-      if check_every <= 0
-        run_once()
-      else
-        Thread.new do
-          @running = true
-          while (@running)
-            begin
-              run_once()
-            rescue Exception => ex
-              error "Exception in OmlSqlSource#run: #{ex}"
-              debug "Exception in OmlSqlSource#run: #{ex.backtrace.join("\n\t")}"
-            end
-            sleep check_every
-          end
-        end
-      end
-    end
-
-    def run_once()
-      unless @db
-        @db = SQLite3::Database.new(@db_file)
-        @db.type_translation = true
-      end
-
-      # first find tables
-      @db.execute( "SELECT * FROM sqlite_master WHERE type='table';") do |r|
-        table_name = r[1]
-        report_new_table(table_name, @table_opts) unless table_name.start_with?('_')
-      end
-    end
-
-
-    protected
-
-    # THis method is being called for every table detected in the database.
-    # It creates a new +OmlSqlRow+ object with +opts+ as the only argument.
-    # The tables is then streamed as a tuple stream.
-    # After the stream has been created, each block registered with
-    # +on_new_stream+ is then called with the new stream as its single
-    # argument.
-    #
-    def report_new_table(table_name, opts = {})
-      return if @tables.key?(table_name) # check if already reported before
-      debug "Found table: #{table_name}"
-      t = @tables[table_name] = OmlSqlRow.new(table_name, @db_file, self, opts)
-      @on_new_stream_procs.each_value do |proc|
-        proc.call(t)
-      end
-    end
-    
-  end
-  
-
-
-end
-
-if $0 == __FILE__
-
-  require 'omf-oml/table'
-  ep = OMF::OML::OmlSqlSource.new('brooklynDemo.sq3')
-  ep.on_new_stream() do |s|
-    puts ">>>>>>>>>>>> New stream #{s.stream_name}: #{s.names.join(', ')}"
-    case s.stream_name
-    when 'wimaxmonitor_wimaxstatus'
-      select = [:oml_ts_server, :sender_hostname, :frequency, :signal, :rssi, :cinr, :avg_tx_pw]
-    when 'GPSlogger_gps_data'
-      select = [:oml_ts_server, :oml_sender_id, :lat, :lon]
-    end
-
-    s.on_new_vector() do |v|
-      puts "New vector(#{s.stream_name}): #{v.select(*select).join('|')}"      
-    end
-  end
-  ep.run()
-
-end
-

data/lib/omf-oml/table.rb

@@ -1,131 +0,0 @@
-
-require 'monitor'
-
-require 'omf_common/lobject'
-require 'omf_oml'
-require 'omf-oml/schema'
-autoload :OmlIndexedTable, 'omf-oml/indexed_table'
-
-module OMF::OML
-          
-  # This class represents a database like table holding a sequence of OML measurements (rows) according
-  # a common schema.
-  #
-  class OmlTable < OMF::Common::LObject
-    include MonitorMixin
-    
-    attr_reader :name
-    attr_accessor :max_size
-    attr_reader :schema
-    
-    # 
-    # tname - Name of table
-    # schema - OmlSchema or Array containing [name, type*] for every column in table
-    #   TODO: define format of TYPE
-    # opts -
-    #   :max_size - keep table to that size by dropping older rows
-    #   :index - only keep the latest inserted row for a unique col value - messes with row order
-    #
-    def initialize(tname, schema, opts = {}, &on_before_row_added)
-      super tname
-      
-      #@endpoint = endpoint
-      @name = tname
-      @schema = OmlSchema.create(schema)
-      @opts = opts
-      if (index = opts[:index])
-        @indexed_rows = {}
-        @index_col = @schema.index_for_col(index)
-      end
-      @on_before_row_added = on_before_row_added
-      @rows = []
-      @max_size = opts[:max_size]
-      @on_row_added = {}
-    end
-    
-    def rows
-      @indexed_rows ? @indexed_rows.values : @rows
-    end
-    
-    # Register +callback+ to be called to process any newly
-    # offered row before it being added to internal storage.
-    # The callback's argument is the new row (TODO: in what form)
-    # and should return what is being added instead of the original
-    # row. If the +callback+ returns nil, nothing is being added.
-    #
-    def on_before_row_added(&callback)
-      @on_before_row_added = callback
-    end
-    
-    def on_row_added(key, &proc)
-      #debug "on_row_added: #{proc.inspect}"
-      if proc
-        @on_row_added[key] = proc
-      else
-        @on_row_added.delete key
-      end
-    end
-    
-    # NOTE: +on_row_added+ callbacks are done within the monitor. 
-    #
-    def add_row(row, needs_casting = false)
-      synchronize do
-        _add_row(row, needs_casting)
-      end
-    end
-    
-    def indexed_by(col_name)
-      OmlIndexedTable.new(col_name, self)
-    end
-    
-    # Add an array of rows to this table
-    #
-    def add_rows(rows, needs_casting = false)
-      synchronize do
-        rows.each { |row| _add_row(row, needs_casting) }
-      end
-    end
-    
-    def describe()
-      rows
-    end
-    
-    def data_sources
-      self
-    end
-    
-    private
-    
-    # NOT synchronized
-    #
-    def _add_row(row, needs_casting = false)
-      if needs_casting
-        row = @schema.cast_row(row)
-      end
-      #puts row.inspect
-      if @on_before_row_added
-        row = @on_before_row_added.call(row)
-      end
-      return unless row 
-
-      if @indexed_rows
-        @indexed_rows[row[@index_col]] = row
-      else
-        @rows << row
-        if @max_size && @max_size > 0 && (s = @rows.size) > @max_size
-          @rows.shift # not necessarily fool proof, but fast
-        end
-      end
-      _notify_row_added(row)
-    end
-    
-    def _notify_row_added(row)
-      @on_row_added.each_value do |proc|
-        #puts "call: #{proc.inspect}"
-        proc.call(row)
-      end
-    end
-    
-  end # OMLTable
-
-end

data/lib/omf-oml/tuple.rb

@@ -1,97 +0,0 @@
-
-require 'omf_common/lobject'
-require 'omf_oml'
-require 'omf-oml/schema'
-
-module OMF::OML
-
-  # This class represents a tuple with an associated schema.
-  # It provides various methods to access the tuple elements.
-  #
-  # NOTE: Do not store the tuple itself, but make a copy as the instance may be
-  # reused over various rows by the sender.
-  #
-  # Use +OmlTuple+ if the schema is an OML one. +OmlTuple+ has additional convenience
-  # methods.
-  #
-  class Tuple < OMF::Common::LObject
-
-    # Return a specific element of the tuple identified either
-    # by it's name, or its col index
-    #
-    def [](name_or_index)
-      @vprocs[name_or_index].call(@raw)
-    end
-
-    # Return the elements of the tuple as an array
-    def to_a()
-      res = []
-      r = @raw
-      @schema.each do |col|
-        res << @vprocs[col[:name]].call(r)
-      end
-      res
-    end
-
-    # Return an array including the values for the names elements
-    # given as parameters.
-    #
-    def select(*col_names)
-      r = @raw
-      col_names.collect do |n|
-        p = @vprocs[n]
-        p ? p.call(r) : nil
-      end
-    end
-
-    attr_reader :schema
-    attr_reader :stream_name
-
-    def initialize(name, schema)
-      # if schema.kind_of? Array
-        # schema = OmlSchema.new(schema)
-      # end
-      @stream_name = name
-      @schema = OmlSchema.create(schema)
-
-      @raw = []
-#      puts "SCHEMA: #{schema.inspect}"
-
-      super name
-      process_schema(@schema)
-    end
-
-
-    # Parse the array of strings into the proper typed vector elements
-    #
-    # NOTE: We assume that each element is only called at most once, with some
-    # never called. We therefore delay type-casting to the get function without
-    # keeping the casted values (would increase lookup time)
-    #
-    def parse_tuple(els)
-      @raw = els
-      @on_new_vector_proc.each_value do |proc|
-        proc.call(self)
-      end
-    end
-
-    protected
-    def process_schema(schema)
-      i = 0
-      @vprocs = {}
-      schema.each_column do |col| #
-        name = col[:name] || raise("Ill-formed schema '#{schema}'")
-        type = col[:type] || raise("Ill-formed schema '#{schema}'")
-        @vprocs[name] = @vprocs[i] = case type
-          when :string
-            lambda do |r| r[i] end
-          when :double 
-            lambda do |r| r[i].to_f end
-          else raise "Unrecognized Schema type '#{type}'"
-        end
-        i += 1
-      end
-    end
-
-  end # Tuple
-end # OMF::OML

data/lib/omf_oml.rb

@@ -1,4 +0,0 @@
-
-module OMF
-  module OML; end
-end