omf_oml 0.9.3

data/lib/omf_oml/sql_row.rb

@@ -0,0 +1,302 @@
+
+require 'omf_oml/tuple'
+
+module OMF::OML
+        
+
+  
+  # Read the content of a table and feeds it out as a tuple store.
+  # After creation of the object. The actual tuple feed is started
+  # with a call to +run+.
+  #
+  class OmlSqlRow < OmlTuple
+    
+    # *opts:  
+    #   - offset: Ignore first +offset+ rows. If negative or zero serve +offset+ rows initially
+    #   - limit: Number of rows to fetch each time [1000]
+    #   - check_interval: Interval in seconds when to check for new data. If 0, only run once.
+    #
+    def initialize(table_name, db_file, source, opts = {})
+      @sname = table_name
+      @db_file = db_file
+      @source = source
+      
+      unless @offset = opts[:offset]
+        @offset = 0
+      end
+      @limit = opts[:limit]
+      @limit = 1000 unless @limit
+      
+      @check_interval = opts[:check_interval]
+      @check_interval = 0 unless @check_interval
+      
+
+      @on_new_vector_proc = {}
+
+      schema = find_schema
+      super table_name, schema 
+    end
+    
+    
+    # Return a specific element of the vector identified either
+    # by it's name, or its col index
+    #
+    def [](name_or_index)
+      @vprocs[name_or_index].call(@raw)
+    end
+    
+    # Return the elements of the vector as an array
+    def to_a(include_oml_internals = false)
+      include_oml_internals ? @row.dup : @row[4 .. -1]
+    end
+    
+    # Return an array including the values for the names elements
+    # given as parameters.
+    #
+    def select(*col_names)
+      r = @row
+      col_names.collect do |n|
+        p = @vprocs[n]
+        #puts "#{n}::#{p}"
+        p ? p.call(r) : nil
+      end
+    end
+        
+    def ts
+      self[:oml_ts_server]
+    end
+    
+    def seq_no
+      self[:oml_seq]
+    end    
+    
+    # Register a proc to be called when a new tuple arrived
+    # on this stream.
+    #
+    def on_new_tuple(key = :_, &proc)
+      if proc
+        @on_new_vector_proc[key] = proc
+      else
+        @on_new_vector_proc.delete key
+      end
+      run() unless @on_new_vector_proc.empty?
+    end
+
+    # Create and return an +OmlTable+ which captures this tuple stream
+    #
+    # The argument to this method are either a list of columns to 
+    # to capture in the table, or an array of column names and
+    # an option hash  or just 
+    # the option hash to be provided to the +OmlTable+ constructor.
+    #
+    # If a block is provided, any arriving tuple is executed by the block
+    # which is expected to return an array which is added to the table
+    # or nil in which case nothing is added. If a selector array is given the 
+    # block is called with an array of values in the order of the columns
+    # listed in the selector. Otherwise, the block is called directly 
+    # with the tuple.
+    #
+    # opts:
+    #   :schema - use this schema instead for the table
+    #   :name   - name to use for table
+    #   ....    - remaining options to be passed to table constructur
+    #
+    def capture_in_table(*args, &block)
+      if args.length == 1
+        if args[0].kind_of?(Array)
+          select = args[0]
+        elsif args[0].kind_of?(Hash)
+          opts = args[0]
+        end
+      elsif args.length == 2 && args[1].kind_of?(Hash)
+        select = args[0]
+        opts = args[1]
+      else
+        opts = {}
+        select = args
+      end
+      
+      if (tschema = opts.delete(:schema))
+        # unless tschema[0].kind_of? Hash
+          # tschema = tschema.collect do |cname| {:name => cname} end
+        # end 
+      else
+        tschema = select.collect do |cname| {:name => cname} end
+      end
+      tname = opts.delete(:name) || stream_name
+      t = OMF::OML::OmlTable.new(tname, tschema, opts)
+      if block
+        self.on_new_tuple() do |v|
+          #puts "New vector(#{tname}): #{v.schema.inspect} ---- #{v.select(*select).size} <#{v.select(*select).join('|')}>"
+          if select
+            row = block.call(v.select(*select))
+          else
+            row = block.call(v)
+          end             
+          if row
+            raise "Expected kind of Array, but got '#{row.inspect}'" unless row.kind_of?(Array)
+            t.add_row(row)
+          end  
+        end
+      else
+        self.on_new_tuple() do |v|
+          #puts "New vector(#{tname}): #{v.select(*select).join('|')}"
+          t.add_row(v.select(*select))   
+        end
+      end
+      t
+    end
+    
+    def to_table(name = nil, opts = {})
+      unless name
+        name = @sname
+      end
+      t = OMF::OML::OmlTable.new(name, self.schema)
+      include_oml_internals = opts[:include_oml_internals] || true
+      self.on_new_tuple() do |v|
+        r = v.to_a(include_oml_internals)
+        t.add_row(r)   
+      end
+      t
+    end
+
+
+    protected
+        
+    def find_schema()
+      stmt = _statement
+      cnames = stmt.columns
+      ctypes = stmt.types
+      schema = []
+      #schema << {:name => :oml_sender, :type => 'STRING'}
+      cnames.size.times do |i|
+        name = cnames[i].to_sym
+        schema << {:name => name, :type => ctypes[i]}
+      end
+      # Rename first col
+      first = schema[0]
+      raise "BUG: Should be 'name'" if first[:name] != :name
+      first[:name] = :oml_sender
+      
+      OmlSchema.new(schema)
+    end
+    
+    # override
+    def process_schema(schema)
+      i = 0
+      @vprocs = {}
+      schema.each_column do |col|
+        name = col[:name]
+        j = i + 0
+        l = @vprocs[name] = lambda do |r| r[j] end
+        @vprocs[i - 4] = l if i > 4
+        i += 1
+      end
+    end
+    
+    def run(in_thread = true)
+      return if @running
+      if in_thread
+        if @db
+          # force opening of database in new thread
+          begin
+            @db.close
+          rescue Exception
+            # ALERT: issues with finalising statments, don't know how to deal with it
+          end
+          @db = nil
+          @stmt = nil
+        end
+        Thread.new do
+          begin
+            _run
+          rescue Exception => ex
+            error "Exception in OmlSqlRow: #{ex}"
+            debug "Exception in OmlSqlRow: #{ex.backtrace.join("\n\t")}"
+          end
+        end
+      else
+        _run
+      end
+    end
+    
+    private
+    
+    def _run
+      if @check_interval <= 0
+        _run_once
+      else
+        @running = true
+        while (@running)
+          begin 
+            unless _run_once
+              # All rows read, wait a bit for news to show up
+              sleep @check_interval
+            end
+          rescue Exception => ex
+            warn ex
+            debug "\t", ex.backtrace.join("\n\t")
+          end
+        end 
+      end
+    end
+      
+    # Run a query on database an serve all rows found one at a time.
+    # Return true if there might be more rows in the database
+    def _run_once
+      row_cnt = 0
+      _statement.execute(@limit, @offset).each do |r|
+        @row = r
+        @on_new_vector_proc.each_value do |proc|
+          proc.call(self)
+        end
+        row_cnt += 1
+      end
+      @offset += row_cnt
+      debug "Read #{row_cnt}/#{@offset} rows from '#{@sname}'"
+      row_cnt >= @limit # there could be more to read     
+    end
+    
+    def _statement
+      unless @stmt
+        db = @db = SQLite3::Database.new(@db_file)
+        @db.type_translation = true   
+        table_name = t = @sname  
+        if @offset < 0
+          cnt = db.execute("select count(*) from #{table_name};")[0][0].to_i
+          #debug "CNT: #{cnt}.#{cnt.class} offset: #{@offset}"
+          @offset = cnt + @offset # @offset was negative here
+          debug("Initial offset #{@offset} in '#{table_name}' with #{cnt} rows")
+          @offset = 0 if @offset < 0
+        end
+        #@stmt = db.prepare("SELECT * FROM #{table_name} LIMIT ? OFFSET ?;")
+        @stmt = db.prepare("SELECT _senders.name, #{t}.* FROM #{t} JOIN _senders WHERE #{t}.oml_sender_id = _senders.id LIMIT ? OFFSET ?;")
+      end
+      @stmt
+    end
+  end # OmlSqlRow
+
+
+end
+
+if $0 == __FILE__
+
+  require 'omf_oml/table'
+  ep = OMF::OML::OmlSqlSource.new('brooklynDemo.sq3')
+  ep.on_new_stream() do |s|
+    puts ">>>>>>>>>>>> New stream #{s.stream_name}: #{s.names.join(', ')}"
+    case s.stream_name
+    when 'wimaxmonitor_wimaxstatus'
+      select = [:oml_ts_server, :sender_hostname, :frequency, :signal, :rssi, :cinr, :avg_tx_pw]
+    when 'GPSlogger_gps_data'
+      select = [:oml_ts_server, :oml_sender_id, :lat, :lon]
+    end
+
+    s.on_new_vector() do |v|
+      puts "New vector(#{s.stream_name}): #{v.select(*select).join('|')}"      
+    end
+  end
+  ep.run()
+
+end
+

data/lib/omf_oml/sql_source.rb

@@ -0,0 +1,131 @@
+
+require 'sqlite3'
+
+require 'omf_common/lobject'
+require 'omf_oml/endpoint'
+require 'omf_oml/tuple'
+require 'omf_oml/sql_row'
+
+module OMF::OML
+
+  # This class fetches the content of an sqlite3 database and serves it as multiple
+  # OML streams.
+  #
+  # After creating the object, the @run@ method needs to be called to
+  # start producing the streams.
+  #
+  class OmlSqlSource < OMF::Common::LObject
+
+    # +opts+ - passed on as +opts+ to the OmlSqlRow constructor
+    #
+    def initialize(db_file, opts = {})
+      raise "Can't find database '#{db_file}'" unless File.readable?(db_file)
+      @db_file = db_file
+      @running = false
+      @on_new_stream_procs = {}
+      @tables = {}
+      @table_opts = opts
+    end
+
+    # Register a proc to be called when a new stream was
+    # discovered on this endpoint.
+    #
+    def on_new_stream(key = :_, &proc)
+      if proc
+        @on_new_stream_procs[key] = proc
+      else
+        @on_new_stream_procs.delete key
+      end
+    end
+
+
+    # def report_new_stream(stream)
+      # @on_new_stream_procs.each_value do |proc|
+        # proc.call(stream)
+      # end
+    # end
+
+    # Start checking the database for tables and create a new stream
+    # by calling the internal +report_new_table+ method.
+    # If +check_every+ > 0 continue checking every +check_every+ seconds
+    # for new tables in the database, otherwise it's only checked once
+    #
+    #
+    def run(check_every = -1)
+      if check_every <= 0
+        run_once()
+      else
+        Thread.new do
+          @running = true
+          while (@running)
+            begin
+              run_once()
+            rescue Exception => ex
+              error "Exception in OmlSqlSource#run: #{ex}"
+              debug "Exception in OmlSqlSource#run: #{ex.backtrace.join("\n\t")}"
+            end
+            sleep check_every
+          end
+        end
+      end
+    end
+
+    def run_once()
+      unless @db
+        @db = SQLite3::Database.new(@db_file)
+        @db.type_translation = true
+      end
+
+      # first find tables
+      @db.execute( "SELECT * FROM sqlite_master WHERE type='table';") do |r|
+        table_name = r[1]
+        report_new_table(table_name, @table_opts) unless table_name.start_with?('_')
+      end
+    end
+
+
+    protected
+
+    # THis method is being called for every table detected in the database.
+    # It creates a new +OmlSqlRow+ object with +opts+ as the only argument.
+    # The tables is then streamed as a tuple stream.
+    # After the stream has been created, each block registered with
+    # +on_new_stream+ is then called with the new stream as its single
+    # argument.
+    #
+    def report_new_table(table_name, opts = {})
+      return if @tables.key?(table_name) # check if already reported before
+      debug "Found table: #{table_name}"
+      t = @tables[table_name] = OmlSqlRow.new(table_name, @db_file, self, opts)
+      @on_new_stream_procs.each_value do |proc|
+        proc.call(t)
+      end
+    end
+    
+  end
+  
+
+
+end
+
+if $0 == __FILE__
+
+  require 'omf_oml/table'
+  ep = OMF::OML::OmlSqlSource.new('brooklynDemo.sq3')
+  ep.on_new_stream() do |s|
+    puts ">>>>>>>>>>>> New stream #{s.stream_name}: #{s.names.join(', ')}"
+    case s.stream_name
+    when 'wimaxmonitor_wimaxstatus'
+      select = [:oml_ts_server, :sender_hostname, :frequency, :signal, :rssi, :cinr, :avg_tx_pw]
+    when 'GPSlogger_gps_data'
+      select = [:oml_ts_server, :oml_sender_id, :lat, :lon]
+    end
+
+    s.on_new_vector() do |v|
+      puts "New vector(#{s.stream_name}): #{v.select(*select).join('|')}"      
+    end
+  end
+  ep.run()
+
+end
+

data/lib/omf_oml/table.rb

@@ -0,0 +1,227 @@
+
+require 'monitor'
+
+require 'omf_common/lobject'
+require 'omf_oml'
+require 'omf_oml/schema'
+
+
+
+module OMF::OML
+          
+  # This class represents a database like table holding a sequence of OML measurements (rows) according
+  # a common schema.
+  #
+  class OmlTable < OMF::Common::LObject
+    
+    def self.create(tname, schema, opts = {}, &on_before_row_added)
+      if (index = opts.delete(:index))
+        require 'omf_oml/indexed_table'
+        OmlIndexedTable.new(tname, index, schema, &on_before_row_added)        
+      else
+        OmlTable.new(tname, schema, opts, &on_before_row_added)
+      end
+    end
+    include MonitorMixin
+    
+    attr_reader :name
+    attr_accessor :max_size
+    attr_reader :schema
+    attr_reader :offset
+    
+    # 
+    # tname - Name of table
+    # schema - OmlSchema or Array containing [name, type*] for every column in table
+    #   Table adds a '__id__' column at the beginning which keeps track of the rows unique id
+    # opts -
+    #   :max_size - keep table to that size by dropping older rows
+    #   :index - only keep the latest inserted row for a unique col value - messes with row order
+    #
+    def initialize(tname, schema, opts = {}, &on_before_row_added)
+      super tname
+      
+      #@endpoint = endpoint
+      @name = tname
+      @schema = OmlSchema.create(schema)
+      unless @schema.name_at(0) == :__id__
+        @schema.insert_column_at(0, [:__id__, 'int'])
+      end
+      @opts = opts
+      if (index = opts[:index])
+        throw "No longer supported, use IndexedTable instead"
+        # @indexed_rows = {}
+        # @index_col = @schema.index_for_col(index)
+      end
+      @on_before_row_added = on_before_row_added
+      @offset = 0 # number of rows skipped before the first one recorded here
+      @rows = []
+      @row_id = 0 # Each new row is assigned an id
+      @max_size = opts[:max_size]
+      @on_content_changed = {}
+    end
+    
+    def rows
+      #@indexed_rows ? @indexed_rows.values : @rows
+      @rows
+    end
+    
+    # Register +callback+ to be called to process any newly
+    # offered row before it being added to internal storage.
+    # The callback's argument is the new row (TODO: in what form)
+    # and should return what is being added instead of the original
+    # row. If the +callback+ returns nil, nothing is being added.
+    #
+    def on_before_row_added(&callback)
+      @on_before_row_added = callback
+    end
+    
+    # Register callback for when the content of the table is changes. The key
+    # allows for the callback to be removed by calling this method
+    # without a block. . If the 
+    # optional 'offset' value is set to zero or a positive value,
+    # then the currently stored values starting at this index are being 
+    # immediately sent to 'proc'. The 'proc' is expected to receive two
+    # parameters, an 'action' and the content changed. The 'action' is either
+    # ':added', or ':removed' and the content is an array of rows.
+    #
+    def on_content_changed(key, offset = -1, &proc)
+      #puts ">>>>>>> #{offset}"
+      if proc
+        @on_content_changed[key] = proc
+        if offset >= 0
+          #with_offset = proc.arity == 2
+          proc.call(:added, @rows[offset .. -1])
+          #.each_with_index do |r, i|
+            # with_offset ? proc.call(r, offset + i) : proc.call(r)
+          # end
+        end
+      else
+        @on_content_changed.delete key
+      end
+    end
+    
+    # NOTE: +on_row_added+ callbacks are done within the monitor. 
+    #
+    def add_row(row, needs_casting = false)
+      synchronize do
+        if row = _add_row(row, needs_casting)
+          _notify_content_changed(:added, [row])
+        end
+      end
+    end
+    
+    # Return a new table which shadows this table but only contains
+    # rows with unique values in the column 'col_name' and of these the 
+    # latest added rows to this table.
+    #
+    # col_name - Name of column to use for indexing
+    #
+    def indexed_by(col_name)
+      require 'omf_oml/indexed_table'
+      OmlIndexedTable.shadow(self, col_name)
+    end
+    
+    # Add an array of rows to this table
+    #
+    def add_rows(rows, needs_casting = false)
+      synchronize do
+        added = rows.map { |row| _add_row(row, needs_casting) }
+        added = added.compact
+        unless added.empty?
+          _notify_content_changed(:added, added)
+        end
+      end
+    end
+    
+    # Return a new table which only contains the rows of this
+    # table whose value in column 'col_name' is equal to 'col_value'
+    #
+    def create_sliced_table(col_name, col_value, table_opts = {})
+      sname = "#{@name}_slice_#{Kernel.rand}"
+
+      st = self.class.new(name, @schema, table_opts)
+      st.instance_variable_set(:@sname, sname)
+      st.instance_variable_set(:@master_ds, self)
+      def st.release
+        @master_ds.on_content_changed(@sname) # release callback
+      end
+
+      index = @schema.index_for_col(col_name)
+      on_content_changed(sname, 0) do |action, rows|
+        if action == :removed
+          warn "No support for removing rows from sliced table '#{sname}'."
+          next
+        end
+        rows.each do |row|
+          if row[index] == col_value
+            row = row[1 .. -1] # remove the row_id
+            debug "Add row '#{row.inspect}'"
+            st.add_row(row)
+          end
+        end
+      end
+      debug "Created sliced table from '#{@name}' (rows: #{st.rows.length}-#{@rows.length})"
+      st
+    end
+    
+    def describe()
+      rows
+    end
+    
+    def data_sources
+      self
+    end
+    
+    private
+    
+    # NOT synchronized
+    #
+    def _add_row(row, needs_casting = false)
+      if needs_casting
+        row = @schema.cast_row(row)
+      end
+      #puts row.inspect
+      if @on_before_row_added
+        row = @on_before_row_added.call(row)
+      end
+      return nil unless row 
+
+      row.insert(0, @row_id += 1)
+      _add_row_finally(row)
+    end
+    
+    # Finally add 'row' to internal storage. This would be hte method to
+    # overide in sub classes as this is thread safe and all other pre-storage
+    # test have been performed. Should return the row added, or nil if nothing
+    # was ultimately added.
+    #
+    def _add_row_finally(row)
+      # if @indexed_rows
+        # @indexed_rows[row[@index_col]] = row
+        # return
+      # end
+      
+      @rows << row
+      if @max_size && @max_size > 0 && (s = @rows.size) > @max_size
+        if (removed_row = @rows.shift) # not necessarily fool proof, but fast
+          _notify_content_changed(:removed, [removed_row])
+        end
+        @offset = @offset + 1
+      end
+      row
+    end
+    
+    def _notify_content_changed(action, rows)
+      @on_content_changed.each_value do |proc|
+        #puts "call: #{proc.inspect}"
+        #if proc.arity == 1
+          proc.call(action, rows)
+        #else
+          #proc.call(row, @offset)
+        #end
+      end
+    end
+
+  end # OMLTable
+
+end