omf_oml 0.9.7 → 0.9.8

data/lib/omf_oml/csv_table.rb

@@ -0,0 +1,80 @@
+
+require 'csv'
+require 'omf_oml'
+require 'omf_oml/table'
+
+
+
+module OMF::OML
+
+  # This class represents a table whose content is initially stored in a
+  # CSV file.
+  #
+  class OmlCsvTable < OmlTable
+
+    # @param opts
+    #
+    def self.create(tname, file_name, opts = {}, &on_before_row_added)
+      self.new(tname, file_name, opts, &on_before_row_added)
+    end
+
+    #
+    # tname - Name of table
+    # schema - OmlSchema or Array containing [name, type*] for every column in table
+    #   Table adds a '__id__' column at the beginning which keeps track of the rows unique id
+    # @params opts
+    # @opts :max_size - keep table to that size by dropping older rows
+    # @opts :index - only keep the latest inserted row for a unique col value - messes with row order
+    # @opts :has_csv_header If true use first row in file as schema descriptor
+    # @opts :schema Schema associated with this table
+    #
+    def initialize(tname, file_name, opts = {}, &on_before_row_added)
+      unless File.readable?(file_name)
+        raise "Can't read CSV file '#{file_name}'"
+      end
+      csv_opts = {}
+      csv_opts[:headers] = (opts.delete(:has_csv_header) == true)
+      unless csv_opts[:headers]
+        raise "Current implementation only works with CSV files which inlcude a schema description in the first line"
+      end
+
+      encoding =  opts.delete(:encoding)
+      mode =  "rb"
+      mode << ":#{encoding}" if encoding
+      csv = CSV.open(file_name, mode, csv_opts)
+
+      unless schema = opts.delete(:schema)
+        unless csv_opts[:headers]
+          raise "No schema given and ':has_csv_header' not set to capture schema from file header"
+        end
+        first_row = csv.shift.fields # force reading the first row to have the header parsed
+        #csv.shift.each do |h, v| puts "#{h} => #{v.class}" end
+        schema = csv.headers.map do |c|
+          c = c.encode('UTF-8', :invalid => :replace, :undef => :replace, :replace => '?')
+          name, type = c.split(':')
+          [name, type || 'string']
+        end
+      end
+      super tname, schema, opts
+
+      if first_row # from getting the CSV header
+        first_row.insert(0, @row_id += 1) if @add_index
+        @rows = [@schema.cast_row(first_row)]
+      end
+
+      # This assumes that CSV reader is setup with proper schema converters
+      csv.each do |r|
+        # Convert any strange strings into a clean ruby string
+        row = r.fields.map do |e|
+          e ? e.encode('UTF-8', :invalid => :replace, :undef => :replace, :replace => '?') : nil
+        end
+        row.insert(0, @row_id += 1) if @add_index
+        @rows << @schema.cast_row(row)
+      end
+
+    end
+
+
+
+  end # class
+end # module

data/lib/omf_oml/endpoint.rb

@@ -1,5 +1,5 @@
 
-require 'omf_common/lobject'
+require 'omf_base/lobject'
 require 'omf_oml'
 
 require 'omf_oml/oml_tuple'
@@ -10,7 +10,7 @@ module OMF::OML
   # be visualized. After creating the object, the @run@ method needs to be called to
   # start processing the stream.
   #
-  class OmlEndpoint < OMF::Common::LObject
+  class OmlEndpoint < OMF::Base::LObject
 
     # Register a proc to be called when a new stream was
     # discovered on this endpoint.
@@ -77,7 +77,7 @@ module OMF::OML
   # and instance and then immediately calls the +run+ methods.
   #
   #
-  class OmlSession < OMF::Common::LObject  # :nodoc
+  class OmlSession < OMF::Base::LObject  # :nodoc
 
     # Return the value for the respective @key@ in the protocol header.
     #

data/lib/omf_oml/network.rb

@@ -1,7 +1,7 @@
 require 'monitor'
 require 'json'
 require 'set'
-require 'omf_common/lobject'
+require 'omf_base/lobject'
 require 'omf_oml'
 
 
@@ -19,7 +19,7 @@ module OMF::OML
   # This class represents a network consisting of nodes and links with their respective
   # attributes.
   #
-  class OmlNetwork < OMF::Common::LObject
+  class OmlNetwork < OMF::Base::LObject
     include MonitorMixin
 
     @@name2network = {}
@@ -310,7 +310,7 @@ module OMF::OML
 
   # This class represents an abstract network element and shouldn't be used directly.
   #
-  class NetworkElement < OMF::Common::LObject
+  class NetworkElement < OMF::Base::LObject
 
     attr_reader :name
     attr_reader :el_id
@@ -449,7 +449,7 @@ end
 
 if $0 == __FILE__
   require 'json'
-  include OMF::Common::OML
+  include OMF::Base::OML
 
   nw = OmlNetwork.new
 

data/lib/omf_oml/schema.rb

@@ -1,12 +1,12 @@
 
-require 'omf_common/lobject'
+require 'omf_base/lobject'
 require 'omf_oml'
 
 module OMF::OML
   
   # This class represents the schema of an OML measurement stream.
   #
-  class OmlSchema < OMF::Common::LObject
+  class OmlSchema < OMF::Base::LObject
     
     CLASS2TYPE = {
       TrueClass => 'boolean',

data/lib/omf_oml/sequel/sequel_server.rb

@@ -4,7 +4,7 @@ require 'time'
 require 'logger'
 require 'sequel'
 
-require 'omf_common/lobject'
+require 'omf_base/lobject'
 require 'omf_oml'
 
 # module OMF::OML
@@ -14,7 +14,7 @@ require 'omf_oml'
 module OMF::OML::Sequel
   module Server
 
-    class Query < OMF::Common::LObject
+    class Query < OMF::Base::LObject
 
       def self.parse(xmls, repoFactory = RepositoryFactory.new, logger = Logger.new(STDOUT))
         if xmls.kind_of? String
@@ -222,7 +222,7 @@ module OMF::OML::Sequel
 
     end # Query
 
-    class RepositoryFactory < OMF::Common::LObject
+    class RepositoryFactory < OMF::Base::LObject
 
       def initialize(opts = {})
         @opts = opts

data/lib/omf_oml/sql_row.rb

@@ -1,52 +1,52 @@
 
-require 'omf_oml/tuple'
+require 'omf_oml/oml_tuple'
 require 'omf_oml/table'
 
 module OMF::OML
-        
 
-  
+
+
   # Read the content of a table and feeds it out as a tuple store.
   # After creation of the object. The actual tuple feed is started
   # with a call to +run+.
   #
   class OmlSqlRow < OmlTuple
     attr_reader :row
-    
-    # 
+
+    #
     # Create a representation of a row in a database. Can be used to fill a table.
     #
     # @param [String] sql_table_name - name of SQL table in respective SQL database
     # @param [OmlSchema] schema - the schema describing the tuple
-    # @param [Sequel] db - Database 
-    # @param [Hash] opts:  
+    # @param [Sequel::Dataset] query - Databasequery to execute
+    # @param [Hash] opts:
     #   - offset: Ignore first +offset+ rows. If negative or zero serve +offset+ rows initially
     #   - limit: Number of rows to fetch each time [1000]
-    #   - check_interval: Interval in seconds when to check for new data. If 0, only run once.
+    #   - check_interval: Interval in seconds when to check for new data. If < 0, only run once.
     #   - query_interval: Interval between consecutive queries when processing large result set.
     #
-    def initialize(sql_table_name, schema, db, opts = {})
+    def initialize(sql_table_name, schema, query, opts = {})
       @sname = sql_table_name
       @schema = schema
       raise "Expected OmlSchema but got '#{schema.class}" unless schema.is_a? OmlSchema
-      @db = db
-      
+      @query = query
+
       unless @offset = opts[:offset]
         @offset = 0
       end
       @limit = opts[:limit]
       @limit = 1000 unless @limit
-      
+
       @check_interval = opts[:check_interval]
-      @check_interval = 0 unless @check_interval
+      @check_interval = -1 unless @check_interval
       @query_interval = opts[:query_interval]
-      
+
       @on_new_vector_proc = {}
 
-      super opts[:name] || sql_table_name, schema 
+      super opts[:name] || sql_table_name, schema
     end
-    
-    
+
+
     # Return a specific element of the vector identified either
     # by it's name, or its col index
     #
@@ -56,18 +56,18 @@ module OMF::OML
       else
         unless @row.key? name_or_index
           raise "Unknown column name '#{name_or_index}'"
-        end 
+        end
         @row[name_or_index]
       end
     end
-    
-    # Return the elements of the row as an array using the 
+
+    # Return the elements of the row as an array using the
     # associated schema or 'schema' if non-nil.
     #
     def to_a(schema = nil)
       a = (schema || @schema).hash_to_row(@row) # don't need type conversion as sequel is doing this for us
     end
-    
+
     # Return an array including the values for the names elements
     # given as parameters.
     #
@@ -76,19 +76,19 @@ module OMF::OML
       col_names.collect do |n|
         unless @row.key? n
           raise "Unknown column name '#{n}'"
-        end 
+        end
         @row[n]
       end
     end
-        
+
     def ts
       self[:oml_ts_server]
     end
-    
+
     def seq_no
       self[:oml_seq]
-    end    
-    
+    end
+
     # Register a proc to be called when a new tuple arrived
     # on this stream.
     #
@@ -149,14 +149,14 @@ module OMF::OML
       self.on_new_tuple(t) do |v|
         r = v.to_a(schema)
         #puts r.inspect
-        t.add_row(r)   
+        t.add_row(r)
       end
       t
     end
 
 
     protected
-    
+
     def run(in_thread = true)
       return if @running
       if in_thread
@@ -172,16 +172,17 @@ module OMF::OML
         _run
       end
     end
-    
+
     private
-    
+
     def _run
       if @check_interval <= 0
-        while _run_once; end
+        #while _run_once; end
+        _run_once
       else
         @running = true
         while (@running)
-          begin 
+          begin
             unless _run_once
               # All rows read, wait a bit for news to show up
               sleep @check_interval
@@ -190,22 +191,22 @@ module OMF::OML
             warn ex
             debug "\t", ex.backtrace.join("\n\t")
           end
-        end 
+        end
       end
     end
-      
+
     # Run a query on database an serve all rows found one at a time.
     # Return true if there might be more rows in the database
     def _run_once
       row_cnt = 0
       t = table_name = @sname
       if (@offset < 0)
-        cnt = @db[table_name.to_sym].count()
+        cnt = @query.count()
         @offset = cnt + @offset # @offset was negative here
         debug("Initial offset #{@offset} in '#{table_name}' with #{cnt} rows")
         @offset = 0 if @offset < 0
       end
-      @db["SELECT _senders.name as oml_sender, #{t}.* FROM #{t} INNER JOIN _senders ON (_senders.id = #{t}.oml_sender_id) LIMIT #{@limit} OFFSET #{@offset};"].each do |r|
+      @query.limit(@limit, @offset).each do |r|
         @row = r
         @on_new_vector_proc.each_value do |proc|
           proc.call(self)
@@ -216,10 +217,10 @@ module OMF::OML
       debug "Read #{row_cnt} (total #{@offset}) rows from '#{@sname}'" if row_cnt > 0
       if more_to_read = row_cnt >= @limit # there could be more to read
         sleep @query_interval if @query_interval # don't hammer database
-      end 
-      more_to_read    
+      end
+      more_to_read
     end
-    
+
   end # OmlSqlRow
 
 
@@ -227,8 +228,19 @@ end
 
 if $0 == __FILE__
 
+  OMF::Base::Loggable.init_log('sql_row_test')
+
+  require 'omf_oml/sql_source'
+  db_file = File.join(File.dirname(__FILE__), '../../test/data/brooklynDemo.sq3')
+  ss = OMF::OML::OmlSqlSource.new('sqlite://' + File.absolute_path(db_file))
+
+  r = ss.create_stream('wimaxmonitor_wimaxstatus')
+  puts r
+
+  exit
+
   require 'omf_oml/table'
-  ep = OMF::OML::OmlSqlSource.new('brooklynDemo.sq3')
+  ep = OMF::OML::OmlSqlSource.new(File.join(File.dirname(__FILE__), '../../test/data/brooklynDemo.sq3'))
   ep.on_new_stream() do |s|
     puts ">>>>>>>>>>>> New stream #{s.stream_name}: #{s.names.join(', ')}"
     case s.stream_name
@@ -239,7 +251,7 @@ if $0 == __FILE__
     end
 
     s.on_new_vector() do |v|
-      puts "New vector(#{s.stream_name}): #{v.select(*select).join('|')}"      
+      puts "New vector(#{s.stream_name}): #{v.select(*select).join('|')}"
     end
   end
   ep.run()

data/lib/omf_oml/sql_source.rb

@@ -1,7 +1,7 @@
 
 require 'sequel'
 
-require 'omf_common/lobject'
+require 'omf_base/lobject'
 require 'omf_oml/endpoint'
 require 'omf_oml/tuple'
 require 'omf_oml/sql_row'
@@ -14,12 +14,12 @@ module OMF::OML
   # After creating the object, the @run@ method needs to be called to
   # start producing the streams.
   #
-  class OmlSqlSource < OMF::Common::LObject
-    
+  class OmlSqlSource < OMF::Base::LObject
+
     # Sequel adaptors sometimes don't return a :type identifier,
     # but always return the :db_type. This is a list of maps which may not work
     # for all adaptors
-    # 
+    #
     FALLBACK_MAPPING = {
       'UNSIGNED INTEGER' => :integer
     }
@@ -49,42 +49,44 @@ module OMF::OML
         @on_new_stream_procs.delete key
       end
     end
-    
-    # Return a table (more precisely an OmlTable instance) fed from 
+
+    # Return a table (more precisely an OmlTable instance) fed from
     # the content of a table 'table_name' in this database.
     #
     # table_name - Name of table in the SQL database
-    # opts - 
-    #   :include_oml_internals - Include OML 'header' columns [true]
+    # opts -
     #   :name - name used for returned OML Table [table_name]
     #   All other options defined for OmlSqlRow#new
     #
     def create_table(table_name, opts = {})
       tn = opts.delete(:name) || table_name
       schema = _schema_for_table(table_name)
-      r = OmlSqlRow.new(table_name, schema, @db, opts)
+      r = OmlSqlRow.new(table_name, schema, _def_query_for_table(table_name), opts)
       r.to_table(tn, opts)
     end
-    
+
     # Call 'block' for every row in 'table_name' table.
     #
     # table_name - Name of table in the SQL database
-    # opts - 
-    #   :include_oml_internals[boolean] - Include OML 'header' columns [true]
+    # opts -
     #   :schema[Schema] Schema to use for creating row
     #   All other options defined for OmlSqlRow#new
+    # returns OmlSqlRow
     #
     def create_stream(table_name, opts = {}, &block)
       rschema = opts.delete(:schema)
       schema = _schema_for_table(table_name)
-      r = OmlSqlRow.new(table_name, schema, @db, opts)
-      ropts = {}
-      ropts[:schema] = rschema if rschema
-      r.to_stream(ropts, &block)
+      r = OmlSqlRow.new(table_name, schema, _def_query_for_table(table_name), opts)
+      if block
+        ropts = {}
+        ropts[:schema] = rschema if rschema
+        r.to_stream(ropts, &block)
+      end
+      r
     end
 
     #
-    # Run a query on the database and return the result as an OmlTable. The provided schema 
+    # Run a query on the database and return the result as an OmlTable. The provided schema
     # needs to describe the SQL queries result set. Unfortunately we can only do very little
     # sanity checks here
     #
@@ -96,6 +98,33 @@ module OMF::OML
       tbl
     end
 
+    #
+    # Run a query on the database and return the result as an OmlTable. The provided schema
+    # needs to describe the SQL queries result set. Unfortunately we can only do very little
+    # sanity checks here. The query will be defined in the provided block which is passed in
+    # the Sequel Database object and is expected to return a Sequel Dataset instance.
+    #
+    def query2(table_name, schema, &block)
+      tbl = OmlTable.create(table_name, schema)
+      q = block.call(@db)
+      unless q.is_a? Sequel::Dataset
+        raise "Expected a Sequel::Dataset object, but got '#{q.class}'"
+      end
+      q.each do |row|
+        tbl << tbl.schema.hash_to_row(row)
+      end
+      tbl
+    end
+
+    # Return a Sequel Dataset from 'table_name'. See Sequel documentation on
+    # what one can do with that.
+    #
+    # db_table_name Name of table in database
+    #
+    def dataset(db_table_name)
+      @db.from(db_table_name)
+    end
+
     # Start checking the database for tables and create a new stream
     # by calling the internal +report_new_table+ method.
     # If +check_every+ > 0 continue checking every +check_every+ seconds
@@ -130,11 +159,6 @@ module OMF::OML
         report_new_table(table_name) unless table_name.start_with?('_')
       end
       @tables
-      
-      # postgresql
-      # SELECT tablename FROM pg_tables
-      # WHERE tablename NOT LIKE �pg\\_%�
-      # AND tablename NOT LIKE �sql\\_%�;       
     end
 
 
@@ -158,7 +182,7 @@ module OMF::OML
       end
       table
     end
-    
+
     def _schema_for_table(table_name)
       begin
         schema_descr = @db.schema(table_name).map do |col_name, cd|
@@ -173,30 +197,54 @@ module OMF::OML
         raise "Problems reading schema of table '#{table_name}'. Does it exist? (#{@db.tables})"
       end
     end
+
+    def _def_query_for_table(table_name)
+      t = table_name
+      @db["SELECT _senders.name as oml_sender, a.* FROM #{t} AS a INNER JOIN _senders ON (_senders.id = a.oml_sender_id);"]
+    end
   end
-  
+
 
 
 end
 
 if $0 == __FILE__
+  OMF::Base::Loggable.init_log('sql_source_test')
 
   require 'omf_oml/table'
-  ep = OMF::OML::OmlSqlSource.new('brooklynDemo.sq3')
-  ep.on_new_stream() do |s|
-    puts ">>>>>>>>>>>> New stream #{s.stream_name}: #{s.names.join(', ')}"
-    case s.stream_name
-    when 'wimaxmonitor_wimaxstatus'
-      select = [:oml_ts_server, :sender_hostname, :frequency, :signal, :rssi, :cinr, :avg_tx_pw]
-    when 'GPSlogger_gps_data'
-      select = [:oml_ts_server, :oml_sender_id, :lat, :lon]
-    end
+  db_file = File.join(File.dirname(__FILE__), '../../test/data/brooklynDemo.sq3')
+  ep = OMF::OML::OmlSqlSource.new('sqlite://' + File.absolute_path(db_file), limit: 10)
+
+  def on_new_stream(ep)
+    ep.on_new_stream() do |s|
+      puts ">>>>>>>>>>>> New stream #{s.stream_name}: #{s.schema.names}"
+      case s.stream_name
+      when 'wimaxmonitor_wimaxstatus'
+        select = [:oml_ts_server, :sender_hostname, :frequency, :signal, :rssi, :cinr, :avg_tx_pwr]
+      when 'GPSlogger_gps_data'
+        select = [:oml_ts_server, :oml_sender_id, :lat, :lon]
+      end
 
-    s.on_new_vector() do |v|
-      puts "New vector(#{s.stream_name}): #{v.select(*select).join('|')}"      
+      s.on_new_tuple() do |v|
+        begin
+          puts "New vector(#{s.stream_name}): #{v.select(*select).join('|')}"
+        rescue Exception => ex
+          puts "ERROR: #{ex}"
+          abort
+        end
+      end
     end
+    ep.run()
   end
-  ep.run()
+
+  t = ep.query2('gps', [[:lat, :float], [:lon, :float]]) do |db|
+    db.from('GPSlogger_gps_data').select(:lat, :lon).limit(2)
+  end
+  puts t.rows.inspect
+  puts t.schema
+
+  # Raw query on database
+  puts ep.dataset('GPSlogger_gps_data').select(:lat, :lon).limit(2).all
 
 end
 

data/lib/omf_oml/table.rb

@@ -1,7 +1,7 @@
 
 require 'monitor'
 
-require 'omf_common/lobject'
+require 'omf_base/lobject'
 require 'omf_oml'
 require 'omf_oml/schema'
 
@@ -12,8 +12,11 @@ module OMF::OML
   # This class represents a database like table holding a sequence of OML measurements (rows) according
   # a common schema.
   #
-  class OmlTable < OMF::Common::LObject
+  class OmlTable < OMF::Base::LObject
 
+    # @param opts
+    # @opts :index Creates an index table (all rows have distinct values at column 'index')
+    #
     def self.create(tname, schema, opts = {}, &on_before_row_added)
       if (index = opts.delete(:index))
         require 'omf_oml/indexed_table'

data/lib/omf_oml/tuple.rb

@@ -1,5 +1,5 @@
 
-require 'omf_common/lobject'
+require 'omf_base/lobject'
 require 'omf_oml'
 require 'omf_oml/schema'
 
@@ -14,7 +14,7 @@ module OMF::OML
   # Use +OmlTuple+ if the schema is an OML one. +OmlTuple+ has additional convenience
   # methods.
   #
-  class Tuple < OMF::Common::LObject
+  class Tuple < OMF::Base::LObject
 
     # Return a specific element of the tuple identified either
     # by it's name, or its col index

data/lib/omf_oml/version.rb

@@ -1,6 +1,6 @@
 
 module OMF
   module OML
-    VERSION = '0.9.7'
+    VERSION = '0.9.8'
   end
 end

data/omf_oml.gemspec

@@ -23,5 +23,6 @@ Gem::Specification.new do |s|
   # specify any dependencies here; for example:
 #  s.add_development_dependency "minitest", "~> 2.11.3"
   #s.add_runtime_dependency "sqlite3", "~> 1.3.6"
+  s.add_runtime_dependency 'omf_base'
   s.add_runtime_dependency 'sequel', '~> 3.45'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: omf_oml
 version: !ruby/object:Gem::Version
-  version: 0.9.7
+  version: 0.9.8
   prerelease: 
 platform: ruby
 authors:
@@ -9,8 +9,24 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-05-31 00:00:00.000000000 Z
+date: 2013-08-29 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: omf_base
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: sequel
   requirement: !ruby/object:Gem::Requirement
@@ -39,6 +55,7 @@ files:
 - Gemfile
 - Rakefile
 - lib/omf_oml.rb
+- lib/omf_oml/csv_table.rb
 - lib/omf_oml/endpoint.rb
 - lib/omf_oml/indexed_table.rb
 - lib/omf_oml/network.rb
@@ -51,6 +68,7 @@ files:
 - lib/omf_oml/tuple.rb
 - lib/omf_oml/version.rb
 - omf_oml.gemspec
+- test/data/brooklynDemo.sq3
 homepage: https://www.mytestbed.net
 licenses: []
 post_install_message: 
@@ -71,9 +89,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: omf_oml
-rubygems_version: 1.8.24
+rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
 summary: Glue between OMF and OML.
-test_files: []
-has_rdoc: 
+test_files:
+- test/data/brooklynDemo.sq3