omf_oml 0.9.3 → 0.9.4

data/lib/omf_oml/schema.rb

@@ -25,14 +25,18 @@ module OMF::OML
       'int32' => :integer,
       'int64' => :integer,
       'bigint' => :integer,
+      'smallint' => :integer,
       'unsigned integer' => :integer,
+      'unsigned bigint' => :integer,
       'float' => :float,
       'real' => :float,
       'double' => :float,
+      'double precision' => :float,
       'text' => :string,
       'string' => :string,
       'date' => :date,
       'dateTime'.downcase => :dateTime, # should be 'datetime' but we downcase the string for comparison
+      'timestamp' => :dateTime, # Postgreql specific, not sure if this works as it.
       'key' => :key,      
     }
 
@@ -108,7 +112,7 @@ module OMF::OML
           type = :string
         end
       else
-        warn "Missing type definition in '#{col[:name]}', default to 'string'"          
+        warn "Missing type definition in '#{col[:name]}', default to 'string' (#{col.inspect})"          
         type = :string
       end
       col[:type] = type
@@ -144,15 +148,16 @@ module OMF::OML
     # hrow - Hash describing a row
     # set_nil_when_missing - If true, set any columns not described in hrow to nil
     #
-    def hash_to_row(hrow, set_nil_when_missing = false)
+    def hash_to_row(hrow, set_nil_when_missing = false, call_type_conversion = true)
+      #puts "HASH2A => #{hrow.inspect}"
       r = @schema.collect do |cdescr|
         cname = cdescr[:name]
         next nil if cname == :__id__
-        unless hrow.key? cname
+        unless (v = hrow[cname]) ||  (v = hrow[cname.to_sym])
           next nil if set_nil_when_missing
           raise "Missing record element '#{cname}' in record '#{hrow}'"
         end
-        cdescr[:type_conversion].call(hrow[cname])
+        call_type_conversion ? cdescr[:type_conversion].call(v) : v
       end
       r.shift # remove __id__ columns
       #puts "#{r.inspect} -- #{@schema.map {|c| c[:name]}.inspect}"
@@ -161,14 +166,15 @@ module OMF::OML
     
     # Cast each element in 'row' into its proper type according to this schema
     #
-    def cast_row(raw_row)
-      unless raw_row.length == @schema.length
-        raise "Row needs to have same size as schema (#{raw_row.inspect})"
+    def cast_row(raw_row, ignore_first_column = false)
+      start_index = ignore_first_column ? 1 : 0
+      unless raw_row.length == @schema.length - start_index
+        raise "Row needs to have same size as schema (#{raw_row.inspect}-#{describe}) "
       end
       # This can be done more elegantly in 1.9
       row = []
       raw_row.each_with_index do |el, i|
-        row << @schema[i][:type_conversion].call(el)
+        row << @schema[i + start_index][:type_conversion].call(el)
       end
       row
     end

data/lib/omf_oml/sql_row.rb

@@ -10,15 +10,23 @@ module OMF::OML
   # with a call to +run+.
   #
   class OmlSqlRow < OmlTuple
+    attr_reader :row
     
-    # *opts:  
+    # 
+    # Create a representation of a row in a database. Can be used to fill a table.
+    #
+    # table_name - name table in respective SQL database
+    # schema_raw - a hash returned by the sequel library's #schema method
+    # db_opts - Enough information to open a Sequel database adapter
+    # source - Reference to the SqlSource which created this instance
+    # opts:  
     #   - offset: Ignore first +offset+ rows. If negative or zero serve +offset+ rows initially
     #   - limit: Number of rows to fetch each time [1000]
     #   - check_interval: Interval in seconds when to check for new data. If 0, only run once.
     #
-    def initialize(table_name, db_file, source, opts = {})
+    def initialize(table_name, schema_raw, db_opts, source, opts = {})
       @sname = table_name
-      @db_file = db_file
+      @db_opts = db_opts
       @source = source
       
       unless @offset = opts[:offset]
@@ -33,7 +41,7 @@ module OMF::OML
 
       @on_new_vector_proc = {}
 
-      schema = find_schema
+      schema = parse_schema(schema_raw)
       super table_name, schema 
     end
     
@@ -47,7 +55,8 @@ module OMF::OML
     
     # Return the elements of the vector as an array
     def to_a(include_oml_internals = false)
-      include_oml_internals ? @row.dup : @row[4 .. -1]
+      a = @schema.hash_to_row(@row, false, false) # don't need type conversion as sequel is doing this for us
+      include_oml_internals ? a : a[4 .. -1]
     end
     
     # Return an array including the values for the names elements
@@ -163,22 +172,16 @@ module OMF::OML
 
     protected
         
-    def find_schema()
-      stmt = _statement
-      cnames = stmt.columns
-      ctypes = stmt.types
-      schema = []
-      #schema << {:name => :oml_sender, :type => 'STRING'}
-      cnames.size.times do |i|
-        name = cnames[i].to_sym
-        schema << {:name => name, :type => ctypes[i]}
+    def parse_schema(raw)
+      #puts ">> PARSE SCHEMA"
+      sd = raw.collect do |col| 
+        name, opts = col
+        #puts col.inspect
+        {:name => name, :type => opts[:db_type]}
       end
-      # Rename first col
-      first = schema[0]
-      raise "BUG: Should be 'name'" if first[:name] != :name
-      first[:name] = :oml_sender
-      
-      OmlSchema.new(schema)
+      # Query we are using is adding the 'oml_sender_name' to the front of the table
+      sd.insert(0, :name => 'oml_sender', :type => :string)
+      OmlSchema.new(sd)
     end
     
     # override
@@ -197,16 +200,6 @@ module OMF::OML
     def run(in_thread = true)
       return if @running
       if in_thread
-        if @db
-          # force opening of database in new thread
-          begin
-            @db.close
-          rescue Exception
-            # ALERT: issues with finalising statments, don't know how to deal with it
-          end
-          @db = nil
-          @stmt = nil
-        end
         Thread.new do
           begin
             _run
@@ -223,6 +216,8 @@ module OMF::OML
     private
     
     def _run
+      @db = Sequel.connect(@db_opts)
+      
       if @check_interval <= 0
         _run_once
       else
@@ -245,7 +240,16 @@ module OMF::OML
     # Return true if there might be more rows in the database
     def _run_once
       row_cnt = 0
-      _statement.execute(@limit, @offset).each do |r|
+      t = table_name = @sname
+      if (@offset < 0)
+        cnt = @db[table_name.to_sym].count()
+        @offset = cnt + @offset # @offset was negative here
+        debug("Initial offset #{@offset} in '#{table_name}' with #{cnt} rows")
+        @offset = 0 if @offset < 0
+      end
+      @db["SELECT _senders.name as oml_sender, #{t}.* FROM #{t} INNER JOIN _senders ON (_senders.id = #{t}.oml_sender_id) LIMIT #{@limit} OFFSET #{@offset};"].each do |r|
+      #@db["SELECT _senders.name as oml_sender, #{t}.* FROM #{t} JOIN _senders WHERE #{t}.oml_sender_id = _senders.id LIMIT #{@limit} OFFSET #{@offset};"].each do |r|
+        #puts "ROW>>> #{r.inspect}"
         @row = r
         @on_new_vector_proc.each_value do |proc|
           proc.call(self)
@@ -253,27 +257,27 @@ module OMF::OML
         row_cnt += 1
       end
       @offset += row_cnt
-      debug "Read #{row_cnt}/#{@offset} rows from '#{@sname}'"
+      debug "Read #{row_cnt}/#{@offset} rows from '#{@sname}'" if row_cnt > 0
       row_cnt >= @limit # there could be more to read     
     end
     
-    def _statement
-      unless @stmt
-        db = @db = SQLite3::Database.new(@db_file)
-        @db.type_translation = true   
-        table_name = t = @sname  
-        if @offset < 0
-          cnt = db.execute("select count(*) from #{table_name};")[0][0].to_i
-          #debug "CNT: #{cnt}.#{cnt.class} offset: #{@offset}"
-          @offset = cnt + @offset # @offset was negative here
-          debug("Initial offset #{@offset} in '#{table_name}' with #{cnt} rows")
-          @offset = 0 if @offset < 0
-        end
-        #@stmt = db.prepare("SELECT * FROM #{table_name} LIMIT ? OFFSET ?;")
-        @stmt = db.prepare("SELECT _senders.name, #{t}.* FROM #{t} JOIN _senders WHERE #{t}.oml_sender_id = _senders.id LIMIT ? OFFSET ?;")
-      end
-      @stmt
-    end
+    # def _statement
+      # unless @stmt
+        # db = @db = SQLite3::Database.new(@db_file)
+        # @db.type_translation = true   
+        # table_name = t = @sname  
+        # if @offset < 0
+          # cnt = db.execute("select count(*) from #{table_name};")[0][0].to_i
+          # #debug "CNT: #{cnt}.#{cnt.class} offset: #{@offset}"
+          # @offset = cnt + @offset # @offset was negative here
+          # debug("Initial offset #{@offset} in '#{table_name}' with #{cnt} rows")
+          # @offset = 0 if @offset < 0
+        # end
+        # #@stmt = db.prepare("SELECT * FROM #{table_name} LIMIT ? OFFSET ?;")
+        # @stmt = db.prepare("SELECT _senders.name, #{t}.* FROM #{t} JOIN _senders WHERE #{t}.oml_sender_id = _senders.id LIMIT ? OFFSET ?;")
+      # end
+      # @stmt
+    # end
   end # OmlSqlRow
 
 

data/lib/omf_oml/sql_source.rb

@@ -1,5 +1,5 @@
 
-require 'sqlite3'
+require 'sequel'
 
 require 'omf_common/lobject'
 require 'omf_oml/endpoint'
@@ -16,15 +16,19 @@ module OMF::OML
   #
   class OmlSqlSource < OMF::Common::LObject
 
-    # +opts+ - passed on as +opts+ to the OmlSqlRow constructor
+    # db_opts - Options used to create a Sequel adapter
     #
-    def initialize(db_file, opts = {})
-      raise "Can't find database '#{db_file}'" unless File.readable?(db_file)
-      @db_file = db_file
+    # Sequel.connect(:adapter=>'postgres', :host=>'norbit.npc.nicta.com.au', :user=>'oml2', :password=>'omlisgoodforyou', :database=>'openflow-demo')
+    #
+    def initialize(db_opts, row_opts = {})
       @running = false
       @on_new_stream_procs = {}
       @tables = {}
-      @table_opts = opts
+      @db_opts = db_opts
+      debug "Opening DB (#{db_opts})"
+      @db = Sequel.connect(db_opts)
+      debug "DB: #{@db.inspect}"
+      @row_opts = row_opts
     end
 
     # Register a proc to be called when a new stream was
@@ -38,13 +42,6 @@ module OMF::OML
       end
     end
 
-
-    # def report_new_stream(stream)
-      # @on_new_stream_procs.each_value do |proc|
-        # proc.call(stream)
-      # end
-    # end
-
     # Start checking the database for tables and create a new stream
     # by calling the internal +report_new_table+ method.
     # If +check_every+ > 0 continue checking every +check_every+ seconds
@@ -71,16 +68,18 @@ module OMF::OML
     end
 
     def run_once()
-      unless @db
-        @db = SQLite3::Database.new(@db_file)
-        @db.type_translation = true
-      end
-
+      debug "Finding tables #{@db.tables}"
       # first find tables
-      @db.execute( "SELECT * FROM sqlite_master WHERE type='table';") do |r|
-        table_name = r[1]
-        report_new_table(table_name, @table_opts) unless table_name.start_with?('_')
+      @db.tables.each do |tn|
+        table_name = tn.to_s
+        report_new_table(table_name) unless table_name.start_with?('_')
       end
+      @tables
+      
+      # postgresql
+      # SELECT tablename FROM pg_tables
+      # WHERE tablename NOT LIKE �pg\\_%�
+      # AND tablename NOT LIKE �sql\\_%�;       
     end
 
 
@@ -93,13 +92,15 @@ module OMF::OML
     # +on_new_stream+ is then called with the new stream as its single
     # argument.
     #
-    def report_new_table(table_name, opts = {})
-      return if @tables.key?(table_name) # check if already reported before
-      debug "Found table: #{table_name}"
-      t = @tables[table_name] = OmlSqlRow.new(table_name, @db_file, self, opts)
-      @on_new_stream_procs.each_value do |proc|
-        proc.call(t)
+    def report_new_table(table_name)
+      unless table =  @tables[table_name] # check if already reported before
+        debug "Found table: #{table_name}"
+        table = @tables[table_name] = OmlSqlRow.new(table_name, @db.schema(table_name), @db_opts, self, @row_opts)
+        @on_new_stream_procs.each_value do |proc|
+          proc.call(table)
+        end
       end
+      table
     end
     
   end

data/lib/omf_oml/table.rb

@@ -178,7 +178,7 @@ module OMF::OML
     #
     def _add_row(row, needs_casting = false)
       if needs_casting
-        row = @schema.cast_row(row)
+        row = @schema.cast_row(row, true)
       end
       #puts row.inspect
       if @on_before_row_added

data/lib/omf_oml/version.rb

@@ -1,6 +1,6 @@
 
 module OMF
   module OML
-    VERSION = '0.9.3'
+    VERSION = '0.9.4'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: omf_oml
 version: !ruby/object:Gem::Version
-  version: 0.9.3
+  version: 0.9.4
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-10-15 00:00:00.000000000 Z
+date: 2012-11-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sqlite3
-  requirement: &13713900 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,7 +21,12 @@ dependencies:
         version: 1.3.6
   type: :runtime
   prerelease: false
-  version_requirements: *13713900
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.3.6
 description: ! "Glue functionality between OMF and OMF related libraries, such as
   OMF Web and Labwiki, and\n    OML."
 email:
@@ -66,8 +71,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: omf_oml
-rubygems_version: 1.8.10
+rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: Glue between OMF and OML.
 test_files: []
+has_rdoc: