omf_oml 0.9.5 → 0.9.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +1 -0
- data/lib/omf_oml/schema.rb +86 -57
- data/lib/omf_oml/sql_row.rb +66 -132
- data/lib/omf_oml/sql_source.rb +74 -4
- data/lib/omf_oml/table.rb +24 -3
- data/lib/omf_oml/version.rb +1 -1
- data/omf_oml.gemspec +2 -1
- metadata +5 -5
data/.gitignore
CHANGED
data/lib/omf_oml/schema.rb
CHANGED
@@ -34,11 +34,20 @@ module OMF::OML
|
|
34
34
|
'double precision' => :float,
|
35
35
|
'text' => :string,
|
36
36
|
'string' => :string,
|
37
|
+
'varargs' => :string,
|
37
38
|
'date' => :date,
|
38
39
|
'dateTime'.downcase => :dateTime, # should be 'datetime' but we downcase the string for comparison
|
39
40
|
'timestamp' => :dateTime, # Postgreql specific, not sure if this works as it.
|
40
41
|
'key' => :key,
|
41
42
|
}
|
43
|
+
|
44
|
+
OML_INTERNALS = [
|
45
|
+
:oml_sender,
|
46
|
+
:oml_sender_id,
|
47
|
+
:oml_seq,
|
48
|
+
:oml_ts_client,
|
49
|
+
:oml_ts_server
|
50
|
+
]
|
42
51
|
|
43
52
|
def self.create(schema_description)
|
44
53
|
if schema_description.kind_of? self
|
@@ -83,62 +92,13 @@ module OMF::OML
|
|
83
92
|
end
|
84
93
|
|
85
94
|
def insert_column_at(index, col)
|
86
|
-
|
87
|
-
col = [col]
|
88
|
-
end
|
89
|
-
if col.kind_of? Array
|
90
|
-
# should be [name, type]
|
91
|
-
if col.length == 1
|
92
|
-
col = {:name => col[0].to_sym,
|
93
|
-
:type => :string,
|
94
|
-
:title => col[0].to_s.split('_').collect {|s| s.capitalize}.join(' ')}
|
95
|
-
elsif col.length == 2
|
96
|
-
col = {:name => col[0].to_sym,
|
97
|
-
:type => col[1].to_sym,
|
98
|
-
:title => col[0].to_s.split('_').collect {|s| s.capitalize}.join(' ')}
|
99
|
-
elsif col.length == 3
|
100
|
-
col = {:name => col[0].to_sym, :type => col[1].to_sym, :title => col[2]}
|
101
|
-
else
|
102
|
-
throw "Simple column schema should consist of [name, type, title] array, but found '#{col.inspect}'"
|
103
|
-
end
|
104
|
-
elsif col.kind_of? Hash
|
105
|
-
# ensure there is a :title property
|
106
|
-
unless col[:title]
|
107
|
-
col[:title] = col[:name].to_s.split('_').collect {|s| s.capitalize}.join(' ')
|
108
|
-
end
|
109
|
-
end
|
110
|
-
|
111
|
-
# should normalize type
|
112
|
-
if type = col[:type]
|
113
|
-
unless type = ANY2TYPE[type.to_s.downcase]
|
114
|
-
warn "Unknown type definition '#{col[:type]}', default to 'string'"
|
115
|
-
type = :string
|
116
|
-
end
|
117
|
-
else
|
118
|
-
warn "Missing type definition in '#{col[:name]}', default to 'string' (#{col.inspect})"
|
119
|
-
type = :string
|
120
|
-
end
|
121
|
-
col[:type] = type
|
122
|
-
|
123
|
-
col[:type_conversion] = case type
|
124
|
-
when :string
|
125
|
-
lambda do |r| r end
|
126
|
-
when :key
|
127
|
-
lambda do |r| r end
|
128
|
-
when :integer
|
129
|
-
lambda do |r| r.to_i end
|
130
|
-
when :float
|
131
|
-
lambda do |r| r.to_f end
|
132
|
-
when :date
|
133
|
-
lambda do |r| Date.parse(r) end
|
134
|
-
when :dateTime
|
135
|
-
lambda do |r| Time.parse(r) end
|
136
|
-
else raise "Unrecognized Schema type '#{type}'"
|
137
|
-
end
|
138
|
-
|
139
|
-
@schema.insert(index, col)
|
95
|
+
@schema.insert(index, _create_col_descr(col))
|
140
96
|
end
|
141
97
|
|
98
|
+
def replace_column_at(index, col)
|
99
|
+
@schema[index] = _create_col_descr(col)
|
100
|
+
end
|
101
|
+
|
142
102
|
def each_column(&block)
|
143
103
|
@schema.each do |c|
|
144
104
|
block.call(c)
|
@@ -151,7 +111,7 @@ module OMF::OML
|
|
151
111
|
# hrow - Hash describing a row
|
152
112
|
# set_nil_when_missing - If true, set any columns not described in hrow to nil
|
153
113
|
#
|
154
|
-
def hash_to_row(hrow, set_nil_when_missing = false, call_type_conversion =
|
114
|
+
def hash_to_row(hrow, set_nil_when_missing = false, call_type_conversion = false)
|
155
115
|
#puts "HASH2A => #{hrow.inspect}"
|
156
116
|
remove_first_col = false
|
157
117
|
r = @schema.collect do |cdescr|
|
@@ -194,12 +154,22 @@ module OMF::OML
|
|
194
154
|
describe.to_json(*opt)
|
195
155
|
end
|
196
156
|
|
197
|
-
def clone()
|
157
|
+
def clone(exclude_oml_internals = false)
|
198
158
|
c = self.dup
|
199
|
-
|
159
|
+
schema = @schema.clone
|
160
|
+
if exclude_oml_internals
|
161
|
+
schema = schema.select do |cd|
|
162
|
+
not OML_INTERNALS.include?(cd[:name])
|
163
|
+
end
|
164
|
+
end
|
165
|
+
c.instance_variable_set('@schema', schema)
|
200
166
|
c
|
201
167
|
end
|
202
168
|
|
169
|
+
def to_s
|
170
|
+
"OmlSchema: #{@schema.map {|c| "#{c[:name]}:#{c[:type]}"}.join(', ')}"
|
171
|
+
end
|
172
|
+
|
203
173
|
protected
|
204
174
|
|
205
175
|
# schema_description - Array containing [name, type*] for every column in table
|
@@ -214,6 +184,65 @@ module OMF::OML
|
|
214
184
|
debug "schema: '#{describe.inspect}'"
|
215
185
|
|
216
186
|
end
|
187
|
+
|
188
|
+
# Create a column descriptor from whatever is given by user
|
189
|
+
#
|
190
|
+
def _create_col_descr(col)
|
191
|
+
if col.kind_of?(Symbol) || col.kind_of?(String)
|
192
|
+
col = [col]
|
193
|
+
end
|
194
|
+
if col.kind_of? Array
|
195
|
+
# should be [name, type]
|
196
|
+
if col.length == 1
|
197
|
+
col = {:name => col[0].to_sym,
|
198
|
+
:type => :string,
|
199
|
+
:title => col[0].to_s.split('_').collect {|s| s.capitalize}.join(' ')}
|
200
|
+
elsif col.length == 2
|
201
|
+
col = {:name => col[0].to_sym,
|
202
|
+
:type => col[1].to_sym,
|
203
|
+
:title => col[0].to_s.split('_').collect {|s| s.capitalize}.join(' ')}
|
204
|
+
elsif col.length == 3
|
205
|
+
col = {:name => col[0].to_sym, :type => col[1].to_sym, :title => col[2]}
|
206
|
+
else
|
207
|
+
throw "Simple column schema should consist of [name, type, title] array, but found '#{col.inspect}'"
|
208
|
+
end
|
209
|
+
elsif col.kind_of? Hash
|
210
|
+
# ensure there is a :title property
|
211
|
+
unless col[:title]
|
212
|
+
col[:title] = col[:name].to_s.split('_').collect {|s| s.capitalize}.join(' ')
|
213
|
+
end
|
214
|
+
end
|
215
|
+
|
216
|
+
# should normalize type
|
217
|
+
if type = col[:type]
|
218
|
+
tn = type.to_s.split('(')[0] # take care of types like varargs(..)
|
219
|
+
unless type = ANY2TYPE[type.to_s.downcase]
|
220
|
+
warn "Unknown type definition '#{tn}', default to 'string'"
|
221
|
+
type = :string
|
222
|
+
end
|
223
|
+
else
|
224
|
+
warn "Missing type definition in '#{col[:name]}', default to 'string' (#{col.inspect})"
|
225
|
+
type = :string
|
226
|
+
end
|
227
|
+
col[:type] = type
|
228
|
+
|
229
|
+
col[:type_conversion] = case type
|
230
|
+
when :string
|
231
|
+
lambda do |r| r end
|
232
|
+
when :key
|
233
|
+
lambda do |r| r end
|
234
|
+
when :integer
|
235
|
+
lambda do |r| r.to_i end
|
236
|
+
when :float
|
237
|
+
lambda do |r| r.to_f end
|
238
|
+
when :date
|
239
|
+
lambda do |r| Date.parse(r) end
|
240
|
+
when :dateTime
|
241
|
+
lambda do |r| Time.parse(r) end
|
242
|
+
else raise "Unrecognized Schema type '#{type}'"
|
243
|
+
end
|
244
|
+
col
|
245
|
+
end
|
217
246
|
end # OmlSchema
|
218
247
|
|
219
248
|
end
|
data/lib/omf_oml/sql_row.rb
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
|
2
2
|
require 'omf_oml/tuple'
|
3
|
+
require 'omf_oml/table'
|
3
4
|
|
4
5
|
module OMF::OML
|
5
6
|
|
@@ -15,19 +16,20 @@ module OMF::OML
|
|
15
16
|
#
|
16
17
|
# Create a representation of a row in a database. Can be used to fill a table.
|
17
18
|
#
|
18
|
-
#
|
19
|
-
#
|
20
|
-
#
|
21
|
-
#
|
22
|
-
# opts:
|
19
|
+
# @param [String] sql_table_name - name of SQL table in respective SQL database
|
20
|
+
# @param [OmlSchema] schema - the schema describing the tuple
|
21
|
+
# @param [Sequel] db - Database
|
22
|
+
# @param [Hash] opts:
|
23
23
|
# - offset: Ignore first +offset+ rows. If negative or zero serve +offset+ rows initially
|
24
24
|
# - limit: Number of rows to fetch each time [1000]
|
25
25
|
# - check_interval: Interval in seconds when to check for new data. If 0, only run once.
|
26
|
+
# - query_interval: Interval between consecutive queries when processing large result set.
|
26
27
|
#
|
27
|
-
def initialize(
|
28
|
-
@sname =
|
29
|
-
@
|
30
|
-
|
28
|
+
def initialize(sql_table_name, schema, db, opts = {})
|
29
|
+
@sname = sql_table_name
|
30
|
+
@schema = schema
|
31
|
+
raise "Expected OmlSchema but got '#{schema.class}" unless schema.is_a? OmlSchema
|
32
|
+
@db = db
|
31
33
|
|
32
34
|
unless @offset = opts[:offset]
|
33
35
|
@offset = 0
|
@@ -37,12 +39,11 @@ module OMF::OML
|
|
37
39
|
|
38
40
|
@check_interval = opts[:check_interval]
|
39
41
|
@check_interval = 0 unless @check_interval
|
42
|
+
@query_interval = opts[:query_interval]
|
40
43
|
|
41
|
-
|
42
44
|
@on_new_vector_proc = {}
|
43
45
|
|
44
|
-
schema
|
45
|
-
super table_name, schema
|
46
|
+
super opts[:name] || sql_table_name, schema
|
46
47
|
end
|
47
48
|
|
48
49
|
|
@@ -60,10 +61,11 @@ module OMF::OML
|
|
60
61
|
end
|
61
62
|
end
|
62
63
|
|
63
|
-
# Return the elements of the
|
64
|
-
|
65
|
-
|
66
|
-
|
64
|
+
# Return the elements of the row as an array using the
|
65
|
+
# associated schema or 'schema' if non-nil.
|
66
|
+
#
|
67
|
+
def to_a(schema = nil)
|
68
|
+
a = (schema || @schema).hash_to_row(@row) # don't need type conversion as sequel is doing this for us
|
67
69
|
end
|
68
70
|
|
69
71
|
# Return an array including the values for the names elements
|
@@ -72,9 +74,10 @@ module OMF::OML
|
|
72
74
|
def select(*col_names)
|
73
75
|
r = @row
|
74
76
|
col_names.collect do |n|
|
75
|
-
|
76
|
-
|
77
|
-
|
77
|
+
unless @row.key? n
|
78
|
+
raise "Unknown column name '#{n}'"
|
79
|
+
end
|
80
|
+
@row[n]
|
78
81
|
end
|
79
82
|
end
|
80
83
|
|
@@ -98,79 +101,54 @@ module OMF::OML
|
|
98
101
|
run() unless @on_new_vector_proc.empty?
|
99
102
|
end
|
100
103
|
|
101
|
-
#
|
102
|
-
#
|
103
|
-
# The argument to this method are either a list of columns to
|
104
|
-
# to capture in the table, or an array of column names and
|
105
|
-
# an option hash or just
|
106
|
-
# the option hash to be provided to the +OmlTable+ constructor.
|
104
|
+
# Return a table which will capture the content of this tuple stream.
|
107
105
|
#
|
108
|
-
#
|
109
|
-
#
|
110
|
-
#
|
111
|
-
#
|
112
|
-
# listed in the selector. Otherwise, the block is called directly
|
113
|
-
# with the tuple.
|
106
|
+
# @param [string] name - Name to use for returned table
|
107
|
+
# @param [Hash] opts Options to be passed on to Table constructor
|
108
|
+
# @opts [boolean] opts :include_oml_internals If true will also include OML header columns
|
109
|
+
# @opts [OmlSchema] opts :schema use specific schema for table (Needs to be a subset of the tuples schema)
|
114
110
|
#
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
if args[0].kind_of?(Array)
|
123
|
-
select = args[0]
|
124
|
-
elsif args[0].kind_of?(Hash)
|
125
|
-
opts = args[0]
|
111
|
+
def to_stream(opts = {}, &block)
|
112
|
+
unless schema = opts.delete(:schema)
|
113
|
+
include_oml_internals = (opts[:include_oml_internals] != false)
|
114
|
+
schema = self.schema.clone(!include_oml_internals)
|
115
|
+
if include_oml_internals
|
116
|
+
# replace sender_id by sender ... see _run_once
|
117
|
+
schema.replace_column_at 0, :oml_sender
|
126
118
|
end
|
127
|
-
elsif args.length == 2 && args[1].kind_of?(Hash)
|
128
|
-
select = args[0]
|
129
|
-
opts = args[1]
|
130
|
-
else
|
131
|
-
opts = {}
|
132
|
-
select = args
|
133
119
|
end
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
# end
|
139
|
-
else
|
140
|
-
tschema = select.collect do |cname| {:name => cname} end
|
141
|
-
end
|
142
|
-
tname = opts.delete(:name) || stream_name
|
143
|
-
t = OMF::OML::OmlTable.new(tname, tschema, opts)
|
144
|
-
if block
|
145
|
-
self.on_new_tuple() do |v|
|
146
|
-
#puts "New vector(#{tname}): #{v.schema.inspect} ---- #{v.select(*select).size} <#{v.select(*select).join('|')}>"
|
147
|
-
if select
|
148
|
-
row = block.call(v.select(*select))
|
149
|
-
else
|
150
|
-
row = block.call(v)
|
151
|
-
end
|
152
|
-
if row
|
153
|
-
raise "Expected kind of Array, but got '#{row.inspect}'" unless row.kind_of?(Array)
|
154
|
-
t.add_row(row)
|
155
|
-
end
|
156
|
-
end
|
157
|
-
else
|
158
|
-
self.on_new_tuple() do |v|
|
159
|
-
#puts "New vector(#{tname}): #{v.select(*select).join('|')}"
|
160
|
-
t.add_row(v.select(*select))
|
161
|
-
end
|
120
|
+
self.on_new_tuple(rand()) do |t|
|
121
|
+
#v = t.to_a(schema)
|
122
|
+
v = t.row
|
123
|
+
block.arity == 1 ? block.call(v) : block.call(v, schema)
|
162
124
|
end
|
163
|
-
|
125
|
+
schema
|
164
126
|
end
|
165
|
-
|
127
|
+
|
128
|
+
# Return a table which will capture the content of this tuple stream.
|
129
|
+
#
|
130
|
+
# @param [string] name - Name to use for returned table
|
131
|
+
# @param [Hash] opts Options to be passed on to Table constructor
|
132
|
+
# @opts [boolean] opts :include_oml_internals If true will also include OML header columns
|
133
|
+
# @opts [OmlSchema] opts :schema use specific schema for table (Needs to be a subset of the tuples schema)
|
134
|
+
#
|
166
135
|
def to_table(name = nil, opts = {})
|
167
136
|
unless name
|
168
137
|
name = @sname
|
169
138
|
end
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
139
|
+
unless schema = opts.delete(:schema)
|
140
|
+
include_oml_internals = (opts[:include_oml_internals] != false)
|
141
|
+
schema = self.schema.clone(!include_oml_internals)
|
142
|
+
if include_oml_internals
|
143
|
+
# replace sender_id by sender ... see _run_once
|
144
|
+
schema.replace_column_at 0, :oml_sender
|
145
|
+
end
|
146
|
+
end
|
147
|
+
t = OMF::OML::OmlTable.new(name, schema, opts)
|
148
|
+
#puts ">>>>SCHEMA>>> #{schema.inspect}"
|
149
|
+
self.on_new_tuple(t) do |v|
|
150
|
+
r = v.to_a(schema)
|
151
|
+
#puts r.inspect
|
174
152
|
t.add_row(r)
|
175
153
|
end
|
176
154
|
t
|
@@ -178,32 +156,6 @@ module OMF::OML
|
|
178
156
|
|
179
157
|
|
180
158
|
protected
|
181
|
-
|
182
|
-
def parse_schema(raw)
|
183
|
-
#puts ">> PARSE SCHEMA"
|
184
|
-
sd = raw.collect do |col|
|
185
|
-
name, opts = col
|
186
|
-
#puts col.inspect
|
187
|
-
{:name => name, :type => opts[:db_type]}
|
188
|
-
end
|
189
|
-
# Query we are using is adding the 'oml_sender_name' to the front of the table
|
190
|
-
sd.insert(0, :name => :oml_sender, :type => :string)
|
191
|
-
OmlSchema.new(sd)
|
192
|
-
end
|
193
|
-
|
194
|
-
# override
|
195
|
-
def process_schema(schema)
|
196
|
-
i = 0
|
197
|
-
@vprocs = {}
|
198
|
-
schema.each_column do |col|
|
199
|
-
name = col[:name]
|
200
|
-
j = i + 0
|
201
|
-
puts "SCHEMA: '#{name.inspect}'-#{j}"
|
202
|
-
l = @vprocs[name] = lambda do |r| r[j] end
|
203
|
-
@vprocs[i - 4] = l if i > 4
|
204
|
-
i += 1
|
205
|
-
end
|
206
|
-
end
|
207
159
|
|
208
160
|
def run(in_thread = true)
|
209
161
|
return if @running
|
@@ -224,10 +176,8 @@ module OMF::OML
|
|
224
176
|
private
|
225
177
|
|
226
178
|
def _run
|
227
|
-
@db = Sequel.connect(@db_opts)
|
228
|
-
|
229
179
|
if @check_interval <= 0
|
230
|
-
_run_once
|
180
|
+
while _run_once; end
|
231
181
|
else
|
232
182
|
@running = true
|
233
183
|
while (@running)
|
@@ -256,8 +206,6 @@ module OMF::OML
|
|
256
206
|
@offset = 0 if @offset < 0
|
257
207
|
end
|
258
208
|
@db["SELECT _senders.name as oml_sender, #{t}.* FROM #{t} INNER JOIN _senders ON (_senders.id = #{t}.oml_sender_id) LIMIT #{@limit} OFFSET #{@offset};"].each do |r|
|
259
|
-
#@db["SELECT _senders.name as oml_sender, #{t}.* FROM #{t} JOIN _senders WHERE #{t}.oml_sender_id = _senders.id LIMIT #{@limit} OFFSET #{@offset};"].each do |r|
|
260
|
-
#puts "ROW>>> #{r.inspect}"
|
261
209
|
@row = r
|
262
210
|
@on_new_vector_proc.each_value do |proc|
|
263
211
|
proc.call(self)
|
@@ -265,27 +213,13 @@ module OMF::OML
|
|
265
213
|
row_cnt += 1
|
266
214
|
end
|
267
215
|
@offset += row_cnt
|
268
|
-
debug "Read #{row_cnt}
|
269
|
-
row_cnt >= @limit # there could be more to read
|
216
|
+
debug "Read #{row_cnt} (total #{@offset}) rows from '#{@sname}'" if row_cnt > 0
|
217
|
+
if more_to_read = row_cnt >= @limit # there could be more to read
|
218
|
+
sleep @query_interval if @query_interval # don't hammer database
|
219
|
+
end
|
220
|
+
more_to_read
|
270
221
|
end
|
271
222
|
|
272
|
-
# def _statement
|
273
|
-
# unless @stmt
|
274
|
-
# db = @db = SQLite3::Database.new(@db_file)
|
275
|
-
# @db.type_translation = true
|
276
|
-
# table_name = t = @sname
|
277
|
-
# if @offset < 0
|
278
|
-
# cnt = db.execute("select count(*) from #{table_name};")[0][0].to_i
|
279
|
-
# #debug "CNT: #{cnt}.#{cnt.class} offset: #{@offset}"
|
280
|
-
# @offset = cnt + @offset # @offset was negative here
|
281
|
-
# debug("Initial offset #{@offset} in '#{table_name}' with #{cnt} rows")
|
282
|
-
# @offset = 0 if @offset < 0
|
283
|
-
# end
|
284
|
-
# #@stmt = db.prepare("SELECT * FROM #{table_name} LIMIT ? OFFSET ?;")
|
285
|
-
# @stmt = db.prepare("SELECT _senders.name, #{t}.* FROM #{t} JOIN _senders WHERE #{t}.oml_sender_id = _senders.id LIMIT ? OFFSET ?;")
|
286
|
-
# end
|
287
|
-
# @stmt
|
288
|
-
# end
|
289
223
|
end # OmlSqlRow
|
290
224
|
|
291
225
|
|
data/lib/omf_oml/sql_source.rb
CHANGED
@@ -8,13 +8,21 @@ require 'omf_oml/sql_row'
|
|
8
8
|
|
9
9
|
module OMF::OML
|
10
10
|
|
11
|
-
# This class fetches the content of an
|
11
|
+
# This class fetches the content of an SQL database and serves it as multiple
|
12
12
|
# OML streams.
|
13
13
|
#
|
14
14
|
# After creating the object, the @run@ method needs to be called to
|
15
15
|
# start producing the streams.
|
16
16
|
#
|
17
17
|
class OmlSqlSource < OMF::Common::LObject
|
18
|
+
|
19
|
+
# Sequel adaptors sometimes don't return a :type identifier,
|
20
|
+
# but always return the :db_type. This is a list of maps which may not work
|
21
|
+
# for all adaptors
|
22
|
+
#
|
23
|
+
FALLBACK_MAPPING = {
|
24
|
+
'UNSIGNED INTEGER' => :integer
|
25
|
+
}
|
18
26
|
|
19
27
|
# db_opts - Options used to create a Sequel adapter
|
20
28
|
#
|
@@ -41,13 +49,58 @@ module OMF::OML
|
|
41
49
|
@on_new_stream_procs.delete key
|
42
50
|
end
|
43
51
|
end
|
52
|
+
|
53
|
+
# Return a table (more precisely an OmlTable instance) fed from
|
54
|
+
# the content of a table 'table_name' in this database.
|
55
|
+
#
|
56
|
+
# table_name - Name of table in the SQL database
|
57
|
+
# opts -
|
58
|
+
# :include_oml_internals - Include OML 'header' columns [true]
|
59
|
+
# :name - name used for returned OML Table [table_name]
|
60
|
+
# All other options defined for OmlSqlRow#new
|
61
|
+
#
|
62
|
+
def create_table(table_name, opts = {})
|
63
|
+
tn = opts.delete(:name) || table_name
|
64
|
+
schema = _schema_for_table(table_name)
|
65
|
+
r = OmlSqlRow.new(table_name, schema, @db, opts)
|
66
|
+
r.to_table(tn, opts)
|
67
|
+
end
|
68
|
+
|
69
|
+
# Call 'block' for every row in 'table_name' table.
|
70
|
+
#
|
71
|
+
# table_name - Name of table in the SQL database
|
72
|
+
# opts -
|
73
|
+
# :include_oml_internals[boolean] - Include OML 'header' columns [true]
|
74
|
+
# :schema[Schema] Schema to use for creating row
|
75
|
+
# All other options defined for OmlSqlRow#new
|
76
|
+
#
|
77
|
+
def create_stream(table_name, opts = {}, &block)
|
78
|
+
rschema = opts.delete(:schema)
|
79
|
+
schema = _schema_for_table(table_name)
|
80
|
+
r = OmlSqlRow.new(table_name, schema, @db, opts)
|
81
|
+
ropts = {}
|
82
|
+
ropts[:schema] = rschema if rschema
|
83
|
+
r.to_stream(ropts, &block)
|
84
|
+
end
|
85
|
+
|
86
|
+
#
|
87
|
+
# Run a query on the database and return the result as an OmlTable. The provided schema
|
88
|
+
# needs to describe the SQL queries result set. Unfortunately we can only do very little
|
89
|
+
# sanity checks here
|
90
|
+
#
|
91
|
+
def query(sql, table_name, schema)
|
92
|
+
tbl = OmlTable.create(table_name, schema)
|
93
|
+
@db.fetch(sql).each do |row|
|
94
|
+
tbl << schema.hash_to_row(row)
|
95
|
+
end
|
96
|
+
tbl
|
97
|
+
end
|
44
98
|
|
45
99
|
# Start checking the database for tables and create a new stream
|
46
100
|
# by calling the internal +report_new_table+ method.
|
47
101
|
# If +check_every+ > 0 continue checking every +check_every+ seconds
|
48
102
|
# for new tables in the database, otherwise it's only checked once
|
49
103
|
#
|
50
|
-
#
|
51
104
|
def run(check_every = -1)
|
52
105
|
if check_every <= 0
|
53
106
|
run_once()
|
@@ -67,6 +120,8 @@ module OMF::OML
|
|
67
120
|
end
|
68
121
|
end
|
69
122
|
|
123
|
+
protected
|
124
|
+
|
70
125
|
def run_once()
|
71
126
|
debug "Finding tables #{@db.tables}"
|
72
127
|
# first find tables
|
@@ -83,7 +138,6 @@ module OMF::OML
|
|
83
138
|
end
|
84
139
|
|
85
140
|
|
86
|
-
protected
|
87
141
|
|
88
142
|
# THis method is being called for every table detected in the database.
|
89
143
|
# It creates a new +OmlSqlRow+ object with +opts+ as the only argument.
|
@@ -95,7 +149,9 @@ module OMF::OML
|
|
95
149
|
def report_new_table(table_name)
|
96
150
|
unless table = @tables[table_name] # check if already reported before
|
97
151
|
debug "Found table: #{table_name}"
|
98
|
-
|
152
|
+
schema = _schema_for_table(table_name)
|
153
|
+
table = @tables[table_name] = OmlSqlRow.new(table_name, schema, @db, @row_opts)
|
154
|
+
#table = @tables[table_name] = OmlSqlRow.new(table_name, @db.schema(table_name), @db_opts, self, @row_opts)
|
99
155
|
@on_new_stream_procs.each_value do |proc|
|
100
156
|
proc.call(table)
|
101
157
|
end
|
@@ -103,6 +159,20 @@ module OMF::OML
|
|
103
159
|
table
|
104
160
|
end
|
105
161
|
|
162
|
+
def _schema_for_table(table_name)
|
163
|
+
begin
|
164
|
+
schema_descr = @db.schema(table_name).map do |col_name, cd|
|
165
|
+
unless type = cd[:type] || FALLBACK_MAPPING[cd[:db_type]]
|
166
|
+
warn "Can't find ruby type for database type '#{cd[:db_type]}'"
|
167
|
+
end
|
168
|
+
{name: col_name, type: type}
|
169
|
+
end
|
170
|
+
#puts "SCHEMA_DESCR>>>> #{schema_descr}"
|
171
|
+
schema = OmlSchema.new(schema_descr)
|
172
|
+
rescue Sequel::Error => ex
|
173
|
+
raise "Problems reading schema of table '#{table_name}'. Does it exist? (#{@db.tables})"
|
174
|
+
end
|
175
|
+
end
|
106
176
|
end
|
107
177
|
|
108
178
|
|
data/lib/omf_oml/table.rb
CHANGED
@@ -43,7 +43,9 @@ module OMF::OML
|
|
43
43
|
#@endpoint = endpoint
|
44
44
|
@name = tname
|
45
45
|
@schema = OmlSchema.create(schema)
|
46
|
+
@add_index = false
|
46
47
|
unless @schema.name_at(0) == :__id__
|
48
|
+
@add_index = true
|
47
49
|
@schema.insert_column_at(0, [:__id__, 'int'])
|
48
50
|
end
|
49
51
|
@opts = opts
|
@@ -100,6 +102,14 @@ module OMF::OML
|
|
100
102
|
end
|
101
103
|
end
|
102
104
|
|
105
|
+
def on_row_added(key, &block)
|
106
|
+
on_content_changed(key) do |action, rows|
|
107
|
+
if action == :added
|
108
|
+
rows.each {|r| block.call(r)}
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
103
113
|
# NOTE: +on_row_added+ callbacks are done within the monitor.
|
104
114
|
#
|
105
115
|
def add_row(row, needs_casting = false)
|
@@ -110,6 +120,10 @@ module OMF::OML
|
|
110
120
|
end
|
111
121
|
end
|
112
122
|
|
123
|
+
def <<(row)
|
124
|
+
add_row(row)
|
125
|
+
end
|
126
|
+
|
113
127
|
# Return a new table which shadows this table but only contains
|
114
128
|
# rows with unique values in the column 'col_name' and of these the
|
115
129
|
# latest added rows to this table.
|
@@ -164,6 +178,12 @@ module OMF::OML
|
|
164
178
|
st
|
165
179
|
end
|
166
180
|
|
181
|
+
# Return table as an array of rows
|
182
|
+
#
|
183
|
+
def to_a
|
184
|
+
@rows.dup
|
185
|
+
end
|
186
|
+
|
167
187
|
def describe()
|
168
188
|
rows
|
169
189
|
end
|
@@ -177,6 +197,7 @@ module OMF::OML
|
|
177
197
|
# NOT synchronized
|
178
198
|
#
|
179
199
|
def _add_row(row, needs_casting = false)
|
200
|
+
throw "Expected array, but got '#{row}'" unless row.is_a?(Array)
|
180
201
|
if needs_casting
|
181
202
|
row = @schema.cast_row(row, true)
|
182
203
|
end
|
@@ -186,12 +207,12 @@ module OMF::OML
|
|
186
207
|
end
|
187
208
|
return nil unless row
|
188
209
|
|
189
|
-
row.insert(0, @row_id += 1)
|
210
|
+
row.insert(0, @row_id += 1) if @add_index
|
190
211
|
_add_row_finally(row)
|
191
212
|
end
|
192
213
|
|
193
|
-
# Finally add 'row' to internal storage. This would be
|
194
|
-
#
|
214
|
+
# Finally add 'row' to internal storage. This would be the method to
|
215
|
+
# override in sub classes as this is thread safe and all other pre-storage
|
195
216
|
# test have been performed. Should return the row added, or nil if nothing
|
196
217
|
# was ultimately added.
|
197
218
|
#
|
data/lib/omf_oml/version.rb
CHANGED
data/omf_oml.gemspec
CHANGED
@@ -22,5 +22,6 @@ Gem::Specification.new do |s|
|
|
22
22
|
|
23
23
|
# specify any dependencies here; for example:
|
24
24
|
# s.add_development_dependency "minitest", "~> 2.11.3"
|
25
|
-
s.add_runtime_dependency "sqlite3", "~> 1.3.6"
|
25
|
+
#s.add_runtime_dependency "sqlite3", "~> 1.3.6"
|
26
|
+
s.add_runtime_dependency 'sequel', '~> 3.45'
|
26
27
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: omf_oml
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.9.
|
4
|
+
version: 0.9.6
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,16 +9,16 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2013-03-30 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
|
-
name:
|
15
|
+
name: sequel
|
16
16
|
requirement: !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
19
|
- - ~>
|
20
20
|
- !ruby/object:Gem::Version
|
21
|
-
version:
|
21
|
+
version: '3.45'
|
22
22
|
type: :runtime
|
23
23
|
prerelease: false
|
24
24
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -26,7 +26,7 @@ dependencies:
|
|
26
26
|
requirements:
|
27
27
|
- - ~>
|
28
28
|
- !ruby/object:Gem::Version
|
29
|
-
version:
|
29
|
+
version: '3.45'
|
30
30
|
description: ! "Glue functionality between OMF and OMF related libraries, such as
|
31
31
|
OMF Web and Labwiki, and\n OML."
|
32
32
|
email:
|