omf_oml 0.9.5 → 0.9.6
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +1 -0
- data/lib/omf_oml/schema.rb +86 -57
- data/lib/omf_oml/sql_row.rb +66 -132
- data/lib/omf_oml/sql_source.rb +74 -4
- data/lib/omf_oml/table.rb +24 -3
- data/lib/omf_oml/version.rb +1 -1
- data/omf_oml.gemspec +2 -1
- metadata +5 -5
data/.gitignore
CHANGED
data/lib/omf_oml/schema.rb
CHANGED
@@ -34,11 +34,20 @@ module OMF::OML
|
|
34
34
|
'double precision' => :float,
|
35
35
|
'text' => :string,
|
36
36
|
'string' => :string,
|
37
|
+
'varargs' => :string,
|
37
38
|
'date' => :date,
|
38
39
|
'dateTime'.downcase => :dateTime, # should be 'datetime' but we downcase the string for comparison
|
39
40
|
'timestamp' => :dateTime, # Postgreql specific, not sure if this works as it.
|
40
41
|
'key' => :key,
|
41
42
|
}
|
43
|
+
|
44
|
+
OML_INTERNALS = [
|
45
|
+
:oml_sender,
|
46
|
+
:oml_sender_id,
|
47
|
+
:oml_seq,
|
48
|
+
:oml_ts_client,
|
49
|
+
:oml_ts_server
|
50
|
+
]
|
42
51
|
|
43
52
|
def self.create(schema_description)
|
44
53
|
if schema_description.kind_of? self
|
@@ -83,62 +92,13 @@ module OMF::OML
|
|
83
92
|
end
|
84
93
|
|
85
94
|
def insert_column_at(index, col)
|
86
|
-
|
87
|
-
col = [col]
|
88
|
-
end
|
89
|
-
if col.kind_of? Array
|
90
|
-
# should be [name, type]
|
91
|
-
if col.length == 1
|
92
|
-
col = {:name => col[0].to_sym,
|
93
|
-
:type => :string,
|
94
|
-
:title => col[0].to_s.split('_').collect {|s| s.capitalize}.join(' ')}
|
95
|
-
elsif col.length == 2
|
96
|
-
col = {:name => col[0].to_sym,
|
97
|
-
:type => col[1].to_sym,
|
98
|
-
:title => col[0].to_s.split('_').collect {|s| s.capitalize}.join(' ')}
|
99
|
-
elsif col.length == 3
|
100
|
-
col = {:name => col[0].to_sym, :type => col[1].to_sym, :title => col[2]}
|
101
|
-
else
|
102
|
-
throw "Simple column schema should consist of [name, type, title] array, but found '#{col.inspect}'"
|
103
|
-
end
|
104
|
-
elsif col.kind_of? Hash
|
105
|
-
# ensure there is a :title property
|
106
|
-
unless col[:title]
|
107
|
-
col[:title] = col[:name].to_s.split('_').collect {|s| s.capitalize}.join(' ')
|
108
|
-
end
|
109
|
-
end
|
110
|
-
|
111
|
-
# should normalize type
|
112
|
-
if type = col[:type]
|
113
|
-
unless type = ANY2TYPE[type.to_s.downcase]
|
114
|
-
warn "Unknown type definition '#{col[:type]}', default to 'string'"
|
115
|
-
type = :string
|
116
|
-
end
|
117
|
-
else
|
118
|
-
warn "Missing type definition in '#{col[:name]}', default to 'string' (#{col.inspect})"
|
119
|
-
type = :string
|
120
|
-
end
|
121
|
-
col[:type] = type
|
122
|
-
|
123
|
-
col[:type_conversion] = case type
|
124
|
-
when :string
|
125
|
-
lambda do |r| r end
|
126
|
-
when :key
|
127
|
-
lambda do |r| r end
|
128
|
-
when :integer
|
129
|
-
lambda do |r| r.to_i end
|
130
|
-
when :float
|
131
|
-
lambda do |r| r.to_f end
|
132
|
-
when :date
|
133
|
-
lambda do |r| Date.parse(r) end
|
134
|
-
when :dateTime
|
135
|
-
lambda do |r| Time.parse(r) end
|
136
|
-
else raise "Unrecognized Schema type '#{type}'"
|
137
|
-
end
|
138
|
-
|
139
|
-
@schema.insert(index, col)
|
95
|
+
@schema.insert(index, _create_col_descr(col))
|
140
96
|
end
|
141
97
|
|
98
|
+
def replace_column_at(index, col)
|
99
|
+
@schema[index] = _create_col_descr(col)
|
100
|
+
end
|
101
|
+
|
142
102
|
def each_column(&block)
|
143
103
|
@schema.each do |c|
|
144
104
|
block.call(c)
|
@@ -151,7 +111,7 @@ module OMF::OML
|
|
151
111
|
# hrow - Hash describing a row
|
152
112
|
# set_nil_when_missing - If true, set any columns not described in hrow to nil
|
153
113
|
#
|
154
|
-
def hash_to_row(hrow, set_nil_when_missing = false, call_type_conversion =
|
114
|
+
def hash_to_row(hrow, set_nil_when_missing = false, call_type_conversion = false)
|
155
115
|
#puts "HASH2A => #{hrow.inspect}"
|
156
116
|
remove_first_col = false
|
157
117
|
r = @schema.collect do |cdescr|
|
@@ -194,12 +154,22 @@ module OMF::OML
|
|
194
154
|
describe.to_json(*opt)
|
195
155
|
end
|
196
156
|
|
197
|
-
def clone()
|
157
|
+
def clone(exclude_oml_internals = false)
|
198
158
|
c = self.dup
|
199
|
-
|
159
|
+
schema = @schema.clone
|
160
|
+
if exclude_oml_internals
|
161
|
+
schema = schema.select do |cd|
|
162
|
+
not OML_INTERNALS.include?(cd[:name])
|
163
|
+
end
|
164
|
+
end
|
165
|
+
c.instance_variable_set('@schema', schema)
|
200
166
|
c
|
201
167
|
end
|
202
168
|
|
169
|
+
def to_s
|
170
|
+
"OmlSchema: #{@schema.map {|c| "#{c[:name]}:#{c[:type]}"}.join(', ')}"
|
171
|
+
end
|
172
|
+
|
203
173
|
protected
|
204
174
|
|
205
175
|
# schema_description - Array containing [name, type*] for every column in table
|
@@ -214,6 +184,65 @@ module OMF::OML
|
|
214
184
|
debug "schema: '#{describe.inspect}'"
|
215
185
|
|
216
186
|
end
|
187
|
+
|
188
|
+
# Create a column descriptor from whatever is given by user
|
189
|
+
#
|
190
|
+
def _create_col_descr(col)
|
191
|
+
if col.kind_of?(Symbol) || col.kind_of?(String)
|
192
|
+
col = [col]
|
193
|
+
end
|
194
|
+
if col.kind_of? Array
|
195
|
+
# should be [name, type]
|
196
|
+
if col.length == 1
|
197
|
+
col = {:name => col[0].to_sym,
|
198
|
+
:type => :string,
|
199
|
+
:title => col[0].to_s.split('_').collect {|s| s.capitalize}.join(' ')}
|
200
|
+
elsif col.length == 2
|
201
|
+
col = {:name => col[0].to_sym,
|
202
|
+
:type => col[1].to_sym,
|
203
|
+
:title => col[0].to_s.split('_').collect {|s| s.capitalize}.join(' ')}
|
204
|
+
elsif col.length == 3
|
205
|
+
col = {:name => col[0].to_sym, :type => col[1].to_sym, :title => col[2]}
|
206
|
+
else
|
207
|
+
throw "Simple column schema should consist of [name, type, title] array, but found '#{col.inspect}'"
|
208
|
+
end
|
209
|
+
elsif col.kind_of? Hash
|
210
|
+
# ensure there is a :title property
|
211
|
+
unless col[:title]
|
212
|
+
col[:title] = col[:name].to_s.split('_').collect {|s| s.capitalize}.join(' ')
|
213
|
+
end
|
214
|
+
end
|
215
|
+
|
216
|
+
# should normalize type
|
217
|
+
if type = col[:type]
|
218
|
+
tn = type.to_s.split('(')[0] # take care of types like varargs(..)
|
219
|
+
unless type = ANY2TYPE[type.to_s.downcase]
|
220
|
+
warn "Unknown type definition '#{tn}', default to 'string'"
|
221
|
+
type = :string
|
222
|
+
end
|
223
|
+
else
|
224
|
+
warn "Missing type definition in '#{col[:name]}', default to 'string' (#{col.inspect})"
|
225
|
+
type = :string
|
226
|
+
end
|
227
|
+
col[:type] = type
|
228
|
+
|
229
|
+
col[:type_conversion] = case type
|
230
|
+
when :string
|
231
|
+
lambda do |r| r end
|
232
|
+
when :key
|
233
|
+
lambda do |r| r end
|
234
|
+
when :integer
|
235
|
+
lambda do |r| r.to_i end
|
236
|
+
when :float
|
237
|
+
lambda do |r| r.to_f end
|
238
|
+
when :date
|
239
|
+
lambda do |r| Date.parse(r) end
|
240
|
+
when :dateTime
|
241
|
+
lambda do |r| Time.parse(r) end
|
242
|
+
else raise "Unrecognized Schema type '#{type}'"
|
243
|
+
end
|
244
|
+
col
|
245
|
+
end
|
217
246
|
end # OmlSchema
|
218
247
|
|
219
248
|
end
|
data/lib/omf_oml/sql_row.rb
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
|
2
2
|
require 'omf_oml/tuple'
|
3
|
+
require 'omf_oml/table'
|
3
4
|
|
4
5
|
module OMF::OML
|
5
6
|
|
@@ -15,19 +16,20 @@ module OMF::OML
|
|
15
16
|
#
|
16
17
|
# Create a representation of a row in a database. Can be used to fill a table.
|
17
18
|
#
|
18
|
-
#
|
19
|
-
#
|
20
|
-
#
|
21
|
-
#
|
22
|
-
# opts:
|
19
|
+
# @param [String] sql_table_name - name of SQL table in respective SQL database
|
20
|
+
# @param [OmlSchema] schema - the schema describing the tuple
|
21
|
+
# @param [Sequel] db - Database
|
22
|
+
# @param [Hash] opts:
|
23
23
|
# - offset: Ignore first +offset+ rows. If negative or zero serve +offset+ rows initially
|
24
24
|
# - limit: Number of rows to fetch each time [1000]
|
25
25
|
# - check_interval: Interval in seconds when to check for new data. If 0, only run once.
|
26
|
+
# - query_interval: Interval between consecutive queries when processing large result set.
|
26
27
|
#
|
27
|
-
def initialize(
|
28
|
-
@sname =
|
29
|
-
@
|
30
|
-
|
28
|
+
def initialize(sql_table_name, schema, db, opts = {})
|
29
|
+
@sname = sql_table_name
|
30
|
+
@schema = schema
|
31
|
+
raise "Expected OmlSchema but got '#{schema.class}" unless schema.is_a? OmlSchema
|
32
|
+
@db = db
|
31
33
|
|
32
34
|
unless @offset = opts[:offset]
|
33
35
|
@offset = 0
|
@@ -37,12 +39,11 @@ module OMF::OML
|
|
37
39
|
|
38
40
|
@check_interval = opts[:check_interval]
|
39
41
|
@check_interval = 0 unless @check_interval
|
42
|
+
@query_interval = opts[:query_interval]
|
40
43
|
|
41
|
-
|
42
44
|
@on_new_vector_proc = {}
|
43
45
|
|
44
|
-
schema
|
45
|
-
super table_name, schema
|
46
|
+
super opts[:name] || sql_table_name, schema
|
46
47
|
end
|
47
48
|
|
48
49
|
|
@@ -60,10 +61,11 @@ module OMF::OML
|
|
60
61
|
end
|
61
62
|
end
|
62
63
|
|
63
|
-
# Return the elements of the
|
64
|
-
|
65
|
-
|
66
|
-
|
64
|
+
# Return the elements of the row as an array using the
|
65
|
+
# associated schema or 'schema' if non-nil.
|
66
|
+
#
|
67
|
+
def to_a(schema = nil)
|
68
|
+
a = (schema || @schema).hash_to_row(@row) # don't need type conversion as sequel is doing this for us
|
67
69
|
end
|
68
70
|
|
69
71
|
# Return an array including the values for the names elements
|
@@ -72,9 +74,10 @@ module OMF::OML
|
|
72
74
|
def select(*col_names)
|
73
75
|
r = @row
|
74
76
|
col_names.collect do |n|
|
75
|
-
|
76
|
-
|
77
|
-
|
77
|
+
unless @row.key? n
|
78
|
+
raise "Unknown column name '#{n}'"
|
79
|
+
end
|
80
|
+
@row[n]
|
78
81
|
end
|
79
82
|
end
|
80
83
|
|
@@ -98,79 +101,54 @@ module OMF::OML
|
|
98
101
|
run() unless @on_new_vector_proc.empty?
|
99
102
|
end
|
100
103
|
|
101
|
-
#
|
102
|
-
#
|
103
|
-
# The argument to this method are either a list of columns to
|
104
|
-
# to capture in the table, or an array of column names and
|
105
|
-
# an option hash or just
|
106
|
-
# the option hash to be provided to the +OmlTable+ constructor.
|
104
|
+
# Return a table which will capture the content of this tuple stream.
|
107
105
|
#
|
108
|
-
#
|
109
|
-
#
|
110
|
-
#
|
111
|
-
#
|
112
|
-
# listed in the selector. Otherwise, the block is called directly
|
113
|
-
# with the tuple.
|
106
|
+
# @param [string] name - Name to use for returned table
|
107
|
+
# @param [Hash] opts Options to be passed on to Table constructor
|
108
|
+
# @opts [boolean] opts :include_oml_internals If true will also include OML header columns
|
109
|
+
# @opts [OmlSchema] opts :schema use specific schema for table (Needs to be a subset of the tuples schema)
|
114
110
|
#
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
if args[0].kind_of?(Array)
|
123
|
-
select = args[0]
|
124
|
-
elsif args[0].kind_of?(Hash)
|
125
|
-
opts = args[0]
|
111
|
+
def to_stream(opts = {}, &block)
|
112
|
+
unless schema = opts.delete(:schema)
|
113
|
+
include_oml_internals = (opts[:include_oml_internals] != false)
|
114
|
+
schema = self.schema.clone(!include_oml_internals)
|
115
|
+
if include_oml_internals
|
116
|
+
# replace sender_id by sender ... see _run_once
|
117
|
+
schema.replace_column_at 0, :oml_sender
|
126
118
|
end
|
127
|
-
elsif args.length == 2 && args[1].kind_of?(Hash)
|
128
|
-
select = args[0]
|
129
|
-
opts = args[1]
|
130
|
-
else
|
131
|
-
opts = {}
|
132
|
-
select = args
|
133
119
|
end
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
# end
|
139
|
-
else
|
140
|
-
tschema = select.collect do |cname| {:name => cname} end
|
141
|
-
end
|
142
|
-
tname = opts.delete(:name) || stream_name
|
143
|
-
t = OMF::OML::OmlTable.new(tname, tschema, opts)
|
144
|
-
if block
|
145
|
-
self.on_new_tuple() do |v|
|
146
|
-
#puts "New vector(#{tname}): #{v.schema.inspect} ---- #{v.select(*select).size} <#{v.select(*select).join('|')}>"
|
147
|
-
if select
|
148
|
-
row = block.call(v.select(*select))
|
149
|
-
else
|
150
|
-
row = block.call(v)
|
151
|
-
end
|
152
|
-
if row
|
153
|
-
raise "Expected kind of Array, but got '#{row.inspect}'" unless row.kind_of?(Array)
|
154
|
-
t.add_row(row)
|
155
|
-
end
|
156
|
-
end
|
157
|
-
else
|
158
|
-
self.on_new_tuple() do |v|
|
159
|
-
#puts "New vector(#{tname}): #{v.select(*select).join('|')}"
|
160
|
-
t.add_row(v.select(*select))
|
161
|
-
end
|
120
|
+
self.on_new_tuple(rand()) do |t|
|
121
|
+
#v = t.to_a(schema)
|
122
|
+
v = t.row
|
123
|
+
block.arity == 1 ? block.call(v) : block.call(v, schema)
|
162
124
|
end
|
163
|
-
|
125
|
+
schema
|
164
126
|
end
|
165
|
-
|
127
|
+
|
128
|
+
# Return a table which will capture the content of this tuple stream.
|
129
|
+
#
|
130
|
+
# @param [string] name - Name to use for returned table
|
131
|
+
# @param [Hash] opts Options to be passed on to Table constructor
|
132
|
+
# @opts [boolean] opts :include_oml_internals If true will also include OML header columns
|
133
|
+
# @opts [OmlSchema] opts :schema use specific schema for table (Needs to be a subset of the tuples schema)
|
134
|
+
#
|
166
135
|
def to_table(name = nil, opts = {})
|
167
136
|
unless name
|
168
137
|
name = @sname
|
169
138
|
end
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
139
|
+
unless schema = opts.delete(:schema)
|
140
|
+
include_oml_internals = (opts[:include_oml_internals] != false)
|
141
|
+
schema = self.schema.clone(!include_oml_internals)
|
142
|
+
if include_oml_internals
|
143
|
+
# replace sender_id by sender ... see _run_once
|
144
|
+
schema.replace_column_at 0, :oml_sender
|
145
|
+
end
|
146
|
+
end
|
147
|
+
t = OMF::OML::OmlTable.new(name, schema, opts)
|
148
|
+
#puts ">>>>SCHEMA>>> #{schema.inspect}"
|
149
|
+
self.on_new_tuple(t) do |v|
|
150
|
+
r = v.to_a(schema)
|
151
|
+
#puts r.inspect
|
174
152
|
t.add_row(r)
|
175
153
|
end
|
176
154
|
t
|
@@ -178,32 +156,6 @@ module OMF::OML
|
|
178
156
|
|
179
157
|
|
180
158
|
protected
|
181
|
-
|
182
|
-
def parse_schema(raw)
|
183
|
-
#puts ">> PARSE SCHEMA"
|
184
|
-
sd = raw.collect do |col|
|
185
|
-
name, opts = col
|
186
|
-
#puts col.inspect
|
187
|
-
{:name => name, :type => opts[:db_type]}
|
188
|
-
end
|
189
|
-
# Query we are using is adding the 'oml_sender_name' to the front of the table
|
190
|
-
sd.insert(0, :name => :oml_sender, :type => :string)
|
191
|
-
OmlSchema.new(sd)
|
192
|
-
end
|
193
|
-
|
194
|
-
# override
|
195
|
-
def process_schema(schema)
|
196
|
-
i = 0
|
197
|
-
@vprocs = {}
|
198
|
-
schema.each_column do |col|
|
199
|
-
name = col[:name]
|
200
|
-
j = i + 0
|
201
|
-
puts "SCHEMA: '#{name.inspect}'-#{j}"
|
202
|
-
l = @vprocs[name] = lambda do |r| r[j] end
|
203
|
-
@vprocs[i - 4] = l if i > 4
|
204
|
-
i += 1
|
205
|
-
end
|
206
|
-
end
|
207
159
|
|
208
160
|
def run(in_thread = true)
|
209
161
|
return if @running
|
@@ -224,10 +176,8 @@ module OMF::OML
|
|
224
176
|
private
|
225
177
|
|
226
178
|
def _run
|
227
|
-
@db = Sequel.connect(@db_opts)
|
228
|
-
|
229
179
|
if @check_interval <= 0
|
230
|
-
_run_once
|
180
|
+
while _run_once; end
|
231
181
|
else
|
232
182
|
@running = true
|
233
183
|
while (@running)
|
@@ -256,8 +206,6 @@ module OMF::OML
|
|
256
206
|
@offset = 0 if @offset < 0
|
257
207
|
end
|
258
208
|
@db["SELECT _senders.name as oml_sender, #{t}.* FROM #{t} INNER JOIN _senders ON (_senders.id = #{t}.oml_sender_id) LIMIT #{@limit} OFFSET #{@offset};"].each do |r|
|
259
|
-
#@db["SELECT _senders.name as oml_sender, #{t}.* FROM #{t} JOIN _senders WHERE #{t}.oml_sender_id = _senders.id LIMIT #{@limit} OFFSET #{@offset};"].each do |r|
|
260
|
-
#puts "ROW>>> #{r.inspect}"
|
261
209
|
@row = r
|
262
210
|
@on_new_vector_proc.each_value do |proc|
|
263
211
|
proc.call(self)
|
@@ -265,27 +213,13 @@ module OMF::OML
|
|
265
213
|
row_cnt += 1
|
266
214
|
end
|
267
215
|
@offset += row_cnt
|
268
|
-
debug "Read #{row_cnt}
|
269
|
-
row_cnt >= @limit # there could be more to read
|
216
|
+
debug "Read #{row_cnt} (total #{@offset}) rows from '#{@sname}'" if row_cnt > 0
|
217
|
+
if more_to_read = row_cnt >= @limit # there could be more to read
|
218
|
+
sleep @query_interval if @query_interval # don't hammer database
|
219
|
+
end
|
220
|
+
more_to_read
|
270
221
|
end
|
271
222
|
|
272
|
-
# def _statement
|
273
|
-
# unless @stmt
|
274
|
-
# db = @db = SQLite3::Database.new(@db_file)
|
275
|
-
# @db.type_translation = true
|
276
|
-
# table_name = t = @sname
|
277
|
-
# if @offset < 0
|
278
|
-
# cnt = db.execute("select count(*) from #{table_name};")[0][0].to_i
|
279
|
-
# #debug "CNT: #{cnt}.#{cnt.class} offset: #{@offset}"
|
280
|
-
# @offset = cnt + @offset # @offset was negative here
|
281
|
-
# debug("Initial offset #{@offset} in '#{table_name}' with #{cnt} rows")
|
282
|
-
# @offset = 0 if @offset < 0
|
283
|
-
# end
|
284
|
-
# #@stmt = db.prepare("SELECT * FROM #{table_name} LIMIT ? OFFSET ?;")
|
285
|
-
# @stmt = db.prepare("SELECT _senders.name, #{t}.* FROM #{t} JOIN _senders WHERE #{t}.oml_sender_id = _senders.id LIMIT ? OFFSET ?;")
|
286
|
-
# end
|
287
|
-
# @stmt
|
288
|
-
# end
|
289
223
|
end # OmlSqlRow
|
290
224
|
|
291
225
|
|
data/lib/omf_oml/sql_source.rb
CHANGED
@@ -8,13 +8,21 @@ require 'omf_oml/sql_row'
|
|
8
8
|
|
9
9
|
module OMF::OML
|
10
10
|
|
11
|
-
# This class fetches the content of an
|
11
|
+
# This class fetches the content of an SQL database and serves it as multiple
|
12
12
|
# OML streams.
|
13
13
|
#
|
14
14
|
# After creating the object, the @run@ method needs to be called to
|
15
15
|
# start producing the streams.
|
16
16
|
#
|
17
17
|
class OmlSqlSource < OMF::Common::LObject
|
18
|
+
|
19
|
+
# Sequel adaptors sometimes don't return a :type identifier,
|
20
|
+
# but always return the :db_type. This is a list of maps which may not work
|
21
|
+
# for all adaptors
|
22
|
+
#
|
23
|
+
FALLBACK_MAPPING = {
|
24
|
+
'UNSIGNED INTEGER' => :integer
|
25
|
+
}
|
18
26
|
|
19
27
|
# db_opts - Options used to create a Sequel adapter
|
20
28
|
#
|
@@ -41,13 +49,58 @@ module OMF::OML
|
|
41
49
|
@on_new_stream_procs.delete key
|
42
50
|
end
|
43
51
|
end
|
52
|
+
|
53
|
+
# Return a table (more precisely an OmlTable instance) fed from
|
54
|
+
# the content of a table 'table_name' in this database.
|
55
|
+
#
|
56
|
+
# table_name - Name of table in the SQL database
|
57
|
+
# opts -
|
58
|
+
# :include_oml_internals - Include OML 'header' columns [true]
|
59
|
+
# :name - name used for returned OML Table [table_name]
|
60
|
+
# All other options defined for OmlSqlRow#new
|
61
|
+
#
|
62
|
+
def create_table(table_name, opts = {})
|
63
|
+
tn = opts.delete(:name) || table_name
|
64
|
+
schema = _schema_for_table(table_name)
|
65
|
+
r = OmlSqlRow.new(table_name, schema, @db, opts)
|
66
|
+
r.to_table(tn, opts)
|
67
|
+
end
|
68
|
+
|
69
|
+
# Call 'block' for every row in 'table_name' table.
|
70
|
+
#
|
71
|
+
# table_name - Name of table in the SQL database
|
72
|
+
# opts -
|
73
|
+
# :include_oml_internals[boolean] - Include OML 'header' columns [true]
|
74
|
+
# :schema[Schema] Schema to use for creating row
|
75
|
+
# All other options defined for OmlSqlRow#new
|
76
|
+
#
|
77
|
+
def create_stream(table_name, opts = {}, &block)
|
78
|
+
rschema = opts.delete(:schema)
|
79
|
+
schema = _schema_for_table(table_name)
|
80
|
+
r = OmlSqlRow.new(table_name, schema, @db, opts)
|
81
|
+
ropts = {}
|
82
|
+
ropts[:schema] = rschema if rschema
|
83
|
+
r.to_stream(ropts, &block)
|
84
|
+
end
|
85
|
+
|
86
|
+
#
|
87
|
+
# Run a query on the database and return the result as an OmlTable. The provided schema
|
88
|
+
# needs to describe the SQL queries result set. Unfortunately we can only do very little
|
89
|
+
# sanity checks here
|
90
|
+
#
|
91
|
+
def query(sql, table_name, schema)
|
92
|
+
tbl = OmlTable.create(table_name, schema)
|
93
|
+
@db.fetch(sql).each do |row|
|
94
|
+
tbl << schema.hash_to_row(row)
|
95
|
+
end
|
96
|
+
tbl
|
97
|
+
end
|
44
98
|
|
45
99
|
# Start checking the database for tables and create a new stream
|
46
100
|
# by calling the internal +report_new_table+ method.
|
47
101
|
# If +check_every+ > 0 continue checking every +check_every+ seconds
|
48
102
|
# for new tables in the database, otherwise it's only checked once
|
49
103
|
#
|
50
|
-
#
|
51
104
|
def run(check_every = -1)
|
52
105
|
if check_every <= 0
|
53
106
|
run_once()
|
@@ -67,6 +120,8 @@ module OMF::OML
|
|
67
120
|
end
|
68
121
|
end
|
69
122
|
|
123
|
+
protected
|
124
|
+
|
70
125
|
def run_once()
|
71
126
|
debug "Finding tables #{@db.tables}"
|
72
127
|
# first find tables
|
@@ -83,7 +138,6 @@ module OMF::OML
|
|
83
138
|
end
|
84
139
|
|
85
140
|
|
86
|
-
protected
|
87
141
|
|
88
142
|
# THis method is being called for every table detected in the database.
|
89
143
|
# It creates a new +OmlSqlRow+ object with +opts+ as the only argument.
|
@@ -95,7 +149,9 @@ module OMF::OML
|
|
95
149
|
def report_new_table(table_name)
|
96
150
|
unless table = @tables[table_name] # check if already reported before
|
97
151
|
debug "Found table: #{table_name}"
|
98
|
-
|
152
|
+
schema = _schema_for_table(table_name)
|
153
|
+
table = @tables[table_name] = OmlSqlRow.new(table_name, schema, @db, @row_opts)
|
154
|
+
#table = @tables[table_name] = OmlSqlRow.new(table_name, @db.schema(table_name), @db_opts, self, @row_opts)
|
99
155
|
@on_new_stream_procs.each_value do |proc|
|
100
156
|
proc.call(table)
|
101
157
|
end
|
@@ -103,6 +159,20 @@ module OMF::OML
|
|
103
159
|
table
|
104
160
|
end
|
105
161
|
|
162
|
+
def _schema_for_table(table_name)
|
163
|
+
begin
|
164
|
+
schema_descr = @db.schema(table_name).map do |col_name, cd|
|
165
|
+
unless type = cd[:type] || FALLBACK_MAPPING[cd[:db_type]]
|
166
|
+
warn "Can't find ruby type for database type '#{cd[:db_type]}'"
|
167
|
+
end
|
168
|
+
{name: col_name, type: type}
|
169
|
+
end
|
170
|
+
#puts "SCHEMA_DESCR>>>> #{schema_descr}"
|
171
|
+
schema = OmlSchema.new(schema_descr)
|
172
|
+
rescue Sequel::Error => ex
|
173
|
+
raise "Problems reading schema of table '#{table_name}'. Does it exist? (#{@db.tables})"
|
174
|
+
end
|
175
|
+
end
|
106
176
|
end
|
107
177
|
|
108
178
|
|
data/lib/omf_oml/table.rb
CHANGED
@@ -43,7 +43,9 @@ module OMF::OML
|
|
43
43
|
#@endpoint = endpoint
|
44
44
|
@name = tname
|
45
45
|
@schema = OmlSchema.create(schema)
|
46
|
+
@add_index = false
|
46
47
|
unless @schema.name_at(0) == :__id__
|
48
|
+
@add_index = true
|
47
49
|
@schema.insert_column_at(0, [:__id__, 'int'])
|
48
50
|
end
|
49
51
|
@opts = opts
|
@@ -100,6 +102,14 @@ module OMF::OML
|
|
100
102
|
end
|
101
103
|
end
|
102
104
|
|
105
|
+
def on_row_added(key, &block)
|
106
|
+
on_content_changed(key) do |action, rows|
|
107
|
+
if action == :added
|
108
|
+
rows.each {|r| block.call(r)}
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
103
113
|
# NOTE: +on_row_added+ callbacks are done within the monitor.
|
104
114
|
#
|
105
115
|
def add_row(row, needs_casting = false)
|
@@ -110,6 +120,10 @@ module OMF::OML
|
|
110
120
|
end
|
111
121
|
end
|
112
122
|
|
123
|
+
def <<(row)
|
124
|
+
add_row(row)
|
125
|
+
end
|
126
|
+
|
113
127
|
# Return a new table which shadows this table but only contains
|
114
128
|
# rows with unique values in the column 'col_name' and of these the
|
115
129
|
# latest added rows to this table.
|
@@ -164,6 +178,12 @@ module OMF::OML
|
|
164
178
|
st
|
165
179
|
end
|
166
180
|
|
181
|
+
# Return table as an array of rows
|
182
|
+
#
|
183
|
+
def to_a
|
184
|
+
@rows.dup
|
185
|
+
end
|
186
|
+
|
167
187
|
def describe()
|
168
188
|
rows
|
169
189
|
end
|
@@ -177,6 +197,7 @@ module OMF::OML
|
|
177
197
|
# NOT synchronized
|
178
198
|
#
|
179
199
|
def _add_row(row, needs_casting = false)
|
200
|
+
throw "Expected array, but got '#{row}'" unless row.is_a?(Array)
|
180
201
|
if needs_casting
|
181
202
|
row = @schema.cast_row(row, true)
|
182
203
|
end
|
@@ -186,12 +207,12 @@ module OMF::OML
|
|
186
207
|
end
|
187
208
|
return nil unless row
|
188
209
|
|
189
|
-
row.insert(0, @row_id += 1)
|
210
|
+
row.insert(0, @row_id += 1) if @add_index
|
190
211
|
_add_row_finally(row)
|
191
212
|
end
|
192
213
|
|
193
|
-
# Finally add 'row' to internal storage. This would be
|
194
|
-
#
|
214
|
+
# Finally add 'row' to internal storage. This would be the method to
|
215
|
+
# override in sub classes as this is thread safe and all other pre-storage
|
195
216
|
# test have been performed. Should return the row added, or nil if nothing
|
196
217
|
# was ultimately added.
|
197
218
|
#
|
data/lib/omf_oml/version.rb
CHANGED
data/omf_oml.gemspec
CHANGED
@@ -22,5 +22,6 @@ Gem::Specification.new do |s|
|
|
22
22
|
|
23
23
|
# specify any dependencies here; for example:
|
24
24
|
# s.add_development_dependency "minitest", "~> 2.11.3"
|
25
|
-
s.add_runtime_dependency "sqlite3", "~> 1.3.6"
|
25
|
+
#s.add_runtime_dependency "sqlite3", "~> 1.3.6"
|
26
|
+
s.add_runtime_dependency 'sequel', '~> 3.45'
|
26
27
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: omf_oml
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.9.
|
4
|
+
version: 0.9.6
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,16 +9,16 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2013-03-30 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
|
-
name:
|
15
|
+
name: sequel
|
16
16
|
requirement: !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
19
|
- - ~>
|
20
20
|
- !ruby/object:Gem::Version
|
21
|
-
version:
|
21
|
+
version: '3.45'
|
22
22
|
type: :runtime
|
23
23
|
prerelease: false
|
24
24
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -26,7 +26,7 @@ dependencies:
|
|
26
26
|
requirements:
|
27
27
|
- - ~>
|
28
28
|
- !ruby/object:Gem::Version
|
29
|
-
version:
|
29
|
+
version: '3.45'
|
30
30
|
description: ! "Glue functionality between OMF and OMF related libraries, such as
|
31
31
|
OMF Web and Labwiki, and\n OML."
|
32
32
|
email:
|