shalmaneser-rosy 1.2.0.rc4

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,465 @@
1
+ # RosyEval
2
+ # KE May 05
3
+ #
4
+ # Evaluation for Rosy:
5
+ # Precision, Recall, F-score
6
+ # Output to evaluation file,
7
+ # plus optional output of evaluation log file.
8
+ #
9
+ # Builds on the general Salsa Eval package
10
+
11
+ # Salsa packages
12
+ require "common/Eval"
13
+ require "common/ruby_class_extensions"
14
+
15
+ # Rosy packages
16
+ require "rosy/RosyIterator"
17
+ require "rosy/RosySplit"
18
+ require "rosy/RosyTask"
19
+ require "rosy/RosyPruning"
20
+
21
+ # Frprep packages
22
+ require "common/prep_config_data"
23
+
24
+ #######################################################################
25
+ # This class is a subclass of the general evaluation class
26
+ # Eval, which makes evaluation results readable via
27
+ # readable object variables
28
+ #
29
+ # step: can be argrec, arglab, onestep, as usual, but also
30
+ # - "all":
31
+ # evaluate argrec and arglab together.
32
+ # When argrec == NONE, use the argrec value, else use the arglab value
33
+ # - "prune":
34
+ # evaluate the pruning column as if it were an argrec assignment
35
+ #
36
+ # When step == argrec or prune, evaluate _only_ the target class FE
37
+ # Otherwise, evaluate all target classes
38
+ class RosyEval < Eval
39
+ def initialize(exp, # RosyConfigData object: experiment file
40
+ ttt_obj, # RosyTrainingTestTable object
41
+ step, # string: argrec, arglab, onestep, all, prune
42
+ splitID, # string: splitlog ID, or nil
43
+ testID, # string: test ID, or nil
44
+ outfilename, # string: name of file to print output to
45
+ logfilename, # string: name of file to print eval log to (may be nil)
46
+ dont_adjoin_frprep_exp) # string: if non-nil, don't re-adjoin frprep experiment obj
47
+ @exp = exp
48
+ @step = step
49
+
50
+ if outfilename
51
+ $stderr.puts "Rosy evaluation: printing results to " + outfilename
52
+ end
53
+ if logfilename
54
+ $stderr.puts "and printing an evaluation log to " + logfilename
55
+ end
56
+
57
+ ##
58
+ # add preprocessing information to the experiment file object
59
+ unless dont_adjoin_frprep_exp
60
+ if splitID
61
+ # use split data
62
+ preproc_expname = @exp.get("preproc_descr_file_train")
63
+ else
64
+ # use test data
65
+ preproc_expname = @exp.get("preproc_descr_file_test")
66
+ end
67
+ if not(preproc_expname)
68
+ $stderr.puts "Please set the name of the preprocessing exp. file name"
69
+ $stderr.puts "in the experiment file."
70
+ exit 1
71
+ elsif not(File.readable?(preproc_expname))
72
+ $stderr.puts "Error in the experiment file:"
73
+ $stderr.puts "Parameter preproc_descr_file_train has to be a readable file."
74
+ exit 1
75
+ end
76
+ preproc_exp = FrPrepConfigData.new(preproc_expname)
77
+ @exp.adjoin(preproc_exp)
78
+ end
79
+
80
+ ##
81
+ # evaluate which labels?
82
+ if ["argrec", "prune"].include? @step
83
+ # evaluate only the label "FE"
84
+ super(outfilename, logfilename, "FE")
85
+ else
86
+ # evaluate all target classes
87
+ super(outfilename, logfilename)
88
+ end
89
+
90
+ ##
91
+ # what are classifier columns?
92
+ case @step
93
+ when "all"
94
+ # read one argrec and one arglab classifier run column
95
+ @classif_column_argrec = ttt_obj.existing_runlog("argrec", "test", testID,splitID)
96
+ @classif_column_arglab = ttt_obj.existing_runlog("arglab", "test", testID,splitID)
97
+ @columns = ["gold", @classif_column_argrec, @classif_column_arglab]
98
+
99
+ if @classif_column_argrec.nil? or @classif_column_arglab.nil?
100
+ # no run found for the given specifications
101
+ $stderr.puts "Couldn't determine the run to evaluate."
102
+ $stderr.puts "There were either none or too many possible runs given your specification.\n"
103
+ $stderr.puts "Here is a list of all runs the system knows for this experiment ID:\n\n"
104
+ $stderr.puts ttt_obj.runlog_to_s("test", testID, splitID)
105
+ exit 1
106
+ end
107
+
108
+ when "prune"
109
+ # read pruning column, evaluate as a kind of argrec assignment
110
+ unless Pruning.prune?(@exp)
111
+ raise "Error: Pruning evaluation without pruning column. Skipping."
112
+ end
113
+ @classif_column = Pruning.colname(@exp)
114
+ @columns = ["gold", @classif_column]
115
+
116
+ else
117
+ # read the classifier run column for the current step
118
+ @classif_column = ttt_obj.existing_runlog(@step, "test", testID,splitID)
119
+ @columns = ["gold", @classif_column]
120
+
121
+ if @classif_column.nil?
122
+ # no run found for the given specifications
123
+ $stderr.puts "Couldn't determine the run to evaluate."
124
+ $stderr.puts "There were either none or too many possible runs given your specification.\n"
125
+ $stderr.puts "Here is a list of all runs the system knows for this experiment ID:\n\n"
126
+ $stderr.puts ttt_obj.runlog_to_s("test", testID, splitID)
127
+ exit 1
128
+ end
129
+ end
130
+
131
+ ##
132
+ # make object for iterating through groups and making views
133
+ case @step
134
+ when "all"
135
+ # all: no step in particular
136
+ @iterator = RosyIterator.new(ttt_obj, exp, "test",
137
+ "step" => nil,
138
+ "testID" => testID,
139
+ "splitID" => splitID,
140
+ "xwise" => "frame")
141
+ when "prune"
142
+ # prune: use argrec
143
+ @iterator = RosyIterator.new(ttt_obj, exp, "test",
144
+ "step" => "argrec",
145
+ "testID" => testID,
146
+ "splitID" => splitID)
147
+
148
+ else
149
+ # use the given step
150
+ @iterator = RosyIterator.new(ttt_obj, exp, "test",
151
+ "step" => @step,
152
+ "testID" => testID,
153
+ "splitID" => splitID)
154
+ end
155
+
156
+ ##
157
+ # xwise
158
+ if @step == "all"
159
+ # argrec and arglab may have different xwises,
160
+ # which would create trouble.
161
+ # just use "frame" instead
162
+ @xwise = ["frame"]
163
+ else
164
+ # evaluate as you have trained and tested
165
+ @xwise = @iterator.get_xwise_column_names()
166
+ end
167
+
168
+ ##
169
+ # split? then include FE labels from unparsed sentences
170
+ # in count of gold labels
171
+ if splitID
172
+ # get a FailedParses object for this split
173
+ @failed_parses_split = FailedParses.new()
174
+ fp_filename = File.new_filename(@exp.instantiate("rosy_dir",
175
+ "exp_ID" => @exp.get("experiment_ID")),
176
+ @exp.instantiate("failed_file",
177
+ "exp_ID" => @exp.get("experiment_ID"),
178
+ "split_ID" => splitID,
179
+ "dataset" => "test"))
180
+ @failed_parses_split.load(fp_filename)
181
+ end
182
+
183
+ # announce the task
184
+ $stderr.puts "---------"
185
+ $stderr.print "Rosy experiment #{@exp.get("experiment_ID")}: Evaluating "
186
+ if splitID
187
+ $stderr.puts "on split dataset #{splitID}"
188
+ else
189
+ $stderr.puts "on test dataset #{testID}"
190
+ end
191
+ $stderr.puts "---------"
192
+ end
193
+
194
+ ###
195
+ protected
196
+
197
+ ###
198
+ # each_group
199
+ #
200
+ # yield each group name in turn
201
+ def each_group()
202
+
203
+ @view = nil
204
+
205
+ # for the sake of the failed parses module:
206
+ # it can split the failed parses by frame, target and target_pos,
207
+ # but if our "xwise" splits the data along any further columns,
208
+ # the failed parses module cannot know how to split up its failed parses.
209
+ # so see whether we've got any column names besides the three named above
210
+ # in our xwise,
211
+ # and if so, count the groups and split the failed parses evenly between them
212
+ normal_xwise_cols = ["frame", "target", "target_pos"] & @xwise
213
+ extra_xwise_cols = @xwise - normal_xwise_cols
214
+
215
+ # num_groups_for_normalxwise: hash: normal_xwise_values(string) -> num. of
216
+ # groups with these normal xwise values(integer)
217
+ # where the key normal_xwise_values is a conjunction of
218
+ # strings <col_name>=<value> joined by commas,
219
+ # and the column names are in alphabetical order
220
+ num_groups_for_normalxwise = Hash.new(0)
221
+
222
+ unless extra_xwise_cols.empty?
223
+ # we do have extra columns
224
+
225
+ # for each value sequence for normal_xwise_cols: find out how many values
226
+ # of extra xwise col.s there are
227
+ @iterator.each_group() { |group_descr_hash, group_name|
228
+
229
+ # make the hash key
230
+ key = normal_xwise_cols.sort.map { |col_name|
231
+ col_name + "=" + group_descr_hash[col_name]
232
+ }.join(",")
233
+
234
+ # record one occurrence of this hash key
235
+ num_groups_for_normalxwise[key] += 1
236
+ }
237
+ end
238
+
239
+ @iterator.each_group() { |group_descr_hash, group_name|
240
+
241
+ if @exp.get("verbose")
242
+ $stderr.puts group_name
243
+ end
244
+
245
+ # construct view for the current group
246
+ @view = @iterator.get_a_view_for_current_group(@columns)
247
+
248
+ ##
249
+ # get counts of FE labels from unparsed sentences:
250
+
251
+ # first take apart the group label to find
252
+ # the frame name, target name, target POS name in this group
253
+ # (all but one may be nil)
254
+ frame = target = target_pos = nil
255
+
256
+ # get a description of this group, array of pairs [column name, value]
257
+ # where column name is the name of one database column
258
+ @xwise.interleave(group_name.split()).each { |col_name, col_value|
259
+ case col_name
260
+ when "frame"
261
+ frame = col_value
262
+ when "target"
263
+ target = col_value
264
+ when "target_pos"
265
+ target_pos = col_value
266
+ else
267
+ # additional database columns: handled below
268
+ end
269
+ }
270
+
271
+ # do we have additional column names in "xwise", besides 'frame', 'target', 'target_pos'?
272
+ if extra_xwise_cols.empty?
273
+ split_between_groups = 1
274
+ else
275
+ key = normal_xwise_cols.sort.map { |col_name|
276
+ col_name + "=" + group_descr_hash[col_name]
277
+ }.join(",")
278
+ split_between_groups = num_groups_for_normalxwise[key]
279
+
280
+ # sanity check
281
+ if split_between_groups == 0
282
+ raise "shouldn't be here"
283
+ end
284
+ end
285
+
286
+ # failed_fes returns: hash that maps FE names [String] onto numbers of failed FEs [Int]
287
+ if @failed_parses_split
288
+ @failed_parses_split.failed_fes(frame, target, target_pos).each_pair { |fe, count|
289
+ # add this number of gold labels we failed to find
290
+ # to the number of gold labels that Eval counts
291
+
292
+ # if argrec, map all non-NONE FEs to "FE"
293
+ if @step == "argrec" and fe != @exp.get("noval")
294
+ fe = "FE"
295
+ end
296
+ inject_gold_counts(group_name, fe, (count.to_f / split_between_groups.to_f).round)
297
+ }
298
+ end
299
+
300
+ # yield the name of the group to the Eval object for evaluation
301
+ yield group_name
302
+ @view.close()
303
+ }
304
+ end
305
+
306
+ ###
307
+ # each_instance
308
+ #
309
+ # given a group name, yield each instance of this group in turn,
310
+ # or rather: yield pairs [gold_class(string), assigned_class(string)]
311
+ #
312
+ # this method depends on each_group() having been called before and
313
+ # having initialized @view to the right view object
314
+ def each_instance(group) # string: group name
315
+ case @step
316
+ when "all"
317
+ # step "all":
318
+ # if the argrec label is "NONE", use that as the assigned label.
319
+ # else use the arglab-label
320
+ @view.each_hash { |row|
321
+ if row[@classif_column_argrec] == @exp.get("noval")
322
+ yield [ row["gold"], row[@classif_column_argrec] ]
323
+ else
324
+ yield [ row["gold"], row[@classif_column_arglab] ]
325
+ end
326
+ }
327
+
328
+ when "prune"
329
+ # step "prune":
330
+ # if the pruning column has entry 1, regard as assignment "FE",
331
+ # else regard as assignment "NONE".
332
+ @view.each_hash { |row|
333
+ if row[@classif_column] == "1"
334
+ yield [ row["gold"], "FE" ]
335
+ else
336
+ yield [ row["gold"], @exp.get("noval") ]
337
+ end
338
+ }
339
+
340
+ else
341
+ # argrec, arglab, onestep:
342
+ # just yield pairs [goldlabel, classif_column_label]
343
+ # as given in the view
344
+
345
+ @view.each_hash { |row|
346
+ yield [row["gold"], row[@classif_column]]
347
+ }
348
+ end
349
+
350
+ end
351
+ end
352
+
353
+ ###########################################################33
354
+ # This is the class to be called from rosy.rb
355
+ ###########################################################33
356
+ class RosyEvalTask < RosyTask
357
+
358
+ def initialize(exp, # RosyConfigData object: experiment description
359
+ opts, # hash: runtime argument option (string) -> value (string)
360
+ ttt_obj) # RosyTrainingTestTable object
361
+
362
+ #####
363
+ # In enduser mode, this whole task is unavailable
364
+ in_enduser_mode_unavailable()
365
+
366
+ @exp = exp
367
+ @ttt_obj = ttt_obj
368
+
369
+ ##
370
+ # check runtime options
371
+ @step = "both"
372
+ @splitID = nil
373
+ @testID = default_test_ID()
374
+
375
+ opts.each do |opt,arg|
376
+ case opt
377
+ when "--step"
378
+ unless ["argrec", "arglab", "both", "onestep"].include? arg
379
+ raise "Classification step must be one of: argrec, arglab, both, onestep. I got: " + arg.to_s
380
+ end
381
+ @step = arg
382
+ when "--logID"
383
+ @splitID = arg
384
+ when "--testID"
385
+ @testID = arg
386
+ else
387
+ # this is an option that is okay but has already been read and used by rosy.rb
388
+ end
389
+ end
390
+ end
391
+
392
+ def perform()
393
+ dont_adjoin_frprep_exp = nil
394
+ original_step = @step
395
+
396
+ if ["both", "argrec", "onestep"].include? original_step and
397
+ Pruning.prune?(@exp)
398
+ # evaluate pruning
399
+ $stderr.puts "Rosy evaluating pruning"
400
+ @step = "prune"
401
+ perform_aux()
402
+ dont_adjoin_frprep_exp = "dont_adjoin_frprep_exp"
403
+ end
404
+
405
+ if original_step == "both"
406
+ # both? then do first argrec, then arglab
407
+ $stderr.puts "Rosy evaluating step argrec"
408
+ @step = "argrec"
409
+ perform_aux(dont_adjoin_frprep_exp)
410
+
411
+
412
+ $stderr.puts "Rosy evaluating step arglab"
413
+ @step = "arglab"
414
+ perform_aux("dont_adjoin_frprep_exp")
415
+
416
+ # KE Jan 30, 2006: evaluation "all" deactivated until we've
417
+ # figured out how to evaluate accuracy for the NONE class
418
+ # $stderr.puts "Rosy overall evaluation"
419
+ # @step = "all"
420
+ # perform_aux("dont_adjoin_frprep_exp")
421
+
422
+ else
423
+ # not both? then just do one
424
+ @step = original_step
425
+ perform_aux(dont_adjoin_frprep_exp)
426
+ end
427
+ end
428
+
429
+ ###############3
430
+ private
431
+
432
+ # perform_aux: do the actual work of the perform() method
433
+ # moved here because of the possibility of having @step=="both",
434
+ # which makes it necessary to perform two eval steps one after the other
435
+ def perform_aux(dont_adjoin_frprep_exp = nil) # string passed on to RosyEval initialize method
436
+ # construct names for evaluation output file
437
+ # and evaluation log file (which classifies each instances as correct/incorrect/unassigned)
438
+ if @splitID
439
+ outfilename_id = "split" + @splitID
440
+ else
441
+ outfilename_id = "test" + @testID
442
+ end
443
+ @outfilename = File.new_filename(@exp.instantiate("rosy_dir",
444
+ "exp_ID" => @exp.get("experiment_ID")),
445
+ @exp.instantiate("eval_file",
446
+ "exp_ID" => @exp.get("experiment_ID"),
447
+ "test_ID" => outfilename_id,
448
+ "step" => @step))
449
+
450
+ if @exp.get("print_eval_log")
451
+ @logfilename = File.new_filename(@exp.instantiate("rosy_dir",
452
+ "exp_ID" => @exp.get("experiment_ID")),
453
+ @exp.instantiate("log_file",
454
+ "exp_ID" => @exp.get("experiment_ID"),
455
+ "test_ID" => outfilename_id,
456
+ "step" => @step))
457
+ else
458
+ @logfilename = nil
459
+ end
460
+ @eval_obj = RosyEval.new(@exp, @ttt_obj, @step, @splitID, @testID,
461
+ @outfilename, @logfilename,
462
+ dont_adjoin_frprep_exp)
463
+ @eval_obj.compute()
464
+ end
465
+ end