shalmaneser 1.2.0.rc4 → 1.2.rc5

Sign up to get free protection for your applications and to get access to all the features.
Files changed (115) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +47 -18
  3. data/bin/shalmaneser +8 -2
  4. data/doc/index.md +1 -0
  5. data/lib/shalmaneser/opt_parser.rb +68 -67
  6. metadata +49 -119
  7. data/bin/fred +0 -16
  8. data/bin/frprep +0 -34
  9. data/bin/rosy +0 -17
  10. data/lib/common/AbstractSynInterface.rb +0 -1229
  11. data/lib/common/Counter.rb +0 -18
  12. data/lib/common/EnduserMode.rb +0 -27
  13. data/lib/common/Eval.rb +0 -480
  14. data/lib/common/FixSynSemMapping.rb +0 -196
  15. data/lib/common/Graph.rb +0 -345
  16. data/lib/common/ISO-8859-1.rb +0 -24
  17. data/lib/common/ML.rb +0 -186
  18. data/lib/common/Mallet.rb +0 -236
  19. data/lib/common/Maxent.rb +0 -229
  20. data/lib/common/Optimise.rb +0 -195
  21. data/lib/common/Parser.rb +0 -213
  22. data/lib/common/RegXML.rb +0 -269
  23. data/lib/common/RosyConventions.rb +0 -171
  24. data/lib/common/STXmlTerminalOrder.rb +0 -194
  25. data/lib/common/SalsaTigerRegXML.rb +0 -2347
  26. data/lib/common/SalsaTigerXMLHelper.rb +0 -99
  27. data/lib/common/SynInterfaces.rb +0 -282
  28. data/lib/common/TabFormat.rb +0 -721
  29. data/lib/common/Tiger.rb +0 -1448
  30. data/lib/common/Timbl.rb +0 -144
  31. data/lib/common/Tree.rb +0 -61
  32. data/lib/common/config_data.rb +0 -470
  33. data/lib/common/config_format_element.rb +0 -220
  34. data/lib/common/headz.rb +0 -338
  35. data/lib/common/option_parser.rb +0 -13
  36. data/lib/common/prep_config_data.rb +0 -62
  37. data/lib/common/prep_helper.rb +0 -1330
  38. data/lib/common/ruby_class_extensions.rb +0 -310
  39. data/lib/db/db_interface.rb +0 -48
  40. data/lib/db/db_mysql.rb +0 -145
  41. data/lib/db/db_sqlite.rb +0 -280
  42. data/lib/db/db_table.rb +0 -239
  43. data/lib/db/db_wrapper.rb +0 -176
  44. data/lib/db/sql_query.rb +0 -243
  45. data/lib/ext/maxent/Classify.class +0 -0
  46. data/lib/ext/maxent/Train.class +0 -0
  47. data/lib/fred/Baseline.rb +0 -150
  48. data/lib/fred/FileZipped.rb +0 -31
  49. data/lib/fred/FredBOWContext.rb +0 -877
  50. data/lib/fred/FredConventions.rb +0 -232
  51. data/lib/fred/FredDetermineTargets.rb +0 -319
  52. data/lib/fred/FredEval.rb +0 -312
  53. data/lib/fred/FredFeatureExtractors.rb +0 -322
  54. data/lib/fred/FredFeatures.rb +0 -1061
  55. data/lib/fred/FredFeaturize.rb +0 -602
  56. data/lib/fred/FredNumTrainingSenses.rb +0 -27
  57. data/lib/fred/FredParameters.rb +0 -402
  58. data/lib/fred/FredSplit.rb +0 -84
  59. data/lib/fred/FredSplitPkg.rb +0 -180
  60. data/lib/fred/FredTest.rb +0 -606
  61. data/lib/fred/FredTrain.rb +0 -144
  62. data/lib/fred/PlotAndREval.rb +0 -480
  63. data/lib/fred/fred.rb +0 -47
  64. data/lib/fred/fred_config_data.rb +0 -185
  65. data/lib/fred/md5.rb +0 -23
  66. data/lib/fred/opt_parser.rb +0 -250
  67. data/lib/frprep/Ampersand.rb +0 -39
  68. data/lib/frprep/CollinsInterface.rb +0 -1165
  69. data/lib/frprep/Counter.rb +0 -18
  70. data/lib/frprep/FNCorpusXML.rb +0 -643
  71. data/lib/frprep/FNDatabase.rb +0 -144
  72. data/lib/frprep/FrameXML.rb +0 -513
  73. data/lib/frprep/Graph.rb +0 -345
  74. data/lib/frprep/MiniparInterface.rb +0 -1388
  75. data/lib/frprep/RegXML.rb +0 -269
  76. data/lib/frprep/STXmlTerminalOrder.rb +0 -194
  77. data/lib/frprep/SleepyInterface.rb +0 -384
  78. data/lib/frprep/TntInterface.rb +0 -44
  79. data/lib/frprep/TreetaggerInterface.rb +0 -327
  80. data/lib/frprep/do_parses.rb +0 -143
  81. data/lib/frprep/frprep.rb +0 -693
  82. data/lib/frprep/interfaces/berkeley_interface.rb +0 -372
  83. data/lib/frprep/interfaces/stanford_interface.rb +0 -353
  84. data/lib/frprep/interpreters/berkeley_interpreter.rb +0 -22
  85. data/lib/frprep/interpreters/stanford_interpreter.rb +0 -22
  86. data/lib/frprep/one_parsed_file.rb +0 -28
  87. data/lib/frprep/opt_parser.rb +0 -94
  88. data/lib/frprep/ruby_class_extensions.rb +0 -310
  89. data/lib/rosy/AbstractFeatureAndExternal.rb +0 -242
  90. data/lib/rosy/ExternalConfigData.rb +0 -58
  91. data/lib/rosy/FailedParses.rb +0 -130
  92. data/lib/rosy/FeatureInfo.rb +0 -242
  93. data/lib/rosy/GfInduce.rb +0 -1115
  94. data/lib/rosy/GfInduceFeature.rb +0 -148
  95. data/lib/rosy/InputData.rb +0 -294
  96. data/lib/rosy/RosyConfusability.rb +0 -338
  97. data/lib/rosy/RosyEval.rb +0 -465
  98. data/lib/rosy/RosyFeatureExtractors.rb +0 -1609
  99. data/lib/rosy/RosyFeaturize.rb +0 -281
  100. data/lib/rosy/RosyInspect.rb +0 -336
  101. data/lib/rosy/RosyIterator.rb +0 -478
  102. data/lib/rosy/RosyPhase2FeatureExtractors.rb +0 -230
  103. data/lib/rosy/RosyPruning.rb +0 -165
  104. data/lib/rosy/RosyServices.rb +0 -744
  105. data/lib/rosy/RosySplit.rb +0 -232
  106. data/lib/rosy/RosyTask.rb +0 -19
  107. data/lib/rosy/RosyTest.rb +0 -829
  108. data/lib/rosy/RosyTrain.rb +0 -234
  109. data/lib/rosy/RosyTrainingTestTable.rb +0 -787
  110. data/lib/rosy/TargetsMostFrequentFrame.rb +0 -60
  111. data/lib/rosy/View.rb +0 -418
  112. data/lib/rosy/opt_parser.rb +0 -379
  113. data/lib/rosy/rosy.rb +0 -78
  114. data/lib/rosy/rosy_config_data.rb +0 -121
  115. data/lib/shalmaneser/version.rb +0 -3
@@ -1,220 +0,0 @@
1
-
2
- ##############################
3
- # ConfigFormatelement is an auxiliary class
4
- # of ConfigData.
5
- # It keeps track of feature patterns with variables in them
6
- # that can be instantiated.
7
- # @author Andrei Beliankou
8
- #
9
- class ConfigFormatElement
10
-
11
- # given a pattern and a list of variable names,
12
- # analyze the pattern and remember the variable names
13
- #
14
- def initialize(string, # string: feature name, may include names of variables.
15
- # they are included in <>
16
- variables) # list of variable names that can occur
17
-
18
- @variables = variables
19
-
20
- # pattern: this is what the 'string' is split into,
21
- # an array of elements that are either fixed parts or variables.
22
- # fixed part: pair [item:string, "string"]
23
- # variable: pair [variable_name:string, "variable"]
24
- @pattern = Array.new
25
- state = "out"
26
- item = ""
27
-
28
- # analyze string,
29
- # split into variables and fixed parts
30
- string.split(//).each { |char|
31
-
32
- case state
33
- when "in"
34
- case char
35
- when "<"
36
- raise "Duplicate < in " + string
37
- when ">"
38
- unless @variables.include? item
39
- raise "Unknown variable " + item
40
- end
41
- @pattern << [item, "variable"]
42
- item = ""
43
- state = "out"
44
- else
45
- item << char
46
- state = "in"
47
- end
48
-
49
- when "out"
50
- case char
51
- when "<"
52
- unless item.empty?
53
- @pattern << [item, "string"]
54
- item = ""
55
- end
56
- state = "in"
57
- when ">"
58
- raise "Unexpected > in " + string
59
- else
60
- item << char
61
- state = "out"
62
- end
63
-
64
- else
65
- raise "Shouldn't be here"
66
- end
67
- }
68
-
69
- # read through the whole of "string"
70
- # end state has to be "out"
71
- unless state == "out"
72
- raise "Unclosed < in " + string
73
- end
74
-
75
- # last bit still to be recorded?
76
- unless item.empty?
77
- @pattern << [item, "string"]
78
- end
79
-
80
- # make regexp for matching this pattern
81
- @regexp = make_regexp(@pattern)
82
- end
83
-
84
- # instantiate: given pairs of variable names and variable values,
85
- # instantiate @pattern to a string in which var names are replaced
86
- # by their values
87
- #
88
- # returns: string
89
- def instantiate(var_hash) # hash variable name(string) => variable value(string)
90
-
91
- # instantiate the pattern
92
- return @pattern.map { |item, string_or_var|
93
-
94
- case string_or_var
95
- when "string"
96
- item
97
-
98
- when "variable"
99
-
100
- if var_hash[item].nil?
101
- raise "Missing variable instantiation: " + item
102
- end
103
- var_hash[item]
104
-
105
- else
106
- raise "Shouldn't be here"
107
- end
108
- }.join
109
- end
110
-
111
- # match()
112
- #
113
- # given a string, try to match it against the @pattern
114
- # while setting the variables given in 'fillers' to
115
- # the values given in that hash.
116
- #
117
- # returns: if the string matches, a hash variable name => value
118
- # that includes the fillers given as a parameter as well as
119
- # values for all other variables mentioned in @pattern,
120
- # or false if no match.
121
- def match(string, # a string
122
- fillers = nil) # hash variable name(string) => value(string)
123
-
124
- # have we been given partial info about variables?
125
- if fillers
126
- match = make_regexp(@pattern, fillers).match(string)
127
- # $stderr.print "matching " + make_regexp(@pattern, fillers).source +
128
- # " against " + string + " "
129
- # if match.nil?
130
- # $stderr.puts "no"
131
- # else
132
- # $stderr.puts "yes"
133
- # end
134
- else
135
- match = @regexp.match(string)
136
- end
137
-
138
- if match.nil?
139
- # no match via the regular expression
140
- return false
141
- end
142
-
143
- # regular expression matched.
144
- # construct return value in hash
145
- # retv: variable name(string) => value(string)
146
- retv = Hash.new()
147
- if fillers
148
- # include given fillers in retv hash
149
- fillers.each_pair { |name, val| retv[name] = val }
150
- end
151
-
152
- # now put values for other variables in @pattern into retv
153
- index = 1
154
- @pattern.to_a.select { |item, string_or_var|
155
- string_or_var == "variable"
156
- }.select { |item, string_or_var|
157
- fillers.nil? or
158
- fillers[item].nil?
159
- }.each { |item, string_or_var|
160
- # for all items on the pattern list
161
- # that are variables and
162
- # haven't been filled by the "fillers" list already:
163
- # fill from matches
164
-
165
- if match[index].nil?
166
- raise "Match, but not enough matched elements? Strange."
167
- end
168
-
169
- if retv[item].nil?
170
- retv[item] = match[index]
171
- else
172
- unless retv[item] == match[index]
173
- return false
174
- end
175
- end
176
-
177
- index += 1
178
- }
179
-
180
- return retv
181
- end
182
-
183
- # used_variables
184
- #
185
- # returns: an array of variable names used in @pattern
186
- def used_variables()
187
- return @pattern.select { |item, string_or_var|
188
- string_or_var == "variable"
189
- }.map { |item, string_or_var| item}
190
- end
191
-
192
- ####################
193
- private
194
-
195
- # make_regexp:
196
- # make regular expression from a pattern
197
- # together with some variable fillers
198
- #
199
- # returns: Regexp object
200
- def make_regexp(pattern, # array of pairs [string, "string"] or [string, "variable"]
201
- fillers = nil) # hash variable name(string) => value(string)
202
- return (Regexp.new "^" +
203
- pattern.map { |item, string_or_var|
204
- case string_or_var
205
- when "variable"
206
- if fillers and
207
- fillers[item]
208
- Regexp.escape(fillers[item])
209
- else
210
- "(.+)"
211
- end
212
- when "string"
213
- Regexp.escape(item)
214
- else
215
- raise "Shouldn't be here"
216
- end
217
- }.join + "$")
218
- end
219
-
220
- end
@@ -1,338 +0,0 @@
1
- # name: Module Headz
2
- # auth: albu@coli.uni-sb.de
3
- #
4
- # modified KE Sept 04:
5
- # changed from old Sentence pkg to new SalsaTigerSentence pkg
6
- #
7
- # modified KE April 05:
8
- # suppress the flood of warnings
9
- #
10
- # modified SP June 05: added some more cases; change to SalsTigerRegXML
11
- #
12
- #
13
- # INIT: REXML TIGER sentence,
14
- # FUNC: syn_nodes(term/non_term) -> heads
15
- #
16
- #
17
- # usage:
18
- #
19
- # h = Headz.new()
20
- #
21
- # hash = h.get_sem_head(node) # node is a SalsaTigerXmlNode obj
22
- #
23
- # head = hash["head"]
24
- # prep = hash["prep"]
25
- #
26
- # if h.complex(head)
27
- # print "preposition of conjunction involved"
28
- # end
29
-
30
- require 'common/SalsaTigerRegXML'
31
-
32
- class Headz
33
-
34
- def initialize()
35
- @Helpers = HeadzHelpers.new()
36
- @Verbose = false #KE 13.4.05: please not that many messages!
37
- end
38
-
39
- # head of one node
40
- def get_sem_head(node)
41
- gsh(node)
42
- end
43
-
44
- # all headz of top-nodes covering fe
45
- def get_fe_heads(fe)
46
- if (const = fe.children())
47
- const.map { |node|
48
- get_sem_head(node)
49
- }
50
- else
51
- $stderr.puts "Headz.get_sem_head: no children for FE #{fe}"
52
- []
53
- end
54
- end
55
-
56
- def gsh (node)
57
- if !node then
58
- if @Verbose then $stderr.puts "Headz.gsh: no input node" end
59
- return {}
60
-
61
- elsif node.is_terminal? then return Hash['head'=>node]
62
-
63
- else
64
- case node.category
65
- when 'AP'
66
- return gsh(@Helpers.get_dtr(node,'HD'))
67
-
68
- when 'AVP'
69
- return gsh(@Helpers.get_dtr(node,'HD'))
70
- when 'CAP', 'CAVP', 'CNP', 'CPP', 'CS', 'CVP'
71
- conjs = @Helpers.get_conjuncts(node)
72
- head = gsh(conjs.shift)
73
- if head
74
- head.update(Hash["conj"=>gsh_conjs(conjs)])
75
- end
76
- return head
77
-
78
- when 'NM'
79
- return gsh(@Helpers.get_rightmost_dtr(node,'NMC'))
80
- when 'NP'
81
- nk = @Helpers.get_rightmost_dtr(node,'NK')
82
- if nk
83
- return gsh(nk)
84
- else
85
- return gsh(@Helpers.get_rightmost_dtr(node, "NN"))
86
- end
87
-
88
- when 'PN'
89
- pncs = @Helpers.get_dtrs(node,'PNC')
90
- head = gsh(pncs.last)
91
- if head
92
- head.update(Hash["pncs"=>pncs])
93
- end
94
- return head
95
-
96
- when 'PP'
97
- return pp(node)
98
-
99
- when 'S'
100
- return s(node)
101
- when 'VROOT'
102
- dtrs = @Helpers.get_dtrs(node,'--')
103
-
104
- # discourse level node with sentence nodes below?
105
- # or conjunction with sentence nodes below?
106
- discourselevel_dtr = dtrs.detect { |n| n.category == "DL"}
107
- co_dtr = dtrs.detect { |n| n.category == "CO" }
108
- if discourselevel_dtr
109
- dtrs = discourselevel_dtr.children()
110
- elsif co_dtr
111
- dtrs = co_dtr.children()
112
- end
113
-
114
-
115
- # take first sentence node
116
- sent_dtr = dtrs.detect {|n| n.category =~ /^C?S/}
117
- if sent_dtr
118
- return gsh(sent_dtr)
119
- else
120
- # $stderr.puts "headz Warning: no sentence found below VROOT! Node #{node.id()}"
121
- return nil
122
- end
123
-
124
- when 'VP'
125
- return vp(node)
126
-
127
- when 'MTA'
128
- return gsh(@Helpers.get_rightmost_dtr(node,'ADC'))
129
-
130
- when 'VZ'
131
- return gsh(@Helpers.get_dtr(node,'HD'))
132
- else
133
- if @Verbose
134
- $stderr.puts " Headz.gsh: no rule for #{node.category}"
135
- end
136
- {}
137
- end
138
- end
139
- end
140
-
141
- # flatten the processed conjs to a list of (head) Hashes
142
- # containing no conj features themselves
143
- def gsh_conjs(conjs)
144
- flat = Array.new
145
-
146
- conjs.each {|conj|
147
- current = gsh(conj)
148
- @Helpers.descend(current,flat)
149
- }
150
-
151
- flat
152
- end
153
-
154
- #####################################3
155
- def pp(node)
156
-
157
- prep = node.terminals_sorted().detect { |n|
158
- (pt = n.part_of_speech()) and
159
- (pt =~ /^APPR/ or
160
- pt =~ /^PWAV/ or
161
- pt =~ /^C?PP/
162
- )
163
- }
164
-
165
- if (lastnk = @Helpers.get_rightmost_dtr(node,'NK'))
166
- head = gsh(lastnk)
167
- if head and prep
168
- head.update(Hash['prep'=>prep])
169
- end
170
-
171
- elsif (re = @Helpers.get_dtr(node,'RE'))
172
- head = gsh(re)
173
- if head and prep
174
- head.update(Hash['prep'=>prep])
175
- end
176
- else
177
- if @Verbose then $stderr.puts " pp: no rule for #{node}" end
178
- end
179
-
180
- head
181
- end
182
-
183
- ################
184
- def s(node)
185
- head = @Helpers.get_dtr(node,'HD')
186
- if !head
187
- # $stderr.puts " s: no head for #{node}"
188
- return Hash[]
189
- end
190
-
191
- if head.outdeg() == 0
192
- return gsh(head)
193
- end
194
-
195
- oc = @Helpers.get_dtr(node,'OC')
196
- case head.category
197
- when 'VVFIN'
198
- if svp = @Helpers.get_dtr(node,'SVP') then
199
- h = gsh(head)
200
- if h
201
- return h.update(Hash['svp'=>gsh(svp), 'oc'=>gsh(oc)])
202
- else
203
- return h
204
- end
205
- else
206
- return gsh(head)
207
- end
208
-
209
- when 'VAFIN'
210
- if oc && headd = @Helpers.get_dtr(oc,'HD')
211
- h = gsh(headd)
212
- if h
213
- return h.update(Hash['oc'=>gsh(oc)])
214
- else
215
- return h
216
- end
217
-
218
- elsif pd = @Helpers.get_dtr(node,'PD') && head = @Helpers.get_dtr(pd,'HD')
219
- return gsh(head)
220
-
221
- else
222
- if @Verbose then $stderr.puts " s: no rule for #{node}" end
223
- end
224
- else
225
- if @Verbose then $stderr.puts " s: no rule for #{node}" end
226
- end
227
- end
228
-
229
- ################
230
- def vp(node)
231
- head = gsh(@Helpers.get_dtr(node,'HD'))
232
- tmp = @Verbose
233
- @Verbose = false
234
- newHash = Hash.new
235
- ["da","oa"].each { |type|
236
- if (dtr = @Helpers.get_dtr(node,type.upcase))
237
- newHash[type] = gsh(dtr)
238
- end
239
- }
240
- @Verbose = tmp
241
- if head
242
- return head.update(newHash)
243
- else
244
- return newHash
245
- end
246
- end
247
-
248
- ################
249
- # Access
250
- def head(h)
251
- return h['head']
252
- end
253
-
254
- def complex(h)
255
- prep(h) or conj(h)
256
- end
257
-
258
- def prep(h)
259
- return h['prep']
260
- end
261
-
262
- def conj(h)
263
- return h['conj']
264
- end
265
-
266
-
267
-
268
- end # Class Headz
269
-
270
-
271
- class HeadzHelpers
272
- @Verbose = true
273
-
274
- # Conjunction
275
-
276
- def get_conjuncts(node)
277
- conjuncts = get_dtrs(node,'CJ')
278
- end
279
-
280
- # flatten
281
- def descend(current,flat)
282
- if current.nil?
283
- return flat
284
- end
285
-
286
- if current.has_key?("conj") then
287
- tmp = current.delete("conj")
288
- flat.push current
289
- tmp.each {|item|
290
- descend(item,flat)}
291
- else
292
- flat.push current
293
- end
294
- end
295
-
296
- # Zugriff
297
-
298
- def get_dtr(node,label)
299
- if (dtrs = node.children_by_edgelabels([label]))
300
- dtrs.first
301
- else
302
- if @Verbose then $stderr.puts " SelectHeadDtr: no #{label} dtr for #{node}" end
303
- nil
304
- end
305
- end
306
-
307
- def get_dtrs(node,label)
308
- if ! dtrs = node.children_by_edgelabels([label])
309
- if @Verbose then $stderr.puts " SelectHeadDtr: no #{label} dtr for #{node}" end
310
- else
311
- dtrs
312
- end
313
- end
314
-
315
- def get_rightmost_dtr(node,label)
316
- children = node.children_by_edgelabels([label])
317
- if re = children.last then re
318
- else
319
- if @Verbose then $stderr.puts " SelectHeadDtr: no #{label} dtrs for #{node}" end
320
- nil
321
- end
322
- end
323
-
324
- # def l2h(list)
325
- # h = Hash.new
326
- # while (list.length > 1) do
327
- # h[list.shift] = list.shift
328
- # end
329
- # if list.length == 1 then
330
- # $stderr.puts "l2h: odd number of elems: " + list.join(" / ")
331
- # end
332
- # h
333
- # end
334
-
335
- end # Class HeadzHelpers
336
-
337
-
338
-