opener-opinion-detector-base 2.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (261) hide show
  1. checksums.yaml +7 -0
  2. data/README.md +101 -0
  3. data/bin/opinion-detector-base +19 -0
  4. data/core/annotation.cfg.erb +9 -0
  5. data/core/packages/KafNafParser-1.4.tar.gz +0 -0
  6. data/core/packages/VUA_pylib-1.5.tar.gz +0 -0
  7. data/core/python-scripts/LICENSE +339 -0
  8. data/core/python-scripts/README.md +226 -0
  9. data/core/python-scripts/classify_kaf_naf_file.py +499 -0
  10. data/core/python-scripts/cross_validation.py +634 -0
  11. data/core/python-scripts/generate_folds.py +134 -0
  12. data/core/python-scripts/models.cfg +10 -0
  13. data/core/python-scripts/my_templates/README +33 -0
  14. data/core/python-scripts/my_templates/templates_exp.only0.txt +6 -0
  15. data/core/python-scripts/my_templates/templates_exp.pol0.txt +10 -0
  16. data/core/python-scripts/my_templates/templates_exp.red.txt +7 -0
  17. data/core/python-scripts/my_templates/templates_exp.txt +10 -0
  18. data/core/python-scripts/my_templates/templates_holder.only0.txt +11 -0
  19. data/core/python-scripts/my_templates/templates_holder.red.txt +9 -0
  20. data/core/python-scripts/my_templates/templates_holder.txt +10 -0
  21. data/core/python-scripts/my_templates/templates_target.only0.txt +11 -0
  22. data/core/python-scripts/my_templates/templates_target.red.txt +9 -0
  23. data/core/python-scripts/my_templates/templates_target.txt +10 -0
  24. data/core/python-scripts/run_all_experiments.sh +49 -0
  25. data/core/python-scripts/run_basic.py +20 -0
  26. data/core/python-scripts/run_experiment.sh +42 -0
  27. data/core/python-scripts/scripts/__init__.py +1 -0
  28. data/core/python-scripts/scripts/config_manager.py +314 -0
  29. data/core/python-scripts/scripts/crfutils.py +215 -0
  30. data/core/python-scripts/scripts/extract_feats_relations.py +295 -0
  31. data/core/python-scripts/scripts/extract_features.py +376 -0
  32. data/core/python-scripts/scripts/feats_to_crf.exp.py +105 -0
  33. data/core/python-scripts/scripts/lexicons.py +44 -0
  34. data/core/python-scripts/scripts/link_entities_distance.py +77 -0
  35. data/core/python-scripts/scripts/relation_classifier.py +250 -0
  36. data/core/python-scripts/train.py +566 -0
  37. data/core/site-packages/pre_build/KafNafParser-1.4-py2.7.egg-info/PKG-INFO +10 -0
  38. data/core/site-packages/pre_build/KafNafParser-1.4-py2.7.egg-info/SOURCES.txt +22 -0
  39. data/core/site-packages/pre_build/KafNafParser-1.4-py2.7.egg-info/dependency_links.txt +1 -0
  40. data/core/site-packages/pre_build/KafNafParser-1.4-py2.7.egg-info/installed-files.txt +47 -0
  41. data/core/site-packages/pre_build/KafNafParser-1.4-py2.7.egg-info/top_level.txt +1 -0
  42. data/core/site-packages/pre_build/KafNafParser/KafNafParserMod.py +390 -0
  43. data/core/site-packages/pre_build/KafNafParser/KafNafParserMod.pyc +0 -0
  44. data/core/site-packages/pre_build/KafNafParser/__init__.py +14 -0
  45. data/core/site-packages/pre_build/KafNafParser/__init__.pyc +0 -0
  46. data/core/site-packages/pre_build/KafNafParser/constituency_data.py +125 -0
  47. data/core/site-packages/pre_build/KafNafParser/constituency_data.pyc +0 -0
  48. data/core/site-packages/pre_build/KafNafParser/coreference_data.py +52 -0
  49. data/core/site-packages/pre_build/KafNafParser/coreference_data.pyc +0 -0
  50. data/core/site-packages/pre_build/KafNafParser/dependency_data.py +78 -0
  51. data/core/site-packages/pre_build/KafNafParser/dependency_data.pyc +0 -0
  52. data/core/site-packages/pre_build/KafNafParser/entity_data.py +59 -0
  53. data/core/site-packages/pre_build/KafNafParser/entity_data.pyc +0 -0
  54. data/core/site-packages/pre_build/KafNafParser/external_references_data.py +41 -0
  55. data/core/site-packages/pre_build/KafNafParser/external_references_data.pyc +0 -0
  56. data/core/site-packages/pre_build/KafNafParser/feature_extractor/__init__.py +2 -0
  57. data/core/site-packages/pre_build/KafNafParser/feature_extractor/__init__.pyc +0 -0
  58. data/core/site-packages/pre_build/KafNafParser/feature_extractor/constituency.py +205 -0
  59. data/core/site-packages/pre_build/KafNafParser/feature_extractor/constituency.pyc +0 -0
  60. data/core/site-packages/pre_build/KafNafParser/feature_extractor/dependency.py +309 -0
  61. data/core/site-packages/pre_build/KafNafParser/feature_extractor/dependency.pyc +0 -0
  62. data/core/site-packages/pre_build/KafNafParser/features_data.py +131 -0
  63. data/core/site-packages/pre_build/KafNafParser/features_data.pyc +0 -0
  64. data/core/site-packages/pre_build/KafNafParser/header_data.py +127 -0
  65. data/core/site-packages/pre_build/KafNafParser/header_data.pyc +0 -0
  66. data/core/site-packages/pre_build/KafNafParser/opinion_data.py +211 -0
  67. data/core/site-packages/pre_build/KafNafParser/opinion_data.pyc +0 -0
  68. data/core/site-packages/pre_build/KafNafParser/references_data.py +23 -0
  69. data/core/site-packages/pre_build/KafNafParser/references_data.pyc +0 -0
  70. data/core/site-packages/pre_build/KafNafParser/span_data.py +63 -0
  71. data/core/site-packages/pre_build/KafNafParser/span_data.pyc +0 -0
  72. data/core/site-packages/pre_build/KafNafParser/term_data.py +111 -0
  73. data/core/site-packages/pre_build/KafNafParser/term_data.pyc +0 -0
  74. data/core/site-packages/pre_build/KafNafParser/term_sentiment_data.py +42 -0
  75. data/core/site-packages/pre_build/KafNafParser/term_sentiment_data.pyc +0 -0
  76. data/core/site-packages/pre_build/KafNafParser/text_data.py +99 -0
  77. data/core/site-packages/pre_build/KafNafParser/text_data.pyc +0 -0
  78. data/core/site-packages/pre_build/VUA_pylib-1.5-py2.7.egg-info/PKG-INFO +10 -0
  79. data/core/site-packages/pre_build/VUA_pylib-1.5-py2.7.egg-info/SOURCES.txt +14 -0
  80. data/core/site-packages/pre_build/VUA_pylib-1.5-py2.7.egg-info/dependency_links.txt +1 -0
  81. data/core/site-packages/pre_build/VUA_pylib-1.5-py2.7.egg-info/installed-files.txt +23 -0
  82. data/core/site-packages/pre_build/VUA_pylib-1.5-py2.7.egg-info/top_level.txt +1 -0
  83. data/core/site-packages/pre_build/VUA_pylib/__init__.py +1 -0
  84. data/core/site-packages/pre_build/VUA_pylib/__init__.pyc +0 -0
  85. data/core/site-packages/pre_build/VUA_pylib/common/__init__.py +1 -0
  86. data/core/site-packages/pre_build/VUA_pylib/common/__init__.pyc +0 -0
  87. data/core/site-packages/pre_build/VUA_pylib/common/common.py +28 -0
  88. data/core/site-packages/pre_build/VUA_pylib/common/common.pyc +0 -0
  89. data/core/site-packages/pre_build/VUA_pylib/corpus_reader/__init__.py +1 -0
  90. data/core/site-packages/pre_build/VUA_pylib/corpus_reader/__init__.pyc +0 -0
  91. data/core/site-packages/pre_build/VUA_pylib/corpus_reader/google_web_nl.py +156 -0
  92. data/core/site-packages/pre_build/VUA_pylib/corpus_reader/google_web_nl.pyc +0 -0
  93. data/core/site-packages/pre_build/VUA_pylib/io_utils/__init__.py +1 -0
  94. data/core/site-packages/pre_build/VUA_pylib/io_utils/__init__.pyc +0 -0
  95. data/core/site-packages/pre_build/VUA_pylib/io_utils/feature_file.py +121 -0
  96. data/core/site-packages/pre_build/VUA_pylib/io_utils/feature_file.pyc +0 -0
  97. data/core/site-packages/pre_build/VUA_pylib/lexicon/__init__.py +1 -0
  98. data/core/site-packages/pre_build/VUA_pylib/lexicon/__init__.pyc +0 -0
  99. data/core/site-packages/pre_build/VUA_pylib/lexicon/lexicon.py +72 -0
  100. data/core/site-packages/pre_build/VUA_pylib/lexicon/lexicon.pyc +0 -0
  101. data/core/site-packages/pre_build/VUKafParserPy-1.0-py2.7.egg-info/PKG-INFO +10 -0
  102. data/core/site-packages/pre_build/VUKafParserPy-1.0-py2.7.egg-info/SOURCES.txt +7 -0
  103. data/core/site-packages/pre_build/VUKafParserPy-1.0-py2.7.egg-info/dependency_links.txt +1 -0
  104. data/core/site-packages/pre_build/VUKafParserPy-1.0-py2.7.egg-info/installed-files.txt +11 -0
  105. data/core/site-packages/pre_build/VUKafParserPy-1.0-py2.7.egg-info/top_level.txt +1 -0
  106. data/core/site-packages/pre_build/VUKafParserPy/KafDataObjectsMod.py +165 -0
  107. data/core/site-packages/pre_build/VUKafParserPy/KafDataObjectsMod.pyc +0 -0
  108. data/core/site-packages/pre_build/VUKafParserPy/KafParserMod.py +439 -0
  109. data/core/site-packages/pre_build/VUKafParserPy/KafParserMod.pyc +0 -0
  110. data/core/site-packages/pre_build/VUKafParserPy/__init__.py +7 -0
  111. data/core/site-packages/pre_build/VUKafParserPy/__init__.pyc +0 -0
  112. data/core/vendor/src/crfsuite/AUTHORS +1 -0
  113. data/core/vendor/src/crfsuite/COPYING +27 -0
  114. data/core/vendor/src/crfsuite/ChangeLog +103 -0
  115. data/core/vendor/src/crfsuite/INSTALL +236 -0
  116. data/core/vendor/src/crfsuite/Makefile.am +19 -0
  117. data/core/vendor/src/crfsuite/Makefile.in +783 -0
  118. data/core/vendor/src/crfsuite/README +183 -0
  119. data/core/vendor/src/crfsuite/aclocal.m4 +9018 -0
  120. data/core/vendor/src/crfsuite/autogen.sh +38 -0
  121. data/core/vendor/src/crfsuite/compile +143 -0
  122. data/core/vendor/src/crfsuite/config.guess +1502 -0
  123. data/core/vendor/src/crfsuite/config.h.in +198 -0
  124. data/core/vendor/src/crfsuite/config.sub +1714 -0
  125. data/core/vendor/src/crfsuite/configure +14273 -0
  126. data/core/vendor/src/crfsuite/configure.in +149 -0
  127. data/core/vendor/src/crfsuite/crfsuite.sln +42 -0
  128. data/core/vendor/src/crfsuite/depcomp +630 -0
  129. data/core/vendor/src/crfsuite/example/chunking.py +49 -0
  130. data/core/vendor/src/crfsuite/example/crfutils.py +179 -0
  131. data/core/vendor/src/crfsuite/example/ner.py +270 -0
  132. data/core/vendor/src/crfsuite/example/pos.py +78 -0
  133. data/core/vendor/src/crfsuite/example/template.py +88 -0
  134. data/core/vendor/src/crfsuite/frontend/Makefile.am +29 -0
  135. data/core/vendor/src/crfsuite/frontend/Makefile.in +640 -0
  136. data/core/vendor/src/crfsuite/frontend/dump.c +116 -0
  137. data/core/vendor/src/crfsuite/frontend/frontend.vcxproj +129 -0
  138. data/core/vendor/src/crfsuite/frontend/iwa.c +273 -0
  139. data/core/vendor/src/crfsuite/frontend/iwa.h +65 -0
  140. data/core/vendor/src/crfsuite/frontend/learn.c +439 -0
  141. data/core/vendor/src/crfsuite/frontend/main.c +137 -0
  142. data/core/vendor/src/crfsuite/frontend/option.c +93 -0
  143. data/core/vendor/src/crfsuite/frontend/option.h +86 -0
  144. data/core/vendor/src/crfsuite/frontend/readdata.h +38 -0
  145. data/core/vendor/src/crfsuite/frontend/reader.c +136 -0
  146. data/core/vendor/src/crfsuite/frontend/tag.c +427 -0
  147. data/core/vendor/src/crfsuite/genbinary.sh.in +15 -0
  148. data/core/vendor/src/crfsuite/include/Makefile.am +11 -0
  149. data/core/vendor/src/crfsuite/include/Makefile.in +461 -0
  150. data/core/vendor/src/crfsuite/include/crfsuite.h +1063 -0
  151. data/core/vendor/src/crfsuite/include/crfsuite.hpp +555 -0
  152. data/core/vendor/src/crfsuite/include/crfsuite_api.hpp +400 -0
  153. data/core/vendor/src/crfsuite/include/os.h +61 -0
  154. data/core/vendor/src/crfsuite/install-sh +520 -0
  155. data/core/vendor/src/crfsuite/lib/cqdb/COPYING +28 -0
  156. data/core/vendor/src/crfsuite/lib/cqdb/Makefile.am +21 -0
  157. data/core/vendor/src/crfsuite/lib/cqdb/Makefile.in +549 -0
  158. data/core/vendor/src/crfsuite/lib/cqdb/cqdb.vcxproj +86 -0
  159. data/core/vendor/src/crfsuite/lib/cqdb/include/cqdb.h +524 -0
  160. data/core/vendor/src/crfsuite/lib/cqdb/src/cqdb.c +587 -0
  161. data/core/vendor/src/crfsuite/lib/cqdb/src/lookup3.c +976 -0
  162. data/core/vendor/src/crfsuite/lib/crf/Makefile.am +46 -0
  163. data/core/vendor/src/crfsuite/lib/crf/Makefile.in +721 -0
  164. data/core/vendor/src/crfsuite/lib/crf/crf.vcxproj +216 -0
  165. data/core/vendor/src/crfsuite/lib/crf/src/crf1d.h +353 -0
  166. data/core/vendor/src/crfsuite/lib/crf/src/crf1d_context.c +705 -0
  167. data/core/vendor/src/crfsuite/lib/crf/src/crf1d_encode.c +943 -0
  168. data/core/vendor/src/crfsuite/lib/crf/src/crf1d_feature.c +352 -0
  169. data/core/vendor/src/crfsuite/lib/crf/src/crf1d_model.c +994 -0
  170. data/core/vendor/src/crfsuite/lib/crf/src/crf1d_tag.c +550 -0
  171. data/core/vendor/src/crfsuite/lib/crf/src/crfsuite.c +492 -0
  172. data/core/vendor/src/crfsuite/lib/crf/src/crfsuite_internal.h +236 -0
  173. data/core/vendor/src/crfsuite/lib/crf/src/crfsuite_train.c +272 -0
  174. data/core/vendor/src/crfsuite/lib/crf/src/dataset.c +106 -0
  175. data/core/vendor/src/crfsuite/lib/crf/src/dictionary.c +118 -0
  176. data/core/vendor/src/crfsuite/lib/crf/src/holdout.c +80 -0
  177. data/core/vendor/src/crfsuite/lib/crf/src/logging.c +91 -0
  178. data/core/vendor/src/crfsuite/lib/crf/src/logging.h +48 -0
  179. data/core/vendor/src/crfsuite/lib/crf/src/params.c +335 -0
  180. data/core/vendor/src/crfsuite/lib/crf/src/params.h +80 -0
  181. data/core/vendor/src/crfsuite/lib/crf/src/quark.c +172 -0
  182. data/core/vendor/src/crfsuite/lib/crf/src/quark.h +46 -0
  183. data/core/vendor/src/crfsuite/lib/crf/src/rumavl.c +1107 -0
  184. data/core/vendor/src/crfsuite/lib/crf/src/rumavl.h +160 -0
  185. data/core/vendor/src/crfsuite/lib/crf/src/train_arow.c +408 -0
  186. data/core/vendor/src/crfsuite/lib/crf/src/train_averaged_perceptron.c +242 -0
  187. data/core/vendor/src/crfsuite/lib/crf/src/train_l2sgd.c +507 -0
  188. data/core/vendor/src/crfsuite/lib/crf/src/train_lbfgs.c +338 -0
  189. data/core/vendor/src/crfsuite/lib/crf/src/train_passive_aggressive.c +435 -0
  190. data/core/vendor/src/crfsuite/lib/crf/src/vecmath.h +341 -0
  191. data/core/vendor/src/crfsuite/ltmain.sh +8413 -0
  192. data/core/vendor/src/crfsuite/missing +376 -0
  193. data/core/vendor/src/crfsuite/swig/Makefile.am +13 -0
  194. data/core/vendor/src/crfsuite/swig/Makefile.in +365 -0
  195. data/core/vendor/src/crfsuite/swig/crfsuite.cpp +2 -0
  196. data/core/vendor/src/crfsuite/swig/export.i +32 -0
  197. data/core/vendor/src/crfsuite/swig/python/README +92 -0
  198. data/core/vendor/src/crfsuite/swig/python/crfsuite.py +329 -0
  199. data/core/vendor/src/crfsuite/swig/python/export_wrap.cpp +14355 -0
  200. data/core/vendor/src/crfsuite/swig/python/export_wrap.h +63 -0
  201. data/core/vendor/src/crfsuite/swig/python/prepare.sh +9 -0
  202. data/core/vendor/src/crfsuite/swig/python/sample_tag.py +52 -0
  203. data/core/vendor/src/crfsuite/swig/python/sample_train.py +68 -0
  204. data/core/vendor/src/crfsuite/swig/python/setup.py +44 -0
  205. data/core/vendor/src/crfsuite/win32/stdint.h +679 -0
  206. data/core/vendor/src/liblbfgs/AUTHORS +1 -0
  207. data/core/vendor/src/liblbfgs/COPYING +22 -0
  208. data/core/vendor/src/liblbfgs/ChangeLog +120 -0
  209. data/core/vendor/src/liblbfgs/INSTALL +231 -0
  210. data/core/vendor/src/liblbfgs/Makefile.am +10 -0
  211. data/core/vendor/src/liblbfgs/Makefile.in +638 -0
  212. data/core/vendor/src/liblbfgs/NEWS +0 -0
  213. data/core/vendor/src/liblbfgs/README +71 -0
  214. data/core/vendor/src/liblbfgs/aclocal.m4 +6985 -0
  215. data/core/vendor/src/liblbfgs/autogen.sh +38 -0
  216. data/core/vendor/src/liblbfgs/config.guess +1411 -0
  217. data/core/vendor/src/liblbfgs/config.h.in +64 -0
  218. data/core/vendor/src/liblbfgs/config.sub +1500 -0
  219. data/core/vendor/src/liblbfgs/configure +21146 -0
  220. data/core/vendor/src/liblbfgs/configure.in +107 -0
  221. data/core/vendor/src/liblbfgs/depcomp +522 -0
  222. data/core/vendor/src/liblbfgs/include/lbfgs.h +745 -0
  223. data/core/vendor/src/liblbfgs/install-sh +322 -0
  224. data/core/vendor/src/liblbfgs/lbfgs.sln +26 -0
  225. data/core/vendor/src/liblbfgs/lib/Makefile.am +24 -0
  226. data/core/vendor/src/liblbfgs/lib/Makefile.in +499 -0
  227. data/core/vendor/src/liblbfgs/lib/arithmetic_ansi.h +133 -0
  228. data/core/vendor/src/liblbfgs/lib/arithmetic_sse_double.h +294 -0
  229. data/core/vendor/src/liblbfgs/lib/arithmetic_sse_float.h +298 -0
  230. data/core/vendor/src/liblbfgs/lib/lbfgs.c +1371 -0
  231. data/core/vendor/src/liblbfgs/lib/lib.vcxproj +95 -0
  232. data/core/vendor/src/liblbfgs/ltmain.sh +6426 -0
  233. data/core/vendor/src/liblbfgs/missing +353 -0
  234. data/core/vendor/src/liblbfgs/sample/Makefile.am +15 -0
  235. data/core/vendor/src/liblbfgs/sample/Makefile.in +433 -0
  236. data/core/vendor/src/liblbfgs/sample/sample.c +81 -0
  237. data/core/vendor/src/liblbfgs/sample/sample.cpp +126 -0
  238. data/core/vendor/src/liblbfgs/sample/sample.vcxproj +105 -0
  239. data/core/vendor/src/svm_light/LICENSE.txt +59 -0
  240. data/core/vendor/src/svm_light/Makefile +105 -0
  241. data/core/vendor/src/svm_light/kernel.h +40 -0
  242. data/core/vendor/src/svm_light/svm_classify.c +197 -0
  243. data/core/vendor/src/svm_light/svm_common.c +985 -0
  244. data/core/vendor/src/svm_light/svm_common.h +301 -0
  245. data/core/vendor/src/svm_light/svm_hideo.c +1062 -0
  246. data/core/vendor/src/svm_light/svm_learn.c +4147 -0
  247. data/core/vendor/src/svm_light/svm_learn.h +169 -0
  248. data/core/vendor/src/svm_light/svm_learn_main.c +397 -0
  249. data/core/vendor/src/svm_light/svm_loqo.c +211 -0
  250. data/ext/hack/Rakefile +17 -0
  251. data/ext/hack/support.rb +88 -0
  252. data/lib/opener/opinion_detectors/base.rb +112 -0
  253. data/lib/opener/opinion_detectors/base/version.rb +7 -0
  254. data/lib/opener/opinion_detectors/configuration_creator.rb +86 -0
  255. data/lib/opener/opinion_detectors/de.rb +7 -0
  256. data/lib/opener/opinion_detectors/en.rb +7 -0
  257. data/lib/opener/opinion_detectors/it.rb +7 -0
  258. data/lib/opener/opinion_detectors/nl.rb +6 -0
  259. data/opener-opinion-detector-base.gemspec +35 -0
  260. data/pre_build_requirements.txt +3 -0
  261. metadata +374 -0
@@ -0,0 +1,492 @@
1
+ /*
2
+ * CRFsuite library.
3
+ *
4
+ * Copyright (c) 2007-2010, Naoaki Okazaki
5
+ * All rights reserved.
6
+ *
7
+ * Redistribution and use in source and binary forms, with or without
8
+ * modification, are permitted provided that the following conditions are met:
9
+ * * Redistributions of source code must retain the above copyright
10
+ * notice, this list of conditions and the following disclaimer.
11
+ * * Redistributions in binary form must reproduce the above copyright
12
+ * notice, this list of conditions and the following disclaimer in the
13
+ * documentation and/or other materials provided with the distribution.
14
+ * * Neither the names of the authors nor the names of its contributors
15
+ * may be used to endorse or promote products derived from this
16
+ * software without specific prior written permission.
17
+ *
18
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
22
+ * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
23
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
24
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
25
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
26
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
27
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
28
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
+ */
30
+
31
+ /* $Id$ */
32
+
33
+ #include <os.h>
34
+
35
+ #include <stdarg.h>
36
+ #include <stdio.h>
37
+ #include <stdlib.h>
38
+ #include <string.h>
39
+
40
+ #include <crfsuite.h>
41
+ #include "logging.h"
42
+
43
+ int crf1de_create_instance(const char *iid, void **ptr);
44
+ int crfsuite_dictionary_create_instance(const char *interface, void **ptr);
45
+ int crf1m_create_instance_from_file(const char *filename, void **ptr);
46
+
47
+ int crfsuite_create_instance(const char *iid, void **ptr)
48
+ {
49
+ int ret =
50
+ crf1de_create_instance(iid, ptr) == 0 ||
51
+ crfsuite_dictionary_create_instance(iid, ptr) == 0;
52
+
53
+ return ret;
54
+ }
55
+
56
+ int crfsuite_create_instance_from_file(const char *filename, void **ptr)
57
+ {
58
+ int ret = crf1m_create_instance_from_file(filename, ptr);
59
+ return ret;
60
+ }
61
+
62
+
63
+
64
+ void crfsuite_attribute_init(crfsuite_attribute_t* cont)
65
+ {
66
+ memset(cont, 0, sizeof(*cont));
67
+ cont->value = 1;
68
+ }
69
+
70
+ void crfsuite_attribute_set(crfsuite_attribute_t* cont, int aid, floatval_t value)
71
+ {
72
+ crfsuite_attribute_init(cont);
73
+ cont->aid = aid;
74
+ cont->value = value;
75
+ }
76
+
77
+ void crfsuite_attribute_copy(crfsuite_attribute_t* dst, const crfsuite_attribute_t* src)
78
+ {
79
+ dst->aid = src->aid;
80
+ dst->value = src->value;
81
+ }
82
+
83
+ void crfsuite_attribute_swap(crfsuite_attribute_t* x, crfsuite_attribute_t* y)
84
+ {
85
+ crfsuite_attribute_t tmp = *x;
86
+ x->aid = y->aid;
87
+ x->value = y->value;
88
+ y->aid = tmp.aid;
89
+ y->value = tmp.value;
90
+ }
91
+
92
+
93
+
94
+ void crfsuite_item_init(crfsuite_item_t* item)
95
+ {
96
+ memset(item, 0, sizeof(*item));
97
+ }
98
+
99
+ void crfsuite_item_init_n(crfsuite_item_t* item, int num_contents)
100
+ {
101
+ crfsuite_item_init(item);
102
+ item->num_contents = num_contents;
103
+ item->cap_contents = num_contents;
104
+ item->contents = (crfsuite_attribute_t*)calloc(num_contents, sizeof(crfsuite_attribute_t));
105
+ }
106
+
107
+ void crfsuite_item_finish(crfsuite_item_t* item)
108
+ {
109
+ free(item->contents);
110
+ crfsuite_item_init(item);
111
+ }
112
+
113
+ void crfsuite_item_copy(crfsuite_item_t* dst, const crfsuite_item_t* src)
114
+ {
115
+ int i;
116
+
117
+ dst->num_contents = src->num_contents;
118
+ dst->cap_contents = src->cap_contents;
119
+ dst->contents = (crfsuite_attribute_t*)calloc(dst->num_contents, sizeof(crfsuite_attribute_t));
120
+ for (i = 0;i < dst->num_contents;++i) {
121
+ crfsuite_attribute_copy(&dst->contents[i], &src->contents[i]);
122
+ }
123
+ }
124
+
125
+ void crfsuite_item_swap(crfsuite_item_t* x, crfsuite_item_t* y)
126
+ {
127
+ crfsuite_item_t tmp = *x;
128
+ x->num_contents = y->num_contents;
129
+ x->cap_contents = y->cap_contents;
130
+ x->contents = y->contents;
131
+ y->num_contents = tmp.num_contents;
132
+ y->cap_contents = tmp.cap_contents;
133
+ y->contents = tmp.contents;
134
+ }
135
+
136
+ int crfsuite_item_append_attribute(crfsuite_item_t* item, const crfsuite_attribute_t* cont)
137
+ {
138
+ if (item->cap_contents <= item->num_contents) {
139
+ item->cap_contents = (item->cap_contents + 1) * 2;
140
+ item->contents = (crfsuite_attribute_t*)realloc(
141
+ item->contents, sizeof(crfsuite_attribute_t) * item->cap_contents);
142
+ }
143
+ crfsuite_attribute_copy(&item->contents[item->num_contents++], cont);
144
+ return 0;
145
+ }
146
+
147
+ int crfsuite_item_empty(crfsuite_item_t* item)
148
+ {
149
+ return (item->num_contents == 0);
150
+ }
151
+
152
+
153
+
154
+
155
+ void crfsuite_instance_init(crfsuite_instance_t* inst)
156
+ {
157
+ memset(inst, 0, sizeof(*inst));
158
+ }
159
+
160
+ void crfsuite_instance_init_n(crfsuite_instance_t* inst, int num_items)
161
+ {
162
+ crfsuite_instance_init(inst);
163
+ inst->num_items = num_items;
164
+ inst->cap_items = num_items;
165
+ inst->items = (crfsuite_item_t*)calloc(num_items, sizeof(crfsuite_item_t));
166
+ inst->labels = (int*)calloc(num_items, sizeof(int));
167
+ }
168
+
169
+ void crfsuite_instance_finish(crfsuite_instance_t* inst)
170
+ {
171
+ int i;
172
+
173
+ for (i = 0;i < inst->num_items;++i) {
174
+ crfsuite_item_finish(&inst->items[i]);
175
+ }
176
+ free(inst->labels);
177
+ free(inst->items);
178
+ crfsuite_instance_init(inst);
179
+ }
180
+
181
+ void crfsuite_instance_copy(crfsuite_instance_t* dst, const crfsuite_instance_t* src)
182
+ {
183
+ int i;
184
+
185
+ dst->num_items = src->num_items;
186
+ dst->cap_items = src->cap_items;
187
+ dst->items = (crfsuite_item_t*)calloc(dst->num_items, sizeof(crfsuite_item_t));
188
+ dst->labels = (int*)calloc(dst->num_items, sizeof(int));
189
+ dst->group = src->group;
190
+ for (i = 0;i < dst->num_items;++i) {
191
+ crfsuite_item_copy(&dst->items[i], &src->items[i]);
192
+ dst->labels[i] = src->labels[i];
193
+ }
194
+ }
195
+
196
+ void crfsuite_instance_swap(crfsuite_instance_t* x, crfsuite_instance_t* y)
197
+ {
198
+ crfsuite_instance_t tmp = *x;
199
+ x->num_items = y->num_items;
200
+ x->cap_items = y->cap_items;
201
+ x->items = y->items;
202
+ x->labels = y->labels;
203
+ x->group = y->group;
204
+ y->num_items = tmp.num_items;
205
+ y->cap_items = tmp.cap_items;
206
+ y->items = tmp.items;
207
+ y->labels = tmp.labels;
208
+ y->group = tmp.group;
209
+ }
210
+
211
+ int crfsuite_instance_append(crfsuite_instance_t* inst, const crfsuite_item_t* item, int label)
212
+ {
213
+ if (inst->cap_items <= inst->num_items) {
214
+ inst->cap_items = (inst->cap_items + 1) * 2;
215
+ inst->items = (crfsuite_item_t*)realloc(inst->items, sizeof(crfsuite_item_t) * inst->cap_items);
216
+ inst->labels = (int*)realloc(inst->labels, sizeof(int) * inst->cap_items);
217
+ }
218
+ crfsuite_item_copy(&inst->items[inst->num_items], item);
219
+ inst->labels[inst->num_items] = label;
220
+ ++inst->num_items;
221
+ return 0;
222
+ }
223
+
224
+ int crfsuite_instance_empty(crfsuite_instance_t* inst)
225
+ {
226
+ return (inst->num_items == 0);
227
+ }
228
+
229
+
230
+
231
+
232
+ void crfsuite_data_init(crfsuite_data_t* data)
233
+ {
234
+ memset(data, 0, sizeof(*data));
235
+ }
236
+
237
+ void crfsuite_data_init_n(crfsuite_data_t* data, int n)
238
+ {
239
+ crfsuite_data_init(data);
240
+ data->num_instances = n;
241
+ data->cap_instances = n;
242
+ data->instances = (crfsuite_instance_t*)calloc(n, sizeof(crfsuite_instance_t));
243
+ }
244
+
245
+ void crfsuite_data_finish(crfsuite_data_t* data)
246
+ {
247
+ int i;
248
+
249
+ for (i = 0;i < data->num_instances;++i) {
250
+ crfsuite_instance_finish(&data->instances[i]);
251
+ }
252
+ free(data->instances);
253
+ crfsuite_data_init(data);
254
+ }
255
+
256
+ void crfsuite_data_copy(crfsuite_data_t* dst, const crfsuite_data_t* src)
257
+ {
258
+ int i;
259
+
260
+ dst->num_instances = src->num_instances;
261
+ dst->cap_instances = src->cap_instances;
262
+ dst->instances = (crfsuite_instance_t*)calloc(dst->num_instances, sizeof(crfsuite_instance_t));
263
+ for (i = 0;i < dst->num_instances;++i) {
264
+ crfsuite_instance_copy(&dst->instances[i], &src->instances[i]);
265
+ }
266
+ }
267
+
268
+ void crfsuite_data_swap(crfsuite_data_t* x, crfsuite_data_t* y)
269
+ {
270
+ crfsuite_data_t tmp = *x;
271
+ x->num_instances = y->num_instances;
272
+ x->cap_instances = y->cap_instances;
273
+ x->instances = y->instances;
274
+ y->num_instances = tmp.num_instances;
275
+ y->cap_instances = tmp.cap_instances;
276
+ y->instances = tmp.instances;
277
+ }
278
+
279
+ int crfsuite_data_append(crfsuite_data_t* data, const crfsuite_instance_t* inst)
280
+ {
281
+ if (0 < inst->num_items) {
282
+ if (data->cap_instances <= data->num_instances) {
283
+ data->cap_instances = (data->cap_instances + 1) * 2;
284
+ data->instances = (crfsuite_instance_t*)realloc(
285
+ data->instances, sizeof(crfsuite_instance_t) * data->cap_instances);
286
+ }
287
+ crfsuite_instance_copy(&data->instances[data->num_instances++], inst);
288
+ }
289
+ return 0;
290
+ }
291
+
292
+ int crfsuite_data_maxlength(crfsuite_data_t* data)
293
+ {
294
+ int i, T = 0;
295
+ for (i = 0;i < data->num_instances;++i) {
296
+ if (T < data->instances[i].num_items) {
297
+ T = data->instances[i].num_items;
298
+ }
299
+ }
300
+ return T;
301
+ }
302
+
303
+ int crfsuite_data_totalitems(crfsuite_data_t* data)
304
+ {
305
+ int i, n = 0;
306
+ for (i = 0;i < data->num_instances;++i) {
307
+ n += data->instances[i].num_items;
308
+ }
309
+ return n;
310
+ }
311
+
312
+ static char *safe_strncpy(char *dst, const char *src, size_t n)
313
+ {
314
+ strncpy(dst, src, n-1);
315
+ dst[n-1] = 0;
316
+ return dst;
317
+ }
318
+
319
+ void crfsuite_evaluation_init(crfsuite_evaluation_t* eval, int n)
320
+ {
321
+ memset(eval, 0, sizeof(*eval));
322
+ eval->tbl = (crfsuite_label_evaluation_t*)calloc(n+1, sizeof(crfsuite_label_evaluation_t));
323
+ if (eval->tbl != NULL) {
324
+ eval->num_labels = n;
325
+ }
326
+ }
327
+
328
+ void crfsuite_evaluation_clear(crfsuite_evaluation_t* eval)
329
+ {
330
+ int i;
331
+ for (i = 0;i <= eval->num_labels;++i) {
332
+ memset(&eval->tbl[i], 0, sizeof(eval->tbl[i]));
333
+ }
334
+
335
+ eval->item_total_correct = 0;
336
+ eval->item_total_num = 0;
337
+ eval->item_total_model = 0;
338
+ eval->item_total_observation = 0;
339
+ eval->item_accuracy = 0;
340
+
341
+ eval->inst_total_correct = 0;
342
+ eval->inst_total_num = 0;
343
+ eval->inst_accuracy = 0;
344
+
345
+ eval->macro_precision = 0;
346
+ eval->macro_recall = 0;
347
+ eval->macro_fmeasure = 0;
348
+ }
349
+
350
+ void crfsuite_evaluation_finish(crfsuite_evaluation_t* eval)
351
+ {
352
+ free(eval->tbl);
353
+ memset(eval, 0, sizeof(*eval));
354
+ }
355
+
356
+ int crfsuite_evaluation_accmulate(crfsuite_evaluation_t* eval, const int* reference, const int* prediction, int T)
357
+ {
358
+ int t, nc = 0;
359
+
360
+ for (t = 0;t < T;++t) {
361
+ int lr = reference[t];
362
+ int lt = prediction[t];
363
+
364
+ if (eval->num_labels <= lr || eval->num_labels <= lt) {
365
+ return 1;
366
+ }
367
+
368
+ ++eval->tbl[lr].num_observation;
369
+ ++eval->tbl[lt].num_model;
370
+ if (lr == lt) {
371
+ ++eval->tbl[lr].num_correct;
372
+ ++nc;
373
+ }
374
+ ++eval->item_total_num;
375
+ }
376
+
377
+ if (nc == T) {
378
+ ++eval->inst_total_correct;
379
+ }
380
+ ++eval->inst_total_num;
381
+
382
+ return 0;
383
+ }
384
+
385
+ void crfsuite_evaluation_finalize(crfsuite_evaluation_t* eval)
386
+ {
387
+ int i;
388
+
389
+ for (i = 0;i <= eval->num_labels;++i) {
390
+ crfsuite_label_evaluation_t* lev = &eval->tbl[i];
391
+
392
+ /* Do not evaluate labels that does not in the test data. */
393
+ if (lev->num_observation == 0) {
394
+ continue;
395
+ }
396
+
397
+ /* Sum the number of correct labels for accuracy calculation. */
398
+ eval->item_total_correct += lev->num_correct;
399
+ eval->item_total_model += lev->num_model;
400
+ eval->item_total_observation += lev->num_observation;
401
+
402
+ /* Initialize the precision, recall, and f1-measure values. */
403
+ lev->precision = 0;
404
+ lev->recall = 0;
405
+ lev->fmeasure = 0;
406
+
407
+ /* Compute the precision, recall, and f1-measure values. */
408
+ if (lev->num_model > 0) {
409
+ lev->precision = lev->num_correct / (double)lev->num_model;
410
+ }
411
+ if (lev->num_observation > 0) {
412
+ lev->recall = lev->num_correct / (double)lev->num_observation;
413
+ }
414
+ if (lev->precision + lev->recall > 0) {
415
+ lev->fmeasure = lev->precision * lev->recall * 2 / (lev->precision + lev->recall);
416
+ }
417
+
418
+ /* Exclude unknown labels from calculation of macro-average values. */
419
+ if (i != eval->num_labels) {
420
+ eval->macro_precision += lev->precision;
421
+ eval->macro_recall += lev->recall;
422
+ eval->macro_fmeasure += lev->fmeasure;
423
+ }
424
+ }
425
+
426
+ /* Copute the macro precision, recall, and f1-measure values. */
427
+ eval->macro_precision /= eval->num_labels;
428
+ eval->macro_recall /= eval->num_labels;
429
+ eval->macro_fmeasure /= eval->num_labels;
430
+
431
+ /* Compute the item accuracy. */
432
+ eval->item_accuracy = 0;
433
+ if (0 < eval->item_total_num) {
434
+ eval->item_accuracy = eval->item_total_correct / (double)eval->item_total_num;
435
+ }
436
+
437
+ /* Compute the instance accuracy. */
438
+ eval->inst_accuracy = 0;
439
+ if (0 < eval->inst_total_num) {
440
+ eval->inst_accuracy = eval->inst_total_correct / (double)eval->inst_total_num;
441
+ }
442
+ }
443
+
444
+ void crfsuite_evaluation_output(crfsuite_evaluation_t* eval, crfsuite_dictionary_t* labels, crfsuite_logging_callback cbm, void *instance)
445
+ {
446
+ int i;
447
+ const char *lstr = NULL;
448
+ logging_t lg;
449
+
450
+ lg.func = cbm;
451
+ lg.instance = instance;
452
+
453
+ logging(&lg, "Performance by label (#match, #model, #ref) (precision, recall, F1):\n");
454
+
455
+ for (i = 0;i < eval->num_labels;++i) {
456
+ const crfsuite_label_evaluation_t* lev = &eval->tbl[i];
457
+
458
+ labels->to_string(labels, i, &lstr);
459
+ if (lstr == NULL) lstr = "[UNKNOWN]";
460
+
461
+ if (lev->num_observation == 0) {
462
+ logging(&lg, " %s: (%d, %d, %d) (******, ******, ******)\n",
463
+ lstr, lev->num_correct, lev->num_model, lev->num_observation
464
+ );
465
+ } else {
466
+ logging(&lg, " %s: (%d, %d, %d) (%1.4f, %1.4f, %1.4f)\n",
467
+ lstr, lev->num_correct, lev->num_model, lev->num_observation,
468
+ lev->precision, lev->recall, lev->fmeasure
469
+ );
470
+ }
471
+ labels->free(labels, lstr);
472
+ }
473
+ logging(&lg, "Macro-average precision, recall, F1: (%f, %f, %f)\n",
474
+ eval->macro_precision, eval->macro_recall, eval->macro_fmeasure
475
+ );
476
+ logging(&lg, "Item accuracy: %d / %d (%1.4f)\n",
477
+ eval->item_total_correct, eval->item_total_num, eval->item_accuracy
478
+ );
479
+ logging(&lg, "Instance accuracy: %d / %d (%1.4f)\n",
480
+ eval->inst_total_correct, eval->inst_total_num, eval->inst_accuracy
481
+ );
482
+ }
483
+
484
+ int crfsuite_interlocked_increment(int *count)
485
+ {
486
+ return ++(*count);
487
+ }
488
+
489
+ int crfsuite_interlocked_decrement(int *count)
490
+ {
491
+ return --(*count);
492
+ }