biblicit 1.0 → 2.0.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (406) hide show
  1. data/.gitmodules +3 -0
  2. data/Gemfile +1 -1
  3. data/README.md +125 -30
  4. data/Rakefile +22 -0
  5. data/biblicit.gemspec +9 -7
  6. data/lib/biblicit/cb2bib.rb +10 -11
  7. data/lib/biblicit/citeseer.rb +14 -26
  8. data/lib/biblicit/extractor.rb +40 -19
  9. data/lib/biblicit/parscit.rb +38 -0
  10. data/parscit/.gitignore +8 -0
  11. data/parscit/CHANGELOG +125 -0
  12. data/parscit/COPYING +674 -0
  13. data/parscit/COPYING.LESSER +165 -0
  14. data/parscit/INSTALL +105 -0
  15. data/parscit/README +97 -0
  16. data/{perl/ParsCit/README.TXT → parscit/USAGE} +25 -15
  17. data/parscit/bin/archtest.pl +31 -0
  18. data/parscit/bin/citeExtract.pl +562 -0
  19. data/parscit/bin/conlleval.pl +315 -0
  20. data/parscit/bin/headExtract.pl +40 -0
  21. data/parscit/bin/parsHed/convert2TokenLevel.pl +138 -0
  22. data/parscit/bin/parsHed/keywordGen.pl +308 -0
  23. data/parscit/bin/parsHed/parseXmlHeader.pl +141 -0
  24. data/parscit/bin/parsHed/redo.parsHed.pl +198 -0
  25. data/parscit/bin/parsHed/tr2crfpp_parsHed.pl +521 -0
  26. data/parscit/bin/parseRefStrings.pl +102 -0
  27. data/parscit/bin/phOutput2xml.pl +223 -0
  28. data/parscit/bin/redo.parsCit.pl +105 -0
  29. data/parscit/bin/sectExtract.pl +149 -0
  30. data/parscit/bin/sectLabel/README +110 -0
  31. data/parscit/bin/sectLabel/README.txt +110 -0
  32. data/parscit/bin/sectLabel/genericSect/crossValidation.rb +98 -0
  33. data/parscit/bin/sectLabel/genericSect/extractFeature.rb +104 -0
  34. data/parscit/bin/sectLabel/genericSectExtract.rb +53 -0
  35. data/parscit/bin/sectLabel/getStructureInfo.pl +156 -0
  36. data/parscit/bin/sectLabel/processOmniXML.pl +1427 -0
  37. data/parscit/bin/sectLabel/processOmniXML_new.pl +1025 -0
  38. data/parscit/bin/sectLabel/processOmniXMLv2.pl +1529 -0
  39. data/parscit/bin/sectLabel/processOmniXMLv3.pl +964 -0
  40. data/parscit/bin/sectLabel/redo.sectLabel.pl +219 -0
  41. data/parscit/bin/sectLabel/simplifyOmniXML.pl +382 -0
  42. data/parscit/bin/sectLabel/single2multi.pl +190 -0
  43. data/parscit/bin/sectLabel/tr2crfpp.pl +158 -0
  44. data/parscit/bin/tr2crfpp.pl +260 -0
  45. data/parscit/bin/xml2train.pl +193 -0
  46. data/parscit/lib/CSXUtil/SafeText.pm +130 -0
  47. data/parscit/lib/Omni/Config.pm +93 -0
  48. data/parscit/lib/Omni/Omnicell.pm +263 -0
  49. data/parscit/lib/Omni/Omnicol.pm +292 -0
  50. data/parscit/lib/Omni/Omnidd.pm +328 -0
  51. data/parscit/lib/Omni/Omnidoc.pm +153 -0
  52. data/parscit/lib/Omni/Omniframe.pm +223 -0
  53. data/parscit/lib/Omni/Omniline.pm +423 -0
  54. data/parscit/lib/Omni/Omnipage.pm +282 -0
  55. data/parscit/lib/Omni/Omnipara.pm +232 -0
  56. data/parscit/lib/Omni/Omnirun.pm +303 -0
  57. data/parscit/lib/Omni/Omnitable.pm +336 -0
  58. data/parscit/lib/Omni/Omniword.pm +162 -0
  59. data/parscit/lib/Omni/Traversal.pm +313 -0
  60. data/parscit/lib/ParsCit/.PostProcess.pm.swp +0 -0
  61. data/parscit/lib/ParsCit/Citation.pm +737 -0
  62. data/parscit/lib/ParsCit/CitationContext.pm +220 -0
  63. data/parscit/lib/ParsCit/Config.pm +35 -0
  64. data/parscit/lib/ParsCit/Controller.pm +653 -0
  65. data/parscit/lib/ParsCit/PostProcess.pm +505 -0
  66. data/parscit/lib/ParsCit/PreProcess.pm +1041 -0
  67. data/parscit/lib/ParsCit/Tr2crfpp.pm +1195 -0
  68. data/parscit/lib/ParsHed/Config.pm +49 -0
  69. data/parscit/lib/ParsHed/Controller.pm +143 -0
  70. data/parscit/lib/ParsHed/PostProcess.pm +322 -0
  71. data/parscit/lib/ParsHed/Tr2crfpp.pm +448 -0
  72. data/{perl/ParsCit/lib/ParsCit/Tr2crfpp.pm → parscit/lib/ParsHed/Tr2crfpp_token.pm} +22 -21
  73. data/parscit/lib/SectLabel/AAMatching.pm +1949 -0
  74. data/parscit/lib/SectLabel/Config.pm +88 -0
  75. data/parscit/lib/SectLabel/Controller.pm +332 -0
  76. data/parscit/lib/SectLabel/PostProcess.pm +425 -0
  77. data/parscit/lib/SectLabel/PreProcess.pm +116 -0
  78. data/parscit/lib/SectLabel/Tr2crfpp.pm +1246 -0
  79. data/parscit/resources/parsCit.model +0 -0
  80. data/parscit/resources/parsCit.split.model +0 -0
  81. data/{perl/ParsCit → parscit}/resources/parsCitDict.txt +205 -0
  82. data/parscit/resources/parsHed/bigram +10 -0
  83. data/parscit/resources/parsHed/keywords +10 -0
  84. data/parscit/resources/parsHed/parsHed.model +0 -0
  85. data/parscit/resources/parsHed/parsHed.template +178 -0
  86. data/parscit/resources/sectLabel/affiliation.model +0 -0
  87. data/parscit/resources/sectLabel/author.model +0 -0
  88. data/parscit/resources/sectLabel/funcWord +320 -0
  89. data/parscit/resources/sectLabel/genericSect.model +0 -0
  90. data/parscit/resources/sectLabel/sectLabel.config +42 -0
  91. data/parscit/resources/sectLabel/sectLabel.configXml +42 -0
  92. data/parscit/resources/sectLabel/sectLabel.model +0 -0
  93. data/sh/convert_to_text.sh +20 -0
  94. data/spec/biblicit/extractor_spec.rb +121 -0
  95. data/spec/fixtures/Review_of_Michael_Tyes_Consciousness_Revisited.docx +0 -0
  96. data/spec/fixtures/critical-infrastructures.ps +63951 -0
  97. data/spec/fixtures/txt/E06-1050.txt +867 -0
  98. data/spec/fixtures/txt/sample1.txt +902 -0
  99. data/spec/fixtures/txt/sample2.txt +394 -0
  100. data/spec/spec_helper.rb +3 -0
  101. data/{perl → svm-header-parse}/HeaderParseService/lib/HeaderParse/API/Function.pm +2 -20
  102. data/{perl → svm-header-parse}/HeaderParseService/lib/HeaderParse/API/MultiClassChunking.pm +0 -7
  103. data/{perl → svm-header-parse}/HeaderParseService/lib/HeaderParse/API/Parser.pm +0 -2
  104. data/{perl → svm-header-parse}/HeaderParseService/lib/HeaderParse/API/ParserMethods.pm +0 -7
  105. data/{perl → svm-header-parse}/HeaderParseService/lib/HeaderParse/Config/API_Config.pm +6 -1
  106. data/svm-header-parse/HeaderParseService/tmp/.gitignore +4 -0
  107. data/svm-header-parse/extract.pl +75 -0
  108. metadata +351 -317
  109. data/perl/DocFilter/lib/DocFilter/Config.pm +0 -35
  110. data/perl/DocFilter/lib/DocFilter/Filter.pm +0 -51
  111. data/perl/FileConversionService/README.TXT +0 -11
  112. data/perl/FileConversionService/converters/PDFBox/pdfbox-app-1.7.1.jar +0 -0
  113. data/perl/FileConversionService/lib/CSXUtil/SafeText.pm +0 -140
  114. data/perl/FileConversionService/lib/FileConverter/CheckSum.pm +0 -77
  115. data/perl/FileConversionService/lib/FileConverter/Compression.pm +0 -137
  116. data/perl/FileConversionService/lib/FileConverter/Config.pm +0 -57
  117. data/perl/FileConversionService/lib/FileConverter/Controller.pm +0 -191
  118. data/perl/FileConversionService/lib/FileConverter/JODConverter.pm +0 -61
  119. data/perl/FileConversionService/lib/FileConverter/PDFBox.pm +0 -69
  120. data/perl/FileConversionService/lib/FileConverter/PSConverter.pm +0 -69
  121. data/perl/FileConversionService/lib/FileConverter/PSToText.pm +0 -88
  122. data/perl/FileConversionService/lib/FileConverter/Prescript.pm +0 -68
  123. data/perl/FileConversionService/lib/FileConverter/TET.pm +0 -75
  124. data/perl/FileConversionService/lib/FileConverter/Utils.pm +0 -130
  125. data/perl/HeaderParseService/lib/CSXUtil/SafeText.pm +0 -140
  126. data/perl/HeaderParseService/resources/data/EbizHeaders.txt +0 -24330
  127. data/perl/HeaderParseService/resources/data/EbizHeaders.txt.parsed +0 -27506
  128. data/perl/HeaderParseService/resources/data/EbizHeaders.txt.parsed.old +0 -26495
  129. data/perl/HeaderParseService/resources/data/tagged_headers.txt +0 -40668
  130. data/perl/HeaderParseService/resources/data/test_header.txt +0 -31
  131. data/perl/HeaderParseService/resources/data/test_header.txt.parsed +0 -31
  132. data/perl/HeaderParseService/tmp/tmpVec_1156237246.08016_test1 +0 -23
  133. data/perl/HeaderParseService/tmp/tmpVec_1156237246.08016_test10 +0 -23
  134. data/perl/HeaderParseService/tmp/tmpVec_1156237246.08016_test11 +0 -23
  135. data/perl/HeaderParseService/tmp/tmpVec_1156237246.08016_test12 +0 -23
  136. data/perl/HeaderParseService/tmp/tmpVec_1156237246.08016_test13 +0 -23
  137. data/perl/HeaderParseService/tmp/tmpVec_1156237246.08016_test14 +0 -23
  138. data/perl/HeaderParseService/tmp/tmpVec_1156237246.08016_test15 +0 -23
  139. data/perl/HeaderParseService/tmp/tmpVec_1156237246.08016_test2 +0 -23
  140. data/perl/HeaderParseService/tmp/tmpVec_1156237246.08016_test3 +0 -23
  141. data/perl/HeaderParseService/tmp/tmpVec_1156237246.08016_test4 +0 -23
  142. data/perl/HeaderParseService/tmp/tmpVec_1156237246.08016_test5 +0 -23
  143. data/perl/HeaderParseService/tmp/tmpVec_1156237246.08016_test6 +0 -23
  144. data/perl/HeaderParseService/tmp/tmpVec_1156237246.08016_test7 +0 -23
  145. data/perl/HeaderParseService/tmp/tmpVec_1156237246.08016_test8 +0 -23
  146. data/perl/HeaderParseService/tmp/tmpVec_1156237246.08016_test9 +0 -23
  147. data/perl/HeaderParseService/tmp/tmpVec_914027525.276114_test1 +0 -23
  148. data/perl/HeaderParseService/tmp/tmpVec_914027525.276114_test10 +0 -23
  149. data/perl/HeaderParseService/tmp/tmpVec_914027525.276114_test11 +0 -23
  150. data/perl/HeaderParseService/tmp/tmpVec_914027525.276114_test12 +0 -23
  151. data/perl/HeaderParseService/tmp/tmpVec_914027525.276114_test13 +0 -23
  152. data/perl/HeaderParseService/tmp/tmpVec_914027525.276114_test14 +0 -23
  153. data/perl/HeaderParseService/tmp/tmpVec_914027525.276114_test15 +0 -23
  154. data/perl/HeaderParseService/tmp/tmpVec_914027525.276114_test2 +0 -23
  155. data/perl/HeaderParseService/tmp/tmpVec_914027525.276114_test3 +0 -23
  156. data/perl/HeaderParseService/tmp/tmpVec_914027525.276114_test4 +0 -23
  157. data/perl/HeaderParseService/tmp/tmpVec_914027525.276114_test5 +0 -23
  158. data/perl/HeaderParseService/tmp/tmpVec_914027525.276114_test6 +0 -23
  159. data/perl/HeaderParseService/tmp/tmpVec_914027525.276114_test7 +0 -23
  160. data/perl/HeaderParseService/tmp/tmpVec_914027525.276114_test8 +0 -23
  161. data/perl/HeaderParseService/tmp/tmpVec_914027525.276114_test9 +0 -23
  162. data/perl/ParsCit/crfpp/traindata/parsCit.template +0 -60
  163. data/perl/ParsCit/crfpp/traindata/parsCit.train.data +0 -12104
  164. data/perl/ParsCit/crfpp/traindata/tagged_references.txt +0 -500
  165. data/perl/ParsCit/lib/CSXUtil/SafeText.pm +0 -140
  166. data/perl/ParsCit/lib/ParsCit/Citation.pm +0 -462
  167. data/perl/ParsCit/lib/ParsCit/CitationContext.pm +0 -132
  168. data/perl/ParsCit/lib/ParsCit/Config.pm +0 -46
  169. data/perl/ParsCit/lib/ParsCit/Controller.pm +0 -306
  170. data/perl/ParsCit/lib/ParsCit/PostProcess.pm +0 -367
  171. data/perl/ParsCit/lib/ParsCit/PreProcess.pm +0 -333
  172. data/perl/ParsCit/resources/parsCit.model +0 -0
  173. data/perl/extract.pl +0 -199
  174. data/spec/biblicit/cb2bib_spec.rb +0 -48
  175. data/spec/biblicit/citeseer_spec.rb +0 -40
  176. /data/{perl → svm-header-parse}/HeaderParseService/README.TXT +0 -0
  177. /data/{perl/DocFilter → svm-header-parse/HeaderParseService}/lib/CSXUtil/SafeText.pm +0 -0
  178. /data/{perl → svm-header-parse}/HeaderParseService/lib/HeaderParse/API/AssembleXMLMetadata.pm +0 -0
  179. /data/{perl → svm-header-parse}/HeaderParseService/lib/HeaderParse/API/LoadInformation.pm +0 -0
  180. /data/{perl → svm-header-parse}/HeaderParseService/lib/HeaderParse/API/NamePatternMatch.pm +0 -0
  181. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/50states +0 -0
  182. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/AddrTopWords.txt +0 -0
  183. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/AffiTopWords.txt +0 -0
  184. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/AffiTopWordsAll.txt +0 -0
  185. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/ChineseSurNames.txt +0 -0
  186. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/Csurnames.bin +0 -0
  187. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/Csurnames_spec.bin +0 -0
  188. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/DomainSuffixes.txt +0 -0
  189. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/LabeledHeader +0 -0
  190. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/README +0 -0
  191. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/TrainMulClassLines +0 -0
  192. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/TrainMulClassLines1 +0 -0
  193. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/abstract.txt +0 -0
  194. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/abstractTopWords +0 -0
  195. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/addr.txt +0 -0
  196. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/affi.txt +0 -0
  197. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/affis.bin +0 -0
  198. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/all_namewords_spec.bin +0 -0
  199. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/allnamewords.bin +0 -0
  200. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/cities_US.txt +0 -0
  201. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/cities_world.txt +0 -0
  202. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/city.txt +0 -0
  203. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/cityname.txt +0 -0
  204. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/country_abbr.txt +0 -0
  205. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/countryname.txt +0 -0
  206. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/dateTopWords +0 -0
  207. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/degree.txt +0 -0
  208. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/email.txt +0 -0
  209. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/excludeWords.txt +0 -0
  210. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/female-names +0 -0
  211. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/firstNames.txt +0 -0
  212. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/firstnames.bin +0 -0
  213. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/firstnames_spec.bin +0 -0
  214. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/intro.txt +0 -0
  215. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/keyword.txt +0 -0
  216. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/keywordTopWords +0 -0
  217. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/male-names +0 -0
  218. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/middleNames.txt +0 -0
  219. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/month.txt +0 -0
  220. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/mul +0 -0
  221. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/mul.label +0 -0
  222. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/mul.label.old +0 -0
  223. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/mul.processed +0 -0
  224. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/mulAuthor +0 -0
  225. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/mulClassStat +0 -0
  226. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/nickname.txt +0 -0
  227. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/nicknames.bin +0 -0
  228. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/note.txt +0 -0
  229. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/page.txt +0 -0
  230. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/phone.txt +0 -0
  231. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/postcode.txt +0 -0
  232. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/pubnum.txt +0 -0
  233. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/statename.bin +0 -0
  234. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/statename.txt +0 -0
  235. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/states_and_abbreviations.txt +0 -0
  236. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/stopwords +0 -0
  237. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/stopwords.bin +0 -0
  238. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/surNames.txt +0 -0
  239. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/surnames.bin +0 -0
  240. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/surnames_spec.bin +0 -0
  241. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/A.html +0 -0
  242. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/B.html +0 -0
  243. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/C.html +0 -0
  244. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/D.html +0 -0
  245. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/E.html +0 -0
  246. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/F.html +0 -0
  247. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/G.html +0 -0
  248. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/H.html +0 -0
  249. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/I.html +0 -0
  250. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/J.html +0 -0
  251. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/K.html +0 -0
  252. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/L.html +0 -0
  253. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/M.html +0 -0
  254. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/N.html +0 -0
  255. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/O.html +0 -0
  256. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/P.html +0 -0
  257. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/Q.html +0 -0
  258. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/R.html +0 -0
  259. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/S.html +0 -0
  260. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/T.html +0 -0
  261. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/U.html +0 -0
  262. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/V.html +0 -0
  263. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/W.html +0 -0
  264. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/WCSelect.gif +0 -0
  265. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/X.html +0 -0
  266. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/Y.html +0 -0
  267. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/Z.html +0 -0
  268. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/ae.html +0 -0
  269. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/am.html +0 -0
  270. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/ar.html +0 -0
  271. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/at.html +0 -0
  272. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/au.html +0 -0
  273. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/bd.html +0 -0
  274. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/be.html +0 -0
  275. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/bg.html +0 -0
  276. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/bh.html +0 -0
  277. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/blueribbon.gif +0 -0
  278. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/bm.html +0 -0
  279. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/bn.html +0 -0
  280. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/br.html +0 -0
  281. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/ca.html +0 -0
  282. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/ch.html +0 -0
  283. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/cl.html +0 -0
  284. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/cn.html +0 -0
  285. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/co.html +0 -0
  286. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/cr.html +0 -0
  287. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/cy.html +0 -0
  288. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/cz.html +0 -0
  289. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/de.html +0 -0
  290. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/dean-mainlink.jpg +0 -0
  291. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/dk.html +0 -0
  292. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/ec.html +0 -0
  293. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/ee.html +0 -0
  294. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/eg.html +0 -0
  295. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/es.html +0 -0
  296. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/et.html +0 -0
  297. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/faq.html +0 -0
  298. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/fi.html +0 -0
  299. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/fj.html +0 -0
  300. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/fo.html +0 -0
  301. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/fr.html +0 -0
  302. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/geog.html +0 -0
  303. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/gr.html +0 -0
  304. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/gu.html +0 -0
  305. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/hk.html +0 -0
  306. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/hr.html +0 -0
  307. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/hu.html +0 -0
  308. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/id.html +0 -0
  309. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/ie.html +0 -0
  310. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/il.html +0 -0
  311. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/in.html +0 -0
  312. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/is.html +0 -0
  313. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/it.html +0 -0
  314. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/jm.html +0 -0
  315. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/jo.html +0 -0
  316. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/jp.html +0 -0
  317. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/kaplan.gif +0 -0
  318. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/kr.html +0 -0
  319. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/kw.html +0 -0
  320. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/lb.html +0 -0
  321. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/linkbw2.gif +0 -0
  322. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/lk.html +0 -0
  323. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/lt.html +0 -0
  324. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/lu.html +0 -0
  325. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/lv.html +0 -0
  326. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/ma.html +0 -0
  327. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/maczynski.gif +0 -0
  328. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/mirror.tar +0 -0
  329. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/mk.html +0 -0
  330. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/mo.html +0 -0
  331. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/mseawdm.gif +0 -0
  332. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/mt.html +0 -0
  333. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/mx.html +0 -0
  334. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/my.html +0 -0
  335. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/ni.html +0 -0
  336. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/nl.html +0 -0
  337. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/no.html +0 -0
  338. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/nz.html +0 -0
  339. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/pa.html +0 -0
  340. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/pe.html +0 -0
  341. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/ph.html +0 -0
  342. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/pl.html +0 -0
  343. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/pointcom.gif +0 -0
  344. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/pr.html +0 -0
  345. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/ps.html +0 -0
  346. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/pt.html +0 -0
  347. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/recognition.html +0 -0
  348. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/results.html +0 -0
  349. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/ro.html +0 -0
  350. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/ru.html +0 -0
  351. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/sd.html +0 -0
  352. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/se.html +0 -0
  353. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/sg.html +0 -0
  354. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/si.html +0 -0
  355. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/sk.html +0 -0
  356. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/th.html +0 -0
  357. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/tr.html +0 -0
  358. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/tw.html +0 -0
  359. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/ua.html +0 -0
  360. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/uk.html +0 -0
  361. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/univ-full.html +0 -0
  362. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/univ.html +0 -0
  363. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/uy.html +0 -0
  364. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/ve.html +0 -0
  365. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/yu.html +0 -0
  366. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/za.html +0 -0
  367. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list/zm.html +0 -0
  368. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/university_list.txt +0 -0
  369. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/url.txt +0 -0
  370. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/webTopWords +0 -0
  371. /data/{perl → svm-header-parse}/HeaderParseService/resources/database/words +0 -0
  372. /data/{perl → svm-header-parse}/HeaderParseService/resources/models/10ContextModelfold1 +0 -0
  373. /data/{perl → svm-header-parse}/HeaderParseService/resources/models/10Modelfold1 +0 -0
  374. /data/{perl → svm-header-parse}/HeaderParseService/resources/models/11ContextModelfold1 +0 -0
  375. /data/{perl → svm-header-parse}/HeaderParseService/resources/models/11Modelfold1 +0 -0
  376. /data/{perl → svm-header-parse}/HeaderParseService/resources/models/12ContextModelfold1 +0 -0
  377. /data/{perl → svm-header-parse}/HeaderParseService/resources/models/12Modelfold1 +0 -0
  378. /data/{perl → svm-header-parse}/HeaderParseService/resources/models/13ContextModelfold1 +0 -0
  379. /data/{perl → svm-header-parse}/HeaderParseService/resources/models/13Modelfold1 +0 -0
  380. /data/{perl → svm-header-parse}/HeaderParseService/resources/models/14ContextModelfold1 +0 -0
  381. /data/{perl → svm-header-parse}/HeaderParseService/resources/models/14Modelfold1 +0 -0
  382. /data/{perl → svm-header-parse}/HeaderParseService/resources/models/15ContextModelfold1 +0 -0
  383. /data/{perl → svm-header-parse}/HeaderParseService/resources/models/15Modelfold1 +0 -0
  384. /data/{perl → svm-header-parse}/HeaderParseService/resources/models/1ContextModelfold1 +0 -0
  385. /data/{perl → svm-header-parse}/HeaderParseService/resources/models/1Modelfold1 +0 -0
  386. /data/{perl → svm-header-parse}/HeaderParseService/resources/models/2ContextModelfold1 +0 -0
  387. /data/{perl → svm-header-parse}/HeaderParseService/resources/models/2Modelfold1 +0 -0
  388. /data/{perl → svm-header-parse}/HeaderParseService/resources/models/3ContextModelfold1 +0 -0
  389. /data/{perl → svm-header-parse}/HeaderParseService/resources/models/3Modelfold1 +0 -0
  390. /data/{perl → svm-header-parse}/HeaderParseService/resources/models/4ContextModelfold1 +0 -0
  391. /data/{perl → svm-header-parse}/HeaderParseService/resources/models/4Modelfold1 +0 -0
  392. /data/{perl → svm-header-parse}/HeaderParseService/resources/models/5ContextModelfold1 +0 -0
  393. /data/{perl → svm-header-parse}/HeaderParseService/resources/models/5Modelfold1 +0 -0
  394. /data/{perl → svm-header-parse}/HeaderParseService/resources/models/6ContextModelfold1 +0 -0
  395. /data/{perl → svm-header-parse}/HeaderParseService/resources/models/6Modelfold1 +0 -0
  396. /data/{perl → svm-header-parse}/HeaderParseService/resources/models/7ContextModelfold1 +0 -0
  397. /data/{perl → svm-header-parse}/HeaderParseService/resources/models/7Modelfold1 +0 -0
  398. /data/{perl → svm-header-parse}/HeaderParseService/resources/models/8ContextModelfold1 +0 -0
  399. /data/{perl → svm-header-parse}/HeaderParseService/resources/models/8Modelfold1 +0 -0
  400. /data/{perl → svm-header-parse}/HeaderParseService/resources/models/9ContextModelfold1 +0 -0
  401. /data/{perl → svm-header-parse}/HeaderParseService/resources/models/9Modelfold1 +0 -0
  402. /data/{perl → svm-header-parse}/HeaderParseService/resources/models/NameSpaceModel +0 -0
  403. /data/{perl → svm-header-parse}/HeaderParseService/resources/models/NameSpaceTrainF +0 -0
  404. /data/{perl → svm-header-parse}/HeaderParseService/resources/models/WrapperBaseFeaDict +0 -0
  405. /data/{perl → svm-header-parse}/HeaderParseService/resources/models/WrapperContextFeaDict +0 -0
  406. /data/{perl → svm-header-parse}/HeaderParseService/resources/models/WrapperSpaceAuthorFeaDict +0 -0
@@ -0,0 +1,521 @@
1
+ #!/usr/bin/perl -T
2
+ # Author: Luong Minh Thang <luongmin@comp.nus.edu.sg>, generated at Tue, 02 Jun 2009 01:30:42
3
+
4
+ # Modified from template by Min-Yen Kan <kanmy@comp.nus.edu.sg>
5
+
6
+ =pod HISTORY
7
+ MODIFIED: by Luong Minh thang <luongmin@comp.nus.edu.sg> to generate features at line level for parsHed
8
+ ORIGIN: created from tr2crfpp.pl by Min-Yen Kan <kanmy@comp.nus.edu.sg>
9
+ =cut
10
+
11
+ # I do not know a better solution to find a lib path in -T mode.
12
+ # So if you know a better solution, I'd be glad to hear.
13
+ # See this http://www.perlmonks.org/?node_id=585299 for why I used the below code
14
+ use FindBin;
15
+ my $path;
16
+ BEGIN {
17
+ if ($FindBin::Bin =~ /(.*)/) {
18
+ $path = $1;
19
+ }
20
+ }
21
+ use lib "$path/../../lib";
22
+
23
+ require 5.0;
24
+ use strict 'vars';
25
+ use Getopt::Long;
26
+ use ParsHed::Config;
27
+
28
+ my $parscitHome = "$path/../..";
29
+
30
+ ### USER customizable section
31
+ $0 =~ /([^\/]+)$/; my $progname = $1;
32
+ my $outputVersion = "1.0";
33
+ ### END user customizable section
34
+
35
+ sub License {
36
+ print STDERR "# Copyright 2009 \251 by Luong Minh Thang\n";
37
+ }
38
+
39
+ ### HELP Sub-procedure
40
+ sub Help {
41
+ print STDERR "Generate features for crfpp, similar to tr2crfpp but works at line level with added ngram features\n";
42
+ print STDERR "usage: $progname -h\t[invokes help]\n";
43
+ print STDERR " $progname -in taggedHeaderFile -out outFile [-k keywordFile -b bigramFile]\n";
44
+ print STDERR "Options:\n";
45
+ print STDERR "\t-q\tQuiet Mode (don't echo license)\n";
46
+ print STDERR "\t-k: Default keywordFile $parscitHome/resources/parsHed/keywords\n";
47
+ print STDERR "\t-b: Default bigramFile $parscitHome/resources/parsHed/bigram\n";
48
+ }
49
+ my $QUIET = 0;
50
+ my $HELP = 0;
51
+ my $inFile = undef;
52
+ my $outFile = undef;
53
+ my $dictFile = "$parscitHome/resources/parsCitDict.txt";
54
+ my $keywordFile = "$parscitHome/resources/parsHed/keywords";
55
+ my $bigramFile = "$parscitHome/resources/parsHed/bigram";
56
+ $HELP = 1 unless GetOptions('in=s' => \$inFile,
57
+ 'out=s' => \$outFile,
58
+ 'k=s' => \$keywordFile,
59
+ 'b=s' => \$bigramFile,
60
+ 'h' => \$HELP,
61
+ 'q' => \$QUIET);
62
+
63
+ if ($HELP || !defined $inFile || !defined $outFile) {
64
+ Help();
65
+ exit(0);
66
+ }
67
+
68
+ if (!$QUIET) {
69
+ License();
70
+ }
71
+
72
+ ### Untaint ###
73
+ $inFile = untaintPath($inFile);
74
+ $outFile = untaintPath($outFile);
75
+ $dictFile = untaintPath($dictFile);
76
+ $keywordFile = untaintPath($keywordFile);
77
+ #$bigramFile = untaintPath($bigramFile);
78
+ $ENV{'PATH'} = '/bin:/usr/bin:/usr/local/bin';
79
+ ### End untaint ###
80
+
81
+ my $isGenerateTemplate = 1; #generate template file to stdout
82
+ my @templates = (); # for storing templates info
83
+
84
+ my %dict = ();
85
+ readDict($dictFile);
86
+
87
+ # keyword statistics
88
+ my %keywords = (); #hash of hash $keywords{"affiliation"}->{"Institute"} = 1 indicates "Institute" is a high-frequent word for affiliation tag
89
+ readKeywordDict($keywordFile, \%keywords);
90
+
91
+ # bigram statistics
92
+ my %bigrams = (); #hash of hash $keywords{"abstract"}->{"we-present"} = 1 indicates "we-present" is a high-frequent bigram for abstract tag
93
+ readKeywordDict($bigramFile, \%bigrams);
94
+
95
+ # list of tags trained in parsHed
96
+ # those with value 0 do not have frequent keyword features
97
+ my $tags = $ParsHed::Config::tags;
98
+
99
+ # File IOs
100
+ if (!(-e $inFile)) { die "# $progname crash\t\tFile \"$inFile\" doesn't exist"; }
101
+ open (IF, "<:utf8", $inFile) || die "# $progname crash\t\tCan't open \"$inFile\"";
102
+ open (OF, ">:utf8", $outFile) || die "# $progname crash\t\tCan't open \"$outFile\"";
103
+
104
+ # process input files
105
+ while (<IF>) {
106
+ if (/^\#/) { next; } # skip comments
107
+ elsif (/^\s+$/) { next; } # skip blank lines
108
+ else {
109
+ my @tokens = split(/ +/);
110
+
111
+ my $tag = "";
112
+ my $line = "";
113
+ foreach(@tokens){
114
+ if (/^\s*$/) { # spaces
115
+ next;
116
+ } elsif (/^\<\/([a-z]+)/) { #end tag
117
+ # break field content into multi sub-lines that are separated by +L+
118
+ my @sub_lines = split(/\s*\+L\+\s*/, $line);
119
+ foreach my $sub_line (@sub_lines){
120
+ if($sub_line eq ""){
121
+ next;
122
+ }
123
+
124
+ # generate CRF features
125
+ crfFeature($sub_line, $tag);
126
+
127
+ if($isGenerateTemplate){
128
+ $isGenerateTemplate = 0; #done generate template file
129
+ }
130
+ }
131
+
132
+ $line = ""; # reset
133
+ next;
134
+ } elsif (/^\<([a-z]+)/) { #beginning tag
135
+ $tag = $1;
136
+
137
+ next;
138
+ } else { #contents inside tag
139
+ $line .= "$_ ";
140
+ }
141
+ }
142
+
143
+ print OF "\n"; #blank line separate for each header entry
144
+ }
145
+ }
146
+ close (IF);
147
+ close (OF);
148
+
149
+ ###
150
+ ### END of main program
151
+ ###
152
+
153
+
154
+ ##
155
+ # Main method to extract features
156
+ ##
157
+ sub crfFeature {
158
+ my ($line, $tag) = @_;
159
+
160
+ my @tmpTokens = split(/\s+/, $line);
161
+ #filter out empty token
162
+ my @tokens = ();
163
+ foreach my $token (@tmpTokens){
164
+ if($token ne ""){
165
+ push(@tokens, $token);
166
+ }
167
+ }
168
+
169
+ my @feats = ();
170
+ my $featureCount = 0;
171
+ my %featureCounts = (); # to perform feature linking
172
+ my $token = "";
173
+
174
+ # full form: does not count in crf template file, simply for outputing purpose to get the whole line data
175
+ my $lineFull = join("|||", @tokens);
176
+ push(@feats, "$lineFull");
177
+
178
+ # fullForm token general features: concatenate the wholeline into one token, apply most of Parscit features
179
+ if($ParsHed::Config::isFullFormToken == 1){
180
+ $lineFull =~ s/\d/0/g; # normalize number
181
+ $lineFull =~ s/\|\|\|//g; #strip of the separator we previously added
182
+
183
+ push(@templates, "#fullForm token general features\n");
184
+ $featureCounts{"fullFormToken"}->{"start"} = $featureCount;
185
+ $featureCount = generateTokenFeature($lineFull, "UF", $featureCount, \@feats, \@templates);
186
+ $featureCounts{"fullFormToken"}->{"end"} = $featureCount;
187
+ }
188
+
189
+ # first token general features: apply most of Parscit features
190
+ if($ParsHed::Config::isFirstToken){
191
+ push(@templates, "#first token general features\n");
192
+ $featureCounts{"firstToken"}->{"start"} = $featureCount;
193
+ $featureCount = generateTokenFeature($tokens[0], "U0", $featureCount, \@feats, \@templates);
194
+ $featureCounts{"firstToken"}->{"end"} = $featureCount;
195
+ }
196
+
197
+ # first token keyword features
198
+ if($ParsHed::Config::isKeyword){
199
+ push(@templates, "#first token keyword features\n");
200
+ $featureCounts{"firstKeyword"}->{"start"} = $featureCount;
201
+ $featureCount = generateKeywordFeature($tokens[0], "U1", $featureCount, \@feats, \@templates, \%keywords);
202
+ $featureCounts{"firstKeyword"}->{"end"} = $featureCount;
203
+ }
204
+
205
+ # second token general features: apply most of Parscit features
206
+ if($ParsHed::Config::isSecondToken || $ParsHed::Config::isBigram){
207
+ $token = "EMPTY";
208
+
209
+ if($#tokens > 0){# 1 token
210
+ $token = $tokens[1];
211
+ }
212
+ }
213
+ if($ParsHed::Config::isSecondToken){
214
+ push(@templates, "#second token general features\n");
215
+ $featureCounts{"secondToken"}->{"start"} = $featureCount;
216
+ $featureCount = generateTokenFeature($token, "U2", $featureCount, \@feats, \@templates);
217
+ $featureCounts{"secondToken"}->{"end"} = $featureCount;
218
+ }
219
+
220
+ # bigram features
221
+ if($ParsHed::Config::isBigram){
222
+ my $nGram = $tokens[0];
223
+ $nGram =~ s/^\p{P}+//g; #strip out leading punctuations
224
+ $nGram =~ s/\p{P}+$//g; #strip out trailing punctuations
225
+ $nGram =~ s/\d/0/g; #canocalize number into "0"
226
+ $nGram = lc($nGram);
227
+
228
+ $token =~ s/^\p{P}+//g; #strip out leading punctuations
229
+ $token =~ s/\p{P}+$//g; #strip out trailing punctuations
230
+ $token =~ s/\d/0/g; #canocalize number into "0"
231
+ $token = lc($token);
232
+
233
+ $nGram .= "-$token";
234
+
235
+ push(@templates, "#bigram features\n");
236
+ $featureCounts{"bigram"}->{"start"} = $featureCount;
237
+ $featureCount = generateKeywordFeature($nGram, "U4", $featureCount, \@feats, \@templates, \%bigrams);
238
+ $featureCounts{"bigram"}->{"end"} = $featureCount;
239
+ }
240
+
241
+ # second last token general features: apply most of Parscit features
242
+ if($ParsHed::Config::isSecondLastToken){
243
+ $token = "EMPTY";
244
+ if($#tokens > 0){# only 1 token
245
+ $token = $tokens[$#tokens-1];
246
+ }
247
+ }
248
+ if($ParsHed::Config::isSecondLastToken){
249
+ push(@templates, "#second last token general features\n");
250
+ $featureCounts{"secondLastToken"}->{"start"} = $featureCount;
251
+ $featureCount = generateTokenFeature($token, "U7", $featureCount, \@feats, \@templates);
252
+ $featureCounts{"secondLastToken"}->{"end"} = $featureCount;
253
+ }
254
+
255
+ # last token general features: apply most of Parscit features
256
+ if($ParsHed::Config::isLastToken){
257
+ push(@templates, "#last token general features\n");
258
+ $featureCounts{"lastToken"}->{"start"} = $featureCount;
259
+ $featureCount = generateTokenFeature($tokens[$#tokens], "U8", $featureCount, \@feats, \@templates);
260
+ $featureCounts{"lastToken"}->{"end"} = $featureCount;
261
+ }
262
+
263
+ ## feature linking in terms of template file
264
+ my $i;
265
+ if($ParsHed::Config::isBack1){
266
+ if($isGenerateTemplate){
267
+ # to constraint on last token features at -1 relative position
268
+ $i = 0;
269
+ push(@templates, "#constraint on last token features at -1 relative position \n");
270
+ for(my $j=$featureCounts{"lastToken"}->{"start"}; $j < $featureCounts{"lastToken"}->{"end"}; $j++){
271
+ push(@templates, "UA".$i++.":%x[-1,$j]\n");
272
+ }
273
+ push(@templates, "\n");
274
+ }
275
+ }
276
+ push(@templates, "\n");
277
+
278
+ if($ParsHed::Config::isForw1){
279
+ if($isGenerateTemplate){
280
+ # to constraint on first token features at 1 relative position
281
+ $i = 0;
282
+ push(@templates, "#constraint on first token features at 1 relative position \n");
283
+ for(my $j=$featureCounts{"firstToken"}->{"start"}; $j < $featureCounts{"firstToken"}->{"end"}; $j++){
284
+ push(@templates, "UB".$i++.":%x[1,$j]\n");
285
+ }
286
+ push(@templates, "\n");
287
+ }
288
+ }
289
+ push(@templates, "\n");
290
+ ## end feature linking
291
+
292
+ ##
293
+ # output tag
294
+ ##
295
+ push(@feats, $tag);
296
+
297
+ if($isGenerateTemplate){
298
+ push(@templates, "# Output\nB0\n");
299
+ print STDOUT join ("", @templates);
300
+ }
301
+ print OF join (" ", @feats);
302
+ print OF "\n";
303
+ }
304
+
305
+ sub generateKeywordFeature {
306
+ my ($token, $label, $featureCount, $feats, $templates, $keywords) = @_;
307
+
308
+ my $i = 0;
309
+
310
+ $token =~ s/^\p{P}+//g; #strip out leading punctuations
311
+ $token =~ s/\p{P}+$//g; #strip out trailing punctuations
312
+ $token =~ s/\d/0/g; #canocalize number into "0"
313
+
314
+ foreach(keys %{$tags}){ # 9 tags: feature 1-10
315
+ if($tags->{$_} == 0) { next; }
316
+
317
+ if($keywords->{$_}->{$token}){
318
+ push(@{$feats}, "$_-$token");
319
+ } else {
320
+ push(@{$feats}, "none");
321
+ }
322
+
323
+ if($isGenerateTemplate){
324
+ push(@{$templates}, "$label".$i++.":%x[0,$featureCount]\n");
325
+ $featureCount++;
326
+ }
327
+ }
328
+ push(@{$templates}, "\n");
329
+
330
+ return $featureCount;
331
+ }
332
+
333
+ sub generateTokenFeature {
334
+ my ($token, $label, $featureCount, $feats, $templates) = @_;
335
+
336
+ # prep
337
+ my $word = $token;
338
+ my $wordNP = $token; # no punctuation
339
+ $wordNP =~ s/[^\w]//g;
340
+ if ($wordNP =~ /^\s*$/) { $wordNP = "EMPTY"; }
341
+ my $wordLCNP = lc($wordNP); # lowercased word, no punctuation
342
+ if ($wordLCNP =~ /^\s*$/) { $wordLCNP = "EMPTY"; }
343
+
344
+ ## feature generation
345
+ push(@{$feats}, "TOKEN-$word"); # 0 = lexical word
346
+
347
+ my @chars = split(//,$word);
348
+ my $lastChar = $chars[-1];
349
+ if ($lastChar =~ /[a-z]/) { $lastChar = 'a'; }
350
+ elsif ($lastChar =~ /[A-Z]/) { $lastChar = 'A'; }
351
+ elsif ($lastChar =~ /[0-9]/) { $lastChar = '0'; }
352
+ push(@{$feats}, $lastChar); # 1 = last char
353
+
354
+ push(@{$feats}, $chars[0]); # 2 = first char
355
+ push(@{$feats}, join("",@chars[0..1])); # 3 = first 2 chars
356
+ push(@{$feats}, join("",@chars[0..2])); # 4 = first 3 chars
357
+ push(@{$feats}, join("",@chars[0..3])); # 5 = first 4 chars
358
+
359
+ push(@{$feats}, $chars[-1]); # 6 = last char
360
+ push(@{$feats}, join("",@chars[-2..-1])); # 7 = last 2 chars
361
+ push(@{$feats}, join("",@chars[-3..-1])); # 8 = last 3 chars
362
+ push(@{$feats}, join("",@chars[-4..-1])); # 9 = last 4 chars
363
+
364
+ push(@{$feats}, $wordLCNP); # 10 = lowercased word, no punct
365
+
366
+ # 11 - capitalization
367
+ my $ortho = ($wordNP =~ /^[A-Z]$/) ? "singleCap" :
368
+ ($wordNP =~ /^[A-Z][a-z]+/) ? "InitCap" :
369
+ ($wordNP =~ /^[A-Z]+$/) ? "AllCap" : "others";
370
+ push(@{$feats}, $ortho);
371
+
372
+ # 12 - numbers
373
+ my $num = ($wordNP =~ /^(19|20)[0-9][0-9]$/) ? "year" :
374
+ ($word =~ /[0-9]\-[0-9]/) ? "possiblePage" :
375
+ ($word =~ /[0-9]\([0-9]+\)/) ? "possibleVol" :
376
+ ($wordNP =~ /^[0-9]$/) ? "1dig" :
377
+ ($wordNP =~ /^[0-9][0-9]$/) ? "2dig" :
378
+ ($wordNP =~ /^[0-9][0-9][0-9]$/) ? "3dig" :
379
+ ($wordNP =~ /^[0-9]+$/) ? "4+dig" :
380
+ ($wordNP =~ /^[0-9]+(th|st|nd|rd)$/) ? "ordinal" :
381
+ ($wordNP =~ /[0-9]/) ? "hasDig" : "nonNum";
382
+ push(@{$feats}, $num);
383
+
384
+ # gazetteer (names)
385
+ my $dictStatus = (defined $dict{$wordLCNP}) ? $dict{$wordLCNP} : 0;
386
+ # my $isInDict = ($dictStatus != 0) ? "isInDict" : "no";
387
+ my $isInDict = $dictStatus;
388
+
389
+ my ($publisherName,$placeName,$monthName,$lastName,$femaleName,$maleName);
390
+ if ($dictStatus >= 32) { $dictStatus -= 32; $publisherName = "publisherName" } else { $publisherName = "no"; }
391
+ if ($dictStatus >= 16) { $dictStatus -= 16; $placeName = "placeName" } else { $placeName = "no"; }
392
+ if ($dictStatus >= 8) { $dictStatus -= 8; $monthName = "monthName" } else { $monthName = "no"; }
393
+ if ($dictStatus >= 4) { $dictStatus -= 4; $lastName = "lastName" } else { $lastName = "no"; }
394
+ if ($dictStatus >= 2) { $dictStatus -= 2; $femaleName = "femaleName" } else { $femaleName = "no"; }
395
+ if ($dictStatus >= 1) { $dictStatus -= 1; $maleName = "maleName" } else { $maleName = "no"; }
396
+
397
+ push(@{$feats}, $isInDict); # 13 = name status
398
+ push(@{$feats}, $maleName); # 14 = male name
399
+ push(@{$feats}, $femaleName); # 15 = female name
400
+ push(@{$feats}, $lastName); # 16 = last name
401
+ push(@{$feats}, $monthName); # 17 = month name
402
+ push(@{$feats}, $placeName); # 18 = place name
403
+ push(@{$feats}, $publisherName); # 19 = publisher name
404
+
405
+ #push(@{$feats[$j]}, $hasPossibleEditor); # 20 = possible editor
406
+
407
+ # not accurate ($#tokens counts tags too)
408
+ # my $location = int ($i / $#tokens * 12);
409
+ # push(@{$feats}, $location); # 21 = relative location
410
+
411
+ # 22 - punctuation
412
+ my $punct = ($word =~ /^[\"\'\`]/) ? "leadQuote" :
413
+ ($word =~ /[\"\'\`][^s]?$/) ? "endQuote" :
414
+ ($word =~ /\-.*\-/) ? "multiHyphen" :
415
+ ($word =~ /[\-\,\:\;]$/) ? "contPunct" :
416
+ ($word =~ /[\!\?\.\"\']$/) ? "stopPunct" :
417
+ ($word =~ /^[\(\[\{\<].+[\)\]\}\>].?$/) ? "braces" :
418
+ ($word =~ /^[0-9]{2-5}\([0-9]{2-5}\).?$/) ? "possibleVol" : "others";
419
+ push(@{$feats}, $punct); # 22 = punctuation
420
+
421
+ my $i = 0;
422
+ my $curSize = scalar(@{$feats});
423
+ if($isGenerateTemplate){
424
+ for(my $j=$featureCount; $j < $curSize; $j++){
425
+ push(@{$templates}, "$label".$i++.":%x[0,$j]\n");
426
+ }
427
+ }
428
+
429
+ push(@{$templates}, "\n");
430
+ return $curSize;
431
+ }
432
+
433
+ sub readKeywordDict {
434
+ my ($inFile, $keywords) = @_;
435
+
436
+ open (IF, "<:utf8", $inFile) || die "$progname fatal\t\tCannot open \"$inFile\"!";
437
+ print STDERR "Load keywords from file $inFile\n";
438
+ #process input file
439
+ while(<IF>){
440
+ chomp;
441
+
442
+ if(/^(.+?): (.+)$/){
443
+ my $tag = $1;
444
+ my @tokens = split(/\s+/, $2);
445
+ $keywords->{$tag} = ();
446
+ foreach(@tokens){
447
+ $keywords->{$tag}->{$_} = 1;
448
+ }
449
+ }
450
+ }
451
+
452
+ close (IF);
453
+ }
454
+
455
+ sub readDict {
456
+ my $dictFileLoc = shift @_;
457
+ my $mode = 0;
458
+ print STDERR "Read dict from $dictFileLoc\n";
459
+ open (DATA, $dictFileLoc) || die "$progname fatal\t\tCannot open \"$dictFileLoc\"!";
460
+ while (<DATA>) {
461
+ if (/^\#\# Male/) { $mode = 1; } # male names
462
+ elsif (/^\#\# Female/) { $mode = 2; } # female names
463
+ elsif (/^\#\# Last/) { $mode = 4; } # last names
464
+ elsif (/^\#\# Chinese/) { $mode = 4; } # last names
465
+ elsif (/^\#\# Months/) { $mode = 8; } # month names
466
+ elsif (/^\#\# Place/) { $mode = 16; } # place names
467
+ elsif (/^\#\# Publisher/) { $mode = 32; } # publisher names
468
+ elsif (/^\#/) { next; }
469
+ else {
470
+ chop;
471
+ my $key = $_;
472
+ my $val = 0;
473
+
474
+ if (/\t/) { # has probability
475
+ ($key,$val) = split (/\t/,$_);
476
+ }
477
+ #exit;
478
+
479
+ # already tagged (some entries may appear in same part of lexicon more than once
480
+ # if ($dict{$key} >= $mode) { next; }
481
+ if ($dict{$key} && $dict{$key} >= $mode) { next; } # Thang fix: uninitialized problem
482
+ else { $dict{$key} += $mode; } # not yet tagged
483
+ }
484
+ }
485
+ close (DATA);
486
+ }
487
+
488
+ sub untaintPath {
489
+ my ($path) = @_;
490
+
491
+ if ( $path =~ /^([-_\/\w\.]+)$/ ) {
492
+ $path = $1;
493
+ } else {
494
+ die "Bad path $path\n";
495
+ }
496
+
497
+ return $path;
498
+ }
499
+
500
+ sub untaint {
501
+ my ($s) = @_;
502
+ if ($s =~ /^([\w \-\@\(\),\.\/]+)$/) {
503
+ $s = $1; # $data now untainted
504
+ } else {
505
+ die "Bad data in $s"; # log this somewhere
506
+ }
507
+ return $s;
508
+ }
509
+
510
+ sub execute {
511
+ my ($cmd) = @_;
512
+ print STDERR "Executing: $cmd\n";
513
+ $cmd = untaint($cmd);
514
+ system($cmd);
515
+ }
516
+
517
+ sub newTmpFile {
518
+ my $tmpFile = `date '+%Y%m%d-%H%M%S-$$'`;
519
+ chomp($tmpFile);
520
+ return $tmpFile;
521
+ }
@@ -0,0 +1,102 @@
1
+ #!/usr/bin/perl -CSD
2
+ #
3
+ # Simple command script for executing ParsCit in an
4
+ # offline mode (direct API call instead of going through
5
+ # the web service).
6
+ #
7
+ # Min-Yen Kan (Thu Feb 28 14:10:28 SGT 2008)
8
+ # Derived from citeExtract.pl
9
+ #
10
+ use strict;
11
+ use FindBin;
12
+ use lib "$FindBin::Bin/../lib";
13
+
14
+ use lib "/home/wing.nus/tools/languages/programming/perl-5.10.0/lib/5.10.0";
15
+ use lib "/home/wing.nus/tools/languages/programming/perl-5.10.0/lib/site_perl/5.10.0";
16
+
17
+ use ParsCit::Controller;
18
+ use CSXUtil::SafeText qw(cleanAll cleanXML);
19
+
20
+ my $textFile = $ARGV[0];
21
+ my $outFile = $ARGV[1];
22
+
23
+ if (!defined $textFile) {
24
+ print "Usage: $0 textfile [outfile]\n";
25
+ exit;
26
+ }
27
+
28
+ open (IF, $textFile) || die "Couldn't open text file \"textFile\"!";
29
+ my $normalizedCiteText = "";
30
+ my $line = 0;
31
+ while (<IF>) {
32
+ chop;
33
+ # Tr2cfpp needs an enclosing tag for initial class seed.
34
+ $normalizedCiteText .= "<title> " . $_ . " </title>\n";
35
+ $line++;
36
+ }
37
+ close (IF);
38
+
39
+ if ($line == 0) {
40
+ # Stop - nothing left to do.
41
+ exit();
42
+ }
43
+
44
+ our $msg = "";
45
+ my $tmpFile = ParsCit::Tr2crfpp::PrepData(\$normalizedCiteText, $textFile);
46
+ my $outFile = $tmpFile."_dec";
47
+ my @validCitations = ();
48
+
49
+ my $xml = "";
50
+ $xml .= "<algorithm name=\"$ParsCit::Config::algorithmName\" version=\"$ParsCit::Config::algorithmVersion\">\n";
51
+ $xml .= "<citationList>\n";
52
+ if (ParsCit::Tr2crfpp::Decode($tmpFile, $outFile)) {
53
+ my ($rRawXML, $rCiteInfo, $tstatus, $tmsg) =
54
+ ParsCit::PostProcess::ReadAndNormalize($outFile);
55
+ if ($tstatus <= 0) {
56
+ return ($tstatus, $msg, undef, undef);
57
+ }
58
+ my @citeInfo = @{$rCiteInfo};
59
+ for (my $i=0; $i<=$#citeInfo; $i++) {
60
+ my %citeInfo = %{$citeInfo[$i]};
61
+ $xml .= "<citation>\n";
62
+ foreach my $key (keys %citeInfo) {
63
+ if ($key eq "authors" || $key eq "editors") {
64
+ my $singular = $key;
65
+ chop $singular;
66
+ $xml .= "<$key>\n";
67
+ foreach my $person (@{$citeInfo{$key}}) {
68
+ cleanAll(\$person);
69
+ $xml .= "<$singular>$person</$singular>\n";
70
+ }
71
+ $xml .= "</$key>\n";
72
+ }
73
+ elsif ($key eq "volume")
74
+ {
75
+ if (scalar(@{$citeInfo{$key}}) > 0)
76
+ {
77
+ # Main volume
78
+ cleanAll(\$citeInfo{$key}[ 0 ]);
79
+ $xml .= "<$key>" . $citeInfo{$key}[ 0 ] . "</$key>\n";
80
+
81
+ # Sub-volume, issue
82
+ for (my $i = 1; $i < scalar(@{$citeInfo{$key}}); $i++)
83
+ {
84
+ cleanAll(\$citeInfo{$key}[ $i ]);
85
+ $xml .= "<issue>" . $citeInfo{$key}[ $i ] . "</issue>\n";
86
+ }
87
+ }
88
+ }
89
+ else {
90
+ cleanAll(\$citeInfo{$key});
91
+ $xml .= "<$key>$citeInfo{$key}</$key>\n";
92
+ }
93
+ }
94
+ $xml .= "</citation>\n";
95
+ }
96
+ $xml .= "</citationList>\n</algorithm>\n";
97
+ }
98
+
99
+ unlink($tmpFile);
100
+ unlink($outFile);
101
+
102
+ print $xml;