biblicit 2.0.7 → 2.0.8

data/lib/biblicit/version.rb

@@ -2,6 +2,6 @@
 
 module Biblicit
 
-  VERSION = '2.0.7'
+  VERSION = '2.0.8'
 
 end

data/parscit/bin/sectLabel/genericSect/extractFeature.rb

@@ -2,6 +2,8 @@
 # encoding: UTF-8
 #Author Nguyen Thuy Dung
 require 'find'
+require "#{File.dirname(__FILE__)}/forceUtf8"
+
 #get relative pos in ingeter, values range from 0-10
 def getPos (val)
 	if val == 0
@@ -26,28 +28,6 @@ def getHeader(str)
 	return str.downcase
 end
 
-# Converts a string to UTF-8. If the string is already valid UTF-8, it just
-# marks it as such. If the string isn't valid UTF-8, we assume that it's
-# ISO-8859-1 or Windows-1252 and convert it. If it's not valid in that encoding
-# either, we just strip all non-UTF-8 characters and call it a day.
-#
-# Destructive!
-def force_utf8!(string)
-  string.force_encoding "UTF-8"
-  return string if string.valid_encoding?
-
-  begin
-    string.force_encoding "Windows-1252" # common superset of 8859-1
-    string.encode! "UTF-8"
-  rescue Encoding::InvalidByteSequenceError,
-          Encoding::UndefinedConversionError
-    string.force_encoding "UTF-8"
-    string.encode! "UTF-16",
-      invalid: :replace, undef: :replace, replace: ""
-    string.encode! "UTF-8"
-  end
-end
-
 f = File.open("#{ARGV[0]}")
 hea_array = Array.new
 ahea_array = Array.new

data/parscit/bin/sectLabel/genericSect/forceUtf8.rb

@@ -0,0 +1,24 @@
+# encoding: UTF-8
+
+# Converts a string to UTF-8. If the string is already valid UTF-8, it just
+# marks it as such. If the string isn't valid UTF-8, we assume that it's
+# ISO-8859-1 or Windows-1252 and convert it. If it's not valid in that encoding
+# either, we just strip all non-UTF-8 characters and call it a day.
+#
+# Destructive!
+def force_utf8!(string)
+  string.force_encoding "UTF-8"
+  return string if string.valid_encoding?
+
+  begin
+    string.force_encoding "Windows-1252" # common superset of 8859-1
+    string.encode! "UTF-8"
+  rescue Encoding::InvalidByteSequenceError,
+          Encoding::UndefinedConversionError
+    string.force_encoding "UTF-8"
+    string.encode! "UTF-16",
+      invalid: :replace, undef: :replace, replace: ""
+    string.encode! "UTF-8"
+  end
+end
+

data/parscit/bin/sectLabel/genericSectExtract.rb

@@ -10,6 +10,8 @@ pwd = File.dirname(__FILE__)
 @DATA   = "#{pwd}/../../resources/sectLabel/"
 @TEST_DIR = "/tmp/"
 
+require "#{@SRC}/forceUtf8"
+
 name  = "#{Time.now.to_i}-#{Process.pid}"
 
 cmd = "ruby #{@SRC}/extractFeature.rb #{ARGV[0]} > #{@TEST_DIR}/#{name}.test"
@@ -28,7 +30,7 @@ end
 
 f = File.open("#{@TEST_DIR}/#{name}.out")
 while !f.eof do
-	str = f.gets.chomp.strip
+	str = force_utf8!(f.gets).chomp.strip
 	if str != ""
 		l = str.split(" ")
 		output = l.at(l.length-1)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: biblicit
 version: !ruby/object:Gem::Version
-  version: 2.0.7
+  version: 2.0.8
   prerelease: 
 platform: ruby
 authors:
@@ -401,6 +401,7 @@ files:
 - parscit/bin/sectLabel/README.txt
 - parscit/bin/sectLabel/genericSect/crossValidation.rb
 - parscit/bin/sectLabel/genericSect/extractFeature.rb
+- parscit/bin/sectLabel/genericSect/forceUtf8.rb
 - parscit/bin/sectLabel/genericSectExtract.rb
 - parscit/bin/sectLabel/getStructureInfo.pl
 - parscit/bin/sectLabel/redo.sectLabel.pl