cld3 3.2.4 → 3.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 91da77d7c9125f43c76c2645b04701b4c72a955859cd5cef23b9e846119e1112
-  data.tar.gz: 2090d645b5ae57212b496c41565031624e0987992b1cb616bdbfa5991244f55a
+  metadata.gz: f5b3cc203abda97cb85d5dee0983b7f63c626397b8af8b90e2110bb5fedbbdec
+  data.tar.gz: 197f66798925404ded7af722d0194a705018d6953b11f4576c4e180ea093675d
 SHA512:
-  metadata.gz: e9dd9783d699ee4653b1f808405b655425d1286dd6be663a965e4e448874dfd811eab73dd82f2a0c5e0ef91de8d4df62ffb2228bddb1642a57f9e1d01172115c
-  data.tar.gz: 2d4e703a21feba6b6ef95d5afbeb2f1748e5135dd120ba6a7976383b650659db4501045b2ffded8ff8575312a2a3f6cf092552d5833c5f435a122ae43e7c4a4e
+  metadata.gz: 855e8ee464a2842906bfef211e2afb21820fe9a7449b58d91b9ab1908c997966b9dd4c2d5d51f82ceb84b65b5a118736a5aa4eff6ea9548b9a9abc61b297a9d0
+  data.tar.gz: e38ddfd81489aeb83bccc7b509dd17ea79c56ba641de37cac2d800d3428ed31e5ac57066016bd118e9e71c30c78d31b4c38a266abe012065495558adf07e68f5

data/Gemfile

@@ -1,4 +1,4 @@
-# Copyright 2017 Akihiko Odaki <akihiko.odaki.4i@stu.hosei.ac.jp>
+# Copyright 2017 Akihiko Odaki <akihiko.odaki@gmail.com>
 # All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright 2017 Akihiko Odaki <akihiko.odaki.4i@stu.hosei.ac.jp>
+Copyright 2017 Akihiko Odaki <akihiko.odaki@gmail.com>
 All rights reserved.
 
                                  Apache License
@@ -189,7 +189,7 @@ All rights reserved.
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2017, Akihiko Odaki <akihiko.odaki.4i@stu.hosei.ac.jp>
+   Copyright 2017, Akihiko Odaki <akihiko.odaki@gmail.com>
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

data/README.md

@@ -8,11 +8,11 @@ require 'cld3'
 
 cld3 = CLD3::NNetLanguageIdentifier.new(0, 1000)
 
-cld3.find_language("こんにちは") # => #<struct Struct::Result language=:ja, probability=1.0, reliable?=true, proportion=1.0>
+cld3.find_language("こんにちは") # => #<struct Struct::Result language=:ja, probability=1.0, reliable?=true, proportion=1.0, byte_ranges=[]>
 
-cld3.find_language("This is a pen.") # => #<struct Struct::Result language=:en, probability=0.9999408721923828, reliable?=true, proportion=1.0>
+cld3.find_language("This is a pen.") # => #<struct Struct::Result language=:en, probability=0.9999408721923828, reliable?=true, proportion=1.0, byte_ranges=[]>
 
-cld3.find_language("здравствуйте") # => #<struct Struct::Result language=:ru, probability=0.3140212297439575, reliable?=false, proportion=1.0>
+cld3.find_language("здравствуйте") # => #<struct Struct::Result language=:ru, probability=0.3140212297439575, reliable?=false, proportion=1.0, byte_ranges=[]>
 ```
 
 ## Installation

data/cld3.gemspec

@@ -1,4 +1,4 @@
-# Copyright 2017 Akihiko Odaki <akihiko.odaki.4i@stu.hosei.ac.jp>
+# Copyright 2017 Akihiko Odaki <akihiko.odaki@gmail.com>
 # All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,16 +16,16 @@
 
 Gem::Specification.new do |gem|
   gem.name = "cld3"
-  gem.version = "3.2.4"
+  gem.version = "3.4.2"
   gem.summary = "Compact Language Detector v3 (CLD3)"
   gem.description = "Compact Language Detector v3 (CLD3) is a neural network model for language identification."
   gem.license = "Apache-2.0"
   gem.homepage = "https://github.com/akihikodaki/cld3-ruby"
   gem.author = "Akihiko Odaki"
-  gem.email = "akihiko.odaki.4i@stu.hosei.ac.jp"
-  gem.required_ruby_version = [ ">= 2.3.0", "< 2.7.0" ]
-  gem.add_dependency "ffi", [ ">= 1.1.0", "< 1.11.0" ]
-  gem.add_development_dependency "rspec", [ ">=3.0.0", "< 3.9.0" ]
+  gem.email = "akihiko.odaki@gmail.com"
+  gem.required_ruby_version = [ ">= 2.6.0", "< 3.1.0" ]
+  gem.add_dependency "ffi", [ ">= 1.1.0", "< 1.16.0" ]
+  gem.add_development_dependency "rspec", [ ">=3.0.0", "< 3.11.0" ]
   gem.files = Dir[
     "Gemfile", "LICENSE", "LICENSE_CLD3", "README.md",
     "cld3.gemspec", "ext/**/*", "lib/**/*"

data/ext/cld3/Makefile

@@ -0,0 +1,266 @@
+
+SHELL = /bin/sh
+
+# V=0 quiet, V=1 verbose.  other values don't work.
+V = 1
+Q1 = $(V:1=)
+Q = $(Q1:0=@)
+ECHO1 = $(V:1=@ :)
+ECHO = $(ECHO1:0=@ echo)
+NULLCMD = :
+
+#### Start of system configuration section. ####
+
+srcdir = .
+topdir = /usr/include
+hdrdir = $(topdir)
+arch_hdrdir = /usr/include
+PATH_SEPARATOR = :
+VPATH = $(srcdir):$(arch_hdrdir)/ruby:$(hdrdir)/ruby
+prefix = $(DESTDIR)/usr
+rubysitearchprefix = $(sitearchlibdir)/$(RUBY_BASE_NAME)
+rubyarchprefix = $(DESTDIR)/usr/lib64/ruby
+rubylibprefix = $(exec_prefix)/share/ruby
+exec_prefix = $(DESTDIR)/usr
+vendorarchhdrdir = $(vendorhdrdir)/$(arch)
+sitearchhdrdir = $(sitehdrdir)/$(arch)
+rubyarchhdrdir = $(DESTDIR)/usr/include
+vendorhdrdir = $(rubyhdrdir)/vendor_ruby
+sitehdrdir = $(rubyhdrdir)/site_ruby
+rubyhdrdir = $(DESTDIR)/usr/include
+rubygemsdir = $(DESTDIR)/usr/share/rubygems
+vendorarchdir = $(DESTDIR)/usr/lib64/ruby/vendor_ruby
+vendorlibdir = $(vendordir)
+vendordir = $(DESTDIR)/usr/share/ruby/vendor_ruby
+sitearchdir = $(DESTDIR)/usr/local/lib64/ruby/site_ruby
+sitelibdir = $(sitedir)
+sitedir = $(DESTDIR)/usr/local/share/ruby/site_ruby
+rubyarchdir = $(rubyarchprefix)
+rubylibdir = $(rubylibprefix)
+sitearchincludedir = $(includedir)/$(sitearch)
+archincludedir = $(includedir)/$(arch)
+sitearchlibdir = $(libdir)/$(sitearch)
+archlibdir = $(DESTDIR)/usr/lib64
+ridir = $(datarootdir)/$(RI_BASE_NAME)
+mandir = $(DESTDIR)/usr/share/man
+localedir = $(datarootdir)/locale
+libdir = $(exec_prefix)/lib64
+psdir = $(docdir)
+pdfdir = $(docdir)
+dvidir = $(docdir)
+htmldir = $(docdir)
+infodir = $(DESTDIR)/usr/share/info
+docdir = $(datarootdir)/doc/$(PACKAGE)
+oldincludedir = $(DESTDIR)/usr/include
+includedir = $(DESTDIR)/usr/include
+localstatedir = $(DESTDIR)/var
+sharedstatedir = $(DESTDIR)/var/lib
+sysconfdir = $(DESTDIR)/etc
+datadir = $(DESTDIR)/usr/share
+datarootdir = $(prefix)/share
+libexecdir = $(DESTDIR)/usr/libexec
+sbindir = $(DESTDIR)/usr/sbin
+bindir = $(exec_prefix)/bin
+archdir = $(rubyarchdir)
+
+
+CC_WRAPPER = 
+CC = gcc
+CXX = g++
+LIBRUBY = $(LIBRUBY_SO)
+LIBRUBY_A = lib$(RUBY_SO_NAME)-static.a
+LIBRUBYARG_SHARED = -l$(RUBY_SO_NAME)
+LIBRUBYARG_STATIC = -l$(RUBY_SO_NAME)-static $(MAINLIBS)
+empty =
+OUTFLAG = -o $(empty)
+COUTFLAG = -o $(empty)
+CSRCFLAG = $(empty)
+
+RUBY_EXTCONF_H = 
+cflags   = $(optflags) $(debugflags) $(warnflags)
+cxxflags = 
+optflags = -O3
+debugflags = -ggdb3
+warnflags = -Wall -Wextra -Wdeprecated-declarations -Wduplicated-cond -Wimplicit-function-declaration -Wimplicit-int -Wmisleading-indentation -Wpointer-arith -Wwrite-strings -Wimplicit-fallthrough=0 -Wmissing-noreturn -Wno-cast-function-type -Wno-constant-logical-operand -Wno-long-long -Wno-missing-field-initializers -Wno-overlength-strings -Wno-packed-bitfield-compat -Wno-parentheses-equality -Wno-self-assign -Wno-tautological-compare -Wno-unused-parameter -Wno-unused-value -Wsuggest-attribute=format -Wsuggest-attribute=noreturn -Wunused-variable
+cppflags = 
+CCDLFLAGS = -fPIC
+CFLAGS   = $(CCDLFLAGS) -O2  -fexceptions -g -grecord-gcc-switches -pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -fstack-protector-strong -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1  -mbranch-protection=standard -fasynchronous-unwind-tables -fstack-clash-protection -fPIC  $(ARCH_FLAG)
+INCFLAGS = -I. -I$(arch_hdrdir) -I$(hdrdir)/ruby/backward -I$(hdrdir) -I$(srcdir) 
+DEFS     = 
+CPPFLAGS =   $(DEFS) $(cppflags)
+CXXFLAGS = $(CCDLFLAGS) -O2  -fexceptions -g -grecord-gcc-switches -pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -fstack-protector-strong -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1  -mbranch-protection=standard -fasynchronous-unwind-tables -fstack-clash-protection  -fvisibility=hidden -std=c++11 $(ARCH_FLAG)
+ldflags  = -L. -Wl,-z,relro -Wl,--as-needed  -Wl,-z,now -specs=/usr/lib/rpm/redhat/redhat-hardened-ld  -fstack-protector-strong -rdynamic -Wl,-export-dynamic 
+dldflags = -Wl,-z,relro -Wl,--as-needed  -Wl,-z,now -specs=/usr/lib/rpm/redhat/redhat-hardened-ld  
+ARCH_FLAG = 
+DLDFLAGS = $(ldflags) $(dldflags) $(ARCH_FLAG)
+LDSHARED = $(CC) -shared
+LDSHAREDXX = $(CXX) -shared
+AR = ar
+EXEEXT = 
+
+RUBY_INSTALL_NAME = $(RUBY_BASE_NAME)
+RUBY_SO_NAME = ruby
+RUBYW_INSTALL_NAME = 
+RUBY_VERSION_NAME = $(RUBY_BASE_NAME)-$(ruby_version_dir_name)
+RUBYW_BASE_NAME = rubyw
+RUBY_BASE_NAME = ruby
+
+arch = aarch64-linux
+sitearch = $(arch)
+ruby_version = 2.7.0
+ruby = $(bindir)/$(RUBY_BASE_NAME)
+RUBY = $(ruby)
+ruby_headers = $(hdrdir)/ruby.h $(hdrdir)/ruby/backward.h $(hdrdir)/ruby/ruby.h $(hdrdir)/ruby/defines.h $(hdrdir)/ruby/missing.h $(hdrdir)/ruby/intern.h $(hdrdir)/ruby/st.h $(hdrdir)/ruby/subst.h $(arch_hdrdir)/ruby/config.h
+
+RM = rm -f
+RM_RF = $(RUBY) -run -e rm -- -rf
+RMDIRS = rmdir --ignore-fail-on-non-empty -p
+MAKEDIRS = /usr/bin/mkdir -p
+INSTALL = /usr/bin/install -c
+INSTALL_PROG = $(INSTALL) -m 0755
+INSTALL_DATA = $(INSTALL) -m 644
+COPY = cp
+TOUCH = exit >
+
+#### End of system configuration section. ####
+
+preload = 
+libpath = . $(archlibdir)
+LIBPATH =  -L. -L$(archlibdir)
+DEFFILE = 
+
+CLEANFILES = mkmf.log
+DISTCLEANFILES = 
+DISTCLEANDIRS = 
+
+extout = 
+extout_prefix = 
+target_prefix = 
+LOCAL_LIBS = 
+LIBS =   -lprotobuf -lpthread -lm   -lc
+ORIG_SRCS = base.cc embedding_feature_extractor.cc embedding_network.cc feature_extractor.cc feature_extractor.pb.cc feature_types.cc fixunicodevalue.cc fml_parser.cc generated_entities.cc generated_ulscript.cc getonescriptspan.cc lang_id_nn_params.cc language_identifier_features.cc nnet_language_identifier.cc nnet_language_identifier_c.cc offsetmap.cc registry.cc relevant_script_feature.cc sentence.pb.cc sentence_features.cc task_context.cc task_context_params.cc task_spec.pb.cc text_processing.cc unicodetext.cc utf8statetable.cc utils.cc workspace.cc
+SRCS = $(ORIG_SRCS) 
+OBJS = base.o embedding_feature_extractor.o embedding_network.o feature_extractor.o feature_extractor.pb.o feature_types.o fixunicodevalue.o fml_parser.o generated_entities.o generated_ulscript.o getonescriptspan.o lang_id_nn_params.o language_identifier_features.o nnet_language_identifier.o nnet_language_identifier_c.o offsetmap.o registry.o relevant_script_feature.o sentence.pb.o sentence_features.o task_context.o task_context_params.o task_spec.pb.o text_processing.o unicodetext.o utf8statetable.o utils.o workspace.o
+HDRS = $(srcdir)/base.h $(srcdir)/casts.h $(srcdir)/embedding_feature_extractor.h $(srcdir)/embedding_network.h $(srcdir)/embedding_network_params.h $(srcdir)/feature_extractor.h $(srcdir)/feature_types.h $(srcdir)/float16.h $(srcdir)/fml_parser.h $(srcdir)/language_identifier_features.h $(srcdir)/lang_id_nn_params.h $(srcdir)/nnet_language_identifier.h $(srcdir)/registry.h $(srcdir)/relevant_script_feature.h $(srcdir)/script_detector.h $(srcdir)/sentence_features.h $(srcdir)/simple_adder.h $(srcdir)/fixunicodevalue.h $(srcdir)/generated_ulscript.h $(srcdir)/getonescriptspan.h $(srcdir)/integral_types.h $(srcdir)/offsetmap.h $(srcdir)/port.h $(srcdir)/stringpiece.h $(srcdir)/text_processing.h $(srcdir)/utf8acceptinterchange.h $(srcdir)/utf8prop_lettermarkscriptnum.h $(srcdir)/utf8repl_lettermarklower.h $(srcdir)/utf8scannot_lettermarkspecial.h $(srcdir)/utf8statetable.h $(srcdir)/task_context.h $(srcdir)/task_context_params.h $(srcdir)/unicodetext.h $(srcdir)/utils.h $(srcdir)/workspace.h $(srcdir)/feature_extractor.pb.h $(srcdir)/sentence.pb.h $(srcdir)/task_spec.pb.h
+LOCAL_HDRS = 
+TARGET = libcld3
+TARGET_NAME = libcld3
+TARGET_ENTRY = Init_$(TARGET_NAME)
+DLLIB = $(TARGET).so
+EXTSTATIC = 
+STATIC_LIB = 
+
+TIMESTAMP_DIR = .
+BINDIR        = $(bindir)
+RUBYCOMMONDIR = $(sitedir)$(target_prefix)
+RUBYLIBDIR    = $(sitelibdir)$(target_prefix)
+RUBYARCHDIR   = $(sitearchdir)$(target_prefix)
+HDRDIR        = $(rubyhdrdir)/ruby$(target_prefix)
+ARCHHDRDIR    = $(rubyhdrdir)/$(arch)/ruby$(target_prefix)
+TARGET_SO_DIR =
+TARGET_SO     = $(TARGET_SO_DIR)$(DLLIB)
+CLEANLIBS     = $(TARGET_SO) 
+CLEANOBJS     = *.o  *.bak
+
+all:    $(DLLIB)
+static: $(STATIC_LIB)
+.PHONY: all install static install-so install-rb
+.PHONY: clean clean-so clean-static clean-rb
+
+clean-static::
+clean-rb-default::
+clean-rb::
+clean-so::
+clean: clean-so clean-static clean-rb-default clean-rb
+		-$(Q)$(RM) $(CLEANLIBS) $(CLEANOBJS) $(CLEANFILES) .*.time
+
+distclean-rb-default::
+distclean-rb::
+distclean-so::
+distclean-static::
+distclean: clean distclean-so distclean-static distclean-rb-default distclean-rb
+		-$(Q)$(RM) Makefile $(RUBY_EXTCONF_H) conftest.* mkmf.log
+		-$(Q)$(RM) core ruby$(EXEEXT) *~ $(DISTCLEANFILES)
+		-$(Q)$(RMDIRS) $(DISTCLEANDIRS) 2> /dev/null || true
+
+realclean: distclean
+install: install-so install-rb
+
+install-so: $(DLLIB) $(TIMESTAMP_DIR)/.sitearchdir.time
+	$(INSTALL_PROG) $(DLLIB) $(RUBYARCHDIR)
+clean-static::
+	-$(Q)$(RM) $(STATIC_LIB)
+install-rb: pre-install-rb do-install-rb install-rb-default
+install-rb-default: pre-install-rb-default do-install-rb-default
+pre-install-rb: Makefile
+pre-install-rb-default: Makefile
+do-install-rb:
+do-install-rb-default:
+pre-install-rb-default:
+	@$(NULLCMD)
+$(TIMESTAMP_DIR)/.sitearchdir.time:
+	$(Q) $(MAKEDIRS) $(@D) $(RUBYARCHDIR)
+	$(Q) $(TOUCH) $@
+
+site-install: site-install-so site-install-rb
+site-install-so: install-so
+site-install-rb: install-rb
+
+.SUFFIXES: .c .m .cc .mm .cxx .cpp .o .S
+
+.cc.o:
+	$(ECHO) compiling $(<)
+	$(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -c $(CSRCFLAG)$<
+
+.cc.S:
+	$(ECHO) translating $(<)
+	$(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -S $(CSRCFLAG)$<
+
+.mm.o:
+	$(ECHO) compiling $(<)
+	$(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -c $(CSRCFLAG)$<
+
+.mm.S:
+	$(ECHO) translating $(<)
+	$(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -S $(CSRCFLAG)$<
+
+.cxx.o:
+	$(ECHO) compiling $(<)
+	$(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -c $(CSRCFLAG)$<
+
+.cxx.S:
+	$(ECHO) translating $(<)
+	$(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -S $(CSRCFLAG)$<
+
+.cpp.o:
+	$(ECHO) compiling $(<)
+	$(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -c $(CSRCFLAG)$<
+
+.cpp.S:
+	$(ECHO) translating $(<)
+	$(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -S $(CSRCFLAG)$<
+
+.c.o:
+	$(ECHO) compiling $(<)
+	$(Q) $(CC) $(INCFLAGS) $(CPPFLAGS) $(CFLAGS) $(COUTFLAG)$@ -c $(CSRCFLAG)$<
+
+.c.S:
+	$(ECHO) translating $(<)
+	$(Q) $(CC) $(INCFLAGS) $(CPPFLAGS) $(CFLAGS) $(COUTFLAG)$@ -S $(CSRCFLAG)$<
+
+.m.o:
+	$(ECHO) compiling $(<)
+	$(Q) $(CC) $(INCFLAGS) $(CPPFLAGS) $(CFLAGS) $(COUTFLAG)$@ -c $(CSRCFLAG)$<
+
+.m.S:
+	$(ECHO) translating $(<)
+	$(Q) $(CC) $(INCFLAGS) $(CPPFLAGS) $(CFLAGS) $(COUTFLAG)$@ -S $(CSRCFLAG)$<
+
+$(TARGET_SO): $(OBJS) Makefile
+	$(ECHO) linking shared-object $(DLLIB)
+	-$(Q)$(RM) $(@)
+	$(Q) $(LDSHAREDXX) -o $@ $(OBJS) $(LIBPATH) $(DLDFLAGS) $(LOCAL_LIBS) $(LIBS)
+
+
+
+$(OBJS): $(HDRS) $(ruby_headers)

data/ext/cld3/extconf.rb

@@ -1,4 +1,4 @@
-# Copyright 2017 Akihiko Odaki <akihiko.odaki.4i@stu.hosei.ac.jp>
+# Copyright 2017 Akihiko Odaki <akihiko.odaki@gmail.com>
 # All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -33,7 +33,7 @@ FileUtils.mkdir_p("cld_3/protos")
 FileUtils.mkdir_p("script_span")
 
 [ "feature_extractor", "sentence", "task_spec" ].each {|name|
-  `protoc '#{name}.proto' --cpp_out=.`
+  system "protoc", "#{name}.proto", "--cpp_out=.", exception: true
   ln_fallback("#{name}.pb.h", "cld_3/protos/#{name}.pb.h")
 }
 
@@ -56,4 +56,5 @@ FileUtils.mkdir_p("script_span")
 }
 
 $CXXFLAGS += " -fvisibility=hidden -std=c++11"
+$LIBRUBYARG = ""
 create_makefile("libcld3")

data/ext/cld3/getonescriptspan.h

@@ -93,7 +93,7 @@ class ScriptScanner {
   // again with the first byte of the following range.
   int MapBack(int text_offset);
 
-  const char* GetBufferStart() {return start_byte_;};
+  const char* GetBufferStart() {return start_byte_;}
 
  private:
   // Skip over tags and non-letters

data/ext/cld3/libcld3.def

@@ -0,0 +1,8 @@
+EXPORTS
+  NNetLanguageIdentifier_find_language
+  NNetLanguageIdentifier_find_top_n_most_freq_langs
+  delete_NNetLanguageIdentifier
+  delete_result
+  delete_results
+  new_NNetLanguageIdentifier
+  refer_to_nth_result

data/ext/cld3/mkmf.log

@@ -0,0 +1,37 @@
+"pkg-config --exists protobuf"
+| pkg-config --libs protobuf
+=> "-lprotobuf -lpthread \n"
+"gcc -o conftest -I/usr/include -I/usr/include/ruby/backward -I/usr/include -I.    -O2  -fexceptions -g -grecord-gcc-switches -pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -fstack-protector-strong -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1  -mbranch-protection=standard -fasynchronous-unwind-tables -fstack-clash-protection -fPIC conftest.c  -L. -L/usr/lib64 -L. -Wl,-z,relro -Wl,--as-needed  -Wl,-z,now -specs=/usr/lib/rpm/redhat/redhat-hardened-ld  -fstack-protector-strong -rdynamic -Wl,-export-dynamic     -lruby  -lm   -lc"
+checked program was:
+/* begin */
+1: #include "ruby.h"
+2: 
+3: int main(int argc, char **argv)
+4: {
+5:   return !!argv[argc];
+6: }
+/* end */
+
+"gcc -o conftest -I/usr/include -I/usr/include/ruby/backward -I/usr/include -I.    -O2  -fexceptions -g -grecord-gcc-switches -pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -fstack-protector-strong -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1  -mbranch-protection=standard -fasynchronous-unwind-tables -fstack-clash-protection -fPIC conftest.c  -L. -L/usr/lib64 -L. -Wl,-z,relro -Wl,--as-needed  -Wl,-z,now -specs=/usr/lib/rpm/redhat/redhat-hardened-ld  -fstack-protector-strong -rdynamic -Wl,-export-dynamic     -lruby -lprotobuf -lpthread -lm   -lc"
+checked program was:
+/* begin */
+1: #include "ruby.h"
+2: 
+3: int main(int argc, char **argv)
+4: {
+5:   return !!argv[argc];
+6: }
+/* end */
+
+| pkg-config --cflags-only-I protobuf
+=> "\n"
+| pkg-config --cflags-only-other protobuf
+=> "\n"
+| pkg-config --libs-only-l protobuf
+=> "-lprotobuf -lpthread \n"
+package configuration for protobuf
+incflags: 
+cflags: 
+ldflags: 
+libs: -lprotobuf -lpthread
+

data/ext/cld3/nnet_language_identifier.cc

@@ -47,6 +47,9 @@ struct LangChunksStats {
 
   // Number chunks corresponding to the language.
   int num_chunks = 0;
+
+  // Specifies the byte ranges that language applies to.
+  std::vector<NNetLanguageIdentifier::SpanInfo> byte_ranges;
 };
 
 // Compares two pairs based on their values.
@@ -298,12 +301,16 @@ NNetLanguageIdentifier::FindTopNMostFreqLangs(const string &text,
     total_num_bytes += num_original_span_bytes;
 
     const string selected_text = SelectTextGivenScriptSpan(script_span);
+
     result = FindLanguageOfValidUTF8(selected_text);
     language = result.language;
     lang_stats[language].byte_sum += num_original_span_bytes;
     lang_stats[language].prob_sum +=
         result.probability * num_original_span_bytes;
     lang_stats[language].num_chunks++;
+    // Add SpanInfo. Start and end indices are relative to original input.
+    lang_stats[language].byte_ranges.push_back(SpanInfo(
+        ss.MapBack(0), ss.MapBack(script_span.text_bytes), result.probability));
   }
 
   // Sort the languages based on the number of bytes associated with them.
@@ -329,6 +336,7 @@ NNetLanguageIdentifier::FindTopNMostFreqLangs(const string &text,
     result.probability = stats.prob_sum / stats.byte_sum;
     result.proportion = stats.byte_sum / byte_sum;
     result.is_reliable = ResultIsReliable(language, result.probability);
+    result.byte_ranges = stats.byte_ranges;
     results.push_back(result);
   }
 

data/ext/cld3/nnet_language_identifier.h

@@ -44,6 +44,19 @@ class LanguageIdEmbeddingFeatureExtractor
 // Class for detecting the language of a document.
 class NNetLanguageIdentifier {
  public:
+  // Holds probability that Span, specified by start/end indices, is a given
+  // language. The langauge is not stored here; it can be found in Result, which
+  // holds a vector of SpanInfo.
+  struct SpanInfo {
+    SpanInfo(int start_index_val, int end_index_val, float probability_val)
+        : start_index(start_index_val),
+          end_index(end_index_val),
+          probability(probability_val) {}
+    int start_index = -1;
+    int end_index = -1;
+    float probability = 0.0;
+  };
+
   // Information about a predicted language.
   struct Result {
     string language = kUnknown;
@@ -53,6 +66,9 @@ class NNetLanguageIdentifier {
     // Proportion of bytes associated with the language. If FindLanguage is
     // called, this variable is set to 1.
     float proportion = 0.0;
+
+    // Specifies the byte ranges that |language| applies to.
+    std::vector<SpanInfo> byte_ranges;
   };
 
   NNetLanguageIdentifier();

data/ext/cld3/nnet_language_identifier_c.cc

@@ -1,4 +1,4 @@
-/* Copyright 2017 Akihiko Odaki <akihiko.odaki.4i@stu.hosei.ac.jp>
+/* Copyright 2017 Akihiko Odaki <akihiko.odaki@gmail.com>
 All Rights Reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
@@ -26,42 +26,90 @@ limitations under the License.
   #define EXPORT __attribute__ ((visibility ("default")))
 #endif
 
-struct NNetLanguageIdentifier {
-  chrome_lang_id::NNetLanguageIdentifier context;
-  std::string language;
-};
-
 struct Result {
   struct {
     const char *data;
     std::size_t size;
   } language;
+  struct {
+    const chrome_lang_id::NNetLanguageIdentifier::SpanInfo *data;
+    std::size_t size;
+  } byte_ranges;
   float probability;
   float proportion;
   bool is_reliable;
 };
 
+struct OwningResult {
+  OwningResult(chrome_lang_id::NNetLanguageIdentifier::Result&& result) {
+    references.language = std::move(result.language);
+    references.byte_ranges = std::move(result.byte_ranges);
+    plain.language.data = references.language.data();
+    plain.language.size = references.language.size();
+    plain.byte_ranges.data = references.byte_ranges.data();
+    plain.byte_ranges.size = references.byte_ranges.size();
+    plain.probability = result.probability;
+    plain.proportion = result.proportion;
+    plain.is_reliable = result.is_reliable;
+  }
+
+  Result plain;
+  struct {
+    std::string language;
+    std::vector<chrome_lang_id::NNetLanguageIdentifier::SpanInfo> byte_ranges;
+  } references;
+};
+
 extern "C" {
-  EXPORT Result NNetLanguageIdentifier_find_language(void *pointer,
-                                                     const char *data,
-                                                     std::size_t size) {
-    auto instance = static_cast<NNetLanguageIdentifier *>(pointer);
-    auto result = instance->context.FindLanguage(std::string(data, size));
-    instance->language = std::move(result.language);
-
-    return Result {
-        { instance->language.data(), instance->language.size() },
-        result.probability,
-        result.proportion,
-        result.is_reliable
-    };
+  EXPORT OwningResult *NNetLanguageIdentifier_find_language(
+      chrome_lang_id::NNetLanguageIdentifier *instance,
+      const char *data,
+      std::size_t size) {
+    return new OwningResult(instance->FindLanguage(std::string(data, size)));
   }
 
-  EXPORT void delete_NNetLanguageIdentifier(void *pointer) {
-    delete static_cast<NNetLanguageIdentifier *>(pointer);
+  EXPORT std::vector<chrome_lang_id::NNetLanguageIdentifier::Result>*
+  NNetLanguageIdentifier_find_top_n_most_freq_langs(
+      chrome_lang_id::NNetLanguageIdentifier *instance,
+      const char *data, std::size_t size, int num_langs) {
+    std::string text(data, size);
+    return new auto(instance->FindTopNMostFreqLangs(text, num_langs));
   }
 
-  EXPORT void *new_NNetLanguageIdentifier(int min_num_bytes, int max_num_bytes) {
-    return new NNetLanguageIdentifier{{min_num_bytes, max_num_bytes}, {}};
+  EXPORT void delete_NNetLanguageIdentifier(
+      chrome_lang_id::NNetLanguageIdentifier *pointer) {
+    delete pointer;
+  }
+
+  EXPORT void delete_result(OwningResult *pointer) {
+    delete pointer;
+  }
+
+  EXPORT void delete_results(
+      std::vector<chrome_lang_id::NNetLanguageIdentifier::Result> *pointer) {
+    delete pointer;
+  }
+
+  EXPORT chrome_lang_id::NNetLanguageIdentifier *new_NNetLanguageIdentifier(
+      int min_num_bytes, int max_num_bytes) {
+    return new chrome_lang_id::NNetLanguageIdentifier(
+        min_num_bytes, max_num_bytes);
+  }
+
+  EXPORT Result refer_to_nth_result(
+      std::vector<chrome_lang_id::NNetLanguageIdentifier::Result> *results,
+      std::size_t index) {
+    Result c;
+    auto& cc = (*results)[index];
+
+    c.language.data = cc.language.data();
+    c.language.size = cc.language.size();
+    c.byte_ranges.data = cc.byte_ranges.data();
+    c.byte_ranges.size = cc.byte_ranges.size();
+    c.probability = cc.probability;
+    c.proportion = cc.proportion;
+    c.is_reliable = cc.is_reliable;
+
+    return c;
   }
 }

data/lib/cld3.rb

@@ -1,7 +1,7 @@
 # File including an implementation of CLD3 module. Some documentations are
 # extracted from ext/cld3/ext/src/nnet_language_identifier.h.
 #
-# Copyright 2017 Akihiko Odaki <akihiko.odaki.4i@stu.hosei.ac.jp>
+# Copyright 2017 Akihiko Odaki <akihiko.odaki@gmail.com>
 # All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -49,10 +49,15 @@ module CLD3
     # This is Numeric object.
     RELIABILITY_HR_BS_THRESHOLD = 0.5
 
+    # Holds probability that Span, specified by start/end indices, is a given
+    # language. The langauge is not stored here; it can be found in Result, which
+    # holds an Array of SpanInfo.
+    SpanInfo = Struct.new(:start_index, :end_index, :probability)
+
     # Information about a predicted language.
     # This is an instance of Struct with the following members:
     #
-    # [language]    This is symbol or nil.
+    # [language]    This is symbol.
     #
     # [probability] Language probability. This is Numeric object.
     #
@@ -61,33 +66,95 @@ module CLD3
     # [proportion]  Proportion of bytes associated with the language. If
     #               #find_language is called, this variable is set to 1.
     #               This is Numeric object.
-    Result = Struct.new("Result", :language, :probability, :reliable?, :proportion)
+    #
+    # [byte_ranges] Specifies the byte ranges in UTF-8 that |language| applies to.
+    #               This is an Array of SpanInfo.
+    Result = Struct.new(:language, :probability, :reliable?, :proportion, :byte_ranges)
 
     # The arguments are two String objects.
-    def initialize(minNumBytes = MIN_NUM_BYTES_TO_CONSIDER, maxNumBytes = MAX_NUM_BYTES_TO_CONSIDER)
-      @cc = Unstable::NNetLanguageIdentifier::Pointer.new(Unstable.new_NNetLanguageIdentifier(minNumBytes, maxNumBytes))
+    def initialize(min_num_bytes = MIN_NUM_BYTES_TO_CONSIDER, max_num_bytes = MAX_NUM_BYTES_TO_CONSIDER)
+      @cc = Unstable::NNetLanguageIdentifier::Pointer.new(Unstable.new_NNetLanguageIdentifier(min_num_bytes, max_num_bytes))
     end
 
     # Finds the most likely language for the given text, along with additional
     # information (e.g., probability). The prediction is based on the first N
     # bytes where N is the minumum between the number of interchange valid UTF8
     # bytes and +max_num_bytes_+. If N is less than +min_num_bytes_+ long, then
-    # this function returns nil as language.
+    # this function returns nil.
     # The argument is a String object.
     # The returned value of this function is an instance of Result.
     def find_language(text)
       text_utf8 = text.encode(Encoding::UTF_8)
       pointer = FFI::MemoryPointer.new(:char, text_utf8.bytesize)
-      pointer.put_bytes(0, text_utf8)
 
-      cc_result = Unstable.NNetLanguageIdentifier_find_language(@cc, pointer, text_utf8.bytesize)
-      language = cc_result[:language_data].read_bytes(cc_result[:language_size])
+      begin
+        pointer.put_bytes(0, text_utf8)
+
+        result = Unstable.NNetLanguageIdentifier_find_language(@cc, pointer, text_utf8.bytesize)
+        begin
+          convert_result Unstable::NNetLanguageIdentifier::Result.new(result)
+        ensure
+          Unstable.delete_result result
+        end
+      ensure
+        pointer.free
+      end
+    end
+
+    # Splits the input text (up to the first byte, if any, that is not
+    # interchange valid UTF8) into spans based on the script, predicts a language
+    # for each span, and returns a vector storing the top num_langs most frequent
+    # languages along with additional information (e.g., proportions). The number
+    # of bytes considered for each span is the minimum between the size of the
+    # span and +max_num_bytes_+. If more languages are requested than what is
+    # available in the input, then the number of the returned elements will be
+    # the number of the latter. Also, if the size of the span is less than
+    # +min_num_bytes_+ long, then the span is skipped. If the input text is too
+    # long, only the first +MAX_NUM_INPUT_BYTES_TO_CONSIDER+ bytes are processed.
+    # The first argument is a String object.
+    # The second argument is Numeric object.
+    # The returned value of this functions is an Array of Result instances.
+    def find_top_n_most_freq_langs(text, num_langs)
+      text_utf8 = text.encode(Encoding::UTF_8)
+      pointer = FFI::MemoryPointer.new(:char, text_utf8.bytesize)
+
+      begin
+        pointer.put_bytes(0, text_utf8)
+
+        results = Unstable.NNetLanguageIdentifier_find_top_n_most_freq_langs(@cc, pointer, text_utf8.bytesize, num_langs)
+        begin
+          num_langs.times
+            .lazy
+            .map { |index| convert_result Unstable.refer_to_nth_result(results, index) }
+            .take_while { |result| !result.nil? }
+            .to_a
+        ensure
+          Unstable.delete_results results
+        end
+      ensure
+        pointer.free
+      end
+    end
+
+    private
+
+    def convert_result(result)
+      language = result[:language_data].read_bytes(result[:language_size])
+      return nil if language == "und"
+
+      cursor = result[:byte_ranges_data]
+      byte_ranges = result[:byte_ranges_size].times.map do
+        info = Unstable::NNetLanguageIdentifier::SpanInfo.new(cursor)
+        cursor += Unstable::NNetLanguageIdentifier::SpanInfo.size
+        SpanInfo.new(info[:start_index], info[:end_index], info[:probability])
+      end
 
       Result.new(
-          language == "und" ? nil : language.to_sym,
-          cc_result[:probability],
-          cc_result[:reliable?],
-          cc_result[:proportion])
+          language.to_sym,
+          result[:probability],
+          result[:reliable?],
+          result[:proportion],
+          byte_ranges)
     end
   end
 
@@ -112,7 +179,7 @@ module CLD3
   module Unstable
     extend FFI::Library
 
-    ffi_lib File.join(File.expand_path(File.dirname(__FILE__)), "..", "ext", "cld3", "libcld3." + RbConfig::CONFIG["DLEXT"])
+    ffi_lib File.join(__dir__, "..", "ext", "cld3", "libcld3." + RbConfig::CONFIG["DLEXT"])
 
     module NNetLanguageIdentifier
       class Pointer < FFI::AutoPointer
@@ -121,17 +188,30 @@ module CLD3
         end
       end
 
+      class SpanInfo < FFI::Struct
+        layout :start_index, :int, :end_index, :int, :probability, :float
+      end
+
       class Result < FFI::Struct
-        layout :language_data, :pointer, :language_size, :size_t, :probability, :float, :proportion, :float, :reliable?, :bool
+        layout :language_data, :pointer, :language_size, :size_t, :byte_ranges_data, :pointer, :byte_ranges_size, :size_t, :probability, :float, :proportion, :float, :reliable?, :bool
       end
     end
 
     attach_function :delete_NNetLanguageIdentifier, [ :pointer ], :void
 
+    attach_function :delete_result, [ :pointer ], :void
+
+    attach_function :delete_results, [ :pointer ], :void
+
     attach_function :new_NNetLanguageIdentifier, [ :int, :int ], :pointer
 
+    attach_function :refer_to_nth_result, [ :pointer, :size_t ], NNetLanguageIdentifier::Result.by_value
+
     attach_function :NNetLanguageIdentifier_find_language,
-        [ :pointer, :buffer_in, :size_t ], NNetLanguageIdentifier::Result.by_value
+        [ :pointer, :buffer_in, :size_t ], :pointer
+
+    attach_function :NNetLanguageIdentifier_find_top_n_most_freq_langs,
+        [ :pointer, :buffer_in, :size_t, :int ], :pointer
   end
 
   private_constant :Unstable

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cld3
 version: !ruby/object:Gem::Version
-  version: 3.2.4
+  version: 3.4.2
 platform: ruby
 authors:
 - Akihiko Odaki
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-04-20 00:00:00.000000000 Z
+date: 2021-04-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -19,7 +19,7 @@ dependencies:
         version: 1.1.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 1.11.0
+        version: 1.16.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -29,7 +29,7 @@ dependencies:
         version: 1.1.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 1.11.0
+        version: 1.16.0
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -39,7 +39,7 @@ dependencies:
         version: 3.0.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 3.9.0
+        version: 3.11.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -49,10 +49,10 @@ dependencies:
         version: 3.0.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 3.9.0
+        version: 3.11.0
 description: Compact Language Detector v3 (CLD3) is a neural network model for language
   identification.
-email: akihiko.odaki.4i@stu.hosei.ac.jp
+email: akihiko.odaki@gmail.com
 executables: []
 extensions:
 - ext/cld3/extconf.rb
@@ -63,76 +63,108 @@ files:
 - LICENSE_CLD3
 - README.md
 - cld3.gemspec
+- ext/cld3/Makefile
 - ext/cld3/base.cc
 - ext/cld3/base.h
+- ext/cld3/base.o
 - ext/cld3/casts.h
 - ext/cld3/embedding_feature_extractor.cc
 - ext/cld3/embedding_feature_extractor.h
+- ext/cld3/embedding_feature_extractor.o
 - ext/cld3/embedding_network.cc
 - ext/cld3/embedding_network.h
+- ext/cld3/embedding_network.o
 - ext/cld3/embedding_network_params.h
 - ext/cld3/extconf.rb
 - ext/cld3/feature_extractor.cc
 - ext/cld3/feature_extractor.h
+- ext/cld3/feature_extractor.o
+- ext/cld3/feature_extractor.pb.o
 - ext/cld3/feature_extractor.proto
 - ext/cld3/feature_types.cc
 - ext/cld3/feature_types.h
+- ext/cld3/feature_types.o
 - ext/cld3/fixunicodevalue.cc
 - ext/cld3/fixunicodevalue.h
+- ext/cld3/fixunicodevalue.o
 - ext/cld3/float16.h
 - ext/cld3/fml_parser.cc
 - ext/cld3/fml_parser.h
+- ext/cld3/fml_parser.o
 - ext/cld3/generated_entities.cc
+- ext/cld3/generated_entities.o
 - ext/cld3/generated_ulscript.cc
 - ext/cld3/generated_ulscript.h
+- ext/cld3/generated_ulscript.o
 - ext/cld3/getonescriptspan.cc
 - ext/cld3/getonescriptspan.h
+- ext/cld3/getonescriptspan.o
 - ext/cld3/integral_types.h
 - ext/cld3/lang_id_nn_params.cc
 - ext/cld3/lang_id_nn_params.h
+- ext/cld3/lang_id_nn_params.o
 - ext/cld3/language_identifier_features.cc
 - ext/cld3/language_identifier_features.h
+- ext/cld3/language_identifier_features.o
+- ext/cld3/libcld3.def
+- ext/cld3/libcld3.so
+- ext/cld3/mkmf.log
 - ext/cld3/nnet_language_identifier.cc
 - ext/cld3/nnet_language_identifier.h
+- ext/cld3/nnet_language_identifier.o
 - ext/cld3/nnet_language_identifier_c.cc
+- ext/cld3/nnet_language_identifier_c.o
 - ext/cld3/offsetmap.cc
 - ext/cld3/offsetmap.h
+- ext/cld3/offsetmap.o
 - ext/cld3/port.h
 - ext/cld3/registry.cc
 - ext/cld3/registry.h
+- ext/cld3/registry.o
 - ext/cld3/relevant_script_feature.cc
 - ext/cld3/relevant_script_feature.h
+- ext/cld3/relevant_script_feature.o
 - ext/cld3/script_detector.h
+- ext/cld3/sentence.pb.o
 - ext/cld3/sentence.proto
 - ext/cld3/sentence_features.cc
 - ext/cld3/sentence_features.h
+- ext/cld3/sentence_features.o
 - ext/cld3/simple_adder.h
 - ext/cld3/stringpiece.h
 - ext/cld3/task_context.cc
 - ext/cld3/task_context.h
+- ext/cld3/task_context.o
 - ext/cld3/task_context_params.cc
 - ext/cld3/task_context_params.h
+- ext/cld3/task_context_params.o
+- ext/cld3/task_spec.pb.o
 - ext/cld3/task_spec.proto
 - ext/cld3/text_processing.cc
 - ext/cld3/text_processing.h
+- ext/cld3/text_processing.o
 - ext/cld3/unicodetext.cc
 - ext/cld3/unicodetext.h
+- ext/cld3/unicodetext.o
 - ext/cld3/utf8acceptinterchange.h
 - ext/cld3/utf8prop_lettermarkscriptnum.h
 - ext/cld3/utf8repl_lettermarklower.h
 - ext/cld3/utf8scannot_lettermarkspecial.h
 - ext/cld3/utf8statetable.cc
 - ext/cld3/utf8statetable.h
+- ext/cld3/utf8statetable.o
 - ext/cld3/utils.cc
 - ext/cld3/utils.h
+- ext/cld3/utils.o
 - ext/cld3/workspace.cc
 - ext/cld3/workspace.h
+- ext/cld3/workspace.o
 - lib/cld3.rb
 homepage: https://github.com/akihikodaki/cld3-ruby
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -140,18 +172,18 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.3.0
+      version: 2.6.0
   - - "<"
     - !ruby/object:Gem::Version
-      version: 2.7.0
+      version: 3.1.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
 summary: Compact Language Detector v3 (CLD3)
 test_files: []