text_sentencer_c 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: c8b424a5b6ea7cb3ac0b7138f3f9c2ec480144b6a5a21a706937f41e37e11edd
+  data.tar.gz: 40f8b0e846f631e1142113b0073b47169ef6cb2348bedf9b1a55c5b8662a3ee9
+SHA512:
+  metadata.gz: 68fe2267dbb019c2e3e515527f0cd0026b9a2212ebafafe03b87d3663dc326d66fbe92c5c5fbf909e55b53798158e4f79ff2be79ce7925470025e64f249b61a6
+  data.tar.gz: c03d9f4347e6a3e2dbec7730560526ff5eac208fa553c87be9b9ebf78b0c9036346fc08ac5aa3cc1ea2402b3c8d82f5ddef94097cd1e3d5dc1ad71e0ce51b3bc

data/README.md

@@ -0,0 +1,11 @@
+# text_sentencer_c
+
+A Ruby gem for text segmentation using a C extension.
+It is a re-implementation of 'text_sentencer', which is originally implemented in ruby, using a C extension, for a better performance.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'text_sentencer_c'

data/Rakefile

@@ -0,0 +1,6 @@
+require 'rake/extensiontask'
+
+Rake::ExtensionTask.new('text_sentencer_c') do |ext|
+	ext.lib_dir = 'lib/text_sentencer_c'
+	ext.ext_dir = 'ext/text_sentencer_c'
+end

data/bin/text_sentencer

@@ -0,0 +1,52 @@
+#!/usr/bin/env ruby
+require 'json'
+require 'text_sentencer_c'
+
+rules_filename = nil
+output_mode = :sentences
+
+## command line option processing
+require 'optparse'
+optparse = OptionParser.new do |opts|
+	opts.banner = "Usage: text_sentencer [options]"
+
+	opts.on('-r', '--rules=rules_filename', 'specifies the rules JSON file.') do |c|
+		rules_filename = c
+	end
+		
+	opts.on('-j', '--json_output', 'outputs the result in JSON.') do
+		output_mode = :json
+	end
+
+	opts.on('-h', '--help', 'displays this screen.') do
+		puts opts
+		exit
+	end
+end
+
+optparse.parse!
+
+rules = if rules_filename && File.file?(rules_filename)
+	JSON.parse File.read(rules_filename)
+end
+
+sentencer = TextSentencer.new(rules || TextSentencer::DEFAULT_RULES)
+
+text = ARGF.read
+
+# segments = sentencer.segment(text)
+# p segments
+# segments.each do |b, e|
+#   puts text[b...e]
+# end
+
+annotations = sentencer.annotate(text)
+
+if output_mode == :json
+	puts JSON.pretty_generate(annotations)
+else
+	annotations['blocks']&.each do |d|
+		span = d['span']
+		puts text[span['begin']...span['end']]
+	end
+end

data/ext/text_sentencer_c/Makefile

@@ -0,0 +1,269 @@
+
+SHELL = /bin/sh
+
+# V=0 quiet, V=1 verbose.  other values don't work.
+V = 0
+V0 = $(V:0=)
+Q1 = $(V:1=)
+Q = $(Q1:0=@)
+ECHO1 = $(V:1=@ :)
+ECHO = $(ECHO1:0=@ echo)
+NULLCMD = :
+
+#### Start of system configuration section. ####
+
+srcdir = .
+topdir = /home/jdkim/.rvm/rubies/ruby-3.3.4/include/ruby-3.3.0
+hdrdir = $(topdir)
+arch_hdrdir = /home/jdkim/.rvm/rubies/ruby-3.3.4/include/ruby-3.3.0/x86_64-linux
+PATH_SEPARATOR = :
+VPATH = $(srcdir):$(arch_hdrdir)/ruby:$(hdrdir)/ruby
+prefix = $(DESTDIR)/home/jdkim/.rvm/rubies/ruby-3.3.4
+rubysitearchprefix = $(rubylibprefix)/$(sitearch)
+rubyarchprefix = $(rubylibprefix)/$(arch)
+rubylibprefix = $(libdir)/$(RUBY_BASE_NAME)
+exec_prefix = $(prefix)
+vendorarchhdrdir = $(vendorhdrdir)/$(sitearch)
+sitearchhdrdir = $(sitehdrdir)/$(sitearch)
+rubyarchhdrdir = $(rubyhdrdir)/$(arch)
+vendorhdrdir = $(rubyhdrdir)/vendor_ruby
+sitehdrdir = $(rubyhdrdir)/site_ruby
+rubyhdrdir = $(includedir)/$(RUBY_VERSION_NAME)
+vendorarchdir = $(vendorlibdir)/$(sitearch)
+vendorlibdir = $(vendordir)/$(ruby_version)
+vendordir = $(rubylibprefix)/vendor_ruby
+sitearchdir = $(sitelibdir)/$(sitearch)
+sitelibdir = $(sitedir)/$(ruby_version)
+sitedir = $(rubylibprefix)/site_ruby
+rubyarchdir = $(rubylibdir)/$(arch)
+rubylibdir = $(rubylibprefix)/$(ruby_version)
+sitearchincludedir = $(includedir)/$(sitearch)
+archincludedir = $(includedir)/$(arch)
+sitearchlibdir = $(libdir)/$(sitearch)
+archlibdir = $(libdir)/$(arch)
+ridir = $(datarootdir)/$(RI_BASE_NAME)
+mandir = $(datarootdir)/man
+localedir = $(datarootdir)/locale
+libdir = $(exec_prefix)/lib
+psdir = $(docdir)
+pdfdir = $(docdir)
+dvidir = $(docdir)
+htmldir = $(docdir)
+infodir = $(datarootdir)/info
+docdir = $(datarootdir)/doc/$(PACKAGE)
+oldincludedir = $(DESTDIR)/usr/include
+includedir = $(prefix)/include
+runstatedir = $(localstatedir)/run
+localstatedir = $(prefix)/var
+sharedstatedir = $(prefix)/com
+sysconfdir = $(prefix)/etc
+datadir = $(datarootdir)
+datarootdir = $(prefix)/share
+libexecdir = $(exec_prefix)/libexec
+sbindir = $(exec_prefix)/sbin
+bindir = $(exec_prefix)/bin
+archdir = $(rubyarchdir)
+
+
+CC_WRAPPER = 
+CC = gcc
+CXX = g++
+LIBRUBY = $(LIBRUBY_SO)
+LIBRUBY_A = lib$(RUBY_SO_NAME)-static.a
+LIBRUBYARG_SHARED = -Wl,-rpath,$(libdir) -L$(libdir) -l$(RUBY_SO_NAME)
+LIBRUBYARG_STATIC = -Wl,-rpath,$(libdir) -L$(libdir) -l$(RUBY_SO_NAME)-static $(MAINLIBS)
+empty =
+OUTFLAG = -o $(empty)
+COUTFLAG = -o $(empty)
+CSRCFLAG = $(empty)
+
+RUBY_EXTCONF_H = 
+cflags   = $(optflags) $(debugflags) $(warnflags)
+cxxflags = 
+optflags = -O3 -fno-fast-math
+debugflags = -ggdb3
+warnflags = -Wall -Wextra -Wdeprecated-declarations -Wdiv-by-zero -Wduplicated-cond -Wimplicit-function-declaration -Wimplicit-int -Wpointer-arith -Wwrite-strings -Wold-style-definition -Wimplicit-fallthrough=0 -Wmissing-noreturn -Wno-cast-function-type -Wno-constant-logical-operand -Wno-long-long -Wno-missing-field-initializers -Wno-overlength-strings -Wno-packed-bitfield-compat -Wno-parentheses-equality -Wno-self-assign -Wno-tautological-compare -Wno-unused-parameter -Wno-unused-value -Wsuggest-attribute=format -Wsuggest-attribute=noreturn -Wunused-variable -Wmisleading-indentation -Wundef
+cppflags = 
+CCDLFLAGS = -fPIC
+CFLAGS   = $(CCDLFLAGS) $(cflags)  -fPIC $(ARCH_FLAG)
+INCFLAGS = -I. -I$(arch_hdrdir) -I$(hdrdir)/ruby/backward -I$(hdrdir) -I$(srcdir)
+DEFS     = 
+CPPFLAGS =   $(DEFS) $(cppflags)
+CXXFLAGS = $(CCDLFLAGS)  $(ARCH_FLAG)
+ldflags  = -L. -fstack-protector-strong -rdynamic -Wl,-export-dynamic -Wl,--no-as-needed
+dldflags = -Wl,--compress-debug-sections=zlib 
+ARCH_FLAG = 
+DLDFLAGS = $(ldflags) $(dldflags) $(ARCH_FLAG)
+LDSHARED = $(CC) -shared
+LDSHAREDXX = $(CXX) -shared
+AR = gcc-ar
+EXEEXT = 
+
+RUBY_INSTALL_NAME = $(RUBY_BASE_NAME)
+RUBY_SO_NAME = ruby
+RUBYW_INSTALL_NAME = 
+RUBY_VERSION_NAME = $(RUBY_BASE_NAME)-$(ruby_version)
+RUBYW_BASE_NAME = rubyw
+RUBY_BASE_NAME = ruby
+
+arch = x86_64-linux
+sitearch = $(arch)
+ruby_version = 3.3.0
+ruby = $(bindir)/$(RUBY_BASE_NAME)
+RUBY = $(ruby)
+BUILTRUBY = $(bindir)/$(RUBY_BASE_NAME)
+ruby_headers = $(hdrdir)/ruby.h $(hdrdir)/ruby/backward.h $(hdrdir)/ruby/ruby.h $(hdrdir)/ruby/defines.h $(hdrdir)/ruby/missing.h $(hdrdir)/ruby/intern.h $(hdrdir)/ruby/st.h $(hdrdir)/ruby/subst.h $(arch_hdrdir)/ruby/config.h
+
+RM = rm -f
+RM_RF = rm -fr
+RMDIRS = rmdir --ignore-fail-on-non-empty -p
+MAKEDIRS = /usr/bin/mkdir -p
+INSTALL = /usr/bin/install -c
+INSTALL_PROG = $(INSTALL) -m 0755
+INSTALL_DATA = $(INSTALL) -m 644
+COPY = cp
+TOUCH = exit >
+
+#### End of system configuration section. ####
+
+preload = 
+libpath = . $(libdir)
+LIBPATH =  -L. -L$(libdir) -Wl,-rpath,$(libdir)
+DEFFILE = 
+
+CLEANFILES = mkmf.log
+DISTCLEANFILES = 
+DISTCLEANDIRS = 
+
+extout = 
+extout_prefix = 
+target_prefix = /text_sentencer_c
+LOCAL_LIBS = 
+LIBS = $(LIBRUBYARG_SHARED)  -lm -lpthread  -lc
+ORIG_SRCS = text_sentencer.c
+SRCS = $(ORIG_SRCS) 
+OBJS = text_sentencer.o
+HDRS = $(srcdir)/text_sentencer.h
+LOCAL_HDRS = 
+TARGET = text_sentencer_c
+TARGET_NAME = text_sentencer_c
+TARGET_ENTRY = Init_$(TARGET_NAME)
+DLLIB = $(TARGET).so
+EXTSTATIC = 
+STATIC_LIB = 
+
+TIMESTAMP_DIR = .
+BINDIR        = $(bindir)
+RUBYCOMMONDIR = $(sitedir)$(target_prefix)
+RUBYLIBDIR    = $(sitelibdir)$(target_prefix)
+RUBYARCHDIR   = $(sitearchdir)$(target_prefix)
+HDRDIR        = $(sitehdrdir)$(target_prefix)
+ARCHHDRDIR    = $(sitearchhdrdir)$(target_prefix)
+TARGET_SO_DIR =
+TARGET_SO     = $(TARGET_SO_DIR)$(DLLIB)
+CLEANLIBS     = $(TARGET_SO) false
+CLEANOBJS     = $(OBJS) *.bak
+TARGET_SO_DIR_TIMESTAMP = $(TIMESTAMP_DIR)/.sitearchdir.-.text_sentencer_c.time
+
+all:    $(DLLIB)
+static: $(STATIC_LIB)
+.PHONY: all install static install-so install-rb
+.PHONY: clean clean-so clean-static clean-rb
+
+clean-static::
+clean-rb-default::
+clean-rb::
+clean-so::
+clean: clean-so clean-static clean-rb-default clean-rb
+		-$(Q)$(RM_RF) $(CLEANLIBS) $(CLEANOBJS) $(CLEANFILES) .*.time
+
+distclean-rb-default::
+distclean-rb::
+distclean-so::
+distclean-static::
+distclean: clean distclean-so distclean-static distclean-rb-default distclean-rb
+		-$(Q)$(RM) Makefile $(RUBY_EXTCONF_H) conftest.* mkmf.log
+		-$(Q)$(RM) core ruby$(EXEEXT) *~ $(DISTCLEANFILES)
+		-$(Q)$(RMDIRS) $(DISTCLEANDIRS) 2> /dev/null || true
+
+realclean: distclean
+install: install-so install-rb
+
+install-so: $(DLLIB) $(TARGET_SO_DIR_TIMESTAMP)
+	$(INSTALL_PROG) $(DLLIB) $(RUBYARCHDIR)
+clean-static::
+	-$(Q)$(RM) $(STATIC_LIB)
+install-rb: pre-install-rb do-install-rb install-rb-default
+install-rb-default: pre-install-rb-default do-install-rb-default
+pre-install-rb: Makefile
+pre-install-rb-default: Makefile
+do-install-rb:
+do-install-rb-default:
+pre-install-rb-default:
+	@$(NULLCMD)
+$(TARGET_SO_DIR_TIMESTAMP):
+	$(Q) $(MAKEDIRS) $(@D) $(RUBYARCHDIR)
+	$(Q) $(TOUCH) $@
+
+site-install: site-install-so site-install-rb
+site-install-so: install-so
+site-install-rb: install-rb
+
+.SUFFIXES: .c .m .cc .mm .cxx .cpp .o .S
+
+.cc.o:
+	$(ECHO) compiling $(<)
+	$(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -c $(CSRCFLAG)$<
+
+.cc.S:
+	$(ECHO) translating $(<)
+	$(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -S $(CSRCFLAG)$<
+
+.mm.o:
+	$(ECHO) compiling $(<)
+	$(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -c $(CSRCFLAG)$<
+
+.mm.S:
+	$(ECHO) translating $(<)
+	$(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -S $(CSRCFLAG)$<
+
+.cxx.o:
+	$(ECHO) compiling $(<)
+	$(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -c $(CSRCFLAG)$<
+
+.cxx.S:
+	$(ECHO) translating $(<)
+	$(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -S $(CSRCFLAG)$<
+
+.cpp.o:
+	$(ECHO) compiling $(<)
+	$(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -c $(CSRCFLAG)$<
+
+.cpp.S:
+	$(ECHO) translating $(<)
+	$(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -S $(CSRCFLAG)$<
+
+.c.o:
+	$(ECHO) compiling $(<)
+	$(Q) $(CC) $(INCFLAGS) $(CPPFLAGS) $(CFLAGS) $(COUTFLAG)$@ -c $(CSRCFLAG)$<
+
+.c.S:
+	$(ECHO) translating $(<)
+	$(Q) $(CC) $(INCFLAGS) $(CPPFLAGS) $(CFLAGS) $(COUTFLAG)$@ -S $(CSRCFLAG)$<
+
+.m.o:
+	$(ECHO) compiling $(<)
+	$(Q) $(CC) $(INCFLAGS) $(CPPFLAGS) $(CFLAGS) $(COUTFLAG)$@ -c $(CSRCFLAG)$<
+
+.m.S:
+	$(ECHO) translating $(<)
+	$(Q) $(CC) $(INCFLAGS) $(CPPFLAGS) $(CFLAGS) $(COUTFLAG)$@ -S $(CSRCFLAG)$<
+
+$(TARGET_SO): $(OBJS) Makefile
+	$(ECHO) linking shared-object text_sentencer_c/$(DLLIB)
+	-$(Q)$(RM) $(@)
+	$(Q) $(LDSHARED) -o $@ $(OBJS) $(LIBPATH) $(DLDFLAGS) $(LOCAL_LIBS) $(LIBS)
+
+
+
+$(OBJS): $(HDRS) $(ruby_headers)

data/ext/text_sentencer_c/extconf.rb

@@ -0,0 +1,7 @@
+require 'mkmf'
+
+# Check for ICU
+have_library('icuuc')
+have_library('icui18n')
+
+create_makefile('text_sentencer_c/text_sentencer_c')

data/ext/text_sentencer_c/text_sentencer.c

@@ -0,0 +1,600 @@
+#include "text_sentencer.h"
+#include <ruby/thread.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+// Define the TextSentencer class
+VALUE rb_cTextSentencer;
+
+static void text_sentencer_free(void *p) {
+	TextSentencer *ts = (TextSentencer *)p;
+
+	// Free each compiled regex
+	uregex_close(ts->break_pattern);
+	uregex_close(ts->candidate_pattern);
+
+	for (size_t i = 0; i < ts->num_positive_rules; i++) {
+		uregex_close(ts->positive_rules_pre[i]);
+		uregex_close(ts->positive_rules_post[i]);
+	}
+	for (size_t i = 0; i < ts->num_negative_rules; i++) {
+		uregex_close(ts->negative_rules_pre[i]);
+		uregex_close(ts->negative_rules_post[i]);
+	}
+
+	free(ts->positive_rules_pre);
+	free(ts->positive_rules_post);
+	free(ts->negative_rules_pre);
+	free(ts->negative_rules_post);
+
+	free(ts);
+}
+
+static size_t text_sentencer_memsize(const void *p) {
+	const TextSentencer *ts = (const TextSentencer *)p;
+	
+	// Calculate the size of the TextSentencer structure itself
+	size_t size = sizeof(TextSentencer);
+	
+	// Add the size of the arrays of pointers to URegularExpression
+	size += ts->num_positive_rules * sizeof(URegularExpression *);
+	size += ts->num_negative_rules * sizeof(URegularExpression *);
+	
+	// Add the two main pointers for break_pattern and candidate_pattern
+	size += sizeof(URegularExpression *) * 2;
+
+	return size;
+}
+
+
+static const rb_data_type_t text_sentencer_type = {
+	"TextSentencer",
+	{
+		0,                           // Mark function (not needed here)
+		text_sentencer_free,         // Free function
+		text_sentencer_memsize,      // Size function
+	},
+	NULL, NULL, RUBY_TYPED_FREE_IMMEDIATELY,
+};
+
+/*
+// Helper function to convert UTF-8 string to UTF-16
+static UChar* utf8_to_utf16(const char* utf8, int32_t* utf16_len, UErrorCode* status) {
+	int32_t utf16_capacity = 0;
+	u_strFromUTF8(NULL, 0, &utf16_capacity, utf8, -1, status);
+	if (*status != U_BUFFER_OVERFLOW_ERROR) {
+		return NULL;
+	}
+
+	*status = U_ZERO_ERROR;
+	UChar* utf16 = (UChar*)malloc(utf16_capacity * sizeof(UChar));
+	u_strFromUTF8(utf16, utf16_capacity, utf16_len, utf8, -1, status);
+	return utf16;
+}
+*/
+
+// Helper function to convert UTF-8 string to UTF-16
+static UChar* utf8_to_utf16(const char* utf8, int32_t* utf16_len, UErrorCode* status) {
+	int32_t utf16_capacity = 0;
+
+	// First, get the required capacity for the UTF-16 string
+	u_strFromUTF8(NULL, 0, &utf16_capacity, utf8, -1, status);
+	if (*status != U_BUFFER_OVERFLOW_ERROR && U_FAILURE(*status)) {
+		printf("utf8_to_utf16 returning NULL.\n");
+		return NULL; // Return if an error occurred that is not buffer overflow
+	}
+
+	*status = U_ZERO_ERROR; // Reset the status before the actual conversion
+
+	// Allocate memory for the UTF-16 string
+	UChar* utf16 = (UChar*)malloc(utf16_capacity * sizeof(UChar));
+	if (utf16 == NULL) {
+		*status = U_MEMORY_ALLOCATION_ERROR;
+		printf("utf8_to_utf16 returning NULL.\n");
+		return NULL;
+	}
+
+	// Perform the conversion from UTF-8 to UTF-16
+	u_strFromUTF8(utf16, utf16_capacity, utf16_len, utf8, -1, status);
+	if (U_FAILURE(*status)) {
+		free(utf16); // Free allocated memory if conversion fails
+		printf("utf8_to_utf16 returning NULL.\n");
+		return NULL;
+	}
+
+	return utf16;
+}
+
+/*
+// Utility function to scan text with a regex pattern and return offsets using ICU
+static long scan_offsets(const UChar *text, URegularExpression *pattern, long **span_starts, long **span_ends, UErrorCode *status) {
+	long allocated = 10;
+	long used = 0;
+
+	*span_starts = malloc(allocated * sizeof(long));
+	*span_ends = malloc(allocated * sizeof(long));
+	if (*span_starts == NULL || *span_ends == NULL) {
+		rb_raise(rb_eRuntimeError, "Memory allocation failed");
+	}
+
+	uregex_setText(pattern, text, -1, status);
+	if (U_FAILURE(*status)) {
+		rb_raise(rb_eRuntimeError, "Failed to set text for regex");
+	}
+
+	while (uregex_findNext(pattern, status)) {
+		if (used >= allocated) {
+			allocated *= 2;
+			*span_starts = realloc(*span_starts, allocated * sizeof(long));
+			*span_ends = realloc(*span_ends, allocated * sizeof(long));
+			if (*span_starts == NULL || *span_ends == NULL) {
+				rb_raise(rb_eRuntimeError, "Memory allocation failed");
+			}
+		}
+
+		(*span_starts)[used] = uregex_start(pattern, 0, status);
+		(*span_ends)[used] = uregex_end(pattern, 0, status);
+		if (U_FAILURE(*status)) {
+			rb_raise(rb_eRuntimeError, "Error during regex match: %s", u_errorName(*status));
+		}
+
+		used++;
+	}
+
+	return used;
+}
+*/
+// Utility function to match a regex against a substring using ICU
+static int match_regex(const UChar *text, long start, long end, URegularExpression *pattern, UErrorCode *status) {
+	uregex_setText(pattern, text + start, end - start, status);
+	if (U_FAILURE(*status)) {
+		rb_raise(rb_eRuntimeError, "Failed to set text for regex");
+	}
+
+	return uregex_find(pattern, 0, status);
+}
+
+void print_utf16_hex(const UChar *utf16_text, int32_t utf16_len) {
+	printf("UTF-16 Length: %d\n", utf16_len);
+	printf("UTF-16 Hex Values: ");
+	for (int32_t i = 0; i < utf16_len; i++) {
+		printf("%04X ", utf16_text[i]);  // Print each UChar as a 4-digit hexadecimal
+	}
+	printf("\n");
+}
+
+// Function to print UChar array after converting back to UTF-8
+void print_utf16_as_utf8(const UChar *utf16_text, int32_t utf16_len) {
+	UErrorCode status = U_ZERO_ERROR;
+	int32_t utf8_len = 0;
+
+	// First, determine the length of the UTF-8 string
+	u_strToUTF8(NULL, 0, &utf8_len, utf16_text, utf16_len, &status);
+	if (status != U_BUFFER_OVERFLOW_ERROR) {
+		printf("Error in calculating UTF-8 length: %s\n", u_errorName(status));
+		return;
+	}
+
+	status = U_ZERO_ERROR;  // Reset the error code
+	char *utf8_text = (char *)malloc(utf8_len + 1);  // Allocate memory for the UTF-8 string
+	if (utf8_text == NULL) {
+		printf("Memory allocation failed for UTF-8 string\n");
+		return;
+	}
+
+	// Convert the UTF-16 string back to UTF-8
+	u_strToUTF8(utf8_text, utf8_len + 1, NULL, utf16_text, utf16_len, &status);
+	if (U_FAILURE(status)) {
+		printf("Error in converting UTF-16 to UTF-8: %s\n", u_errorName(status));
+	} else {
+		printf("UTF-8 Text: %s\n", utf8_text);
+	}
+
+	free(utf8_text);  // Free the allocated memory for UTF-8 string
+}
+
+// Utility function to scan text with a regex pattern and return offsets using ICU
+static long scan_offsets(const UChar *text, URegularExpression *pattern, long **span_starts, long **span_ends, UErrorCode *status) {
+	long allocated = 10;
+	long used = 0;
+
+	if (status == NULL || U_FAILURE(*status)) {
+		rb_raise(rb_eRuntimeError, "Invalid or uninitialized UErrorCode provided");
+	}
+
+	*span_starts = malloc(allocated * sizeof(long));
+	*span_ends = malloc(allocated * sizeof(long));
+	if (*span_starts == NULL || *span_ends == NULL) {
+		rb_raise(rb_eRuntimeError, "Memory allocation failed");
+	}
+
+	// Set the text to search with the regex
+	uregex_setText(pattern, text, -1, status);
+	if (U_FAILURE(*status)) {
+		rb_raise(rb_eRuntimeError, "Failed to set text for regex: %s", u_errorName(*status));
+	}
+
+	// Find matches and store offsets
+	while (uregex_findNext(pattern, status)) {
+		if (U_FAILURE(*status)) {
+			rb_raise(rb_eRuntimeError, "Error during regex match: %s", u_errorName(*status));
+		}
+
+		if (used >= allocated) {
+			allocated *= 2;
+			*span_starts = realloc(*span_starts, allocated * sizeof(long));
+			*span_ends = realloc(*span_ends, allocated * sizeof(long));
+			if (*span_starts == NULL || *span_ends == NULL) {
+				rb_raise(rb_eRuntimeError, "Memory allocation failed");
+			}
+		}
+
+		(*span_starts)[used] = uregex_start(pattern, 0, status);
+		(*span_ends)[used] = uregex_end(pattern, 0, status);
+		if (U_FAILURE(*status)) {
+			rb_raise(rb_eRuntimeError, "Error retrieving match offsets: %s", u_errorName(*status));
+		}
+
+		used++;
+	}
+
+	return used;
+}
+
+// Function to print UChar array as hexadecimal
+struct segment_args {
+	// input
+	TextSentencer *ts;
+	const char *text;
+
+	// output
+	long *segment_starts;
+	long *segment_ends;
+	long num_segments;
+};
+
+static void *segment_without_gvl(void *arg) {
+	struct segment_args *args = (struct segment_args *)arg;
+
+	const TextSentencer *ts = args->ts;
+	const char *text = args->text;
+
+	UErrorCode status = U_ZERO_ERROR;
+	int32_t utf16_len;
+	UChar *utf16_text = utf8_to_utf16(text, &utf16_len, &status);
+	if (U_FAILURE(status)) {
+		rb_raise(rb_eRuntimeError, "Failed to convert text to UTF-16: %s", u_errorName(status));
+	}
+
+	// Scan for break patterns
+	long *break_starts, *break_ends;
+	long num_breaks = scan_offsets(utf16_text, ts->break_pattern, &break_starts, &break_ends, &status);
+
+	// Scan for candidate patterns
+	long *candidate_starts, *candidate_ends;
+	long num_candidates = scan_offsets(utf16_text, ts->candidate_pattern, &candidate_starts, &candidate_ends, &status);
+
+	// Filter candidates using positive rules
+	long *filtered_candidate_starts = malloc(num_candidates * sizeof(long));
+	long *filtered_candidate_ends = malloc(num_candidates * sizeof(long));
+	long num_filtered_candidates = 0;
+
+	for (long i = 0; i < num_candidates; ++i) {
+		long start = candidate_starts[i];
+		long end = candidate_ends[i];
+
+		int matched = 0;
+		for (size_t j = 0; j < ts->num_positive_rules; ++j) {
+			if ((start == 0 || match_regex(utf16_text, 0, start, ts->positive_rules_pre[j], &status)) &&
+				(end == utf16_len || match_regex(utf16_text, end, utf16_len, ts->positive_rules_post[j], &status))) {
+				matched = 1;
+				break;
+			}
+		}
+
+		if (matched) {
+			filtered_candidate_starts[num_filtered_candidates] = start;
+			filtered_candidate_ends[num_filtered_candidates] = end;
+			num_filtered_candidates++;
+		}
+	}
+
+	free(candidate_starts);
+	free(candidate_ends);
+
+	// Remove candidates using negative rules
+	long *final_candidate_starts = malloc(num_filtered_candidates * sizeof(long));
+	long *final_candidate_ends = malloc(num_filtered_candidates * sizeof(long));
+	long num_final_candidates = 0;
+
+	for (long i = 0; i < num_filtered_candidates; ++i) {
+		long start = filtered_candidate_starts[i];
+		long end = filtered_candidate_ends[i];
+
+		int reject = 0;
+		for (size_t j = 0; j < ts->num_negative_rules; ++j) {
+			if (match_regex(utf16_text, 0, start, ts->negative_rules_pre[j], &status) &&
+				match_regex(utf16_text, end, utf16_len, ts->negative_rules_post[j], &status)) {
+				reject = 1;
+				break;
+			}
+		}
+
+		if (!reject) {
+			final_candidate_starts[num_final_candidates] = start;
+			final_candidate_ends[num_final_candidates] = end;
+			num_final_candidates++;
+		}
+	}
+
+	free(filtered_candidate_starts);
+	free(filtered_candidate_ends);
+
+	// Merge the breaks and the filtered candidates
+	long total_breaks = num_breaks + num_final_candidates;
+	long *all_break_starts = malloc(total_breaks * sizeof(long));
+	long *all_break_ends = malloc(total_breaks * sizeof(long));
+
+	int i = 0, j = 0, k = 0;
+	while (i < num_breaks && j < num_final_candidates) {
+		if (break_starts[i] <= final_candidate_starts[j]) {
+			all_break_starts[k] = break_starts[i];
+			all_break_ends[k] = break_ends[i];
+			k++;
+			i++;
+
+			// To avoid duplicates
+			if (break_starts[i] == final_candidate_starts[j]) {
+				j++;
+			}
+		} else {
+			all_break_starts[k] = final_candidate_starts[j];
+			all_break_ends[k] = final_candidate_ends[j];
+			k++;
+			j++;
+		}
+	}
+
+	// Store remaining elements of the breaks
+	while (i < num_breaks) {
+		all_break_starts[k] = break_starts[i];
+		all_break_ends[k] = break_ends[i];
+		k++;
+		i++;
+	}
+
+	// Store remaining elements of the final candidates
+	while (j < num_final_candidates) {
+		all_break_starts[k] = final_candidate_starts[j];
+		all_break_ends[k] = final_candidate_ends[j];
+		k++;
+		j++;
+	}
+
+	free(break_starts);
+	free(break_ends);
+	free(final_candidate_starts);
+	free(final_candidate_ends);
+
+	// Convert breaks into sentence segments
+	long num_segments = 0;
+	long *segment_starts = malloc((k + 1) * sizeof(long));
+	long *segment_ends = malloc((k + 1) * sizeof(long));
+
+	long start = 0;
+	for (long i = 0; i < k; i++) {
+		if (all_break_starts[i] == 0) {
+			start = all_break_ends[i];
+			continue;
+		}
+
+		segment_starts[num_segments] = start;
+		segment_ends[num_segments] = all_break_starts[i];
+		num_segments++;
+		start = all_break_ends[i];
+	}
+
+	// Add the last segment
+	if (all_break_ends[k - 1] < utf16_len) {
+		segment_starts[num_segments] = start;
+		segment_ends[num_segments] = utf16_len;
+		num_segments++;
+	}
+
+	free(all_break_starts);
+	free(all_break_ends);
+	free(utf16_text);  // Free the UTF-16 text after processing
+
+	// Pass the results back to the caller
+	args->segment_starts = segment_starts;
+	args->segment_ends = segment_ends;
+	args->num_segments = num_segments;
+
+	return NULL;
+}
+
+VALUE text_sentencer_segment(VALUE self, VALUE rb_text) {
+	TextSentencer *ts;
+	TypedData_Get_Struct(self, TextSentencer, &text_sentencer_type, ts);
+
+	Check_Type(rb_text, T_STRING);
+	const char *text = StringValueCStr(rb_text);
+
+	long *segment_starts = NULL;
+	long *segment_ends = NULL;
+	long num_segments = 0;
+
+	struct segment_args args = {ts, text, segment_starts, segment_ends, num_segments};
+	rb_thread_call_without_gvl(segment_without_gvl, &args, RUBY_UBF_IO, NULL);
+
+	VALUE segments = rb_ary_new();
+	for (long i = 0; i < args.num_segments; ++i) {
+		VALUE segment = rb_ary_new();
+		rb_ary_push(segment, LONG2NUM(args.segment_starts[i]));
+		rb_ary_push(segment, LONG2NUM(args.segment_ends[i]));
+		rb_ary_push(segments, segment);
+	}
+
+	free(args.segment_starts);
+	free(args.segment_ends);
+
+	return segments;
+}
+
+// Annotate method
+VALUE text_sentencer_annotate(VALUE self, VALUE rb_text) {
+	Check_Type(rb_text, T_STRING);
+
+	VALUE segments = text_sentencer_segment(self, rb_text);
+	VALUE blocks = rb_ary_new();
+
+	for (long i = 0; i < RARRAY_LEN(segments); i++) {
+		VALUE span = rb_hash_new();
+		VALUE block = rb_hash_new();
+
+		VALUE segment = rb_ary_entry(segments, i);
+		long start = NUM2LONG(rb_ary_entry(segment, 0));
+		long end = NUM2LONG(rb_ary_entry(segment, 1));
+		
+		rb_hash_aset(span, ID2SYM(rb_intern("begin")), LONG2NUM(start));
+		rb_hash_aset(span, ID2SYM(rb_intern("end")), LONG2NUM(end));
+		
+		rb_hash_aset(block, ID2SYM(rb_intern("span")), span);
+		rb_hash_aset(block, ID2SYM(rb_intern("obj")), rb_str_new_cstr("Sentence"));
+		
+		rb_ary_push(blocks, block);
+	}
+
+	VALUE result = rb_hash_new();
+	rb_hash_aset(result, ID2SYM(rb_intern("text")), rb_text);
+	rb_hash_aset(result, ID2SYM(rb_intern("blocks")), blocks);
+
+	return result;
+}
+
+// Allocate memory for TextSentencer structure
+VALUE text_sentencer_allocate(VALUE klass) {
+	TextSentencer *ts;
+	VALUE obj = TypedData_Make_Struct(klass, TextSentencer, &text_sentencer_type, ts);
+
+	// Initialize fields to avoid uninitialized memory issues
+	ts->break_pattern = NULL;
+	ts->candidate_pattern = NULL;
+	ts->num_positive_rules = 0;
+	ts->positive_rules_pre = NULL;
+	ts->positive_rules_post = NULL;
+	ts->num_negative_rules = 0;
+	ts->negative_rules_pre = NULL;
+	ts->negative_rules_post = NULL;
+
+	return obj;
+}
+
+// Helper function to compile ICU regex
+URegularExpression* compile_pattern(VALUE rb_pattern) {
+	Check_Type(rb_pattern, T_STRING);
+	const char *pattern_str = StringValueCStr(rb_pattern);
+
+	if (strlen(pattern_str) == 0) {
+		rb_raise(rb_eArgError, "Empty regex pattern is not allowed");
+	}
+
+	UErrorCode status = U_ZERO_ERROR;
+
+	// Convert the UTF-8 pattern to UTF-16
+	int32_t pattern_utf16_len = 0;
+	UChar *pattern_utf16 = utf8_to_utf16(pattern_str, &pattern_utf16_len, &status);
+
+	if (U_FAILURE(status)) {
+		rb_raise(rb_eRuntimeError, "Failed to convert regex pattern to UTF-16: %s", u_errorName(status));
+	}
+
+	// Compile the regular expression
+	UParseError parse_error;
+	URegularExpression *regex = uregex_open(pattern_utf16, pattern_utf16_len, 0, &parse_error, &status);
+	free(pattern_utf16);  // Free the UTF-16 pattern after compilation
+
+	if (U_FAILURE(status)) {
+		char error_message[256];
+		snprintf(error_message, sizeof(error_message), "ICU regex compilation failed: %s", u_errorName(status));
+		rb_raise(rb_eRuntimeError, "%s", error_message);
+	}
+
+	return regex;
+}
+
+VALUE text_sentencer_initialize(VALUE self, VALUE rules) {
+	TextSentencer *ts;
+	TypedData_Get_Struct(self, TextSentencer, &text_sentencer_type, ts);
+
+	// Extract and compile the break pattern from the rules hash
+	VALUE rb_break_pattern = rb_hash_aref(rules, ID2SYM(rb_intern("break_pattern")));
+
+	if (!NIL_P(rb_break_pattern)) {
+		ts->break_pattern = compile_pattern(rb_break_pattern);
+	} else {
+		rb_raise(rb_eArgError, "break_pattern is required in rules");
+	}
+
+	// Extract and compile the candidate pattern from the rules hash
+	VALUE rb_candidate_pattern = rb_hash_aref(rules, ID2SYM(rb_intern("candidate_pattern")));
+
+	if (!NIL_P(rb_candidate_pattern)) {
+		ts->candidate_pattern = compile_pattern(rb_candidate_pattern);
+	} else {
+		rb_raise(rb_eArgError, "candidate_pattern is required in rules");
+	}
+
+	// Compile positive rules
+	VALUE rb_positive_rules = rb_hash_aref(rules, ID2SYM(rb_intern("positive_rules")));
+	if (NIL_P(rb_positive_rules) || TYPE(rb_positive_rules) != T_ARRAY) {
+		rb_raise(rb_eArgError, "positive_rules must be an array");
+	}
+
+	ts->num_positive_rules = RARRAY_LEN(rb_positive_rules);
+	ts->positive_rules_pre = malloc(ts->num_positive_rules * sizeof(URegularExpression *));
+	ts->positive_rules_post = malloc(ts->num_positive_rules * sizeof(URegularExpression *));
+	if ((ts->positive_rules_pre == NULL) || (ts->positive_rules_post == NULL)) {
+		rb_raise(rb_eNoMemError, "Failed to allocate memory for positive rules");
+	}
+
+	for (size_t i = 0; i < ts->num_positive_rules; i++) {
+		VALUE rb_rule = rb_ary_entry(rb_positive_rules, i);
+		VALUE rb_rule_pre = rb_ary_entry(rb_rule, 0);
+		VALUE rb_rule_post = rb_ary_entry(rb_rule, 1);
+		ts->positive_rules_pre[i] = compile_pattern(rb_rule_pre);
+		ts->positive_rules_post[i] = compile_pattern(rb_rule_post);
+	}
+
+	// Compile negative rules
+	VALUE rb_negative_rules = rb_hash_aref(rules, ID2SYM(rb_intern("negative_rules")));
+	if (NIL_P(rb_negative_rules) || TYPE(rb_negative_rules) != T_ARRAY) {
+		rb_raise(rb_eArgError, "negative_rules must be an array");
+	}
+
+	ts->num_negative_rules = RARRAY_LEN(rb_negative_rules);
+	ts->negative_rules_pre = malloc(ts->num_negative_rules * sizeof(URegularExpression *));
+	ts->negative_rules_post = malloc(ts->num_negative_rules * sizeof(URegularExpression *));
+	if ((ts->negative_rules_pre == NULL) || (ts->negative_rules_post == NULL)) {
+		rb_raise(rb_eNoMemError, "Failed to allocate memory for negative rules");
+	}
+
+	for (size_t i = 0; i < ts->num_negative_rules; i++) {
+		VALUE rb_rule = rb_ary_entry(rb_negative_rules, i);
+		VALUE rb_rule_pre = rb_ary_entry(rb_rule, 0);
+		VALUE rb_rule_post = rb_ary_entry(rb_rule, 1);
+		ts->negative_rules_pre[i] = compile_pattern(rb_rule_pre);
+		ts->negative_rules_post[i] = compile_pattern(rb_rule_post);
+	}
+
+	return self;
+}
+
+// Module initialization
+void Init_text_sentencer_c(void) {
+	rb_cTextSentencer = rb_define_class("TextSentencer", rb_cObject);
+	rb_define_alloc_func(rb_cTextSentencer, text_sentencer_allocate);
+	rb_define_method(rb_cTextSentencer, "initialize", text_sentencer_initialize, 1);
+	rb_define_method(rb_cTextSentencer, "segment", text_sentencer_segment, 1);
+	rb_define_method(rb_cTextSentencer, "annotate", text_sentencer_annotate, 1);
+}

data/ext/text_sentencer_c/text_sentencer.h

@@ -0,0 +1,31 @@
+#ifndef TEXT_SENTENCER_H
+#define TEXT_SENTENCER_H
+
+#include "ruby.h"
+#include <unicode/uregex.h>
+#include <unicode/ustring.h>
+#include <unicode/utypes.h>
+#include <unicode/utext.h>
+
+
+typedef struct {
+	URegularExpression *break_pattern;			// a single pattern
+	URegularExpression *candidate_pattern;		// a single pattern
+
+	size_t num_positive_rules;
+	URegularExpression **positive_rules_pre;	// an array of pre-patterns
+	URegularExpression **positive_rules_post;	// an array of post-patterns
+
+	size_t num_negative_rules;
+	URegularExpression **negative_rules_pre;	// an array of pre-patterns
+	URegularExpression **negative_rules_post;	// an array of post-patterns
+} TextSentencer;
+
+// Function prototypes
+void Init_text_sentencer_c(void);
+VALUE text_sentencer_allocate(VALUE klass);
+VALUE text_sentencer_initialize(VALUE self, VALUE rules);
+VALUE text_sentencer_segment(VALUE self, VALUE text);
+VALUE text_sentencer_annotate(VALUE self, VALUE text);
+
+#endif // TEXT_SENTENCER_H

data/lib/text_sentencer_c.rb

@@ -0,0 +1,26 @@
+require 'text_sentencer_c/text_sentencer_c'
+
+class TextSentencer
+	DEFAULT_RULES = {
+		break_pattern: "([ \t]*\n+)+[ \t]*", # one or more consecutive blank lines
+		candidate_pattern: "[ \t]+",
+		positive_rules: [
+			["[.!?]$", "^[0-9A-Z]"],
+			[":$", "^[0-9]"],
+			[":$", "^[A-Z][a-z]"]
+		],
+		negative_rules: [
+			['(Mrs|Mmes|Mr|Messrs|Ms|Prof|Dr|Drs|Rev|Hon|Sen|St)\.$', '^[A-Z][a-z]'],
+			['(Sr|Jr)\.$', '^[A-Z][a-z]'],
+			['\b[A-Z][a-z]*\.$', '^[0-9A-Z]'],
+			['(cf|vs)\.$', '^.'],
+			['e\.g\.$', '^.'],
+			['i\.e\.$', '^.'],
+			['(Sec|Chap|Fig|Eq)\.$', '^[0-9A-Z]']
+		]
+	}
+
+	def self.update_rules(rules)
+		DEFAULT_RULES.merge(rules)
+	end
+end

data/text_sentencer_c.gemspec

@@ -0,0 +1,16 @@
+Gem::Specification.new do |spec|
+	spec.name          = "text_sentencer_c"
+	spec.version       = "0.1.0"
+	spec.authors       = ["Jin-Dong Kim"]
+	spec.email         = ["jindong.kim@gmail.com"]
+	spec.summary       = "Text segmentation into sentences"
+	spec.description   = "It is a reimplementation text_sentencer, which is originally written in ruby, using C extension for a better performance. It is a preliminary version, and may not be fully functional."
+	spec.homepage      = "https://github.com/jdkim/text_sentencer_c"
+	spec.license       = "MIT"
+
+	spec.files         = Dir["{lib,ext}/**/*"] + ["text_sentencer_c.gemspec", "Rakefile", "README.md"]
+	spec.executables   = ['text_sentencer']
+	spec.require_paths = ["lib", "ext"]
+
+	spec.extensions    = ["ext/text_sentencer_c/extconf.rb"]
+end

metadata

@@ -0,0 +1,58 @@
+--- !ruby/object:Gem::Specification
+name: text_sentencer_c
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Jin-Dong Kim
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2024-08-28 00:00:00.000000000 Z
+dependencies: []
+description: It is a reimplementation text_sentencer, which is originally written
+  in ruby, using C extension for a better performance. It is a preliminary version,
+  and may not be fully functional.
+email:
+- jindong.kim@gmail.com
+executables:
+- text_sentencer
+extensions:
+- ext/text_sentencer_c/extconf.rb
+extra_rdoc_files: []
+files:
+- README.md
+- Rakefile
+- bin/text_sentencer
+- ext/text_sentencer_c/Makefile
+- ext/text_sentencer_c/extconf.rb
+- ext/text_sentencer_c/text_sentencer.c
+- ext/text_sentencer_c/text_sentencer.h
+- lib/text_sentencer_c.rb
+- lib/text_sentencer_c/text_sentencer_c.so
+- text_sentencer_c.gemspec
+homepage: https://github.com/jdkim/text_sentencer_c
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+- ext
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.11
+signing_key: 
+specification_version: 4
+summary: Text segmentation into sentences
+test_files: []