sentence_it 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 28527ce7fb8f1678ce0f66e885a244c84aea773d1e904596248d9274b602e87d
+  data.tar.gz: 28930eab68014759d472c5d5714c7c93715732fbf2d7af1d8be9c581bb39e25e
+SHA512:
+  metadata.gz: 4a8622f46efb4c2b06b888773107522642a91525fb65156277a1d9d2cf4d2159ce9eb58cc2321613faba682e0fa955b7db18bc3935dd6c686eb8f5c96e5bcca6
+  data.tar.gz: 62cf0ac8a791d6120fdb9cbc59c8cf437e5d8abdf15135dc55827ff0b51670d02cfef5ed70b473ef5fcd13a7f6a05248afbbc270782e866b41d747fcfa30272c

data/README.md

@@ -0,0 +1,11 @@
+# SentenceIt
+
+A Ruby gem for text segmentation using a C extension.
+It is a re-implementation of 'text_sentencer', which is originally implemented in ruby, using a C extension, for a better performance.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'sentencer_it'

data/bin/sentence_it

@@ -0,0 +1,46 @@
+#!/usr/bin/env ruby
+require 'json'
+require 'sentence_it'
+
+rules_filename = nil
+output_mode = :sentences
+
+## command line option processing
+require 'optparse'
+optparse = OptionParser.new do |opts|
+  opts.banner = "Usage: text_sentencer [options]"
+
+  opts.on('-r', '--rules=rules_filename', 'specifies the rules JSON file.') do |c|
+    rules_filename = c
+  end
+    
+  opts.on('-j', '--json_output', 'outputs the result in JSON.') do
+    output_mode = :json
+  end
+
+  opts.on('-h', '--help', 'displays this screen.') do
+    puts opts
+    exit
+  end
+end
+
+optparse.parse!
+
+rules = if rules_filename && File.file?(rules_filename)
+  JSON.parse File.read(rules_filename)
+end
+
+sentencer = SentenceIt.new(rules || SentenceIt::DEFAULT_RULES)
+
+text = ARGF.read
+
+annotations = sentencer.annotate(text)
+
+if output_mode == :json
+  puts JSON.pretty_generate(annotations)
+else
+  annotations['blocks']&.each do |d|
+    span = d['span']
+    puts text[span['begin']...span['end']]
+  end
+end

data/ext/sentence_it/extconf.rb

@@ -0,0 +1,4 @@
+# extconf.rb
+require 'mkmf'
+
+create_makefile('senrence_it/sentence_it')

data/ext/sentence_it/sentence_it.c

@@ -0,0 +1,196 @@
+#include "sentence_it.h"
+#include <string.h>
+#include <regex.h>
+#include <stdio.h>
+
+// Define the default rules
+VALUE default_rules;
+
+// Allocate memory for sentence_it_t structure
+VALUE sentence_it_allocate(VALUE klass) {
+    sentence_it_t *si;
+    return Data_Make_Struct(klass, sentence_it_t, NULL, RUBY_DEFAULT_FREE, si);
+}
+
+// Initialize the Ruby module and class
+void Init_sentence_it() {
+    VALUE cSentenceIt = rb_define_class("SentenceIt", rb_cObject);
+    rb_define_alloc_func(cSentenceIt, sentence_it_allocate);
+    rb_define_method(cSentenceIt, "initialize", sentence_it_initialize, 1);
+    rb_define_method(cSentenceIt, "annotate", sentence_it_annotate, 1);
+    rb_define_method(cSentenceIt, "segment", sentence_it_segment, 1);
+}
+
+// Initialize the SentenceIt object with rules
+VALUE sentence_it_initialize(VALUE self, VALUE rules) {
+    sentence_it_t *si;
+    Data_Get_Struct(self, sentence_it_t, si);
+
+    si->break_pattern = rb_hash_aref(rules, ID2SYM(rb_intern("break_pattern")));
+    si->candidate_pattern = rb_hash_aref(rules, ID2SYM(rb_intern("candidate_pattern")));
+    si->positive_rules = rb_hash_aref(rules, ID2SYM(rb_intern("positive_rules")));
+    si->negative_rules = rb_hash_aref(rules, ID2SYM(rb_intern("negative_rules")));
+
+    return self;
+}
+
+// Utility function to scan text with a regex pattern and return offsets
+static VALUE scan_offsets(VALUE text, const char *pattern) {
+    regex_t regex;
+    regmatch_t pmatch[1];
+    int start = 0;
+    VALUE offsets = rb_ary_new();
+
+    if (regcomp(&regex, pattern, REG_EXTENDED)) {
+        rb_raise(rb_eRuntimeError, "Could not compile regex");
+    }
+
+    while (!regexec(&regex, RSTRING_PTR(text) + start, 1, pmatch, 0)) {
+        VALUE offset = rb_ary_new();
+        rb_ary_push(offset, INT2NUM(start + pmatch[0].rm_so));
+        rb_ary_push(offset, INT2NUM(start + pmatch[0].rm_eo));
+        rb_ary_push(offsets, offset);
+        start += pmatch[0].rm_eo;
+    }
+
+    regfree(&regex);
+    return offsets;
+}
+
+// Implement the segment function
+VALUE sentence_it_segment(VALUE self, VALUE text) {
+    sentence_it_t *si;
+    Data_Get_Struct(self, sentence_it_t, si);
+
+    VALUE breaks;
+    if (RSTRING_LEN(si->break_pattern) == 0) {
+        breaks = rb_ary_new();
+    } else {
+        breaks = scan_offsets(text, RSTRING_PTR(si->break_pattern));
+    }
+
+    VALUE candidates;
+    if (RSTRING_LEN(si->candidate_pattern) == 0) {
+        candidates = rb_ary_new();
+    } else {
+        candidates = scan_offsets(text, RSTRING_PTR(si->candidate_pattern));
+    }
+
+    // Remove candidates that are already in breaks
+    for (int i = 0; i < RARRAY_LEN(breaks); i++) {
+        VALUE break_offset = rb_ary_entry(breaks, i);
+        for (int j = 0; j < RARRAY_LEN(candidates); j++) {
+            VALUE candidate_offset = rb_ary_entry(candidates, j);
+            if (rb_equal(break_offset, candidate_offset)) {
+                rb_ary_delete_at(candidates, j);
+                break;
+            }
+        }
+    }
+
+    // Process candidates
+    for (int i = 0; i < RARRAY_LEN(candidates); i++) {
+        VALUE candidate = rb_ary_entry(candidates, i);
+        int last_end = NUM2INT(rb_ary_entry(candidate, 0));
+        int next_begin = NUM2INT(rb_ary_entry(candidate, 1));
+
+        if (last_end == 0 || next_begin == RSTRING_LEN(text)) {
+            rb_ary_push(breaks, candidate);
+            continue;
+        }
+
+        VALUE last_text = rb_str_substr(text, 0, last_end);
+        VALUE next_text = rb_str_substr(text, next_begin, RSTRING_LEN(text) - next_begin);
+
+        for (int j = 0; j < RARRAY_LEN(si->positive_rules); j++) {
+            VALUE rule = rb_ary_entry(si->positive_rules, j);
+            const char *pattern1 = RSTRING_PTR(rb_ary_entry(rule, 0));
+            const char *pattern2 = RSTRING_PTR(rb_ary_entry(rule, 1));
+
+            regex_t regex1, regex2;
+            regcomp(&regex1, pattern1, REG_EXTENDED | REG_NOSUB);
+            regcomp(&regex2, pattern2, REG_EXTENDED | REG_NOSUB);
+
+            int match1 = !regexec(&regex1, RSTRING_PTR(last_text), 0, NULL, 0);
+            int match2 = !regexec(&regex2, RSTRING_PTR(next_text), 0, NULL, 0);
+
+            regfree(&regex1);
+            regfree(&regex2);
+
+            if (match1 && match2) {
+                int break_p = 1;
+                for (int k = 0; k < RARRAY_LEN(si->negative_rules); k++) {
+                    VALUE neg_rule = rb_ary_entry(si->negative_rules, k);
+                    const char *neg_pattern1 = RSTRING_PTR(rb_ary_entry(neg_rule, 0));
+                    const char *neg_pattern2 = RSTRING_PTR(rb_ary_entry(neg_rule, 1));
+
+                    regex_t neg_regex1, neg_regex2;
+                    regcomp(&neg_regex1, neg_pattern1, REG_EXTENDED | REG_NOSUB);
+                    regcomp(&neg_regex2, neg_pattern2, REG_EXTENDED | REG_NOSUB);
+
+                    int neg_match1 = !regexec(&neg_regex1, RSTRING_PTR(last_text), 0, NULL, 0);
+                    int neg_match2 = !regexec(&neg_regex2, RSTRING_PTR(next_text), 0, NULL, 0);
+
+                    regfree(&neg_regex1);
+                    regfree(&neg_regex2);
+
+                    if (neg_match1 && neg_match2) {
+                        break_p = 0;
+                        break;
+                    }
+                }
+                if (break_p) {
+                    rb_ary_push(breaks, candidate);
+                    break;
+                }
+            }
+        }
+    }
+
+    // Sort breaks
+    rb_funcall(breaks, rb_intern("sort!"), 0);
+
+    VALUE sentences = rb_ary_new();
+    int last_break = 0;
+    for (int i = 0; i < RARRAY_LEN(breaks); i++) {
+        VALUE b = rb_ary_entry(breaks, i);
+        int begin = NUM2INT(rb_ary_entry(b, 0));
+        if (begin > last_break) {
+            VALUE sentence = rb_ary_new3(2, INT2NUM(last_break), INT2NUM(begin));
+            rb_ary_push(sentences, sentence);
+        }
+        last_break = NUM2INT(rb_ary_entry(b, 1));
+    }
+    if (last_break < RSTRING_LEN(text)) {
+        VALUE sentence = rb_ary_new3(2, INT2NUM(last_break), INT2NUM(RSTRING_LEN(text)));
+        rb_ary_push(sentences, sentence);
+    }
+
+    return sentences;
+}
+
+
+// Implement the annotate function
+VALUE sentence_it_annotate(VALUE self, VALUE text) {
+    VALUE segments = sentence_it_segment(self, text);
+    VALUE blocks = rb_ary_new();
+
+    for (int i = 0; i < RARRAY_LEN(segments); i++) {
+        VALUE segment = rb_ary_entry(segments, i);
+        VALUE span = rb_hash_new();
+        rb_hash_aset(span, rb_str_new_cstr("begin"), rb_ary_entry(segment, 0));
+        rb_hash_aset(span, rb_str_new_cstr("end"), rb_ary_entry(segment, 1));
+
+        VALUE block = rb_hash_new();
+        rb_hash_aset(block, rb_str_new_cstr("span"), span);
+        rb_hash_aset(block, rb_str_new_cstr("obj"), rb_str_new_cstr("Sentence"));
+
+        rb_ary_push(blocks, block);
+    }
+
+    VALUE result = rb_hash_new();
+    rb_hash_aset(result, rb_str_new_cstr("text"), text);
+    rb_hash_aset(result, rb_str_new_cstr("blocks"), blocks);
+
+    return result;
+}

data/ext/sentence_it/sentence_it.h

@@ -0,0 +1,21 @@
+#ifndef SENTENCE_IT_H
+#define SENTENCE_IT_H
+
+#include "ruby.h"
+
+// Define a structure to hold the rules
+typedef struct {
+    VALUE break_pattern;
+    VALUE candidate_pattern;
+    VALUE positive_rules;
+    VALUE negative_rules;
+} sentence_it_t;
+
+// Function prototypes
+void Init_sentence_it();
+VALUE sentence_it_allocate(VALUE klass);
+VALUE sentence_it_initialize(VALUE self, VALUE rules);
+VALUE sentence_it_annotate(VALUE self, VALUE text);
+VALUE sentence_it_segment(VALUE self, VALUE text);
+
+#endif // SENTENCE_IT_H

data/lib/sentence_it.rb

@@ -0,0 +1,26 @@
+require 'sentence_it/sentence_it'
+
+class SentenceIt
+	DEFAULT_RULES = {
+		break_pattern: "([ \t]*\n+)+[ \t]*",
+		candidate_pattern: "[ \t]+",
+		positive_rules: [
+		  ['[.!?]', '[0-9A-Z]'],
+		  [':', '[0-9]'],
+		  [':', '[A-Z][a-z]']
+		],
+		negative_rules: [
+		  ['(Mrs|Mmes|Mr|Messrs|Ms|Prof|Dr|Drs|Rev|Hon|Sen|St)\.', '[A-Z][a-z]'],
+		  ['(Sr|Jr)\.', '[A-Z][a-z]'],
+		  ['\b[A-Z][a-z]*\.', '[0-9A-Z]'],
+		  ['(cf|vs)\.', ''],
+		  ['e\.g\.', ''],
+		  ['i\.e\.', ''],
+		  ['(Sec|Chap|Fig|Eq)\.', '[0-9A-Z]']
+		]
+	}
+
+	define self.update_rules(rules)
+		DEFAULT_RULES.merge(rules)
+	end
+end

metadata

@@ -0,0 +1,56 @@
+--- !ruby/object:Gem::Specification
+name: sentence_it
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Jin-Dong Kim
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2024-08-02 00:00:00.000000000 Z
+dependencies: []
+description: It is a reimplementation text_sentencer, which is originally written
+  in ruby, using C extension for a better performance.
+email:
+- jindong.kim@gmail.com
+executables:
+- sentence_it
+extensions:
+- ext/sentence_it/extconf.rb
+extra_rdoc_files: []
+files:
+- README.md
+- bin/sentence_it
+- ext/sentence_it/extconf.rb
+- ext/sentence_it/sentence_it.c
+- ext/sentence_it/sentence_it.h
+- lib/sentence_it.rb
+- lib/sentence_it/sentence_it.so
+homepage: https://github.com/jdkim/sentence_it
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/jdkim/sentence_it
+  source_code_uri: https://github.com/jdkim/sentence_it
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+- ext/sentence_it
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.11
+signing_key: 
+specification_version: 4
+summary: A Ruby gem for text segmentation using a C extension
+test_files: []