text_clean 0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 631e9068f2dffa8f14377a682868a03732144d00
+  data.tar.gz: 9aa6e9bb2d6ede27d7bca149da79e077f9a038e8
+SHA512:
+  metadata.gz: e55cad056b49d56c812df735ef6961dad3be08de91c63026cc1129e76cd90e96de0f9c12ca46928530c3846450a5498c214aabbed64991cea3e8a6a891ecc0cc
+  data.tar.gz: 139483afa8b4cb5204361e22430305196bca352278f23a80eafc1d711732f02b0fac96ffb589782135b77af8d63bb10c255d20ea0fc1ff78800b8084d7a657d5

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/Rakefile

@@ -0,0 +1,6 @@
+require "rake/extensiontask"
+
+Rake::ExtensionTask.new "text_clean" do |ext|
+  ext.lib_dir = "lib/text_clean"
+  ext.source_pattern = "*.{c,cc}"
+end

data/bin/text_clean

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require "text_clean"
+
+puts TextClean.clean(STDIN.read, "\n")

data/bin/text_clean.sh

@@ -0,0 +1,24 @@
+# replace newlines with periods to make regexes easier
+tr '\n' '.'                           |
+# convert sentence endings to periods to be consistent
+tr ':' '.'                            |
+
+tr ';' '.'                            |
+
+tr '?' '.'                            |
+
+tr '!' '.'                            |
+# compact line endings
+sed -E $'s/\.+/./g'                   |
+# join lines ending in hyphenation
+sed -E $'s/- *\.//g'                  |
+# replace periods with newlines
+sed -E $'s/[ \\\t]*\.[ \\\t]*/\\\n/g' |
+# compact whitespace-line characters
+sed -E $'s/[&\/, \\\t\-]+/ /g'        |
+# remove non-alpha characters
+sed -E $'s/[^a-zA-Z\' ]//g'           |
+# strip leading whitespace
+sed -E $'s/^ *//'                     |
+# convert to lowercase
+tr '[:upper:]' '[:lower:]'

data/ext/text_clean/extconf.rb

@@ -0,0 +1,5 @@
+require "mkmf"
+
+dir_config("text_clean")
+
+create_makefile("text_clean")

data/ext/text_clean/text_clean.cc

@@ -0,0 +1,116 @@
+#include <ruby.h>
+#include <ruby/encoding.h>
+#include <stdio.h>
+#include <string.h>
+#include <assert.h>
+
+static rb_encoding* u8_enc;
+static rb_encoding* bin_enc;
+
+/** Transforms text such as the following:
+ *
+ *   And behold, I said, "This is no good!"
+ *   What shall ye say unto these people, there-
+ *   fore?
+ *
+ * Into a cleaned up single line of text, like the following:
+ *
+ *   and behold i said this is no good.what shall ye say unto these people therefore.
+ *
+ * Spaces indicate word boundaries, while periods indicate sentence boundaries.
+ */
+size_t text_clean_cstr(char* text, long len, char line_sep)
+{
+  if (len <= 0) return 0;
+
+  char* eos = text + (size_t)len;
+  char* read;
+  char* write = text;
+  uint8_t just_added_space = true,   // prevent prefix spaces
+          just_added_period = false;
+  for (read = text; read < eos; read++) {
+    char c = *read;
+    if (c >= 'A' && c <= 'Z') {
+      // Change upper case to lowercase
+      c += 32;
+    } else if (c == '\t' || c == ',' || c == '&' || c == '/') {
+      // Change inconsequential punctuation to spaces (i.e. all count as whitespace)
+      c = ' ';
+    } else if (c == '?' || c == '!' || c == ':' || c == ';') {
+      // Change exclamation, question marks to periods (i.e. sentence boundaries)
+      c = '.';
+    }
+
+    // hyphen at end of line joins word fragments
+    if (c == '-') {
+      // double dash?
+      if (*(read + 1) == '-') {
+        *write++ = ' ';
+        read++;
+      } else {
+        // scan ahead to see if this is a hyphen at the end of the line
+        char* scan_ahead;
+        for (scan_ahead = read + 1; scan_ahead < eos; scan_ahead++) {
+          char s = *scan_ahead;
+          if (s != '\t' && s != ' ') {
+            if (s == '\n') {
+              // this is a hyphenated line join, so join the lines
+              read = scan_ahead;
+              break;
+            } else {
+              // not a line join
+              break;
+            }
+          }
+        }
+      }
+    } else if (c == '.' && !just_added_period) {
+      // erase space before period
+      if (just_added_space) write--;
+      *write++ = line_sep;
+      just_added_period = true;
+      just_added_space = false;
+    } else if (c == ' ' && !just_added_space && !just_added_period) {
+      *write++ = ' ';
+      just_added_space = true;
+      just_added_period = false;
+    } else if (c >= 'a' && c <= 'z') {
+      *write++ = c;
+      just_added_space = false;
+      just_added_period = false;
+    }
+  }
+  // erase space at end of text
+  if (just_added_space) write--;
+
+  // Return the new length of the string
+  return (size_t)(write - text);
+}
+
+static VALUE text_clean(VALUE self, VALUE text, VALUE ending) {
+  char*   ptext = RSTRING_PTR(text);
+  long      len = RSTRING_LEN(text);
+  char line_sep = '.';
+
+  if (RSTRING_LEN(ending) == 1) {
+    line_sep = RSTRING_PTR(ending)[0];
+  }
+
+  rb_str_modify(text);
+
+  size_t new_length = text_clean_cstr(ptext, len, line_sep);
+
+  rb_str_set_len(text, (long)new_length);
+
+  return text;
+}
+
+extern "C"
+void Init_text_clean() {
+    VALUE rb_mText = rb_define_module("TextClean");
+
+    u8_enc = rb_utf8_encoding();
+    bin_enc = rb_ascii8bit_encoding();
+
+    rb_define_module_function(rb_mText, "text_clean", RUBY_METHOD_FUNC(text_clean), 2);
+}

data/lib/text_clean.rb

@@ -0,0 +1,9 @@
+require "text_clean/text_clean"
+require "text_clean/version"
+
+module TextClean
+  def self.clean(text, line_sep = "\n")
+    # Call the C function
+    text_clean(text, line_sep)
+  end
+end

data/lib/text_clean/version.rb

@@ -0,0 +1,3 @@
+module TextClean
+  VERSION = '0.1'
+end

data/text_clean.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'text_clean/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "text_clean"
+  spec.version       = TextClean::VERSION
+  spec.authors       = ["Duane Johnson"]
+  spec.email         = ["duane.johnson@gmail.com"]
+  spec.description   = %q{Cleans text by removing punctuation, lowercasing. Very fast.}
+  spec.summary       = %q{Text cleaner}
+  spec.homepage      = "https://github.com/wordtreefoundation"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib", "ext"]
+
+  spec.extensions    = %w[ext/text_clean/extconf.rb]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake", "~> 10.3"
+  spec.add_development_dependency "rake-compiler", "~> 0.9"
+  spec.add_development_dependency "byebug", "~> 3.4"
+end

metadata

@@ -0,0 +1,113 @@
+--- !ruby/object:Gem::Specification
+name: text_clean
+version: !ruby/object:Gem::Version
+  version: '0.1'
+platform: ruby
+authors:
+- Duane Johnson
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-03-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.3'
+- !ruby/object:Gem::Dependency
+  name: rake-compiler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
+description: Cleans text by removing punctuation, lowercasing. Very fast.
+email:
+- duane.johnson@gmail.com
+executables:
+- text_clean
+- text_clean.sh
+extensions:
+- ext/text_clean/extconf.rb
+extra_rdoc_files: []
+files:
+- Gemfile
+- Rakefile
+- bin/text_clean
+- bin/text_clean.sh
+- ext/text_clean/extconf.rb
+- ext/text_clean/text_clean.cc
+- lib/text_clean.rb
+- lib/text_clean/version.rb
+- text_clean.gemspec
+homepage: https://github.com/wordtreefoundation
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+- ext
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Text cleaner
+test_files: []