feedbackmine-ferret_tokenizer 0.1.1

data/Manifest.txt

@@ -0,0 +1,13 @@
+ext/ferret_tokenizer.c
+ext/scanner.rl
+ext/url.rl
+ext/extconf.rb
+ext/scanner.c
+ext/email.rl
+ext/scanner.in
+Rakefile
+Manifest.txt
+README.txt
+test
+test/test.rb
+

data/README.txt

@@ -0,0 +1,20 @@
+== Overview
+A string tokenizer based on Ferret::Analysis::StandardAnalyzer. I made some improvements for better text analysis.
+
+== Install
+
+sudo gem sources -a http://gems.github.com   (you only have to do this once)
+sudo gem install feedbackmine-tokenizer
+
+== Usage
+
+require 'tokenizer'
+t = Tokenizer.new '@feedbackmine, I LOVE this!!! :-)'
+while (tok = t.next)
+  p tok
+end
+
+== Follow us on twitter
+
+http://twitter.com/feedbackmine
+

data/Rakefile

@@ -0,0 +1,25 @@
+require 'rubygems'
+require 'hoe'
+
+EXT = "ext/ferret_tokenizer.#{Hoe::DLEXT}"
+
+Hoe.new('ferret_tokenizer', '0.1.1') do |p|
+  p.author = 'FeedbackMine'
+  p.email = 'feedbackmine@feedbackmine.com'
+  p.url = 'http://www.tweetjobsearch.com'
+  p.summary = 'tokenizer'
+  p.description = 'tokenizer'
+  
+  p.spec_extras[:extensions] = "ext/extconf.rb"
+  p.clean_globs << EXT << "ext/*.o" << "ext/Makefile"
+end
+
+task :test => EXT
+
+file EXT => ["ext/extconf.rb", "ext/ferret_tokenizer.c"] do
+  Dir.chdir "ext" do
+    sh "ragel scanner.rl -o scanner.c"
+    ruby "extconf.rb"
+    sh "make"
+  end
+end

data/ext/email.rl

@@ -0,0 +1,21 @@
+#// email.rl -*-C-*-
+%%{
+    machine Email;
+
+    #// RFC 2822 - matching email addresses
+    NO_WS_CTL       = ( 1..8 | 11 | 12 | 14..31 | 127 );
+    ASCII           = 1..127;
+    atext           = [a-zA-Z0-9!#$%&\'*+\-/=?^_`{|}~];
+    qtext           = ( NO_WS_CTL | 33 | 35..91 | 93..126 );
+    dtext           = ( NO_WS_CTL | 33..90 | 94..126 );
+    dot_atom        = atext+ ('.' atext+)*;
+    text            = ( 1..9 | 11 | 12 | 14..127 );
+    quoted_pair     = '\\' text;
+    quoted_string   = '"' ( qtext | quoted_pair )* '"';
+    domain_literal  = '[' (dtext | quoted_pair)* ']';
+
+    local_part      = dot_atom | quoted_string;
+    domain          = dot_atom | domain_literal;
+
+    email           = local_part '@' domain;
+}%%

data/ext/extconf.rb

@@ -0,0 +1,3 @@
+require 'mkmf'
+$CFLAGS = "#{ENV['CFLAGS']} -Wall -O3 "
+create_makefile('ferret_tokenizer')

data/ext/ferret_tokenizer.c

@@ -0,0 +1,67 @@
+#include <ruby.h>
+
+extern void frt_std_scan(const char *in,
+                  char *out, size_t out_size,
+                  const char **start,
+                  const char **end,
+                  int *token_size);
+
+typedef struct _Tokenizer {
+  char *data;
+} Tokenizer;
+
+Tokenizer *new_tokenizer(void)
+{
+  return (Tokenizer*)malloc(sizeof(Tokenizer));
+}
+
+void free_tokenizer(Tokenizer *t)
+{
+  free(t);
+}
+
+static VALUE tokenizer_alloc(VALUE klass) {
+  Tokenizer *tokenizer;
+  VALUE obj;
+  // Vendor library creates the Jukebox
+  tokenizer = new_tokenizer();
+  // then we wrap it inside a Ruby CDPlayer object
+  obj = Data_Wrap_Struct(klass, 0, free_tokenizer, tokenizer);
+  return obj;
+}
+
+VALUE method_next(VALUE self) {
+	const char *start = NULL;
+  const char *end = NULL;
+  int len;
+  
+  Tokenizer *t;
+  Data_Get_Struct(self, Tokenizer, t);
+
+  char buffer[1024];
+  
+  frt_std_scan(t->data, buffer, sizeof(buffer) - 1,
+                         &start, &end, &len);
+  if (len == 0)
+    return Qnil;
+  
+  t->data = end;
+  return rb_str_new2(buffer);
+}
+
+VALUE method_initialize(VALUE self, VALUE str) {
+	char* s = RSTRING(str)->ptr;
+	Tokenizer *t;
+  Data_Get_Struct(self, Tokenizer, t);
+  t->data = s;
+  return self;
+}
+
+VALUE cTokenizer = Qnil;
+
+void Init_ferret_tokenizer() {
+	cTokenizer = rb_define_class("FerretTokenizer", rb_cObject);
+	rb_define_alloc_func(cTokenizer, tokenizer_alloc);
+	rb_define_method(cTokenizer, "initialize", method_initialize, 1);
+	rb_define_method(cTokenizer, "next", method_next, 0);
+}

data/ext/scanner.c

@@ -0,0 +1,976 @@
+#line 1 "scanner.rl"
+/* scanner.rl -*-C-*- */
+#include <ctype.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#define RET goto ret;
+
+#define STRIP(c) do { \
+    strip_char = c;   \
+    goto ret;         \
+} while(0)
+
+#line 26 "scanner.rl"
+
+
+
+#line 21 "scanner.c"
+static const char _StdTok_actions[] = {
+	0, 1, 0, 1, 1, 1, 2, 1, 
+	3, 1, 4, 1, 5, 1, 21, 1, 
+	23, 1, 24, 1, 25, 1, 26, 1, 
+	27, 1, 28, 1, 29, 1, 30, 1, 
+	31, 1, 32, 1, 33, 1, 34, 1, 
+	35, 1, 36, 1, 37, 1, 38, 1, 
+	39, 1, 40, 1, 41, 1, 42, 1, 
+	43, 1, 44, 1, 45, 1, 46, 2, 
+	1, 22, 2, 5, 6, 2, 5, 7, 
+	2, 5, 8, 2, 5, 9, 2, 5, 
+	10, 2, 5, 11, 2, 5, 12, 2, 
+	5, 13, 2, 5, 14, 2, 5, 15, 
+	2, 5, 16, 2, 5, 17, 2, 5, 
+	18, 2, 5, 19, 2, 5, 20, 3, 
+	5, 1, 16
+};
+
+static const short _StdTok_key_offsets[] = {
+	0, 0, 0, 14, 28, 43, 57, 69, 
+	75, 85, 86, 92, 107, 128, 149, 170, 
+	191, 218, 239, 241, 263, 285, 307, 329, 
+	351, 373, 395, 417, 439, 460, 487, 488, 
+	495, 497, 524, 525, 538, 538, 563, 577, 
+	591, 601, 615, 631, 647, 663, 686, 707, 
+	729, 732, 755, 778, 801, 824, 848, 871, 
+	894, 917, 939, 961, 982, 990, 1013, 1016, 
+	1017, 1018, 1033, 1039, 1064, 1083, 1104, 1125, 
+	1144, 1165, 1187, 1209, 1231, 1253, 1275, 1297, 
+	1319, 1341, 1366, 1382, 1401, 1421, 1446, 1473, 
+	1499, 1525, 1550, 1563, 1589, 1615, 1641, 1667
+};
+
+static const unsigned char _StdTok_trans_keys[] = {
+	33u, 46u, 61u, 64u, 35u, 39u, 42u, 43u, 
+	45u, 57u, 63u, 90u, 94u, 126u, 33u, 45u, 
+	61u, 63u, 35u, 39u, 42u, 43u, 47u, 57u, 
+	65u, 90u, 94u, 126u, 33u, 45u, 61u, 63u, 
+	91u, 35u, 39u, 42u, 43u, 47u, 57u, 65u, 
+	90u, 94u, 126u, 33u, 45u, 61u, 63u, 35u, 
+	39u, 42u, 43u, 47u, 57u, 65u, 90u, 94u, 
+	126u, 92u, 93u, 1u, 8u, 11u, 12u, 14u, 
+	31u, 33u, 90u, 94u, 127u, 1u, 9u, 11u, 
+	12u, 14u, 127u, 34u, 92u, 1u, 8u, 11u, 
+	12u, 14u, 31u, 33u, 127u, 64u, 1u, 9u, 
+	11u, 12u, 14u, 127u, 33u, 45u, 47u, 61u, 
+	63u, 35u, 39u, 42u, 43u, 48u, 57u, 65u, 
+	90u, 94u, 126u, 33u, 42u, 43u, 46u, 61u, 
+	63u, 64u, 35u, 39u, 45u, 47u, 48u, 57u, 
+	65u, 90u, 94u, 96u, 97u, 122u, 123u, 126u, 
+	33u, 45u, 46u, 47u, 61u, 63u, 64u, 35u, 
+	39u, 42u, 43u, 48u, 57u, 65u, 90u, 94u, 
+	96u, 97u, 122u, 123u, 126u, 33u, 45u, 46u, 
+	47u, 61u, 63u, 64u, 35u, 39u, 42u, 43u, 
+	48u, 57u, 65u, 90u, 94u, 96u, 97u, 122u, 
+	123u, 126u, 33u, 45u, 46u, 47u, 61u, 63u, 
+	64u, 35u, 39u, 42u, 43u, 48u, 57u, 65u, 
+	90u, 94u, 96u, 97u, 122u, 123u, 126u, 33u, 
+	45u, 47u, 61u, 63u, 98u, 99u, 101u, 103u, 
+	105u, 109u, 110u, 111u, 35u, 39u, 42u, 43u, 
+	48u, 57u, 65u, 90u, 94u, 96u, 97u, 122u, 
+	123u, 126u, 33u, 45u, 46u, 47u, 61u, 63u, 
+	64u, 35u, 39u, 42u, 43u, 48u, 57u, 65u, 
+	90u, 94u, 96u, 97u, 122u, 123u, 126u, 48u, 
+	57u, 33u, 45u, 46u, 47u, 61u, 63u, 64u, 
+	105u, 35u, 39u, 42u, 43u, 48u, 57u, 65u, 
+	90u, 94u, 96u, 97u, 122u, 123u, 126u, 33u, 
+	45u, 46u, 47u, 61u, 63u, 64u, 111u, 35u, 
+	39u, 42u, 43u, 48u, 57u, 65u, 90u, 94u, 
+	96u, 97u, 122u, 123u, 126u, 33u, 45u, 46u, 
+	47u, 61u, 63u, 64u, 100u, 35u, 39u, 42u, 
+	43u, 48u, 57u, 65u, 90u, 94u, 96u, 97u, 
+	122u, 123u, 126u, 33u, 45u, 46u, 47u, 61u, 
+	63u, 64u, 111u, 35u, 39u, 42u, 43u, 48u, 
+	57u, 65u, 90u, 94u, 96u, 97u, 122u, 123u, 
+	126u, 33u, 45u, 46u, 47u, 61u, 63u, 64u, 
+	110u, 35u, 39u, 42u, 43u, 48u, 57u, 65u, 
+	90u, 94u, 96u, 97u, 122u, 123u, 126u, 33u, 
+	45u, 46u, 47u, 61u, 63u, 64u, 111u, 35u, 
+	39u, 42u, 43u, 48u, 57u, 65u, 90u, 94u, 
+	96u, 97u, 122u, 123u, 126u, 33u, 45u, 46u, 
+	47u, 61u, 63u, 64u, 105u, 35u, 39u, 42u, 
+	43u, 48u, 57u, 65u, 90u, 94u, 96u, 97u, 
+	122u, 123u, 126u, 33u, 45u, 46u, 47u, 61u, 
+	63u, 64u, 101u, 35u, 39u, 42u, 43u, 48u, 
+	57u, 65u, 90u, 94u, 96u, 97u, 122u, 123u, 
+	126u, 33u, 45u, 46u, 47u, 61u, 63u, 64u, 
+	114u, 35u, 39u, 42u, 43u, 48u, 57u, 65u, 
+	90u, 94u, 96u, 97u, 122u, 123u, 126u, 33u, 
+	46u, 61u, 63u, 64u, 35u, 39u, 42u, 43u, 
+	45u, 47u, 48u, 57u, 65u, 90u, 94u, 96u, 
+	97u, 122u, 123u, 126u, 33u, 45u, 47u, 61u, 
+	63u, 98u, 99u, 101u, 103u, 105u, 109u, 110u, 
+	111u, 35u, 39u, 42u, 43u, 48u, 57u, 65u, 
+	90u, 94u, 96u, 97u, 122u, 123u, 126u, 47u, 
+	95u, 44u, 58u, 64u, 90u, 97u, 122u, 40u, 
+	41u, 33u, 45u, 47u, 61u, 63u, 98u, 99u, 
+	101u, 103u, 105u, 109u, 110u, 111u, 35u, 39u, 
+	42u, 43u, 48u, 57u, 65u, 90u, 94u, 96u, 
+	97u, 122u, 123u, 126u, 47u, 45u, 47u, 58u, 
+	64u, 95u, 44u, 46u, 48u, 57u, 65u, 90u, 
+	97u, 122u, 0u, 33u, 34u, 42u, 43u, 45u, 
+	47u, 58u, 61u, 63u, 64u, 102u, 104u, 35u, 
+	39u, 48u, 57u, 65u, 90u, 94u, 96u, 97u, 
+	122u, 123u, 126u, 33u, 46u, 61u, 64u, 35u, 
+	39u, 42u, 43u, 45u, 57u, 63u, 90u, 94u, 
+	126u, 33u, 46u, 61u, 63u, 35u, 39u, 42u, 
+	43u, 45u, 57u, 65u, 90u, 94u, 126u, 34u, 
+	92u, 1u, 8u, 11u, 12u, 14u, 31u, 33u, 
+	127u, 33u, 46u, 61u, 64u, 35u, 39u, 42u, 
+	43u, 45u, 57u, 63u, 90u, 94u, 126u, 33u, 
+	46u, 61u, 64u, 35u, 39u, 42u, 43u, 45u, 
+	47u, 48u, 57u, 63u, 90u, 94u, 126u, 33u, 
+	46u, 61u, 64u, 35u, 39u, 42u, 43u, 45u, 
+	47u, 48u, 57u, 63u, 90u, 94u, 126u, 33u, 
+	46u, 61u, 64u, 35u, 39u, 42u, 43u, 45u, 
+	47u, 48u, 57u, 63u, 90u, 94u, 126u, 33u, 
+	42u, 43u, 45u, 46u, 47u, 58u, 61u, 63u, 
+	64u, 95u, 35u, 39u, 48u, 57u, 65u, 90u, 
+	94u, 96u, 97u, 122u, 123u, 126u, 33u, 46u, 
+	61u, 63u, 64u, 35u, 39u, 42u, 43u, 45u, 
+	47u, 48u, 57u, 65u, 90u, 94u, 96u, 97u, 
+	122u, 123u, 126u, 33u, 45u, 46u, 47u, 58u, 
+	61u, 63u, 64u, 35u, 39u, 42u, 43u, 48u, 
+	57u, 65u, 90u, 94u, 96u, 97u, 122u, 123u, 
+	126u, 47u, 48u, 57u, 33u, 45u, 46u, 47u, 
+	58u, 61u, 63u, 64u, 122u, 35u, 39u, 42u, 
+	43u, 48u, 57u, 65u, 90u, 94u, 96u, 97u, 
+	121u, 123u, 126u, 33u, 45u, 46u, 47u, 58u, 
+	61u, 63u, 64u, 109u, 35u, 39u, 42u, 43u, 
+	48u, 57u, 65u, 90u, 94u, 96u, 97u, 122u, 
+	123u, 126u, 33u, 45u, 46u, 47u, 58u, 61u, 
+	63u, 64u, 117u, 35u, 39u, 42u, 43u, 48u, 
+	57u, 65u, 90u, 94u, 96u, 97u, 122u, 123u, 
+	126u, 33u, 45u, 46u, 47u, 58u, 61u, 63u, 
+	64u, 118u, 35u, 39u, 42u, 43u, 48u, 57u, 
+	65u, 90u, 94u, 96u, 97u, 122u, 123u, 126u, 
+	33u, 45u, 46u, 47u, 58u, 61u, 63u, 64u, 
+	102u, 116u, 35u, 39u, 42u, 43u, 48u, 57u, 
+	65u, 90u, 94u, 96u, 97u, 122u, 123u, 126u, 
+	33u, 45u, 46u, 47u, 58u, 61u, 63u, 64u, 
+	108u, 35u, 39u, 42u, 43u, 48u, 57u, 65u, 
+	90u, 94u, 96u, 97u, 122u, 123u, 126u, 33u, 
+	45u, 46u, 47u, 58u, 61u, 63u, 64u, 116u, 
+	35u, 39u, 42u, 43u, 48u, 57u, 65u, 90u, 
+	94u, 96u, 97u, 122u, 123u, 126u, 33u, 45u, 
+	46u, 47u, 58u, 61u, 63u, 64u, 103u, 35u, 
+	39u, 42u, 43u, 48u, 57u, 65u, 90u, 94u, 
+	96u, 97u, 122u, 123u, 126u, 33u, 45u, 46u, 
+	47u, 61u, 63u, 64u, 95u, 35u, 39u, 42u, 
+	43u, 48u, 57u, 65u, 90u, 94u, 96u, 97u, 
+	122u, 123u, 126u, 33u, 45u, 46u, 47u, 61u, 
+	63u, 64u, 95u, 35u, 39u, 42u, 43u, 48u, 
+	57u, 65u, 90u, 94u, 96u, 97u, 122u, 123u, 
+	126u, 33u, 45u, 46u, 47u, 61u, 63u, 64u, 
+	35u, 39u, 42u, 43u, 48u, 57u, 65u, 90u, 
+	94u, 96u, 97u, 122u, 123u, 126u, 47u, 95u, 
+	44u, 58u, 64u, 90u, 97u, 122u, 33u, 42u, 
+	43u, 45u, 46u, 47u, 58u, 61u, 63u, 64u, 
+	95u, 35u, 39u, 48u, 57u, 65u, 90u, 94u, 
+	96u, 97u, 122u, 123u, 126u, 40u, 41u, 45u, 
+	40u, 41u, 33u, 46u, 61u, 63u, 64u, 35u, 
+	39u, 42u, 43u, 45u, 57u, 65u, 90u, 94u, 
+	126u, 48u, 57u, 65u, 90u, 97u, 122u, 33u, 
+	38u, 39u, 42u, 43u, 45u, 46u, 47u, 58u, 
+	61u, 63u, 64u, 95u, 35u, 37u, 48u, 57u, 
+	65u, 90u, 94u, 96u, 97u, 122u, 123u, 126u, 
+	33u, 46u, 61u, 63u, 64u, 35u, 39u, 42u, 
+	43u, 45u, 57u, 65u, 90u, 94u, 96u, 97u, 
+	122u, 123u, 126u, 33u, 46u, 61u, 63u, 64u, 
+	35u, 39u, 42u, 43u, 45u, 47u, 48u, 57u, 
+	65u, 90u, 94u, 96u, 97u, 122u, 123u, 126u, 
+	33u, 46u, 61u, 63u, 64u, 83u, 115u, 35u, 
+	39u, 42u, 43u, 45u, 57u, 65u, 90u, 94u, 
+	96u, 97u, 122u, 123u, 126u, 33u, 46u, 61u, 
+	63u, 64u, 35u, 39u, 42u, 43u, 45u, 57u, 
+	65u, 90u, 94u, 96u, 97u, 122u, 123u, 126u, 
+	33u, 45u, 46u, 47u, 61u, 63u, 64u, 35u, 
+	39u, 42u, 43u, 48u, 57u, 65u, 90u, 94u, 
+	96u, 97u, 122u, 123u, 126u, 33u, 45u, 46u, 
+	47u, 61u, 63u, 64u, 105u, 35u, 39u, 42u, 
+	43u, 48u, 57u, 65u, 90u, 94u, 96u, 97u, 
+	122u, 123u, 126u, 33u, 45u, 46u, 47u, 61u, 
+	63u, 64u, 111u, 35u, 39u, 42u, 43u, 48u, 
+	57u, 65u, 90u, 94u, 96u, 97u, 122u, 123u, 
+	126u, 33u, 45u, 46u, 47u, 61u, 63u, 64u, 
+	100u, 35u, 39u, 42u, 43u, 48u, 57u, 65u, 
+	90u, 94u, 96u, 97u, 122u, 123u, 126u, 33u, 
+	45u, 46u, 47u, 61u, 63u, 64u, 111u, 35u, 
+	39u, 42u, 43u, 48u, 57u, 65u, 90u, 94u, 
+	96u, 97u, 122u, 123u, 126u, 33u, 45u, 46u, 
+	47u, 61u, 63u, 64u, 110u, 35u, 39u, 42u, 
+	43u, 48u, 57u, 65u, 90u, 94u, 96u, 97u, 
+	122u, 123u, 126u, 33u, 45u, 46u, 47u, 61u, 
+	63u, 64u, 105u, 35u, 39u, 42u, 43u, 48u, 
+	57u, 65u, 90u, 94u, 96u, 97u, 122u, 123u, 
+	126u, 33u, 45u, 46u, 47u, 61u, 63u, 64u, 
+	101u, 35u, 39u, 42u, 43u, 48u, 57u, 65u, 
+	90u, 94u, 96u, 97u, 122u, 123u, 126u, 33u, 
+	45u, 46u, 47u, 61u, 63u, 64u, 114u, 35u, 
+	39u, 42u, 43u, 48u, 57u, 65u, 90u, 94u, 
+	96u, 97u, 122u, 123u, 126u, 33u, 38u, 39u, 
+	42u, 43u, 45u, 46u, 47u, 58u, 61u, 63u, 
+	64u, 95u, 35u, 37u, 48u, 57u, 65u, 90u, 
+	94u, 96u, 97u, 122u, 123u, 126u, 33u, 46u, 
+	61u, 64u, 83u, 115u, 35u, 39u, 42u, 43u, 
+	45u, 57u, 63u, 90u, 94u, 126u, 33u, 45u, 
+	61u, 63u, 91u, 35u, 39u, 42u, 43u, 47u, 
+	57u, 65u, 90u, 94u, 96u, 97u, 122u, 123u, 
+	126u, 33u, 46u, 61u, 63u, 35u, 39u, 42u, 
+	43u, 45u, 47u, 48u, 57u, 65u, 90u, 94u, 
+	96u, 97u, 122u, 123u, 126u, 33u, 38u, 39u, 
+	42u, 43u, 45u, 46u, 47u, 58u, 61u, 63u, 
+	64u, 95u, 35u, 37u, 48u, 57u, 65u, 90u, 
+	94u, 96u, 97u, 122u, 123u, 126u, 33u, 38u, 
+	39u, 42u, 43u, 45u, 46u, 47u, 58u, 61u, 
+	63u, 64u, 95u, 105u, 116u, 35u, 37u, 48u, 
+	57u, 65u, 90u, 94u, 96u, 97u, 122u, 123u, 
+	126u, 33u, 38u, 39u, 42u, 43u, 45u, 46u, 
+	47u, 58u, 61u, 63u, 64u, 95u, 108u, 35u, 
+	37u, 48u, 57u, 65u, 90u, 94u, 96u, 97u, 
+	122u, 123u, 126u, 33u, 38u, 39u, 42u, 43u, 
+	45u, 46u, 47u, 58u, 61u, 63u, 64u, 95u, 
+	101u, 35u, 37u, 48u, 57u, 65u, 90u, 94u, 
+	96u, 97u, 122u, 123u, 126u, 33u, 38u, 39u, 
+	42u, 43u, 45u, 46u, 47u, 58u, 61u, 63u, 
+	64u, 95u, 35u, 37u, 48u, 57u, 65u, 90u, 
+	94u, 96u, 97u, 122u, 123u, 126u, 45u, 47u, 
+	58u, 64u, 95u, 44u, 46u, 48u, 57u, 65u, 
+	90u, 97u, 122u, 33u, 38u, 39u, 42u, 43u, 
+	45u, 46u, 47u, 58u, 61u, 63u, 64u, 95u, 
+	112u, 35u, 37u, 48u, 57u, 65u, 90u, 94u, 
+	96u, 97u, 122u, 123u, 126u, 33u, 38u, 39u, 
+	42u, 43u, 45u, 46u, 47u, 58u, 61u, 63u, 
+	64u, 95u, 116u, 35u, 37u, 48u, 57u, 65u, 
+	90u, 94u, 96u, 97u, 122u, 123u, 126u, 33u, 
+	38u, 39u, 42u, 43u, 45u, 46u, 47u, 58u, 
+	61u, 63u, 64u, 95u, 116u, 35u, 37u, 48u, 
+	57u, 65u, 90u, 94u, 96u, 97u, 122u, 123u, 
+	126u, 33u, 38u, 39u, 42u, 43u, 45u, 46u, 
+	47u, 58u, 61u, 63u, 64u, 95u, 112u, 35u, 
+	37u, 48u, 57u, 65u, 90u, 94u, 96u, 97u, 
+	122u, 123u, 126u, 33u, 38u, 39u, 42u, 43u, 
+	45u, 46u, 47u, 58u, 61u, 63u, 64u, 95u, 
+	115u, 35u, 37u, 48u, 57u, 65u, 90u, 94u, 
+	96u, 97u, 122u, 123u, 126u, 0
+};
+
+static const char _StdTok_single_lengths[] = {
+	0, 0, 4, 4, 5, 4, 2, 0, 
+	2, 1, 0, 5, 7, 7, 7, 7, 
+	13, 7, 0, 8, 8, 8, 8, 8, 
+	8, 8, 8, 8, 5, 13, 1, 1, 
+	2, 13, 1, 5, 0, 13, 4, 4, 
+	2, 4, 4, 4, 4, 11, 5, 8, 
+	1, 9, 9, 9, 9, 10, 9, 9, 
+	9, 8, 8, 7, 2, 11, 3, 1, 
+	1, 5, 0, 13, 5, 5, 7, 5, 
+	7, 8, 8, 8, 8, 8, 8, 8, 
+	8, 13, 6, 5, 4, 13, 15, 14, 
+	14, 13, 5, 14, 14, 14, 14, 14
+};
+
+static const char _StdTok_range_lengths[] = {
+	0, 0, 5, 5, 5, 5, 5, 3, 
+	4, 0, 3, 5, 7, 7, 7, 7, 
+	7, 7, 1, 7, 7, 7, 7, 7, 
+	7, 7, 7, 7, 8, 7, 0, 3, 
+	0, 7, 0, 4, 0, 6, 5, 5, 
+	4, 5, 6, 6, 6, 6, 8, 7, 
+	1, 7, 7, 7, 7, 7, 7, 7, 
+	7, 7, 7, 7, 3, 6, 0, 0, 
+	0, 5, 3, 6, 7, 8, 7, 7, 
+	7, 7, 7, 7, 7, 7, 7, 7, 
+	7, 6, 5, 7, 8, 6, 6, 6, 
+	6, 6, 4, 6, 6, 6, 6, 6
+};
+
+static const short _StdTok_index_offsets[] = {
+	0, 0, 1, 11, 21, 32, 42, 50, 
+	54, 61, 63, 67, 78, 93, 108, 123, 
+	138, 159, 174, 176, 192, 208, 224, 240, 
+	256, 272, 288, 304, 320, 334, 355, 357, 
+	362, 365, 386, 388, 398, 399, 419, 429, 
+	439, 446, 456, 467, 478, 489, 507, 521, 
+	537, 540, 557, 574, 591, 608, 626, 643, 
+	660, 677, 693, 709, 724, 730, 748, 752, 
+	754, 756, 767, 771, 791, 804, 818, 833, 
+	846, 861, 877, 893, 909, 925, 941, 957, 
+	973, 989, 1009, 1021, 1034, 1047, 1067, 1089, 
+	1110, 1131, 1151, 1161, 1182, 1203, 1224, 1245
+};
+
+static const char _StdTok_indicies[] = {
+	0, 2, 3, 2, 4, 2, 2, 2, 
+	2, 2, 1, 2, 2, 2, 2, 2, 
+	2, 2, 2, 2, 1, 5, 5, 5, 
+	5, 6, 5, 5, 5, 5, 5, 1, 
+	5, 5, 5, 5, 5, 5, 5, 5, 
+	5, 7, 8, 9, 6, 6, 6, 6, 
+	6, 1, 6, 6, 6, 1, 12, 13, 
+	11, 11, 11, 11, 10, 4, 10, 11, 
+	11, 11, 10, 2, 2, 2, 2, 2, 
+	2, 2, 15, 2, 2, 14, 2, 16, 
+	2, 3, 2, 2, 4, 2, 2, 17, 
+	17, 2, 17, 2, 1, 2, 18, 3, 
+	2, 2, 2, 4, 2, 2, 19, 19, 
+	2, 19, 2, 1, 2, 18, 3, 2, 
+	2, 2, 4, 2, 2, 20, 20, 2, 
+	20, 2, 1, 2, 18, 21, 2, 2, 
+	2, 4, 2, 2, 20, 20, 2, 20, 
+	2, 1, 2, 2, 2, 2, 2, 23, 
+	24, 25, 26, 27, 28, 29, 30, 2, 
+	2, 20, 22, 2, 22, 2, 1, 2, 
+	18, 21, 2, 2, 2, 4, 2, 2, 
+	20, 31, 2, 31, 2, 1, 33, 32, 
+	2, 18, 21, 2, 2, 2, 4, 34, 
+	2, 2, 20, 31, 2, 31, 2, 1, 
+	2, 18, 21, 2, 2, 2, 4, 35, 
+	2, 2, 20, 31, 2, 31, 2, 1, 
+	2, 18, 21, 2, 2, 2, 4, 36, 
+	2, 2, 20, 31, 2, 31, 2, 1, 
+	2, 18, 21, 2, 2, 2, 4, 37, 
+	2, 2, 20, 31, 2, 31, 2, 1, 
+	2, 18, 21, 2, 2, 2, 4, 38, 
+	2, 2, 20, 31, 2, 31, 2, 1, 
+	2, 18, 21, 2, 2, 2, 4, 31, 
+	2, 2, 20, 20, 2, 20, 2, 32, 
+	2, 18, 21, 2, 2, 2, 4, 39, 
+	2, 2, 20, 31, 2, 31, 2, 1, 
+	2, 18, 21, 2, 2, 2, 4, 40, 
+	2, 2, 20, 31, 2, 31, 2, 1, 
+	2, 18, 21, 2, 2, 2, 4, 41, 
+	2, 2, 20, 31, 2, 31, 2, 1, 
+	2, 3, 2, 2, 4, 2, 2, 2, 
+	42, 42, 2, 42, 2, 1, 2, 2, 
+	2, 2, 2, 23, 24, 25, 26, 27, 
+	28, 29, 30, 2, 2, 44, 22, 2, 
+	22, 2, 43, 45, 1, 46, 46, 46, 
+	46, 1, 47, 48, 10, 2, 2, 2, 
+	2, 2, 50, 51, 52, 53, 54, 55, 
+	56, 57, 2, 2, 20, 49, 2, 49, 
+	2, 1, 59, 58, 60, 61, 46, 46, 
+	60, 46, 60, 60, 60, 58, 62, 63, 
+	65, 66, 67, 68, 68, 67, 70, 67, 
+	71, 72, 74, 75, 67, 69, 73, 67, 
+	73, 67, 64, 65, 3, 2, 4, 2, 
+	2, 2, 2, 2, 76, 5, 78, 5, 
+	5, 5, 5, 5, 5, 5, 77, 12, 
+	13, 11, 11, 11, 11, 79, 2, 3, 
+	2, 4, 2, 2, 2, 2, 2, 1, 
+	2, 3, 2, 4, 2, 2, 2, 80, 
+	2, 2, 79, 2, 82, 2, 4, 2, 
+	2, 2, 80, 2, 2, 81, 2, 3, 
+	2, 4, 2, 2, 2, 15, 2, 2, 
+	83, 2, 16, 2, 85, 86, 2, 87, 
+	2, 2, 4, 89, 2, 69, 88, 2, 
+	88, 2, 84, 2, 3, 2, 2, 4, 
+	2, 2, 2, 17, 17, 2, 17, 2, 
+	90, 2, 18, 21, 92, 93, 2, 2, 
+	4, 2, 2, 20, 20, 2, 20, 2, 
+	91, 94, 33, 91, 2, 18, 21, 92, 
+	93, 2, 2, 4, 31, 2, 2, 20, 
+	20, 2, 20, 2, 91, 2, 18, 21, 
+	92, 93, 2, 2, 4, 31, 2, 2, 
+	20, 20, 2, 20, 2, 91, 2, 18, 
+	21, 92, 93, 2, 2, 4, 31, 2, 
+	2, 20, 20, 2, 20, 2, 91, 2, 
+	18, 21, 92, 93, 2, 2, 4, 31, 
+	2, 2, 20, 20, 2, 20, 2, 91, 
+	2, 18, 21, 92, 93, 2, 2, 4, 
+	95, 31, 2, 2, 20, 20, 2, 20, 
+	2, 91, 2, 18, 21, 92, 93, 2, 
+	2, 4, 31, 2, 2, 20, 20, 2, 
+	20, 2, 91, 2, 18, 21, 92, 93, 
+	2, 2, 4, 31, 2, 2, 20, 20, 
+	2, 20, 2, 91, 2, 18, 21, 92, 
+	93, 2, 2, 4, 31, 2, 2, 20, 
+	20, 2, 20, 2, 91, 2, 85, 21, 
+	2, 2, 2, 4, 89, 2, 2, 19, 
+	19, 2, 19, 2, 84, 2, 89, 3, 
+	2, 2, 2, 4, 89, 2, 2, 42, 
+	42, 2, 42, 2, 84, 2, 18, 21, 
+	2, 2, 2, 4, 2, 2, 44, 20, 
+	2, 20, 2, 83, 96, 46, 46, 46, 
+	46, 91, 2, 16, 2, 85, 21, 2, 
+	87, 2, 2, 4, 89, 2, 88, 88, 
+	2, 88, 2, 84, 47, 48, 97, 79, 
+	47, 98, 48, 99, 2, 3, 2, 71, 
+	4, 2, 2, 2, 2, 2, 100, 101, 
+	101, 101, 1, 2, 103, 104, 16, 2, 
+	85, 105, 2, 87, 2, 2, 107, 89, 
+	2, 106, 108, 2, 108, 2, 102, 2, 
+	3, 2, 2, 4, 2, 2, 2, 110, 
+	2, 110, 2, 109, 2, 3, 2, 2, 
+	4, 2, 2, 2, 110, 110, 2, 110, 
+	2, 109, 2, 3, 2, 2, 4, 113, 
+	113, 2, 2, 2, 112, 2, 112, 2, 
+	111, 2, 3, 2, 2, 4, 2, 2, 
+	2, 112, 2, 112, 2, 1, 2, 18, 
+	105, 2, 2, 2, 4, 2, 2, 20, 
+	31, 2, 31, 2, 114, 2, 18, 105, 
+	2, 2, 2, 4, 34, 2, 2, 20, 
+	31, 2, 31, 2, 114, 2, 18, 105, 
+	2, 2, 2, 4, 35, 2, 2, 20, 
+	31, 2, 31, 2, 114, 2, 18, 105, 
+	2, 2, 2, 4, 36, 2, 2, 20, 
+	31, 2, 31, 2, 114, 2, 18, 105, 
+	2, 2, 2, 4, 37, 2, 2, 20, 
+	31, 2, 31, 2, 114, 2, 18, 105, 
+	2, 2, 2, 4, 38, 2, 2, 20, 
+	31, 2, 31, 2, 114, 2, 18, 105, 
+	2, 2, 2, 4, 39, 2, 2, 20, 
+	31, 2, 31, 2, 114, 2, 18, 105, 
+	2, 2, 2, 4, 40, 2, 2, 20, 
+	31, 2, 31, 2, 114, 2, 18, 105, 
+	2, 2, 2, 4, 41, 2, 2, 20, 
+	31, 2, 31, 2, 114, 2, 103, 115, 
+	16, 2, 85, 21, 2, 87, 2, 2, 
+	107, 89, 2, 106, 106, 2, 106, 2, 
+	102, 2, 3, 2, 4, 116, 116, 2, 
+	2, 2, 2, 2, 111, 5, 5, 5, 
+	5, 6, 5, 5, 5, 117, 5, 117, 
+	5, 109, 5, 78, 5, 5, 5, 5, 
+	5, 117, 117, 5, 117, 5, 77, 2, 
+	103, 104, 16, 2, 85, 21, 2, 87, 
+	2, 2, 107, 89, 2, 106, 108, 2, 
+	108, 2, 102, 2, 103, 104, 16, 2, 
+	85, 105, 2, 87, 2, 2, 107, 89, 
+	118, 119, 2, 106, 108, 2, 108, 2, 
+	102, 2, 103, 104, 16, 2, 85, 21, 
+	2, 87, 2, 2, 107, 89, 120, 2, 
+	106, 108, 2, 108, 2, 102, 2, 103, 
+	104, 16, 2, 85, 21, 2, 87, 2, 
+	2, 107, 89, 121, 2, 106, 108, 2, 
+	108, 2, 102, 2, 103, 104, 16, 2, 
+	85, 21, 2, 122, 2, 2, 107, 89, 
+	2, 106, 108, 2, 108, 2, 102, 60, 
+	123, 46, 46, 60, 46, 60, 60, 60, 
+	91, 2, 103, 104, 16, 2, 85, 21, 
+	2, 87, 2, 2, 107, 89, 121, 2, 
+	106, 108, 2, 108, 2, 102, 2, 103, 
+	104, 16, 2, 85, 105, 2, 87, 2, 
+	2, 107, 89, 124, 2, 106, 108, 2, 
+	108, 2, 102, 2, 103, 104, 16, 2, 
+	85, 21, 2, 87, 2, 2, 107, 89, 
+	125, 2, 106, 108, 2, 108, 2, 102, 
+	2, 103, 104, 16, 2, 85, 21, 2, 
+	87, 2, 2, 107, 89, 126, 2, 106, 
+	108, 2, 108, 2, 102, 2, 103, 104, 
+	16, 2, 85, 21, 2, 122, 2, 2, 
+	107, 89, 121, 2, 106, 108, 2, 108, 
+	2, 102, 0
+};
+
+static const char _StdTok_trans_targs[] = {
+	36, 37, 2, 3, 4, 39, 6, 37, 
+	7, 37, 37, 8, 9, 10, 37, 44, 
+	12, 46, 14, 57, 15, 16, 17, 19, 
+	20, 21, 22, 23, 25, 26, 27, 47, 
+	37, 48, 49, 50, 51, 52, 53, 54, 
+	55, 56, 58, 37, 59, 31, 60, 63, 
+	64, 72, 73, 74, 75, 76, 77, 78, 
+	79, 80, 37, 35, 60, 90, 0, 37, 
+	37, 38, 40, 41, 42, 45, 62, 65, 
+	66, 67, 86, 92, 37, 37, 5, 37, 
+	43, 37, 11, 37, 37, 13, 29, 30, 
+	61, 28, 37, 37, 41, 18, 37, 24, 
+	60, 32, 37, 37, 37, 66, 37, 68, 
+	70, 33, 81, 83, 85, 37, 69, 37, 
+	71, 71, 37, 82, 41, 84, 87, 91, 
+	88, 89, 34, 90, 93, 94, 95
+};
+
+static const char _StdTok_trans_actions[] = {
+	5, 61, 0, 0, 0, 11, 0, 49, 
+	0, 13, 59, 0, 0, 0, 57, 105, 
+	0, 75, 0, 90, 0, 0, 0, 0, 
+	0, 0, 0, 0, 0, 0, 0, 96, 
+	55, 0, 96, 96, 96, 96, 96, 96, 
+	96, 96, 90, 53, 105, 0, 0, 0, 
+	0, 99, 99, 99, 99, 99, 99, 99, 
+	99, 99, 51, 0, 1, 0, 0, 15, 
+	17, 69, 108, 108, 108, 90, 11, 66, 
+	108, 78, 78, 78, 21, 29, 0, 47, 
+	102, 43, 0, 45, 35, 0, 0, 0, 
+	90, 0, 27, 39, 111, 0, 63, 0, 
+	3, 0, 25, 23, 19, 72, 31, 93, 
+	81, 0, 78, 93, 78, 37, 93, 33, 
+	87, 84, 41, 81, 84, 11, 78, 78, 
+	78, 78, 0, 3, 78, 78, 78
+};
+
+static const char _StdTok_to_state_actions[] = {
+	0, 7, 0, 0, 0, 0, 0, 0, 
+	0, 0, 0, 0, 0, 0, 0, 0, 
+	0, 0, 0, 0, 0, 0, 0, 0, 
+	0, 0, 0, 0, 0, 0, 0, 0, 
+	0, 0, 0, 0, 7, 7, 0, 0, 
+	0, 0, 0, 0, 0, 0, 0, 0, 
+	0, 0, 0, 0, 0, 0, 0, 0, 
+	0, 0, 0, 0, 0, 0, 0, 0, 
+	0, 0, 0, 0, 0, 0, 0, 0, 
+	0, 0, 0, 0, 0, 0, 0, 0, 
+	0, 0, 0, 0, 0, 0, 0, 0, 
+	0, 0, 0, 0, 0, 0, 0, 0
+};
+
+static const char _StdTok_from_state_actions[] = {
+	0, 0, 0, 0, 0, 0, 0, 0, 
+	0, 0, 0, 0, 0, 0, 0, 0, 
+	0, 0, 0, 0, 0, 0, 0, 0, 
+	0, 0, 0, 0, 0, 0, 0, 0, 
+	0, 0, 0, 0, 0, 9, 0, 0, 
+	0, 0, 0, 0, 0, 0, 0, 0, 
+	0, 0, 0, 0, 0, 0, 0, 0, 
+	0, 0, 0, 0, 0, 0, 0, 0, 
+	0, 0, 0, 0, 0, 0, 0, 0, 
+	0, 0, 0, 0, 0, 0, 0, 0, 
+	0, 0, 0, 0, 0, 0, 0, 0, 
+	0, 0, 0, 0, 0, 0, 0, 0
+};
+
+static const short _StdTok_eof_trans[] = {
+	0, 0, 2, 2, 2, 8, 2, 2, 
+	11, 11, 11, 15, 2, 2, 2, 2, 
+	2, 2, 33, 2, 2, 2, 2, 2, 
+	33, 2, 2, 2, 2, 44, 2, 2, 
+	11, 2, 59, 59, 0, 0, 77, 78, 
+	80, 2, 80, 82, 84, 85, 91, 92, 
+	92, 92, 92, 92, 92, 92, 92, 92, 
+	92, 85, 85, 84, 92, 85, 80, 99, 
+	100, 101, 2, 103, 110, 110, 112, 2, 
+	115, 115, 115, 115, 115, 115, 115, 115, 
+	115, 103, 112, 110, 78, 103, 103, 103, 
+	103, 103, 92, 103, 103, 103, 103, 103
+};
+
+static const int StdTok_start = 1;
+static const int StdTok_error = 0;
+
+static const int StdTok_en_frt_tokenizer = 37;
+static const int StdTok_en_main = 1;
+
+#line 29 "scanner.rl"
+
+void frt_std_scan(const char *in,
+                  char *out, size_t out_size,
+                  const char **start,
+                  const char **end,
+                  int *token_size)
+{
+    int cs, act, top;
+    int stack[32];
+    char *ts = 0, *te = 0;
+
+    
+#line 579 "scanner.c"
+	{
+	cs = StdTok_start;
+	top = 0;
+	ts = 0;
+	te = 0;
+	act = 0;
+	}
+#line 41 "scanner.rl"
+
+    char *p = (char *)in, *pe = 0, *eof = pe;
+    int skip = 0;
+    int trunc = 0;
+    char strip_char = 0;
+
+    *end = 0;
+    *start = 0;
+    *token_size = 0;
+
+    
+#line 599 "scanner.c"
+	{
+	int _klen;
+	unsigned int _trans;
+	const char *_acts;
+	unsigned int _nacts;
+	const unsigned char *_keys;
+
+	if ( p == pe )
+		goto _test_eof;
+	if ( cs == 0 )
+		goto _out;
+_resume:
+	_acts = _StdTok_actions + _StdTok_from_state_actions[cs];
+	_nacts = (unsigned int) *_acts++;
+	while ( _nacts-- > 0 ) {
+		switch ( *_acts++ ) {
+	case 4:
+#line 1 "scanner.rl"
+	{ts = p;}
+	break;
+#line 620 "scanner.c"
+		}
+	}
+
+	_keys = _StdTok_trans_keys + _StdTok_key_offsets[cs];
+	_trans = _StdTok_index_offsets[cs];
+
+	_klen = _StdTok_single_lengths[cs];
+	if ( _klen > 0 ) {
+		const unsigned char *_lower = _keys;
+		const unsigned char *_mid;
+		const unsigned char *_upper = _keys + _klen - 1;
+		while (1) {
+			if ( _upper < _lower )
+				break;
+
+			_mid = _lower + ((_upper-_lower) >> 1);
+			if ( (*p) < *_mid )
+				_upper = _mid - 1;
+			else if ( (*p) > *_mid )
+				_lower = _mid + 1;
+			else {
+				_trans += (_mid - _keys);
+				goto _match;
+			}
+		}
+		_keys += _klen;
+		_trans += _klen;
+	}
+
+	_klen = _StdTok_range_lengths[cs];
+	if ( _klen > 0 ) {
+		const unsigned char *_lower = _keys;
+		const unsigned char *_mid;
+		const unsigned char *_upper = _keys + (_klen<<1) - 2;
+		while (1) {
+			if ( _upper < _lower )
+				break;
+
+			_mid = _lower + (((_upper-_lower) >> 1) & ~1);
+			if ( (*p) < _mid[0] )
+				_upper = _mid - 2;
+			else if ( (*p) > _mid[1] )
+				_lower = _mid + 2;
+			else {
+				_trans += ((_mid - _keys)>>1);
+				goto _match;
+			}
+		}
+		_trans += _klen;
+	}
+
+_match:
+	_trans = _StdTok_indicies[_trans];
+_eof_trans:
+	cs = _StdTok_trans_targs[_trans];
+
+	if ( _StdTok_trans_actions[_trans] == 0 )
+		goto _again;
+
+	_acts = _StdTok_actions + _StdTok_trans_actions[_trans];
+	_nacts = (unsigned int) *_acts++;
+	while ( _nacts-- > 0 )
+	{
+		switch ( *_acts++ )
+		{
+	case 0:
+#line 14 "scanner.rl"
+	{ skip = p - ts; }
+	break;
+	case 1:
+#line 26 "scanner.rl"
+	{ trunc = 1; }
+	break;
+	case 2:
+#line 25 "scanner.rl"
+	{ p--; {stack[top++] = cs; cs = 37; goto _again;} }
+	break;
+	case 5:
+#line 1 "scanner.rl"
+	{te = p+1;}
+	break;
+	case 6:
+#line 12 "scanner.rl"
+	{act = 1;}
+	break;
+	case 7:
+#line 14 "scanner.rl"
+	{act = 2;}
+	break;
+	case 8:
+#line 16 "scanner.rl"
+	{act = 3;}
+	break;
+	case 9:
+#line 22 "scanner.rl"
+	{act = 6;}
+	break;
+	case 10:
+#line 28 "scanner.rl"
+	{act = 8;}
+	break;
+	case 11:
+#line 29 "scanner.rl"
+	{act = 9;}
+	break;
+	case 12:
+#line 30 "scanner.rl"
+	{act = 10;}
+	break;
+	case 13:
+#line 33 "scanner.rl"
+	{act = 11;}
+	break;
+	case 14:
+#line 36 "scanner.rl"
+	{act = 12;}
+	break;
+	case 15:
+#line 39 "scanner.rl"
+	{act = 13;}
+	break;
+	case 16:
+#line 42 "scanner.rl"
+	{act = 14;}
+	break;
+	case 17:
+#line 45 "scanner.rl"
+	{act = 15;}
+	break;
+	case 18:
+#line 48 "scanner.rl"
+	{act = 16;}
+	break;
+	case 19:
+#line 49 "scanner.rl"
+	{act = 17;}
+	break;
+	case 20:
+#line 53 "scanner.rl"
+	{act = 19;}
+	break;
+	case 21:
+#line 25 "scanner.rl"
+	{te = p+1;{ RET; }}
+	break;
+	case 22:
+#line 42 "scanner.rl"
+	{te = p+1;{ RET; }}
+	break;
+	case 23:
+#line 52 "scanner.rl"
+	{te = p+1;{ return; }}
+	break;
+	case 24:
+#line 53 "scanner.rl"
+	{te = p+1;}
+	break;
+	case 25:
+#line 12 "scanner.rl"
+	{te = p;p--;{ RET; }}
+	break;
+	case 26:
+#line 14 "scanner.rl"
+	{te = p;p--;{ RET; }}
+	break;
+	case 27:
+#line 18 "scanner.rl"
+	{te = p;p--;{ RET; }}
+	break;
+	case 28:
+#line 20 "scanner.rl"
+	{te = p;p--;{ RET; }}
+	break;
+	case 29:
+#line 22 "scanner.rl"
+	{te = p;p--;{ RET; }}
+	break;
+	case 30:
+#line 25 "scanner.rl"
+	{te = p;p--;{ RET; }}
+	break;
+	case 31:
+#line 28 "scanner.rl"
+	{te = p;p--;{ RET; }}
+	break;
+	case 32:
+#line 29 "scanner.rl"
+	{te = p;p--;{ trunc = 1; RET; }}
+	break;
+	case 33:
+#line 36 "scanner.rl"
+	{te = p;p--;{ RET; }}
+	break;
+	case 34:
+#line 39 "scanner.rl"
+	{te = p;p--;{ RET; }}
+	break;
+	case 35:
+#line 42 "scanner.rl"
+	{te = p;p--;{ RET; }}
+	break;
+	case 36:
+#line 45 "scanner.rl"
+	{te = p;p--;{ STRIP('.'); }}
+	break;
+	case 37:
+#line 48 "scanner.rl"
+	{te = p;p--;{ RET; }}
+	break;
+	case 38:
+#line 49 "scanner.rl"
+	{te = p;p--;{ RET; }}
+	break;
+	case 39:
+#line 53 "scanner.rl"
+	{te = p;p--;}
+	break;
+	case 40:
+#line 25 "scanner.rl"
+	{{p = ((te))-1;}{ RET; }}
+	break;
+	case 41:
+#line 28 "scanner.rl"
+	{{p = ((te))-1;}{ RET; }}
+	break;
+	case 42:
+#line 36 "scanner.rl"
+	{{p = ((te))-1;}{ RET; }}
+	break;
+	case 43:
+#line 42 "scanner.rl"
+	{{p = ((te))-1;}{ RET; }}
+	break;
+	case 44:
+#line 48 "scanner.rl"
+	{{p = ((te))-1;}{ RET; }}
+	break;
+	case 45:
+#line 53 "scanner.rl"
+	{{p = ((te))-1;}}
+	break;
+	case 46:
+#line 1 "scanner.rl"
+	{	switch( act ) {
+	case 1:
+	{{p = ((te))-1;} RET; }
+	break;
+	case 2:
+	{{p = ((te))-1;} RET; }
+	break;
+	case 3:
+	{{p = ((te))-1;} RET; }
+	break;
+	case 6:
+	{{p = ((te))-1;} RET; }
+	break;
+	case 8:
+	{{p = ((te))-1;} RET; }
+	break;
+	case 9:
+	{{p = ((te))-1;} trunc = 1; RET; }
+	break;
+	case 10:
+	{{p = ((te))-1;} trunc = 2; RET; }
+	break;
+	case 11:
+	{{p = ((te))-1;} RET; }
+	break;
+	case 12:
+	{{p = ((te))-1;} RET; }
+	break;
+	case 13:
+	{{p = ((te))-1;} RET; }
+	break;
+	case 14:
+	{{p = ((te))-1;} RET; }
+	break;
+	case 15:
+	{{p = ((te))-1;} STRIP('.'); }
+	break;
+	case 16:
+	{{p = ((te))-1;} RET; }
+	break;
+	case 17:
+	{{p = ((te))-1;} RET; }
+	break;
+	case 19:
+	{{p = ((te))-1;}}
+	break;
+	}
+	}
+	break;
+#line 913 "scanner.c"
+		}
+	}
+
+_again:
+	_acts = _StdTok_actions + _StdTok_to_state_actions[cs];
+	_nacts = (unsigned int) *_acts++;
+	while ( _nacts-- > 0 ) {
+		switch ( *_acts++ ) {
+	case 3:
+#line 1 "scanner.rl"
+	{ts = 0;}
+	break;
+#line 926 "scanner.c"
+		}
+	}
+
+	if ( cs == 0 )
+		goto _out;
+	if ( ++p != pe )
+		goto _resume;
+	_test_eof: {}
+	if ( p == eof )
+	{
+	if ( _StdTok_eof_trans[cs] > 0 ) {
+		_trans = _StdTok_eof_trans[cs] - 1;
+		goto _eof_trans;
+	}
+	}
+
+	_out: {}
+	}
+#line 52 "scanner.rl"
+
+    if ( cs == StdTok_error )
+                   fprintf(stderr, "PARSE ERROR\n" );
+    else if ( ts ) fprintf(stderr, "STUFF LEFT: '%s'\n", ts);
+    return;
+
+ ret:
+    {
+        size_t __len = te - ts - skip - trunc;
+        if (__len > out_size)
+            __len = out_size;
+
+        *start = ts;
+        *end   = te;
+
+        if (strip_char) {
+            char *__p = ts + skip;
+            char *__o = out;
+            for (; __p < (ts + skip + __len); ++__p) {
+                if (*__p != strip_char)
+                    *__o++ = *__p;
+            }
+            *token_size = __o - out;
+        }
+        else {
+            memcpy(out, ts + skip, __len);
+            *token_size = __len;
+        }
+
+        out[*token_size] = 0;
+    }
+}

data/ext/scanner.in

@@ -0,0 +1,56 @@
+#// scanner.in -*-C-*-
+
+%%{
+    machine StdTok;
+    include URL "url.rl";
+    include Email "email.rl";
+
+    token = frt_alpha frt_alnum*;
+
+    frt_tokenizer := |*
+        #// question_mark 
+        ('?')+ { RET; };
+        #// exclamation_mark
+        ('!')+ { RET; };
+        #// twitter_user
+        '@' (alnum)+  { RET; };
+        #// smile_face 
+        ':' ('-')? (')')+ { RET; };
+        #// angry_face
+        ':' ('-')? ('(')+ { RET; };
+        #// fword
+        alnum+ ('*')+ alnum+ { RET; };
+    
+        #// Email
+        email { RET; };
+        
+        #// Token, or token with possessive
+        token          { RET; };
+        token [\']     { trunc = 1; RET; };
+        token [\'][sS] { trunc = 2; RET; };
+
+        #// contractions
+        frt_alpha+ [\'] frt_alpha+ { RET; };
+
+        #// Token with hyphens
+        frt_alnum+ ([\-_] frt_alnum+)* { RET; };
+
+        #// Company name
+        token [\&\@] token* { RET; };
+
+        #// URL
+        url { RET; };
+
+        #// Acronym
+        (frt_alpha '.')+ frt_alpha { STRIP('.'); };
+
+        #// Int+float
+        [\-\+]?frt_digit+                { RET; };
+        [\-\+]?frt_digit+ '.' frt_digit+ { RET; };
+
+        #// Ignore whitespace and other crap
+        0 { return; };
+        (any - frt_alnum) {};
+
+        *|;
+}%%

data/ext/scanner.rl

@@ -0,0 +1,83 @@
+/* scanner.rl -*-C-*- */
+#include <ctype.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#define RET goto ret;
+
+#define STRIP(c) do { \
+    strip_char = c;   \
+    goto ret;         \
+} while(0)
+
+%%{
+    machine StdTok;
+    alphtype unsigned char;
+
+    frt_alpha = alpha;
+    frt_alnum = alnum;
+    frt_digit = digit;
+
+    include StdTok "scanner.in";
+
+    main := any @{ fhold; fcall frt_tokenizer; };
+}%%
+
+%% write data nofinal;
+
+void frt_std_scan(const char *in,
+                  char *out, size_t out_size,
+                  const char **start,
+                  const char **end,
+                  int *token_size)
+{
+    int cs, act, top;
+    int stack[32];
+    char *ts = 0, *te = 0;
+
+    %% write init;
+
+    char *p = (char *)in, *pe = 0, *eof = pe;
+    int skip = 0;
+    int trunc = 0;
+    char strip_char = 0;
+
+    *end = 0;
+    *start = 0;
+    *token_size = 0;
+
+    %% write exec;
+
+    if ( cs == StdTok_error )
+                   fprintf(stderr, "PARSE ERROR\n" );
+    else if ( ts ) fprintf(stderr, "STUFF LEFT: '%s'\n", ts);
+    return;
+
+ ret:
+    {
+        size_t __len = te - ts - skip - trunc;
+        if (__len > out_size)
+            __len = out_size;
+
+        *start = ts;
+        *end   = te;
+
+        if (strip_char) {
+            char *__p = ts + skip;
+            char *__o = out;
+            for (; __p < (ts + skip + __len); ++__p) {
+                if (*__p != strip_char)
+                    *__o++ = *__p;
+            }
+            *token_size = __o - out;
+        }
+        else {
+            memcpy(out, ts + skip, __len);
+            *token_size = __len;
+        }
+
+        out[*token_size] = 0;
+    }
+}

data/ext/url.rl

@@ -0,0 +1,27 @@
+#// url.rl -*-C-*-
+
+%%{
+    machine URL;
+
+    uword = [_] | alnum;
+    dword = '-' | uword;
+    dalnum = '-' | alnum;
+    proto = 'http'[s]? | 'ftp' | 'file';
+    urlc  = alnum | [.,\/_\-\@\:];
+
+    url =
+        (
+            proto  [:][/]+ %{ skip = p - ts; } dword+ ([.] uword dword*)+ |
+            alnum+ [:][/]+ urlc+                                          |
+
+            (alnum (dalnum* alnum)? [.])+  #// Subdomains
+                ('com' |'edu'   | 'biz' | 'gov' |
+                 'int' | 'info' | 'mil' | 'net' |
+                 'org' | alpha{2})
+        )
+
+        #// Port
+        ( [:] digit+ )?
+
+        ([/]? @{ trunc = 1; });
+}%%

data/test/test.rb

@@ -0,0 +1,7 @@
+require 'ext/ferret_tokenizer'
+
+t = FerretTokenizer.new '@feedbackmine sh*t, I LOVE this!!! so funny:-) :('
+while (tok = t.next)
+  p tok
+end
+

metadata

@@ -0,0 +1,74 @@
+--- !ruby/object:Gem::Specification 
+name: feedbackmine-ferret_tokenizer
+version: !ruby/object:Gem::Version 
+  version: 0.1.1
+platform: ruby
+authors: 
+- FeedbackMine
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-03-23 00:00:00 -07:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.8.3
+    version: 
+description: A string tokenizer based on Ferret::Analysis::StandardAnalyzer.
+email: feedbackmine@feedbackmine.com
+executables: []
+
+extensions: 
+- ext/extconf.rb
+extra_rdoc_files: 
+- Manifest.txt
+files: 
+- ext/ferret_tokenizer.c
+- ext/scanner.rl
+- ext/url.rl
+- ext/extconf.rb
+- ext/scanner.c
+- ext/email.rl
+- ext/scanner.in
+- Rakefile
+- Manifest.txt
+- README.txt
+- test
+- test/test.rb
+has_rdoc: true
+homepage: http://www.tweetjobsearch.com
+post_install_message: 
+rdoc_options: 
+- --main
+- README.txt
+require_paths: 
+- ext
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: tokenizer
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 2
+summary: tokenizer
+test_files: []
+