isomorfeus-ferret 0.12.2 → 0.12.6

data/ext/isomorfeus_ferret_ext/stem_utilities.c

@@ -5,58 +5,66 @@
 
 #include "stem_header.h"
 
-#define unless(C) if(!(C))
-
 #define CREATE_SIZE 1
 
-symbol * create_s(void) {
+extern symbol * create_s(void) {
     symbol * p;
     void * mem = malloc(HEAD + (CREATE_SIZE + 1) * sizeof(symbol));
     if (mem == NULL) return NULL;
     p = (symbol *) (HEAD + (char *) mem);
     CAPACITY(p) = CREATE_SIZE;
-    SET_SIZE(p, CREATE_SIZE);
+    SET_SIZE(p, 0);
     return p;
 }
 
-void lose_s(symbol * p) {
+extern void lose_s(symbol * p) {
     if (p == NULL) return;
     free((char *) p - HEAD);
 }
 
 /*
-   new_p = skip_utf8(p, c, lb, l, n); skips n characters forwards from p + c
-   if n +ve, or n characters backwards from p + c - 1 if n -ve. new_p is the new
-   position, or 0 on failure.
+   new_p = skip_utf8(p, c, l, n); skips n characters forwards from p + c.
+   new_p is the new position, or -1 on failure.
 
    -- used to implement hop and next in the utf8 case.
 */
 
-int skip_utf8(const symbol * p, int c, int lb, int l, int n) {
+extern int skip_utf8(const symbol * p, int c, int limit, int n) {
     int b;
-    if (n >= 0) {
-        for (; n > 0; n--) {
-            if (c >= l) return -1;
-            b = p[c++];
-            if (b >= 0xC0) {   /* 1100 0000 */
-                while (c < l) {
-                    b = p[c];
-                    if (b >= 0xC0 || b < 0x80) break;
-                    /* break unless b is 10------ */
-                    c++;
-                }
+    if (n < 0) return -1;
+    for (; n > 0; n--) {
+        if (c >= limit) return -1;
+        b = p[c++];
+        if (b >= 0xC0) {   /* 1100 0000 */
+            while (c < limit) {
+                b = p[c];
+                if (b >= 0xC0 || b < 0x80) break;
+                /* break unless b is 10------ */
+                c++;
             }
         }
-    } else {
-        for (; n < 0; n++) {
-            if (c <= lb) return -1;
-            b = p[--c];
-            if (b >= 0x80) {   /* 1000 0000 */
-                while (c > lb) {
-                    b = p[c];
-                    if (b >= 0xC0) break; /* 1100 0000 */
-                    c--;
-                }
+    }
+    return c;
+}
+
+/*
+   new_p = skip_b_utf8(p, c, lb, n); skips n characters backwards from p + c - 1
+   new_p is the new position, or -1 on failure.
+
+   -- used to implement hop and next in the utf8 case.
+*/
+
+extern int skip_b_utf8(const symbol * p, int c, int limit, int n) {
+    int b;
+    if (n < 0) return -1;
+    for (; n > 0; n--) {
+        if (c <= limit) return -1;
+        b = p[--c];
+        if (b >= 0x80) {   /* 1000 0000 */
+            while (c > limit) {
+                b = p[c];
+                if (b >= 0xC0) break; /* 1100 0000 */
+                c--;
             }
         }
     }
@@ -66,156 +74,174 @@ int skip_utf8(const symbol * p, int c, int lb, int l, int n) {
 /* Code for character groupings: utf8 cases */
 
 static int get_utf8(const symbol * p, int c, int l, int * slot) {
-    int b0, b1;
+    int b0, b1, b2;
     if (c >= l) return 0;
     b0 = p[c++];
     if (b0 < 0xC0 || c == l) {   /* 1100 0000 */
-        * slot = b0; return 1;
+        *slot = b0;
+        return 1;
     }
-    b1 = p[c++];
+    b1 = p[c++] & 0x3F;
     if (b0 < 0xE0 || c == l) {   /* 1110 0000 */
-        * slot = (b0 & 0x1F) << 6 | (b1 & 0x3F); return 2;
+        *slot = (b0 & 0x1F) << 6 | b1;
+        return 2;
+    }
+    b2 = p[c++] & 0x3F;
+    if (b0 < 0xF0 || c == l) {   /* 1111 0000 */
+        *slot = (b0 & 0xF) << 12 | b1 << 6 | b2;
+        return 3;
     }
-    * slot = (b0 & 0xF) << 12 | (b1 & 0x3F) << 6 | (p[c] & 0x3F); return 3;
+    *slot = (b0 & 0x7) << 18 | b1 << 12 | b2 << 6 | (p[c] & 0x3F);
+    return 4;
 }
 
 static int get_b_utf8(const symbol * p, int c, int lb, int * slot) {
-    int b0, b1;
+    int a, b;
     if (c <= lb) return 0;
-    b0 = p[--c];
-    if (b0 < 0x80 || c == lb) {   /* 1000 0000 */
-        * slot = b0; return 1;
+    b = p[--c];
+    if (b < 0x80 || c == lb) {   /* 1000 0000 */
+        *slot = b;
+        return 1;
     }
-    b1 = p[--c];
-    if (b1 >= 0xC0 || c == lb) {   /* 1100 0000 */
-        * slot = (b1 & 0x1F) << 6 | (b0 & 0x3F); return 2;
+    a = b & 0x3F;
+    b = p[--c];
+    if (b >= 0xC0 || c == lb) {   /* 1100 0000 */
+        *slot = (b & 0x1F) << 6 | a;
+        return 2;
     }
-    * slot = (p[c] & 0xF) << 12 | (b1 & 0x3F) << 6 | (b0 & 0x3F); return 3;
+    a |= (b & 0x3F) << 6;
+    b = p[--c];
+    if (b >= 0xE0 || c == lb) {   /* 1110 0000 */
+        *slot = (b & 0xF) << 12 | a;
+        return 3;
+    }
+    *slot = (p[--c] & 0x7) << 18 | (b & 0x3F) << 12 | a;
+    return 4;
 }
 
-int in_grouping_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
+extern int in_grouping_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
     do {
-	int ch;
-	int w = get_utf8(z->p, z->c, z->l, & ch);
-	unless (w) return -1;
-	if (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
-	    return w;
-	z->c += w;
+        int ch;
+        int w = get_utf8(z->p, z->c, z->l, & ch);
+        if (!w) return -1;
+        if (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
+            return w;
+        z->c += w;
     } while (repeat);
     return 0;
 }
 
-int in_grouping_b_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
+extern int in_grouping_b_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
     do {
-	int ch;
-	int w = get_b_utf8(z->p, z->c, z->lb, & ch);
-	unless (w) return -1;
-	if (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
-	    return w;
-	z->c -= w;
+        int ch;
+        int w = get_b_utf8(z->p, z->c, z->lb, & ch);
+        if (!w) return -1;
+        if (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
+            return w;
+        z->c -= w;
     } while (repeat);
     return 0;
 }
 
-int out_grouping_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
+extern int out_grouping_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
     do {
-	int ch;
-	int w = get_utf8(z->p, z->c, z->l, & ch);
-	unless (w) return -1;
-	unless (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
-	    return w;
-	z->c += w;
+        int ch;
+        int w = get_utf8(z->p, z->c, z->l, & ch);
+        if (!w) return -1;
+        if (!(ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0))
+            return w;
+        z->c += w;
     } while (repeat);
     return 0;
 }
 
-int out_grouping_b_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
+extern int out_grouping_b_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
     do {
-	int ch;
-	int w = get_b_utf8(z->p, z->c, z->lb, & ch);
-	unless (w) return -1;
-	unless (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
-	    return w;
-	z->c -= w;
+        int ch;
+        int w = get_b_utf8(z->p, z->c, z->lb, & ch);
+        if (!w) return -1;
+        if (!(ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0))
+            return w;
+        z->c -= w;
     } while (repeat);
     return 0;
 }
 
 /* Code for character groupings: non-utf8 cases */
 
-int in_grouping(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
+extern int in_grouping(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
     do {
-	int ch;
-	if (z->c >= z->l) return -1;
-	ch = z->p[z->c];
-	if (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
-	    return 1;
-	z->c++;
+        int ch;
+        if (z->c >= z->l) return -1;
+        ch = z->p[z->c];
+        if (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
+            return 1;
+        z->c++;
     } while (repeat);
     return 0;
 }
 
-int in_grouping_b(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
+extern int in_grouping_b(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
     do {
-	int ch;
-	if (z->c <= z->lb) return -1;
-	ch = z->p[z->c - 1];
-	if (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
-	    return 1;
-	z->c--;
+        int ch;
+        if (z->c <= z->lb) return -1;
+        ch = z->p[z->c - 1];
+        if (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
+            return 1;
+        z->c--;
     } while (repeat);
     return 0;
 }
 
-int out_grouping(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
+extern int out_grouping(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
     do {
-	int ch;
-	if (z->c >= z->l) return -1;
-	ch = z->p[z->c];
-	unless (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
-	    return 1;
-	z->c++;
+        int ch;
+        if (z->c >= z->l) return -1;
+        ch = z->p[z->c];
+        if (!(ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0))
+            return 1;
+        z->c++;
     } while (repeat);
     return 0;
 }
 
-int out_grouping_b(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
+extern int out_grouping_b(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
     do {
-	int ch;
-	if (z->c <= z->lb) return -1;
-	ch = z->p[z->c - 1];
-	unless (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
-	    return 1;
-	z->c--;
+        int ch;
+        if (z->c <= z->lb) return -1;
+        ch = z->p[z->c - 1];
+        if (!(ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0))
+            return 1;
+        z->c--;
     } while (repeat);
     return 0;
 }
 
-int eq_s(struct SN_env * z, int s_size, const symbol * s) {
+extern int eq_s(struct SN_env * z, int s_size, const symbol * s) {
     if (z->l - z->c < s_size || memcmp(z->p + z->c, s, s_size * sizeof(symbol)) != 0) return 0;
     z->c += s_size; return 1;
 }
 
-int eq_s_b(struct SN_env * z, int s_size, const symbol * s) {
+extern int eq_s_b(struct SN_env * z, int s_size, const symbol * s) {
     if (z->c - z->lb < s_size || memcmp(z->p + z->c - s_size, s, s_size * sizeof(symbol)) != 0) return 0;
     z->c -= s_size; return 1;
 }
 
-int eq_v(struct SN_env * z, const symbol * p) {
+extern int eq_v(struct SN_env * z, const symbol * p) {
     return eq_s(z, SIZE(p), p);
 }
 
-int eq_v_b(struct SN_env * z, const symbol * p) {
+extern int eq_v_b(struct SN_env * z, const symbol * p) {
     return eq_s_b(z, SIZE(p), p);
 }
 
-int find_among(struct SN_env * z, const struct among * v, int v_size) {
+extern int find_among(struct SN_env * z, const struct among * v, int v_size) {
 
     int i = 0;
     int j = v_size;
 
     int c = z->c; int l = z->l;
-    symbol * q = z->p + c;
+    const symbol * q = z->p + c;
 
     const struct among * w;
 
@@ -224,7 +250,7 @@ int find_among(struct SN_env * z, const struct among * v, int v_size) {
 
     int first_key_inspected = 0;
 
-    while(1) {
+    while (1) {
         int k = i + ((j - i) >> 1);
         int diff = 0;
         int common = common_i < common_j ? common_i : common_j; /* smaller */
@@ -237,8 +263,13 @@ int find_among(struct SN_env * z, const struct among * v, int v_size) {
                 common++;
             }
         }
-        if (diff < 0) { j = k; common_j = common; }
-                 else { i = k; common_i = common; }
+        if (diff < 0) {
+            j = k;
+            common_j = common;
+        } else {
+            i = k;
+            common_i = common;
+        }
         if (j - i <= 1) {
             if (i > 0) break; /* v->s has been inspected */
             if (j == i) break; /* only one item in v */
@@ -251,7 +282,7 @@ int find_among(struct SN_env * z, const struct among * v, int v_size) {
             first_key_inspected = 1;
         }
     }
-    while(1) {
+    while (1) {
         w = v + i;
         if (common_i >= w->s_size) {
             z->c = c + w->s_size;
@@ -269,13 +300,13 @@ int find_among(struct SN_env * z, const struct among * v, int v_size) {
 
 /* find_among_b is for backwards processing. Same comments apply */
 
-int find_among_b(struct SN_env * z, const struct among * v, int v_size) {
+extern int find_among_b(struct SN_env * z, const struct among * v, int v_size) {
 
     int i = 0;
     int j = v_size;
 
     int c = z->c; int lb = z->lb;
-    symbol * q = z->p + c - 1;
+    const symbol * q = z->p + c - 1;
 
     const struct among * w;
 
@@ -284,7 +315,7 @@ int find_among_b(struct SN_env * z, const struct among * v, int v_size) {
 
     int first_key_inspected = 0;
 
-    while(1) {
+    while (1) {
         int k = i + ((j - i) >> 1);
         int diff = 0;
         int common = common_i < common_j ? common_i : common_j;
@@ -306,7 +337,7 @@ int find_among_b(struct SN_env * z, const struct among * v, int v_size) {
             first_key_inspected = 1;
         }
     }
-    while(1) {
+    while (1) {
         w = v + i;
         if (common_i >= w->s_size) {
             z->c = c - w->s_size;
@@ -345,7 +376,7 @@ static symbol * increase_size(symbol * p, int n) {
    Returns 0 on success, -1 on error.
    Also, frees z->p (and sets it to NULL) on error.
 */
-int replace_s(struct SN_env * z, int c_bra, int c_ket, int s_size, const symbol * s, int * adjptr)
+extern int replace_s(struct SN_env * z, int c_bra, int c_ket, int s_size, const symbol * s, int * adjptr)
 {
     int adjustment;
     int len;
@@ -367,11 +398,10 @@ int replace_s(struct SN_env * z, int c_bra, int c_ket, int s_size, const symbol
         z->l += adjustment;
         if (z->c >= c_ket)
             z->c += adjustment;
-        else
-            if (z->c > c_bra)
-                z->c = c_bra;
+        else if (z->c > c_bra)
+            z->c = c_bra;
     }
-    unless (s_size == 0) memmove(z->p + c_bra, s, s_size * sizeof(symbol));
+    if (s_size) memmove(z->p + c_bra, s, s_size * sizeof(symbol));
     if (adjptr != NULL)
         *adjptr = adjustment;
     return 0;
@@ -394,20 +424,20 @@ static int slice_check(struct SN_env * z) {
     return 0;
 }
 
-int slice_from_s(struct SN_env * z, int s_size, const symbol * s) {
+extern int slice_from_s(struct SN_env * z, int s_size, const symbol * s) {
     if (slice_check(z)) return -1;
     return replace_s(z, z->bra, z->ket, s_size, s, NULL);
 }
 
-int slice_from_v(struct SN_env * z, const symbol * p) {
+extern int slice_from_v(struct SN_env * z, const symbol * p) {
     return slice_from_s(z, SIZE(p), p);
 }
 
-int slice_del(struct SN_env * z) {
+extern int slice_del(struct SN_env * z) {
     return slice_from_s(z, 0, 0);
 }
 
-int insert_s(struct SN_env * z, int bra, int ket, int s_size, const symbol * s) {
+extern int insert_s(struct SN_env * z, int bra, int ket, int s_size, const symbol * s) {
     int adjustment;
     if (replace_s(z, bra, ket, s_size, s, &adjustment))
         return -1;
@@ -416,16 +446,11 @@ int insert_s(struct SN_env * z, int bra, int ket, int s_size, const symbol * s)
     return 0;
 }
 
-int insert_v(struct SN_env * z, int bra, int ket, const symbol * p) {
-    int adjustment;
-    if (replace_s(z, bra, ket, SIZE(p), p, &adjustment))
-        return -1;
-    if (bra <= z->bra) z->bra += adjustment;
-    if (bra <= z->ket) z->ket += adjustment;
-    return 0;
+extern int insert_v(struct SN_env * z, int bra, int ket, const symbol * p) {
+    return insert_s(z, bra, ket, SIZE(p), p);
 }
 
-symbol * slice_to(struct SN_env * z, symbol * p) {
+extern symbol * slice_to(struct SN_env * z, symbol * p) {
     if (slice_check(z)) {
         lose_s(p);
         return NULL;
@@ -443,7 +468,7 @@ symbol * slice_to(struct SN_env * z, symbol * p) {
     return p;
 }
 
-symbol * assign_to(struct SN_env * z, symbol * p) {
+extern symbol * assign_to(struct SN_env * z, symbol * p) {
     int len = z->l;
     if (CAPACITY(p) < len) {
         p = increase_size(p, len);
@@ -455,8 +480,18 @@ symbol * assign_to(struct SN_env * z, symbol * p) {
     return p;
 }
 
+extern int len_utf8(const symbol * p) {
+    int size = SIZE(p);
+    int len = 0;
+    while (size--) {
+        symbol b = *p++;
+        if (b >= 0xC0 || b < 0x80) ++len;
+    }
+    return len;
+}
+
 #if 0
-void debug(struct SN_env * z, int number, int line_count) {
+extern void debug(struct SN_env * z, int number, int line_count) {
     int i;
     int limit = SIZE(z->p);
     /*if (number >= 0) printf("%3d (line %4d): '", number, line_count);*/

data/ext/isomorfeus_ferret_ext/test.c

@@ -774,6 +774,12 @@ static VALUE frb_ts_term_vectors(VALUE v) { return INT2FIX(execute_test(33)); }
 static VALUE frb_ts_test(VALUE v)         { return INT2FIX(execute_test(34)); }
 static VALUE frb_ts_threading(VALUE v)    { return INT2FIX(execute_test(35)); }
 
+static VALUE frb_ts_posh(VALUE v) {
+    const char *posh = POSH_GetArchString();
+    printf("\n%s\n", posh);
+    return Qnil;
+}
+
 static VALUE frb_ts_run_all(VALUE v) {
     int i, test_count;
     int rv = 0;
@@ -845,6 +851,6 @@ void Init_Test(void) {
     rb_define_singleton_method(mTest, "term_vectors",   frb_ts_term_vectors, 0);
     rb_define_singleton_method(mTest, "test",           frb_ts_test, 0);
     rb_define_singleton_method(mTest, "threading",      frb_ts_threading, 0);
-
+    rb_define_singleton_method(mTest, "posh",           frb_ts_posh, 0);
     rb_define_singleton_method(mTest, "run_all",        frb_ts_run_all, 0);
 }

data/ext/isomorfeus_ferret_ext/test_search.c

@@ -556,7 +556,6 @@ static void test_phrase_query(TestCase *tc, void *data)
     check_to_s(tc, phq, field, "\"\"");
     check_to_s(tc, phq, (FrtSymbol)NULL, "field:\"\"");
 
-
     frt_phq_add_term(phq, "quick", 1);
     frt_phq_add_term(phq, "brown", 1);
     frt_phq_add_term(phq, "fox", 1);

data/lib/isomorfeus/ferret/index/index.rb

@@ -39,7 +39,7 @@ module Isomorfeus
         #                         object.  Using a multiple field key will slow
         #                         down indexing so it should not be done if
         #                         performance is a concern. A single field key (or
-        #                         id) should be find however. Also, you must make
+        #                         id) should be fine however. Also, you must make
         #                         sure that your key/keys are either untokenized
         #                         or that they are not broken up by the analyzer.
         # auto_flush::            Default: false. Set this option to true if you

data/lib/isomorfeus/ferret/version.rb

@@ -1,5 +1,5 @@
 module Isomorfeus
   module Ferret
-    VERSION = '0.12.2'
+    VERSION = '0.12.6'
   end
 end