ferret 0.11.5 → 0.11.6

data/ext/analysis.c

@@ -13,7 +13,7 @@
  ****************************************************************************/
 
 INLINE Token *tk_set(Token *tk,
-                     char *text, int tlen, int start, int end, int pos_inc)
+                     char *text, int tlen, off_t start, off_t end, int pos_inc)
 {
     if (tlen >= MAX_WORD_SIZE) {
         tlen = MAX_WORD_SIZE - 1;
@@ -31,16 +31,16 @@ INLINE Token *tk_set_ts(Token *tk,
                         char *start, char *end, char *text, int pos_inc)
 {
     return tk_set(tk, start, (int)(end - start),
-                  (int)(start - text), (int)(end - text), pos_inc);
+                  (off_t)(start - text), (off_t)(end - text), pos_inc);
 }
 
 INLINE Token *tk_set_no_len(Token *tk,
-                            char *text, int start, int end, int pos_inc)
+                            char *text, off_t start, off_t end, int pos_inc)
 {
     return tk_set(tk, text, (int)strlen(text), start, end, pos_inc);
 }
 
-INLINE Token *w_tk_set(Token *tk, wchar_t *text, int start, int end,
+INLINE Token *w_tk_set(Token *tk, wchar_t *text, off_t start, off_t end,
                        int pos_inc)
 {
     int len = wcstombs(tk->text, text, MAX_WORD_SIZE - 1);
@@ -374,8 +374,8 @@ static Token *mb_wst_next_lc(TokenStream *ts)
     }
     *w = 0;
     ts->t = t;
-    return w_tk_set(&(CTS(ts)->token), wbuf, (int)(start - ts->text),
-                    (int)(t - ts->text), 1);
+    return w_tk_set(&(CTS(ts)->token), wbuf, (off_t)(start - ts->text),
+                    (off_t)(t - ts->text), 1);
 }
 
 TokenStream *mb_whitespace_tokenizer_new(bool lowercase)
@@ -512,8 +512,8 @@ Token *mb_lt_next_lc(TokenStream *ts)
     }
     *w = 0;
     ts->t = t;
-    return w_tk_set(&(CTS(ts)->token), wbuf, (int)(start - ts->text),
-                    (int)(t - ts->text), 1);
+    return w_tk_set(&(CTS(ts)->token), wbuf, (off_t)(start - ts->text),
+                    (off_t)(t - ts->text), 1);
 }
 
 TokenStream *mb_letter_tokenizer_new(bool lowercase)
@@ -926,8 +926,8 @@ static Token *std_next(TokenStream *ts)
         }
         ts->t = t + len;
         token[len] = 0;
-        return tk_set(&(CTS(ts)->token), token, len, (int)(start - ts->text),
-               (int)(ts->t - ts->text), 1);
+        return tk_set(&(CTS(ts)->token), token, len, (off_t)(start - ts->text),
+               (off_t)(ts->t - ts->text), 1);
     }
 
     /* now see how long a url we can find. */
@@ -974,8 +974,8 @@ static Token *std_next(TokenStream *ts)
                 }
             }
             tk_set(&(CTS(ts)->token), token, token_i,
-                   (int)(start - ts->text),
-                   (int)(t - ts->text), 1);
+                   (off_t)(start - ts->text),
+                   (off_t)(t - ts->text), 1);
         }
         else { /* just return the url as is */
             tk_set_ts(&(CTS(ts)->token), start, t, ts->text, 1);

data/ext/analysis.h

@@ -16,16 +16,16 @@ typedef struct Token
 {
     char text[MAX_WORD_SIZE];
     int len;
-    int start;
-    int end;
+    off_t start;
+    off_t end;
     int pos_inc;
 } Token;
 
 extern Token *tk_new();
 extern void tk_destroy(void *p);
-extern Token *tk_set(Token *tk, char *text, int tlen, int start, int end,
+extern Token *tk_set(Token *tk, char *text, int tlen, off_t start, off_t end,
                      int pos_inc);
-extern Token *tk_set_no_len(Token *tk, char *text, int start, int end,
+extern Token *tk_set_no_len(Token *tk, char *text, off_t start, off_t end,
                             int pos_inc);
 extern int tk_eq(Token *tk1, Token *tk2);
 extern int tk_cmp(Token *tk1, Token *tk2);

data/ext/index.c

@@ -1450,10 +1450,12 @@ TermVector *fr_read_term_vector(FieldsReader *fr, int field_num)
         if (store_offsets) {
             int num_positions = tv->offset_cnt = is_read_vint(fdt_in);
             Offset *offsets = tv->offsets = ALLOC_N(Offset, num_positions);
-            off_t offset = 0;
+            long long offset = 0;
             for (i = 0; i < num_positions; i++) {
-                offsets[i].start = offset += is_read_vint(fdt_in);
-                offsets[i].end = offset += is_read_vint(fdt_in);
+                offsets[i].start =
+                    (off_t)(offset += (long long)is_read_vll(fdt_in));
+                offsets[i].end =
+                    (off_t)(offset += (long long)is_read_vll(fdt_in));
             }
         }
     }
@@ -1681,13 +1683,13 @@ void fw_add_postings(FieldsWriter *fw,
 
     if (fi_store_offsets(fi)) {
         /* use delta encoding for offsets */
-        int last_end = 0;
+        long long last_end = 0;
         os_write_vint(fdt_out, offset_count);  /* write shared prefix length */
         for (i = 0; i < offset_count; i++) {
-            off_t start = offsets[i].start;
-            off_t end = offsets[i].end;
-            os_write_vint(fdt_out, start - last_end);
-            os_write_vint(fdt_out, end - start);
+            long long start = (long long)offsets[i].start;
+            long long end = (long long)offsets[i].end;
+            os_write_vll(fdt_out, (unsigned long long)(start - last_end));
+            os_write_vll(fdt_out, (unsigned long long)(end - start));
             last_end = end;
         }
     }
@@ -5197,7 +5199,7 @@ static void dw_add_posting(MemoryPool *mp,
     }
 }
 
-static INLINE void dw_add_offsets(DocWriter *dw, int pos, int start, int end)
+static INLINE void dw_add_offsets(DocWriter *dw, int pos, off_t start, off_t end)
 {
     if (pos >= dw->offsets_capa) {
         int old_capa = dw->offsets_capa;

data/ext/r_index.c

@@ -1192,8 +1192,8 @@ static VALUE
 frt_get_tv_offsets(Offset *offset)
 {
     return rb_struct_new(cTVOffsets,
-                         INT2FIX(offset->start),
-                         INT2FIX(offset->end),
+                         ULL2NUM((unsigned long long)offset->start),
+                         ULL2NUM((unsigned long long)offset->end),
                          NULL);
 }
 

data/ext/store.c

@@ -403,6 +403,36 @@ INLINE off_t is_read_voff_t(InStream *is)
     return res;
 }
 
+/* optimized to use unchecked read_byte if there is definitely space */
+INLINE unsigned long long is_read_vll(InStream *is)
+{
+    register unsigned long long res, b;
+    register int shift = 7;
+
+    if (is->buf.pos > (is->buf.len - VINT_MAX_LEN)) {
+        b = is_read_byte(is);
+        res = b & 0x7F;                 /* 0x7F = 0b01111111 */
+
+        while ((b & 0x80) != 0) {       /* 0x80 = 0b10000000 */
+            b = is_read_byte(is);
+            res |= (b & 0x7F) << shift;
+            shift += 7;
+        }
+    }
+    else {                              /* unchecked optimization */
+        b = read_byte(is);
+        res = b & 0x7F;                 /* 0x7F = 0b01111111 */
+
+        while ((b & 0x80) != 0) {       /* 0x80 = 0b10000000 */
+            b = read_byte(is);
+            res |= (b & 0x7F) << shift;
+            shift += 7;
+        }
+    }
+
+    return res;
+}
+
 INLINE void is_skip_vints(InStream *is, register int cnt)
 {
     for (; cnt > 0; cnt--) {
@@ -545,6 +575,25 @@ INLINE void os_write_voff_t(OutStream *os, register off_t num)
     }
 }
 
+/* optimized to use an unchecked write if there is space */
+INLINE void os_write_vll(OutStream *os, register unsigned long long num)
+{
+    if (os->buf.pos > VINT_END) {
+        while (num > 127) {
+            os_write_byte(os, (uchar)((num & 0x7f) | 0x80));
+            num >>= 7;
+        }
+        os_write_byte(os, (uchar)num);
+    }
+    else {
+        while (num > 127) {
+            write_byte(os, (uchar)((num & 0x7f) | 0x80));
+            num >>= 7;
+        }
+        write_byte(os, (uchar)num);
+    }
+}
+
 void os_write_string(OutStream *os, char *str)
 {
     int len = (int)strlen(str);

data/ext/store.h

@@ -176,7 +176,11 @@ struct Store
         CompoundStore *cmpd;    /* for compound_store only */
     } dir;
 
+#ifdef POSH_OS_WIN32
+    int file_mode;
+#else
     mode_t file_mode;
+#endif
     HashSet *locks;
 
     /**
@@ -554,6 +558,16 @@ extern void os_write_vint(OutStream *os, register unsigned int num);
  */
 extern void os_write_voff_t(OutStream *os, register off_t num);
 
+/**
+ * Write an unsigned long long to OutStream in compressed VINT format.
+ * TODO: describe VINT format
+ *
+ * @param os OutStream to write to
+ * @param num the long long to write
+ * @raise IO_ERROR if there is an error writing to the file-system
+ */
+extern void os_write_vll(OutStream *os, register unsigned long long num);
+
 /**
  * Write a string to the OutStream. A string is an integer +length+ in VINT
  * format (see os_write_vint) followed by +length+ bytes. The string can then
@@ -694,6 +708,17 @@ extern INLINE void is_skip_vints(InStream *is, register int cnt);
  */
 extern INLINE off_t is_read_voff_t(InStream *is);
 
+/**
+ * Read a compressed (VINT) unsigned long long from the InStream.
+ * TODO: describe VINT format
+ *
+ * @param is the InStream to read from
+ * @return a long long
+ * @raise IO_ERROR if there is a error reading from the file-system
+ * @raise EOF_ERROR if there is an attempt to read past the end of the file
+ */
+extern INLINE unsigned long long is_read_vll(InStream *is);
+
 /**
  * Read a string from the InStream. A string is an integer +length+ in vint
  * format (see is_read_vint) followed by +length+ bytes. This is the format

data/lib/ferret_version.rb

@@ -1,3 +1,3 @@
 module Ferret
-  VERSION = '0.11.5'
+  VERSION = '0.11.6'
 end

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.9.2
 specification_version: 1
 name: ferret
 version: !ruby/object:Gem::Version 
-  version: 0.11.5
-date: 2007-11-17 00:00:00 +11:00
+  version: 0.11.6
+date: 2007-11-29 00:00:00 +11:00
 summary: Ruby indexing library.
 require_paths: 
 - lib