jk-ferret 0.11.8.2

data/ext/analysis.h

@@ -0,0 +1,266 @@
+#ifndef FRT_ANALYSIS_H
+#define FRT_ANALYSIS_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "global.h"
+#include "hash.h"
+#include "symbol.h"
+#include "multimapper.h"
+#include <wchar.h>
+
+/****************************************************************************
+ *
+ * FrtToken
+ *
+ ****************************************************************************/
+
+typedef struct FrtToken
+{
+    char text[FRT_MAX_WORD_SIZE];
+    int len;
+    off_t start;
+    off_t end;
+    int pos_inc;
+} FrtToken;
+
+extern FrtToken *frt_tk_new();
+extern void frt_tk_destroy(void *p);
+extern FrtToken *frt_tk_set(FrtToken *tk, char *text, int tlen, off_t start, off_t end,
+                     int pos_inc);
+extern FrtToken *frt_tk_set_no_len(FrtToken *tk, char *text, off_t start, off_t end,
+                            int pos_inc);
+extern int frt_tk_eq(FrtToken *tk1, FrtToken *tk2);
+extern int frt_tk_cmp(FrtToken *tk1, FrtToken *tk2);
+
+/****************************************************************************
+ *
+ * FrtTokenStream
+ *
+ ****************************************************************************/
+
+
+typedef struct FrtTokenStream FrtTokenStream;
+struct FrtTokenStream
+{
+    char            *t;             /* ptr used to scan text */
+    char            *text;
+    FrtToken        *(*next)(FrtTokenStream *ts);
+    FrtTokenStream  *(*reset)(FrtTokenStream *ts, char *text);
+    FrtTokenStream  *(*clone_i)(FrtTokenStream *ts);
+    void            (*destroy_i)(FrtTokenStream *ts);
+    int             ref_cnt;
+};
+
+#define frt_ts_new(type) frt_ts_new_i(sizeof(type))
+extern FrtTokenStream *frt_ts_new_i(size_t size);
+extern FrtTokenStream *frt_ts_clone_size(FrtTokenStream *orig_ts, size_t size);
+
+typedef struct FrtCachedTokenStream
+{
+    FrtTokenStream super;
+    FrtToken       token;
+} FrtCachedTokenStream;
+
+typedef struct FrtMultiByteTokenStream
+{
+    FrtCachedTokenStream super;
+    mbstate_t            state;
+} FrtMultiByteTokenStream;
+
+typedef enum 
+{
+    FRT_STT_ASCII,
+    FRT_STT_MB,
+    FRT_STT_UTF8
+} FrtStandardTokenizerType;
+
+typedef struct FrtStandardTokenizer
+{
+    FrtCachedTokenStream     super;
+    FrtStandardTokenizerType type;
+} FrtStandardTokenizer;
+
+typedef struct FrtLegacyStandardTokenizer
+{
+    FrtCachedTokenStream super;
+    bool        (*advance_to_start)(FrtTokenStream *ts);
+    bool        (*is_tok_char)(char *c);
+    int         (*get_alpha)(FrtTokenStream *ts, char *token);
+    int         (*get_apostrophe)(char *input);
+} FrtLegacyStandardTokenizer;
+
+typedef struct FrtTokenFilter
+{
+    FrtTokenStream super;
+    FrtTokenStream *sub_ts;
+} FrtTokenFilter;
+
+extern FrtTokenStream *frt_filter_clone_size(FrtTokenStream *ts, size_t size);
+#define tf_new(type, sub) frt_tf_new_i(sizeof(type), sub)
+extern FrtTokenStream *frt_tf_new_i(size_t size, FrtTokenStream *sub_ts);
+
+typedef struct FrtStopFilter
+{
+    FrtTokenFilter super;
+    FrtHash  *words;
+} FrtStopFilter;
+
+typedef struct FrtMappingFilter
+{
+    FrtTokenFilter  super;
+    FrtMultiMapper *mapper;
+} FrtMappingFilter;
+
+typedef struct FrtHyphenFilter
+{
+    FrtTokenFilter super;
+    char text[FRT_MAX_WORD_SIZE];
+    int start;
+    int pos;
+    int len;
+    FrtToken *tk;
+} FrtHyphenFilter;
+
+typedef struct FrtStemFilter
+{
+    FrtTokenFilter        super;
+    struct sb_stemmer  *stemmer;
+    char               *algorithm;
+    char               *charenc;
+} FrtStemFilter;
+
+#define frt_ts_next(mts) mts->next(mts)
+#define frt_ts_clone(mts) mts->clone_i(mts)
+
+extern void frt_ts_deref(FrtTokenStream *ts);
+
+extern FrtTokenStream *frt_non_tokenizer_new();
+
+extern FrtTokenStream *frt_whitespace_tokenizer_new();
+extern FrtTokenStream *frt_mb_whitespace_tokenizer_new(bool lowercase);
+
+extern FrtTokenStream *frt_letter_tokenizer_new();
+extern FrtTokenStream *frt_mb_letter_tokenizer_new(bool lowercase);
+
+extern FrtTokenStream *frt_standard_tokenizer_new();
+extern FrtTokenStream *frt_mb_standard_tokenizer_new();
+extern FrtTokenStream *frt_utf8_standard_tokenizer_new();
+
+extern FrtTokenStream *frt_legacy_standard_tokenizer_new();
+extern FrtTokenStream *frt_mb_legacy_standard_tokenizer_new();
+
+extern FrtTokenStream *frt_hyphen_filter_new(FrtTokenStream *ts);
+extern FrtTokenStream *frt_lowercase_filter_new(FrtTokenStream *ts);
+extern FrtTokenStream *frt_mb_lowercase_filter_new(FrtTokenStream *ts);
+
+extern const char *FRT_ENGLISH_STOP_WORDS[];
+extern const char *FRT_FULL_ENGLISH_STOP_WORDS[];
+extern const char *FRT_EXTENDED_ENGLISH_STOP_WORDS[];
+extern const char *FRT_FULL_FRENCH_STOP_WORDS[];
+extern const char *FRT_FULL_SPANISH_STOP_WORDS[];
+extern const char *FRT_FULL_PORTUGUESE_STOP_WORDS[];
+extern const char *FRT_FULL_ITALIAN_STOP_WORDS[];
+extern const char *FRT_FULL_GERMAN_STOP_WORDS[];
+extern const char *FRT_FULL_DUTCH_STOP_WORDS[];
+extern const char *FRT_FULL_SWEDISH_STOP_WORDS[];
+extern const char *FRT_FULL_NORWEGIAN_STOP_WORDS[];
+extern const char *FRT_FULL_DANISH_STOP_WORDS[];
+extern const char *FRT_FULL_RUSSIAN_STOP_WORDS[];
+extern const char *FRT_FULL_FINNISH_STOP_WORDS[];
+extern const char *FRT_FULL_HUNGARIAN_STOP_WORDS[];
+extern const char *FRT_FULL_RUSSIAN_STOP_WORDS_KOI8_R[];
+
+extern FrtTokenStream *frt_stop_filter_new_with_words_len(FrtTokenStream *ts,
+                                                   const char **words, int len);
+extern FrtTokenStream *frt_stop_filter_new_with_words(FrtTokenStream *ts,
+                                               const char **words);
+extern FrtTokenStream *frt_stop_filter_new(FrtTokenStream *ts);
+extern FrtTokenStream *frt_stem_filter_new(FrtTokenStream *ts, const char *algorithm,
+                                    const char *charenc);
+
+extern FrtTokenStream *frt_mapping_filter_new(FrtTokenStream *ts);
+extern FrtTokenStream *frt_mapping_filter_add(FrtTokenStream *ts, const char *pattern,
+                                       const char *replacement);
+
+/****************************************************************************
+ *
+ * FrtAnalyzer
+ *
+ ****************************************************************************/
+
+typedef struct FrtAnalyzer
+{
+    FrtTokenStream *current_ts;
+    FrtTokenStream *(*get_ts)(struct FrtAnalyzer *a, FrtSymbol field, char *text);
+    void (*destroy_i)(struct FrtAnalyzer *a);
+    int ref_cnt;
+} FrtAnalyzer;
+
+extern void frt_a_deref(FrtAnalyzer *a);
+
+#define frt_a_get_ts(ma, field, text) ma->get_ts(ma, field, text)
+
+extern FrtAnalyzer *frt_analyzer_new(FrtTokenStream *ts,
+                              void (*destroy)(FrtAnalyzer *a),
+                              FrtTokenStream *(*get_ts)(FrtAnalyzer *a,
+                                                     FrtSymbol field,
+                                                     char *text));
+extern void frt_a_standard_destroy(FrtAnalyzer *a);
+extern FrtAnalyzer *frt_non_analyzer_new();
+
+extern FrtAnalyzer *frt_whitespace_analyzer_new(bool lowercase);
+extern FrtAnalyzer *frt_mb_whitespace_analyzer_new(bool lowercase);
+
+extern FrtAnalyzer *frt_letter_analyzer_new(bool lowercase);
+extern FrtAnalyzer *frt_mb_letter_analyzer_new(bool lowercase);
+
+extern FrtAnalyzer *frt_standard_analyzer_new(bool lowercase);
+extern FrtAnalyzer *frt_mb_standard_analyzer_new(bool lowercase);
+extern FrtAnalyzer *frt_utf8_standard_analyzer_new(bool lowercase);
+
+extern FrtAnalyzer *frt_standard_analyzer_new_with_words(
+    const char **words, bool lowercase);
+extern FrtAnalyzer *frt_standard_analyzer_new_with_words_len(
+    const char **words, int len, bool lowercase);
+extern FrtAnalyzer *frt_mb_standard_analyzer_new_with_words(
+    const char **words, bool lowercase);
+extern FrtAnalyzer *frt_mb_standard_analyzer_new_with_words_len(
+    const char **words, int len, bool lowercase);
+extern FrtAnalyzer *frt_utf8_standard_analyzer_new_with_words(
+    const char **words, bool lowercase);
+extern FrtAnalyzer *frt_utf8_standard_analyzer_new_with_words_len(
+    const char **words, int len, bool lowercase);
+
+extern FrtAnalyzer *frt_legacy_standard_analyzer_new(bool lowercase);
+extern FrtAnalyzer *frt_mb_legacy_standard_analyzer_new(bool lowercase);
+
+extern FrtAnalyzer *frt_legacy_standard_analyzer_new_with_words(
+    const char **words, bool lowercase);
+extern FrtAnalyzer *frt_legacy_standard_analyzer_new_with_words_len(
+    const char **words, int len, bool lowercase);
+extern FrtAnalyzer *frt_mb_legacy_standard_analyzer_new_with_words(
+    const char **words, bool lowercase);
+extern FrtAnalyzer *frt_mb_legacy_standard_analyzer_new_with_words_len(
+    const char **words, int len, bool lowercase);
+
+#define PFA(analyzer) ((FrtPerFieldAnalyzer *)(analyzer))
+typedef struct FrtPerFieldAnalyzer
+{
+    FrtAnalyzer    super;
+    FrtHash  *dict;
+    FrtAnalyzer   *default_a;
+} FrtPerFieldAnalyzer;
+
+extern FrtAnalyzer *frt_per_field_analyzer_new(FrtAnalyzer *a);
+extern void frt_pfa_add_field(FrtAnalyzer *self,
+                              FrtSymbol field,
+                              FrtAnalyzer *analyzer);
+
+#ifdef __cplusplus
+} // extern "C"
+#endif
+
+#endif

data/ext/api.h

@@ -0,0 +1,26 @@
+
+typedef unsigned char symbol;
+
+/* Or replace 'char' above with 'short' for 16 bit characters.
+
+   More precisely, replace 'char' with whatever type guarantees the
+   character width you need. Note however that sizeof(symbol) should divide
+   HEAD, defined in header.h as 2*sizeof(int), without remainder, otherwise
+   there is an alignment problem. In the unlikely event of a problem here,
+   consult Martin Porter.
+
+*/
+
+struct SN_env {
+    symbol * p;
+    int c; int l; int lb; int bra; int ket;
+    symbol * * S;
+    int * I;
+    unsigned char * B;
+};
+
+extern struct SN_env * SN_create_env(int S_size, int I_size, int B_size);
+extern void SN_close_env(struct SN_env * z, int S_size);
+
+extern int SN_set_current(struct SN_env * z, int size, const symbol * s);
+

data/ext/array.c

@@ -0,0 +1,125 @@
+#include "array.h"
+#include <string.h>
+#include "internal.h"
+
+#define META_CNT ARY_META_CNT
+#define DATA_SZ sizeof(int) * META_CNT
+
+void **ary_new_i(int type_size, int init_capa)
+{
+    void **ary;
+    if (init_capa <= 0) {
+        init_capa = ARY_INIT_CAPA;
+    }
+    ary = (void **)&(((int *)ecalloc(DATA_SZ +
+                                        init_capa * type_size))[META_CNT]);
+    ary_type_size(ary) = type_size;
+    ary_capa(ary) = init_capa;
+    return ary;
+}
+
+INLINE void ary_resize_i(void ***ary, int size)
+{
+    size++;
+    if (size > ary_sz(*ary)) {
+        int capa = ary_capa(*ary);
+        if (size >= capa) {
+            int *ary_start = &((int *)*ary)[-META_CNT];
+            while (size >= capa) {
+                capa <<= 1;
+            }
+
+            ary_start = (int *)erealloc(ary_start,
+                                        DATA_SZ + capa * ary_type_size(*ary));
+            *ary = (void **)&(ary_start[META_CNT]);
+            memset(((char *)*ary) + ary_type_size(*ary) * ary_sz(*ary), 0,
+                   (capa - ary_sz(*ary)) * ary_type_size(*ary));
+            ary_capa(*ary) = capa;
+        }
+        ary_sz(*ary) = size;
+    }
+}
+
+void ary_set_i(void ***ary, int index, void *value)
+{
+    if (index < 0) {
+        index += ary_sz(*ary);
+        if (index < 0) {
+            RAISE(INDEX_ERROR, "index %d out array", index);
+        }
+    }
+    ary_resize_i(ary, index);
+    (*ary)[index] = value;
+}
+
+void *ary_get_i(void **ary, int index)
+{
+    if (index < 0) {
+        index += ary_sz(ary);
+    }
+    if (index >= 0 && index < ary_sz(ary)) {
+        return ary[index];
+    }
+    else {
+        return NULL;
+    }
+}
+
+void ary_push_i(void ***ary, void *value)
+{
+    int size = ary_sz(*ary);
+    ary_resize_i(ary, size);
+    (*ary)[size] = value;
+}
+
+void *ary_pop_i(void **ary)
+{
+    void *val = ary[--ary_sz(ary)];
+    ary[ary_sz(ary)] = NULL;
+    return val;
+}
+
+void ary_unshift_i(void ***ary, void *value)
+{
+    int size = ary_sz(*ary);
+    ary_resize_i(ary, size);
+    memmove(*ary + 1, *ary, size * sizeof(void *));
+    (*ary)[0] = value;
+}
+
+void *ary_shift_i(void **ary)
+{
+    void *val = ary[0];
+    int size = --ary_sz(ary);
+    memmove(ary, ary + 1, size * sizeof(void *));
+    ary[size] = NULL;
+    return val;
+}
+
+void *ary_remove_i(void **ary, int index)
+{
+    if (index >= 0 && index < ary_sz(ary)) {
+        void *val = ary[index];
+        memmove(ary + index, ary + index + 1,
+                (ary_sz(ary) - index + 1) * sizeof(void *));
+        ary_sz(ary)--;
+        return val;
+    }
+    else {
+        return NULL;
+    }
+}
+
+void ary_delete_i(void **ary, int index, void (*free_elem)(void *p))
+{
+    free_elem(ary_remove(ary, index));
+}
+
+void ary_destroy_i(void **ary, void (*free_elem)(void *p))
+{
+    int i;
+    for (i = ary_sz(ary) - 1; i >= 0; i--) {
+        free_elem(ary[i]);
+    }
+    ary_free(ary);
+}

data/ext/array.h

@@ -0,0 +1,62 @@
+#ifndef FRT_ARRAY_H
+#define FRT_ARRAY_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "global.h"
+
+#if defined POSH_OS_SOLARIS || defined POSH_OS_SUNOS
+# define FRT_ARY_META_CNT 4
+#else
+# define FRT_ARY_META_CNT 3
+#endif
+
+#define FRT_ARY_INIT_CAPA 8
+#define frt_ary_size(ary)      frt_ary_sz(ary)
+#define frt_ary_sz(ary)        (((int *)ary)[-1])
+#define frt_ary_capa(ary)      (((int *)ary)[-2])
+#define frt_ary_type_size(ary) (((int *)ary)[-3])
+#define frt_ary_start(ary)     ((void **)&(((int *)ary)[-FRT_ARY_META_CNT]))
+#define frt_ary_free(ary)      free(frt_ary_start(ary))
+
+#define frt_ary_new_type_capa(type, init_capa)\
+                                (type *)frt_ary_new_i(sizeof(type), init_capa)
+#define frt_ary_new_type(type)      (type *)frt_ary_new_i(sizeof(type), 0)
+#define frt_ary_new_capa(init_capa) frt_ary_new_i(sizeof(void *), init_capa)
+#define frt_ary_new()               frt_ary_new_i(sizeof(void *), 0)
+#define frt_ary_resize(ary, size)   frt_ary_resize_i(((void ***)(void *)&ary), size)
+#define frt_ary_set(ary, i, val)    frt_ary_set_i(((void ***)(void *)&ary), i, val)
+#define frt_ary_get(ary, i)         frt_ary_get_i(((void **)ary), i)
+#define frt_ary_push(ary, val)      frt_ary_push_i(((void ***)(void *)&ary), val)
+#define frt_ary_pop(ary)            frt_ary_pop_i(((void **)ary))
+#define frt_ary_unshift(ary, val)   frt_ary_unshift_i(((void ***)(void *)&ary), val)
+#define frt_ary_shift(ary)          frt_ary_shift_i(((void **)ary))
+#define frt_ary_remove(ary, i)      frt_ary_remove_i(((void **)ary), i)
+#define frt_ary_delete(ary, i, f)   frt_ary_delete_i(((void **)ary), i, (free_ft)f)
+#define frt_ary_destroy(ary, f)     frt_ary_destroy_i(((void **)ary), (free_ft)f)
+#define frt_ary_rsz(ary, size)      frt_ary_resize(ary, size)
+#define frt_ary_grow(ary)           frt_ary_resize(ary, frt_ary_sz(ary))
+#define frt_ary_last(ary)           ary[frt_ary_sz(ary) - 1]
+#define frt_ary_sort(ary, cmp)      qsort(ary, frt_ary_size(ary), frt_ary_type_size(ary), cmp)
+#define frt_ary_each_rev(ary, i)    for (i = frt_ary_size(ary) - 1; i >= 0; i--)
+#define frt_ary_each(ary, i)        for (i = 0; i < frt_ary_size(ary); i++)
+
+extern void   frt_ary_resize_i(void ***ary, int size);
+extern void **frt_ary_new_i(int type_size, int init_capa);
+extern void   frt_ary_set_i(void ***ary, int index, void *value);
+extern void  *frt_ary_get_i(void **ary, int index);
+extern void   frt_ary_push_i(void ***ary, void *value);
+extern void  *frt_ary_pop_i(void **ary);
+extern void   frt_ary_unshift_i(void ***ary, void *value);
+extern void  *frt_ary_shift_i(void **ary);
+extern void  *frt_ary_remove_i(void **ary, int index);
+extern void   frt_ary_delete_i(void **ary, int index, frt_free_ft p);
+extern void   frt_ary_destroy_i(void **ary, frt_free_ft p);
+
+#ifdef __cplusplus
+} // extern "C"
+#endif
+
+#endif