isomorfeus-ferret 0.12.0

data/ext/isomorfeus_ferret_ext/frt_compound_io.c

@@ -0,0 +1,374 @@
+#include "frt_index.h"
+#include "frt_array.h"
+
+extern void frt_store_destroy(FrtStore *store);
+extern FrtInStream *frt_is_new();
+extern FrtStore *frt_store_new();
+
+/****************************************************************************
+ *
+ * CompoundStore
+ *
+ ****************************************************************************/
+
+typedef struct FileEntry {
+    off_t offset;
+    off_t length;
+} FileEntry;
+
+static void cmpd_touch(FrtStore *store, const char *file_name)
+{
+    store->dir.cmpd->store->touch(store->dir.cmpd->store, file_name);
+}
+
+static int cmpd_exists(FrtStore *store, const char *file_name)
+{
+    if (frt_h_get(store->dir.cmpd->entries, file_name) != NULL) {
+        return true;
+    }
+    else {
+        return false;
+    }
+}
+
+static int cmpd_remove(FrtStore *store, const char *file_name)
+{
+    (void)store;
+    (void)file_name;
+    FRT_RAISE(FRT_UNSUPPORTED_ERROR, "%s", FRT_UNSUPPORTED_ERROR_MSG);
+    return 0;
+}
+
+static void cmpd_rename(FrtStore *store, const char *from, const char *to)
+{
+    (void)store;
+    (void)from;
+    (void)to;
+    FRT_RAISE(FRT_UNSUPPORTED_ERROR, "%s", FRT_UNSUPPORTED_ERROR_MSG);
+}
+
+static int cmpd_count(FrtStore *store)
+{
+    return store->dir.cmpd->entries->size;
+}
+
+static void cmpd_each(FrtStore *store,
+                     void (*func)(const char *fname, void *arg), void *arg)
+{
+    FrtHash *ht = store->dir.cmpd->entries;
+    int i;
+    for (i = 0; i <= ht->mask; i++) {
+        char *fn = (char *)ht->table[i].key;
+        if (fn) {
+            func(fn, arg);
+        }
+    }
+}
+
+
+static void cmpd_clear(FrtStore *store)
+{
+    (void)store;
+    FRT_RAISE(FRT_UNSUPPORTED_ERROR, "%s", FRT_UNSUPPORTED_ERROR_MSG);
+}
+
+static void cmpd_close_i(FrtStore *store)
+{
+    FrtCompoundStore *cmpd = store->dir.cmpd;
+    if (cmpd->stream == NULL) {
+        FRT_RAISE(FRT_IO_ERROR, "Tried to close already closed compound store");
+    }
+
+    frt_h_destroy(cmpd->entries);
+
+    frt_is_close(cmpd->stream);
+    cmpd->stream = NULL;
+    free(store->dir.cmpd);
+    frt_store_destroy(store);
+}
+
+static off_t cmpd_length(FrtStore *store, const char *file_name)
+{
+    FileEntry *fe = (FileEntry *)frt_h_get(store->dir.cmpd->entries, file_name);
+    if (fe != NULL) {
+        return fe->length;
+    }
+    else {
+        return 0;
+    }
+}
+
+static void cmpdi_seek_i(FrtInStream *is, off_t pos)
+{
+    (void)is;
+    (void)pos;
+}
+
+static void cmpdi_close_i(FrtInStream *is)
+{
+    free(is->d.cis);
+}
+
+static off_t cmpdi_length_i(FrtInStream *is)
+{
+    return (is->d.cis->length);
+}
+
+/*
+ * raises: FRT_EOF_ERROR
+ */
+static void cmpdi_read_i(FrtInStream *is, frt_uchar *b, int len)
+{
+    FrtCompoundInStream *cis = is->d.cis;
+    off_t start = frt_is_pos(is);
+
+    if ((start + len) > cis->length) {
+        FRT_RAISE(FRT_EOF_ERROR, "Tried to read past end of file. File length is "
+              "<%"FRT_OFF_T_PFX"d> and tried to read to <%"FRT_OFF_T_PFX"d>",
+              cis->length, start + len);
+    }
+
+    frt_is_seek(cis->sub, cis->offset + start);
+    frt_is_read_bytes(cis->sub, b, len);
+}
+
+static const struct FrtInStreamMethods CMPD_IN_STREAM_METHODS = {
+    cmpdi_read_i,
+    cmpdi_seek_i,
+    cmpdi_length_i,
+    cmpdi_close_i
+};
+
+static FrtInStream *cmpd_create_input(FrtInStream *sub_is, off_t offset, off_t length)
+{
+    FrtInStream *is = frt_is_new();
+    FrtCompoundInStream *cis = FRT_ALLOC(FrtCompoundInStream);
+
+    cis->sub = sub_is;
+    cis->offset = offset;
+    cis->length = length;
+    is->d.cis = cis;
+    is->m = &CMPD_IN_STREAM_METHODS;
+
+    return is;
+}
+
+static FrtInStream *cmpd_open_input(FrtStore *store, const char *file_name)
+{
+    FileEntry *entry;
+    FrtCompoundStore *cmpd = store->dir.cmpd;
+    FrtInStream *is;
+
+    frt_mutex_lock(&store->mutex);
+    if (cmpd->stream == NULL) {
+        frt_mutex_unlock(&store->mutex);
+        FRT_RAISE(FRT_IO_ERROR, "Can't open compound file input stream. Parent "
+              "stream is closed.");
+    }
+
+    entry = (FileEntry *)frt_h_get(cmpd->entries, file_name);
+    if (entry == NULL) {
+        frt_mutex_unlock(&store->mutex);
+        FRT_RAISE(FRT_IO_ERROR, "File %s does not exist: ", file_name);
+    }
+
+    is = cmpd_create_input(cmpd->stream, entry->offset, entry->length);
+    frt_mutex_unlock(&store->mutex);
+
+    return is;
+}
+
+static FrtOutStream *cmpd_new_output(FrtStore *store, const char *file_name)
+{
+    (void)store;
+    (void)file_name;
+    FRT_RAISE(FRT_UNSUPPORTED_ERROR, "%s", FRT_UNSUPPORTED_ERROR_MSG);
+    return NULL;
+}
+
+static FrtLock *cmpd_open_lock_i(FrtStore *store, const char *lock_name)
+{
+    (void)store;
+    (void)lock_name;
+    FRT_RAISE(FRT_UNSUPPORTED_ERROR, "%s", FRT_UNSUPPORTED_ERROR_MSG);
+    return NULL;
+}
+
+static void cmpd_close_lock_i(FrtLock *lock)
+{
+    (void)lock;
+    FRT_RAISE(FRT_UNSUPPORTED_ERROR, "%s", FRT_UNSUPPORTED_ERROR_MSG);
+}
+
+FrtStore *frt_open_cmpd_store(FrtStore *store, const char *name)
+{
+    int count, i;
+    off_t offset;
+    char *fname;
+    FileEntry *volatile entry = NULL;
+    FrtStore *new_store = NULL;
+    FrtCompoundStore *volatile cmpd = NULL;
+    FrtInStream *volatile is = NULL;
+
+    FRT_TRY
+        cmpd = FRT_ALLOC_AND_ZERO(FrtCompoundStore);
+
+        cmpd->store       = store;
+        cmpd->name        = name;
+        cmpd->entries     = frt_h_new_str(&free, &free);
+        is = cmpd->stream = store->open_input(store, cmpd->name);
+
+        /* read the directory and init files */
+        count = frt_is_read_vint(is);
+        entry = NULL;
+        for (i = 0; i < count; i++) {
+            offset = (off_t)frt_is_read_i64(is);
+            fname = frt_is_read_string(is);
+
+            if (entry != NULL) {
+                /* set length of the previous entry */
+                entry->length = offset - entry->offset;
+            }
+
+            entry = FRT_ALLOC(FileEntry);
+            entry->offset = offset;
+            frt_h_set(cmpd->entries, fname, entry);
+        }
+    FRT_XCATCHALL
+        if (is) frt_is_close(is);
+        if (cmpd->entries) frt_h_destroy(cmpd->entries);
+        free(cmpd);
+    FRT_XENDTRY
+
+    /* set the length of the final entry */
+    if (entry != NULL) {
+        entry->length = frt_is_length(is) - entry->offset;
+    }
+
+    new_store               = frt_store_new();
+    new_store->dir.cmpd     = cmpd;
+    new_store->touch        = &cmpd_touch;
+    new_store->exists       = &cmpd_exists;
+    new_store->remove       = &cmpd_remove;
+    new_store->rename       = &cmpd_rename;
+    new_store->count        = &cmpd_count;
+    new_store->clear        = &cmpd_clear;
+    new_store->length       = &cmpd_length;
+    new_store->each         = &cmpd_each;
+    new_store->close_i      = &cmpd_close_i;
+    new_store->new_output   = &cmpd_new_output;
+    new_store->open_input   = &cmpd_open_input;
+    new_store->open_lock_i  = &cmpd_open_lock_i;
+    new_store->close_lock_i = &cmpd_close_lock_i;
+
+    return new_store;
+}
+
+/****************************************************************************
+ *
+ * CompoundWriter
+ *
+ ****************************************************************************/
+
+FrtCompoundWriter *frt_open_cw(FrtStore *store, char *name)
+{
+    FrtCompoundWriter *cw = FRT_ALLOC(FrtCompoundWriter);
+    cw->store = store;
+    cw->name = name;
+    cw->ids = frt_hs_new_str(&free);
+    cw->file_entries = frt_ary_new_type_capa(FrtCWFileEntry, FRT_CW_INIT_CAPA);
+    return cw;
+}
+
+void frt_cw_add_file(FrtCompoundWriter *cw, char *id)
+{
+    id = frt_estrdup(id);
+    if (frt_hs_add(cw->ids, id) != FRT_HASH_KEY_DOES_NOT_EXIST) {
+        FRT_RAISE(FRT_IO_ERROR, "Tried to add file \"%s\" which has already been "
+              "added to the compound store", id);
+    }
+
+    frt_ary_grow(cw->file_entries);
+    frt_ary_last(cw->file_entries).name = id;
+}
+
+static void cw_copy_file(FrtCompoundWriter *cw, FrtCWFileEntry *src, FrtOutStream *os)
+{
+    off_t start_ptr = frt_os_pos(os);
+    off_t end_ptr;
+    off_t remainder, length, len;
+    frt_uchar buffer[FRT_BUFFER_SIZE];
+
+    FrtInStream *is = cw->store->open_input(cw->store, src->name);
+
+    remainder = length = frt_is_length(is);
+
+    while (remainder > 0) {
+        len = FRT_MIN(remainder, FRT_BUFFER_SIZE);
+        frt_is_read_bytes(is, buffer, len);
+        frt_os_write_bytes(os, buffer, len);
+        remainder -= len;
+    }
+
+    /* Verify that remainder is 0 */
+    if (remainder != 0) {
+        FRT_RAISE(FRT_IO_ERROR, "There seems to be an error in the compound file "
+              "should have read to the end but there are <%"FRT_OFF_T_PFX"d> "
+              "bytes left", remainder);
+    }
+
+    /* Verify that the output length diff is equal to original file */
+    end_ptr = frt_os_pos(os);
+    len = end_ptr - start_ptr;
+    if (len != length) {
+        FRT_RAISE(FRT_IO_ERROR, "Difference in compound file output file offsets "
+              "<%"FRT_OFF_T_PFX"d> does not match the original file lenght "
+              "<%"FRT_OFF_T_PFX"d>", len, length);
+    }
+
+    frt_is_close(is);
+}
+
+void frt_cw_close(FrtCompoundWriter *cw)
+{
+    FrtOutStream *os = NULL;
+    int i;
+
+    if (cw->ids->size <= 0) {
+        FRT_RAISE(FRT_STATE_ERROR, "Tried to merge compound file with no entries");
+    }
+
+    os = cw->store->new_output(cw->store, cw->name);
+
+    frt_os_write_vint(os, frt_ary_size(cw->file_entries));
+
+    /* Write the directory with all offsets at 0.
+     * Remember the positions of directory entries so that we can adjust the
+     * offsets later */
+    for (i = 0; i < frt_ary_size(cw->file_entries); i++) {
+        cw->file_entries[i].dir_offset = frt_os_pos(os);
+        frt_os_write_u64(os, 0);  /* for now */
+        frt_os_write_string(os, cw->file_entries[i].name);
+    }
+
+    /* Open the files and copy their data into the stream.  Remember the
+     * locations of each file's data section. */
+    for (i = 0; i < frt_ary_size(cw->file_entries); i++) {
+        cw->file_entries[i].data_offset = frt_os_pos(os);
+        cw_copy_file(cw, &cw->file_entries[i], os);
+    }
+
+    /* Write the data offsets into the directory of the compound stream */
+    for (i = 0; i < frt_ary_size(cw->file_entries); i++) {
+        frt_os_seek(os, cw->file_entries[i].dir_offset);
+        frt_os_write_u64(os, cw->file_entries[i].data_offset);
+    }
+
+    if (os) {
+        frt_os_close(os);
+    }
+
+    frt_hs_destroy(cw->ids);
+    frt_ary_free(cw->file_entries);
+    free(cw);
+}

data/ext/isomorfeus_ferret_ext/frt_config.h

@@ -0,0 +1,44 @@
+#ifndef FRT_DEFINES_H
+#define FRT_DEFINES_H
+
+#include <sys/types.h>
+#include <limits.h>
+#include "frt_posh.h"
+
+#ifndef false
+#define false 0
+#endif
+#ifndef true
+#define true  1
+#endif
+
+#ifndef bool
+typedef unsigned int  bool;
+#endif
+typedef unsigned char frt_uchar;
+
+typedef posh_u16_t frt_u16;
+typedef posh_i16_t frt_i16;
+typedef posh_u32_t frt_u32;
+typedef posh_i32_t frt_i32;
+typedef posh_u64_t frt_u64;
+typedef posh_i64_t frt_i64;
+
+#if ( LONG_MAX == 2147483647 ) && defined(_FILE_OFFSET_BITS) && (_FILE_OFFSET_BITS == 64)
+#define FRT_OFF_T_PFX "ll"
+#else
+#define FRT_OFF_T_PFX "l"
+#endif
+
+#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && !defined(__cplusplus)
+#define FRT_IS_C99
+#define FRT_HAS_ISO_VARARGS
+#define FRT_HAS_VARARGS
+#endif
+
+#if defined(__GNUC__) && !defined(__STRICT_ANSI__) && !defined(__cplusplus)
+#define FRT_HAS_GNUC_VARARGS
+#define FRT_HAS_VARARGS
+#endif
+
+#endif

data/ext/isomorfeus_ferret_ext/frt_document.c

@@ -0,0 +1,134 @@
+#include "frt_document.h"
+#include <string.h>
+
+/****************************************************************************
+ *
+ * FrtDocField
+ *
+ ****************************************************************************/
+
+FrtDocField *frt_df_new(FrtSymbol name)
+{
+    FrtDocField *df = FRT_ALLOC(FrtDocField);
+    df->name = name;
+    df->size = 0;
+    df->capa = FRT_DF_INIT_CAPA;
+    df->data = FRT_ALLOC_N(char *, df->capa);
+    df->lengths = FRT_ALLOC_N(int, df->capa);
+    df->destroy_data = false;
+    df->boost = 1.0f;
+    return df;
+}
+
+FrtDocField *frt_df_add_data_len(FrtDocField *df, char *data, int len)
+{
+    if (df->size >= df->capa) {
+        df->capa <<= 2;
+        FRT_REALLOC_N(df->data, char *, df->capa);
+        FRT_REALLOC_N(df->lengths, int, df->capa);
+    }
+    df->data[df->size] = data;
+    df->lengths[df->size] = len;
+    df->size++;
+    return df;
+}
+
+FrtDocField *frt_df_add_data(FrtDocField *df, char *data)
+{
+    return frt_df_add_data_len(df, data, strlen(data));
+}
+
+void frt_df_destroy(FrtDocField *df)
+{
+    if (df->destroy_data) {
+        int i;
+        for (i = 0; i < df->size; i++) {
+            free(df->data[i]);
+        }
+    }
+    free(df->data);
+    free(df->lengths);
+    free(df);
+}
+
+/*
+ * Format for one item is: name: "data"
+ *        for more items : name: ["data", "data", "data"]
+ */
+char *frt_df_to_s(FrtDocField *df)
+{
+    const char *df_name = rb_id2name(df->name);
+    int i, len = 0, namelen = strlen(df_name);
+    char *str, *s;
+    for (i = 0; i < df->size; i++) {
+        len += df->lengths[i] + 4;
+    }
+    s = str = FRT_ALLOC_N(char, namelen + len + 5);
+    memcpy(s, df_name, namelen);
+    s += namelen;
+    s = frt_strapp(s, ": ");
+
+    if (df->size > 1) {
+        s = frt_strapp(s, "[");
+    }
+    for (i = 0; i < df->size; i++) {
+        if (i != 0) {
+            s = frt_strapp(s, ", ");
+        }
+        s = frt_strapp(s, "\"");
+        memcpy(s, df->data[i], df->lengths[i]);
+        s += df->lengths[i];
+        s = frt_strapp(s, "\"");
+    }
+
+    if (df->size > 1) {
+        s = frt_strapp(s, "]");
+    }
+    *s = 0;
+    return str;
+}
+
+/****************************************************************************
+ *
+ * FrtDocument
+ *
+ ****************************************************************************/
+
+FrtDocument *frt_doc_new()
+{
+    FrtDocument *doc = FRT_ALLOC(FrtDocument);
+    doc->field_dict = frt_h_new_ptr((frt_free_ft)&frt_df_destroy);
+    doc->size = 0;
+    doc->capa = FRT_DOC_INIT_CAPA;
+    doc->fields = FRT_ALLOC_N(FrtDocField *, doc->capa);
+    doc->boost = 1.0f;
+    return doc;
+}
+
+FrtDocField *frt_doc_add_field(FrtDocument *doc, FrtDocField *df)
+{
+    if (!frt_h_set_safe(doc->field_dict, (void *)df->name, df)) {
+        FRT_RAISE(FRT_EXCEPTION, "tried to add %s field which alread existed\n",
+              rb_id2name(df->name));
+    }
+    if (doc->size >= doc->capa) {
+        doc->capa <<= 1;
+        FRT_REALLOC_N(doc->fields, FrtDocField *, doc->capa);
+    }
+    doc->fields[doc->size] = df;
+    doc->size++;
+    return df;
+}
+
+FrtDocField *frt_doc_get_field(FrtDocument *doc, FrtSymbol name)
+{
+    return (FrtDocField *)frt_h_get(doc->field_dict, (void *)name);
+}
+
+void frt_doc_destroy(FrtDocument *doc)
+{
+    frt_h_destroy(doc->field_dict);
+    free(doc->fields);
+    free(doc);
+}
+

data/ext/isomorfeus_ferret_ext/frt_document.h

@@ -0,0 +1,52 @@
+#ifndef FRT_DOCUMENT_H
+#define FRT_DOCUMENT_H
+
+#include "frt_global.h"
+#include "frt_hash.h"
+
+/****************************************************************************
+ *
+ * FrtDocField
+ *
+ ****************************************************************************/
+
+#define FRT_DF_INIT_CAPA 1
+typedef struct FrtDocField
+{
+    FrtSymbol name;
+    int size;
+    int capa;
+    int *lengths;
+    char **data;
+    float boost;
+    bool destroy_data : 1;
+} FrtDocField;
+
+extern FrtDocField *frt_df_new(FrtSymbol name);
+extern FrtDocField *frt_df_add_data(FrtDocField *df, char *data);
+extern FrtDocField *frt_df_add_data_len(FrtDocField *df, char *data, int len);
+extern void frt_df_destroy(FrtDocField *df);
+extern char *frt_df_to_s(FrtDocField *df);
+
+/****************************************************************************
+ *
+ * FrtDocument
+ *
+ ****************************************************************************/
+
+#define FRT_DOC_INIT_CAPA 8
+typedef struct FrtDocument
+{
+    FrtHash *field_dict;
+    int size;
+    int capa;
+    FrtDocField **fields;
+    float boost;
+} FrtDocument;
+
+extern FrtDocument *frt_doc_new();
+extern FrtDocField *frt_doc_add_field(FrtDocument *doc, FrtDocField *df);
+extern FrtDocField *frt_doc_get_field(FrtDocument *doc, FrtSymbol name);
+extern void frt_doc_destroy(FrtDocument *doc);
+
+#endif

data/ext/isomorfeus_ferret_ext/frt_except.c

@@ -0,0 +1,95 @@
+#include <stdarg.h>
+#include "frt_global.h"
+#include "frt_except.h"
+#include "frt_threading.h"
+
+const char *const ERROR_TYPES[] = {
+    "Body",
+    "Finally",
+    "Exception",
+    "IO Error",
+    "File Not Found Error",
+    "Argument Error",
+    "End-of-File Error",
+    "Unsupported Function Error",
+    "State Error",
+    "Parse Error",
+    "Memory Error",
+    "Index Error",
+    "Lock Error"
+};
+
+bool frt_x_do_logging = false;
+bool frt_x_abort_on_exception = true;
+bool frt_x_has_aborted = false;
+FILE *frt_x_exception_stream = NULL;
+
+const char *const FRT_UNSUPPORTED_ERROR_MSG = "Unsupported operation";
+const char *const FRT_EOF_ERROR_MSG = "Read past end of file";
+char frt_xmsg_buffer[FRT_XMSG_BUFFER_SIZE];
+char frt_xmsg_buffer_final[FRT_XMSG_BUFFER_FINAL_SIZE];
+
+static frt_thread_key_t exception_stack_key;
+static frt_thread_once_t exception_stack_key_once = FRT_THREAD_ONCE_INIT;
+
+static void exception_stack_alloc(void)
+{
+    frt_thread_key_create(&exception_stack_key, NULL);
+}
+
+void frt_xpush_context(frt_xcontext_t *context)
+{
+    frt_xcontext_t *top_context;
+    frt_thread_once(&exception_stack_key_once, *exception_stack_alloc);
+    top_context = (frt_xcontext_t *)frt_thread_getspecific(exception_stack_key);
+    context->next = top_context;
+    frt_thread_setspecific(exception_stack_key, context);
+    context->handled = true;
+    context->in_finally = false;
+}
+
+static void frt_xraise_context(frt_xcontext_t *context,
+                                    volatile int excode,
+                                    const char *const msg)
+{
+    context->msg = msg;
+    context->excode = excode;
+    context->handled = false;
+    longjmp(context->jbuf, excode);
+}
+
+void frt_xraise(int excode, const char *const msg)
+{
+    frt_xcontext_t *top_context;
+    frt_thread_once(&exception_stack_key_once, *exception_stack_alloc);
+    top_context = (frt_xcontext_t *)frt_thread_getspecific(exception_stack_key);
+
+    if (!top_context) {
+        FRT_XEXIT(ERROR_TYPES[excode], msg);
+    }
+    else if (!top_context->in_finally) {
+        frt_xraise_context(top_context, excode, msg);
+    }
+    else if (top_context->handled) {
+        top_context->msg = msg;
+        top_context->excode = excode;
+        top_context->handled = false;
+    }
+}
+
+void frt_xpop_context()
+{
+    frt_xcontext_t *top_cxt, *context;
+    frt_thread_once(&exception_stack_key_once, *exception_stack_alloc);
+    top_cxt = (frt_xcontext_t *)frt_thread_getspecific(exception_stack_key);
+    context = top_cxt->next;
+    frt_thread_setspecific(exception_stack_key, context);
+    if (!top_cxt->handled) {
+        if (context) {
+            frt_xraise_context(context, top_cxt->excode, top_cxt->msg);
+        }
+        else {
+            FRT_XEXIT(ERROR_TYPES[top_cxt->excode], top_cxt->msg);
+        }
+    }
+}