jk-ferret 0.11.8.2

data/ext/compound_io.c

@@ -0,0 +1,384 @@
+#include "index.h"
+#include "array.h"
+#include "internal.h"
+
+extern void store_destroy(Store *store);
+extern InStream *is_new();
+extern Store *store_new();
+
+/****************************************************************************
+ *
+ * CompoundStore
+ *
+ ****************************************************************************/
+
+typedef struct FileEntry {
+    off_t offset;
+    off_t length;
+} FileEntry;
+
+static void cmpd_touch(Store *store, const char *file_name)
+{
+    store->dir.cmpd->store->touch(store->dir.cmpd->store, file_name);
+}
+
+static int cmpd_exists(Store *store, const char *file_name)
+{
+    if (h_get(store->dir.cmpd->entries, file_name) != NULL) {
+        return true;
+    }
+    else {
+        return false;
+    }
+}
+
+/**
+ * @throws UNSUPPORTED_ERROR
+ */
+static int cmpd_remove(Store *store, const char *file_name)
+{
+    (void)store;
+    (void)file_name;
+    RAISE(UNSUPPORTED_ERROR, "%s", UNSUPPORTED_ERROR_MSG);
+    return 0;
+}
+
+/**
+ * @throws UNSUPPORTED_ERROR
+ */
+static void cmpd_rename(Store *store, const char *from, const char *to)
+{
+    (void)store;
+    (void)from;
+    (void)to;
+    RAISE(UNSUPPORTED_ERROR, "%s", UNSUPPORTED_ERROR_MSG);
+}
+
+static int cmpd_count(Store *store)
+{
+    return store->dir.cmpd->entries->size;
+}
+
+static void cmpd_each(Store *store,
+                     void (*func)(const char *fname, void *arg), void *arg)
+{
+    Hash *ht = store->dir.cmpd->entries;
+    int i;
+    for (i = 0; i <= ht->mask; i++) {
+        char *fn = (char *)ht->table[i].key;
+        if (fn) {
+            func(fn, arg);
+        }
+    }
+}
+
+
+/**
+ * @throws UNSUPPORTED_ERROR
+ */
+static void cmpd_clear(Store *store)
+{
+    (void)store;
+    RAISE(UNSUPPORTED_ERROR, "%s", UNSUPPORTED_ERROR_MSG);
+}
+
+static void cmpd_close_i(Store *store)
+{
+    CompoundStore *cmpd = store->dir.cmpd;
+    if (cmpd->stream == NULL) {
+        RAISE(IO_ERROR, "Tried to close already closed compound store");
+    }
+
+    h_destroy(cmpd->entries);
+
+    is_close(cmpd->stream);
+    cmpd->stream = NULL;
+    free(store->dir.cmpd);
+    store_destroy(store);
+}
+
+static off_t cmpd_length(Store *store, const char *file_name)
+{
+    FileEntry *fe = (FileEntry *)h_get(store->dir.cmpd->entries, file_name);
+    if (fe != NULL) {
+        return fe->length;
+    }
+    else {
+        return 0;
+    }
+}
+
+static void cmpdi_seek_i(InStream *is, off_t pos)
+{
+    (void)is;
+    (void)pos;
+}
+
+static void cmpdi_close_i(InStream *is)
+{
+    free(is->d.cis);
+}
+
+static off_t cmpdi_length_i(InStream *is)
+{
+    return (is->d.cis->length);
+}
+
+/*
+ * raises: EOF_ERROR
+ */
+static void cmpdi_read_i(InStream *is, uchar *b, int len)
+{
+    CompoundInStream *cis = is->d.cis;
+    off_t start = is_pos(is);
+
+    if ((start + len) > cis->length) {
+        RAISE(EOF_ERROR, "Tried to read past end of file. File length is "
+              "<%"OFF_T_PFX"d> and tried to read to <%"OFF_T_PFX"d>",
+              cis->length, start + len);
+    }
+
+    is_seek(cis->sub, cis->offset + start);
+    is_read_bytes(cis->sub, b, len);
+}
+
+static const struct InStreamMethods CMPD_IN_STREAM_METHODS = {
+    cmpdi_read_i,
+    cmpdi_seek_i,
+    cmpdi_length_i,
+    cmpdi_close_i
+};
+
+static InStream *cmpd_create_input(InStream *sub_is, off_t offset, off_t length)
+{
+    InStream *is = is_new();
+    CompoundInStream *cis = ALLOC(CompoundInStream);
+
+    cis->sub = sub_is;
+    cis->offset = offset;
+    cis->length = length;
+    is->d.cis = cis;
+    is->m = &CMPD_IN_STREAM_METHODS;
+
+    return is;
+}
+
+static InStream *cmpd_open_input(Store *store, const char *file_name)
+{
+    FileEntry *entry;
+    CompoundStore *cmpd = store->dir.cmpd;
+    InStream *is;
+
+    mutex_lock(&store->mutex);
+    if (cmpd->stream == NULL) {
+        mutex_unlock(&store->mutex);
+        RAISE(IO_ERROR, "Can't open compound file input stream. Parent "
+              "stream is closed.");
+    }
+
+    entry = (FileEntry *)h_get(cmpd->entries, file_name);
+    if (entry == NULL) {
+        mutex_unlock(&store->mutex);
+        RAISE(IO_ERROR, "File %s does not exist: ", file_name);
+    }
+
+    is = cmpd_create_input(cmpd->stream, entry->offset, entry->length);
+    mutex_unlock(&store->mutex);
+
+    return is;
+}
+
+static OutStream *cmpd_new_output(Store *store, const char *file_name)
+{
+    (void)store;
+    (void)file_name;
+    RAISE(UNSUPPORTED_ERROR, "%s", UNSUPPORTED_ERROR_MSG);
+    return NULL;
+}
+
+static Lock *cmpd_open_lock_i(Store *store, const char *lock_name)
+{
+    (void)store;
+    (void)lock_name;
+    RAISE(UNSUPPORTED_ERROR, "%s", UNSUPPORTED_ERROR_MSG);
+    return NULL;
+}
+
+static void cmpd_close_lock_i(Lock *lock)
+{
+    (void)lock;
+    RAISE(UNSUPPORTED_ERROR, "%s", UNSUPPORTED_ERROR_MSG);
+}
+
+Store *open_cmpd_store(Store *store, const char *name)
+{
+    int count, i;
+    off_t offset;
+    char *fname;
+    FileEntry *entry = NULL;
+    Store *new_store = NULL;
+    CompoundStore *volatile cmpd = NULL;
+    InStream *volatile is = NULL;
+
+    TRY
+        cmpd = ALLOC_AND_ZERO(CompoundStore);
+
+        cmpd->store       = store;
+        cmpd->name        = name;
+        cmpd->entries     = h_new_str(&free, &free);
+        is = cmpd->stream = store->open_input(store, cmpd->name);
+
+        /* read the directory and init files */
+        count = is_read_vint(is);
+        entry = NULL;
+        for (i = 0; i < count; i++) {
+            offset = (off_t)is_read_i64(is);
+            fname = is_read_string(is);
+
+            if (entry != NULL) {
+                /* set length of the previous entry */
+                entry->length = offset - entry->offset;
+            }
+
+            entry = ALLOC(FileEntry);
+            entry->offset = offset;
+            h_set(cmpd->entries, fname, entry);
+        }
+    XCATCHALL
+        if (is) is_close(is);
+        if (cmpd->entries) h_destroy(cmpd->entries);
+        free(cmpd);
+    XENDTRY
+
+    /* set the length of the final entry */
+    if (entry != NULL) {
+        entry->length = is_length(is) - entry->offset;
+    }
+
+    new_store               = store_new();
+    new_store->dir.cmpd     = cmpd;
+    new_store->touch        = &cmpd_touch;
+    new_store->exists       = &cmpd_exists;
+    new_store->remove       = &cmpd_remove;
+    new_store->rename       = &cmpd_rename;
+    new_store->count        = &cmpd_count;
+    new_store->clear        = &cmpd_clear;
+    new_store->length       = &cmpd_length;
+    new_store->each         = &cmpd_each;
+    new_store->close_i      = &cmpd_close_i;
+    new_store->new_output   = &cmpd_new_output;
+    new_store->open_input   = &cmpd_open_input;
+    new_store->open_lock_i  = &cmpd_open_lock_i;
+    new_store->close_lock_i = &cmpd_close_lock_i;
+
+    return new_store;
+}
+
+/****************************************************************************
+ *
+ * CompoundWriter
+ *
+ ****************************************************************************/
+
+CompoundWriter *open_cw(Store *store, char *name)
+{
+    CompoundWriter *cw = ALLOC(CompoundWriter);
+    cw->store = store;
+    cw->name = name;
+    cw->ids = hs_new_str(&free);
+    cw->file_entries = ary_new_type_capa(CWFileEntry, CW_INIT_CAPA);
+    return cw;
+}
+
+void cw_add_file(CompoundWriter *cw, char *id)
+{
+    id = estrdup(id);
+    if (hs_add(cw->ids, id) != HASH_KEY_DOES_NOT_EXIST) {
+        RAISE(IO_ERROR, "Tried to add file \"%s\" which has already been "
+              "added to the compound store", id);
+    }
+
+    ary_grow(cw->file_entries);
+    ary_last(cw->file_entries).name = id;
+}
+
+static void cw_copy_file(CompoundWriter *cw, CWFileEntry *src, OutStream *os)
+{
+    off_t start_ptr = os_pos(os);
+    off_t end_ptr;
+    off_t remainder, length, len;
+    uchar buffer[BUFFER_SIZE];
+
+    InStream *is = cw->store->open_input(cw->store, src->name);
+
+    remainder = length = is_length(is);
+
+    while (remainder > 0) {
+        len = MIN(remainder, BUFFER_SIZE);
+        is_read_bytes(is, buffer, len);
+        os_write_bytes(os, buffer, len);
+        remainder -= len;
+    }
+
+    /* Verify that remainder is 0 */
+    if (remainder != 0) {
+        RAISE(IO_ERROR, "There seems to be an error in the compound file "
+              "should have read to the end but there are <%"OFF_T_PFX"d> "
+              "bytes left", remainder);
+    }
+
+    /* Verify that the output length diff is equal to original file */
+    end_ptr = os_pos(os);
+    len = end_ptr - start_ptr;
+    if (len != length) {
+        RAISE(IO_ERROR, "Difference in compound file output file offsets "
+              "<%"OFF_T_PFX"d> does not match the original file lenght "
+              "<%"OFF_T_PFX"d>", len, length);
+    }
+
+    is_close(is);
+}
+
+void cw_close(CompoundWriter *cw)
+{
+    OutStream *os = NULL;
+    int i;
+
+    if (cw->ids->size <= 0) {
+        RAISE(STATE_ERROR, "Tried to merge compound file with no entries");
+    }
+
+    os = cw->store->new_output(cw->store, cw->name);
+
+    os_write_vint(os, ary_size(cw->file_entries));
+
+    /* Write the directory with all offsets at 0.
+     * Remember the positions of directory entries so that we can adjust the
+     * offsets later */
+    for (i = 0; i < ary_size(cw->file_entries); i++) {
+        cw->file_entries[i].dir_offset = os_pos(os);
+        os_write_u64(os, 0);  /* for now */
+        os_write_string(os, cw->file_entries[i].name);
+    }
+
+    /* Open the files and copy their data into the stream.  Remember the
+     * locations of each file's data section. */
+    for (i = 0; i < ary_size(cw->file_entries); i++) {
+        cw->file_entries[i].data_offset = os_pos(os);
+        cw_copy_file(cw, &cw->file_entries[i], os);
+    }
+
+    /* Write the data offsets into the directory of the compound stream */
+    for (i = 0; i < ary_size(cw->file_entries); i++) {
+        os_seek(os, cw->file_entries[i].dir_offset);
+        os_write_u64(os, cw->file_entries[i].data_offset);
+    }
+
+    if (os) {
+        os_close(os);
+    }
+
+    hs_destroy(cw->ids);
+    ary_free(cw->file_entries);
+    free(cw);
+}

data/ext/config.h

@@ -0,0 +1,52 @@
+#ifndef FRT_DEFINES_H
+#define FRT_DEFINES_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <sys/types.h>
+#include <limits.h>
+#include "posh.h"
+
+#ifndef false
+#define false 0
+#endif
+#ifndef true
+#define true  1
+#endif
+
+#ifndef __cplusplus
+typedef unsigned int        bool;
+#endif
+typedef unsigned char       frt_uchar;
+
+typedef posh_u16_t frt_u16;
+typedef posh_i16_t frt_i16;
+typedef posh_u32_t frt_u32;
+typedef posh_i32_t frt_i32;
+typedef posh_u64_t frt_u64;
+typedef posh_i64_t frt_i64;
+
+#if ( LONG_MAX == 2147483647 ) && defined(_FILE_OFFSET_BITS) && (_FILE_OFFSET_BITS == 64)
+#define FRT_OFF_T_PFX "ll"
+#else
+#define FRT_OFF_T_PFX "l"
+#endif
+
+#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && !defined(__cplusplus)
+#define FRT_IS_C99
+#define FRT_HAS_ISO_VARARGS
+#define FRT_HAS_VARARGS
+#endif
+
+#if defined(__GNUC__) && !defined(__STRICT_ANSI__) && !defined(__cplusplus)
+#define FRT_HAS_GNUC_VARARGS
+#define FRT_HAS_VARARGS
+#endif
+
+#ifdef __cplusplus
+} // extern "C"
+#endif
+
+#endif

data/ext/document.c

@@ -0,0 +1,159 @@
+#include "document.h"
+#include "symbol.h"
+#include <string.h>
+#include "internal.h"
+
+/****************************************************************************
+ *
+ * DocField
+ *
+ ****************************************************************************/
+
+DocField *df_new(Symbol name)
+{
+    DocField *df = ALLOC(DocField);
+    df->name = name;
+    df->size = 0;
+    df->capa = DF_INIT_CAPA;
+    df->data = ALLOC_N(char *, df->capa);
+    df->lengths = ALLOC_N(int, df->capa);
+    df->destroy_data = false;
+    df->boost = 1.0;
+    return df;
+}
+
+DocField *df_add_data_len(DocField *df, char *data, int len)
+{
+    if (df->size >= df->capa) {
+        df->capa <<= 2;
+        REALLOC_N(df->data, char *, df->capa);
+        REALLOC_N(df->lengths, int, df->capa);
+    }
+    df->data[df->size] = data;
+    df->lengths[df->size] = len;
+    df->size++;
+    return df;
+}
+
+DocField *df_add_data(DocField *df, char *data)
+{
+    return df_add_data_len(df, data, strlen(data));
+}
+
+void df_destroy(DocField *df)
+{
+    if (df->destroy_data) {
+        int i;
+        for (i = 0; i < df->size; i++) {
+            free(df->data[i]);
+        }
+    }
+    free(df->data);
+    free(df->lengths);
+    free(df);
+}
+
+/*
+ * Format for one item is: name: "data"
+ *        for more items : name: ["data", "data", "data"]
+ */
+char *df_to_s(DocField *df)
+{
+#define APPEND(dst, src) ((dst)[0] = (src)[0], 1)
+#define APPEND2(dst, src) (APPEND(dst, src), APPEND(dst+1, src+1), 2)
+
+    int i, len = 0, namelen = sym_len(df->name);
+    char *str, *s;
+    for (i = 0; i < df->size; i++) {
+        len += df->lengths[i] + 4;
+    }
+    s = str = ALLOC_N(char, namelen + len + 5);
+    memcpy(s, df->name, namelen);
+    s += namelen;
+    s += APPEND2(s, ": ");
+
+    if (df->size > 1) {
+        s += APPEND(s, "[");
+    }
+    for (i = 0; i < df->size; i++) {
+        if (i != 0) {
+            s += APPEND2(s, ", ");
+        }
+        s += APPEND(s, "\"");
+        memcpy(s, df->data[i], df->lengths[i]);
+        s += df->lengths[i];
+        s += APPEND(s, "\"");
+    }
+
+    if (df->size > 1) {
+        s += APPEND(s, "]");
+    }
+    *s = 0;
+    return str;
+}
+
+/****************************************************************************
+ *
+ * Document
+ *
+ ****************************************************************************/
+
+Document *doc_new()
+{
+    Document *doc = ALLOC(Document);
+    doc->field_dict = h_new_ptr((free_ft)&df_destroy);
+    doc->size = 0;
+    doc->capa = DOC_INIT_CAPA;
+    doc->fields = ALLOC_N(DocField *, doc->capa);
+    doc->boost = 1.0;
+    return doc;
+}
+
+DocField *doc_add_field(Document *doc, DocField *df)
+{
+    if (!h_set_safe(doc->field_dict, df->name, df)) {
+        RAISE(EXCEPTION, "tried to add %s field which alread existed\n",
+              S(df->name));
+    }
+    if (doc->size >= doc->capa) {
+        doc->capa <<= 1;
+        REALLOC_N(doc->fields, DocField *, doc->capa);
+    }
+    doc->fields[doc->size] = df;
+    doc->size++;
+    return df;
+}
+
+DocField *doc_get_field(Document *doc, Symbol name)
+{
+    return (DocField *)h_get(doc->field_dict, name);
+}
+
+char *doc_to_s(Document *doc)
+{
+    int i;
+    int len = 0;
+    char **fields = ALLOC_N(char *, doc->size);
+    char *buf, *s;
+
+    for (i = 0; i < doc->size; i++) {
+        fields[i] = df_to_s(doc->fields[i]);
+        len += strlen(fields[i]) + 5;
+    }
+    s = buf = ALLOC_N(char, len + 12);
+    s += sprintf(buf, "Document [\n");
+    for (i = 0; i < doc->size; i++) {
+        s += sprintf(s, "  =>%s\n", fields[i]);
+        free(fields[i]);
+    }
+    free(fields);
+    return buf;
+}
+
+void doc_destroy(Document *doc)
+{
+    h_destroy(doc->field_dict);
+    free(doc->fields);
+    free(doc);
+}
+