fast-xml 1.0.1 → 1.1.0

data/ext/fastxml/xh_reader.c

@@ -0,0 +1,528 @@
+#include "xh_config.h"
+#include "xh_core.h"
+
+static void
+xh_common_reader_init(xh_reader_t *reader, VALUE XH_UNUSED(input), xh_char_t *encoding, size_t buf_size)
+{
+    reader->buf_size = buf_size;
+
+    if (encoding[0] != '\0')
+        reader->switch_encoding(reader, encoding, NULL, NULL);
+}
+
+static void
+xh_common_reader_destroy(xh_reader_t *reader)
+{
+#ifdef XH_HAVE_ENCODER
+    xh_buffer_destroy(&reader->enc_buf);
+    if (reader->encoder != NULL)
+        xh_encoder_destroy(reader->encoder);
+#endif
+}
+
+static void
+xh_common_reader_switch_encoding(xh_reader_t *reader, xh_char_t *encoding, xh_char_t **buf, size_t *len)
+{
+    xh_log_debug1("switch encoding to '%s'", encoding);
+
+    if (xh_strcasecmp(encoding, XH_INTERNAL_ENCODING) == 0) {
+#ifdef XH_HAVE_ENCODER
+        if (reader->encoder != NULL) {
+            rb_raise(xh_parse_error_class, "Can't to switch encoding from %s to %s", reader->encoder->fromcode, encoding);
+        }
+#endif
+    }
+    else {
+#ifdef XH_HAVE_ENCODER
+        if (reader->encoder == NULL) {
+            /* create encoder */
+            xh_log_debug1("create a new encoder: %s", encoding);
+
+            reader->encoder = xh_encoder_create(XH_CHAR_CAST XH_INTERNAL_ENCODING, encoding);
+            if (reader->encoder == NULL) {
+                rb_raise(xh_parse_error_class, "Can't create encoder for '%s'", encoding);
+            }
+
+            xh_buffer_init(&reader->enc_buf, reader->buf_size);
+
+            if (len != NULL && *len > 0) {
+                reader->fake_read_pos = *buf;
+                reader->fake_read_len = *len;
+                *len = 0;
+            }
+        }
+        else if (xh_strcasecmp(encoding, reader->encoder->fromcode) != 0) {
+            rb_raise(xh_parse_error_class, "Can't to switch encoding from %s to %s", reader->encoder->fromcode, encoding);
+        }
+#else
+        rb_raise(xh_parse_error_class, "Can't create encoder for '%s'", encoding);
+#endif
+    }
+}
+
+static void
+xh_string_reader_init(xh_reader_t *reader, VALUE input, xh_char_t *encoding, size_t buf_size)
+{
+    size_t     len;
+    xh_char_t *str;
+
+    str = XH_CHAR_CAST RSTRING_PTR(input);
+    len = RSTRING_LEN(input);
+    reader->str = str;
+    reader->len = len;
+
+    reader->main_buf.start = reader->main_buf.cur = str;
+    reader->main_buf.end   = str + len;
+
+    xh_common_reader_init(reader, input, encoding, buf_size);
+}
+
+static size_t
+xh_string_reader_read(xh_reader_t *reader, xh_char_t **buf, xh_char_t *XH_UNUSED(preserve), size_t *off)
+{
+    size_t       len;
+    xh_buffer_t *main_buf;
+
+    *off     = 0;
+    main_buf = &reader->main_buf;
+
+    *buf = xh_buffer_pos(main_buf);
+    len  = xh_buffer_avail(main_buf);
+
+    xh_buffer_seek_eof(main_buf);
+
+    return len;
+}
+
+#ifdef XH_HAVE_ENCODER
+static size_t
+xh_string_reader_read_with_encoding(xh_reader_t *reader, xh_char_t **buf, xh_char_t *preserve, size_t *off)
+{
+    xh_char_t   *old_buf_addr;
+    size_t       src_left, dst_left;
+    xh_buffer_t *main_buf, *enc_buf;
+
+    *off     = 0;
+    main_buf = &reader->main_buf;
+    enc_buf  = &reader->enc_buf;
+
+    xh_log_debug4("enc_buf: %p[%.*s] len: %lu", enc_buf->start, enc_buf->cur - enc_buf->start, enc_buf->cur, enc_buf->cur - enc_buf->start);
+
+    xh_log_debug1("preserve data: %p", preserve);
+    if (preserve == NULL) {
+        xh_buffer_seek_top(enc_buf);
+    }
+    else {
+        *off = preserve - enc_buf->start;
+        xh_log_debug1("off: %lu", *off);
+        if (*off) {
+            xh_log_debug3("memmove dest: %p src %p size: %lu", enc_buf->start, preserve, enc_buf->end - preserve);
+            xh_memmove(enc_buf->start, preserve, enc_buf->end - preserve);
+        }
+        enc_buf->cur -= *off;
+    }
+
+    old_buf_addr = xh_buffer_start(enc_buf);
+    xh_buffer_grow50(enc_buf);
+
+    if (preserve != NULL && xh_buffer_start(enc_buf) != old_buf_addr) {
+        *off += old_buf_addr - xh_buffer_start(enc_buf);
+    }
+
+    *buf = xh_buffer_pos(enc_buf);
+
+    while (enc_buf->cur < enc_buf->end) {
+        if (reader->fake_read_pos != NULL) {
+            main_buf->cur         = reader->fake_read_pos;
+            reader->fake_read_pos = NULL;
+            reader->fake_read_len = 0;
+        }
+
+        xh_log_debug2("main buf cur: %p end: %p", main_buf->cur, main_buf->end);
+        src_left = xh_buffer_avail(main_buf);
+        if (src_left == 0 && reader->encoder->state == XH_ENC_OK) {
+            if (main_buf->cur == main_buf->end)
+                break;
+            rb_raise(xh_parse_error_class, "Truncate char found");
+        }
+
+        dst_left = xh_buffer_avail(enc_buf);
+
+        xh_log_debug4("main_buf: %.*s src_left: %lu dst_left: %lu", src_left, main_buf->cur, src_left, dst_left);
+
+        xh_encoder_encode_string(reader->encoder, &main_buf->cur, &src_left, &enc_buf->cur, &dst_left);
+
+        xh_log_debug3("enc_buf: %.*s len: %lu", enc_buf->cur - enc_buf->start, enc_buf->start, enc_buf->cur - enc_buf->start);
+
+        switch (reader->encoder->state) {
+            case XH_ENC_TRUNCATED_CHAR_FOUND:
+                if (src_left == 0)
+                    rb_raise(xh_parse_error_class, "Truncated char found but buffer is empty");
+                break;
+            case XH_ENC_BUFFER_OVERFLOW:
+            default:
+                goto DONE;
+        }
+    }
+
+DONE:
+    dst_left = enc_buf->cur - *buf;
+    xh_log_debug4("enc_buf: %p[%.*s] len: %lu", enc_buf->start, dst_left, enc_buf->cur, dst_left);
+
+    return dst_left;
+}
+#endif /* XH_HAVE_ENCODER */
+
+static void
+xh_string_reader_switch_encoding(xh_reader_t *reader, xh_char_t *encoding, xh_char_t **buf, size_t *len)
+{
+
+    xh_common_reader_switch_encoding(reader, encoding, buf, len);
+
+#ifdef XH_HAVE_ENCODER
+    reader->read = reader->encoder == NULL
+        ? xh_string_reader_read
+        : xh_string_reader_read_with_encoding;
+#endif
+}
+
+static void
+xh_string_reader_destroy(xh_reader_t *reader)
+{
+    xh_common_reader_destroy(reader);
+}
+
+#ifdef XH_HAVE_MMAP
+static void
+xh_mmaped_file_reader_init(xh_reader_t *reader, VALUE input, xh_char_t *encoding, size_t buf_size)
+{
+    struct stat sb;
+
+    reader->file = XH_CHAR_CAST StringValueCStr(input);
+
+    xh_log_debug1("open file: %s", reader->file);
+
+    reader->fd = open((const char *) reader->file, O_RDONLY);
+    if (reader->fd == -1) {
+        rb_raise(xh_parse_error_class, "Can't open file '%s': %s", reader->file, strerror(errno));
+    }
+
+    if (fstat(reader->fd, &sb) == -1) {
+        rb_raise(xh_parse_error_class, "Can't get stat of file '%s': %s", reader->file, strerror(errno));
+    }
+
+    xh_log_debug1("file size: %lu", sb.st_size);
+
+    if (sb.st_size == 0) {
+        rb_raise(xh_parse_error_class, "File '%s' is empty", reader->file);
+    }
+    reader->len = sb.st_size;
+
+#ifdef WIN32
+    reader->fh = (HANDLE) _get_osfhandle(reader->fd);
+    if (reader->fh == INVALID_HANDLE_VALUE) {
+        rb_raise(xh_parse_error_class, "Can't get file handle of file '%s'", reader->file);
+    }
+
+    xh_log_debug1("create mapping for file %s", reader->file);
+    reader->fm = CreateFileMapping(reader->fh, NULL, PAGE_READONLY, 0, 0, NULL);
+    if (reader->fm == NULL) {
+        rb_raise(xh_parse_error_class, "Can't create file mapping of file '%s'", reader->file);
+    }
+
+    xh_log_debug1("create map view for file %s", reader->file);
+    reader->str = XH_CHAR_CAST MapViewOfFile(reader->fm, FILE_MAP_READ, 0, 0, reader->len);
+    if (reader->str == NULL) {
+        rb_raise(xh_parse_error_class, "Can't create map view of file '%s'", reader->file);
+    }
+#else
+    xh_log_debug1("mmap file %s", reader->file);
+    reader->str = XH_CHAR_CAST mmap((caddr_t) 0, reader->len, PROT_READ, MAP_PRIVATE, reader->fd, 0);
+    if ((caddr_t) reader->str == (caddr_t) (-1)) {
+        rb_raise(xh_parse_error_class, "Can't create map of file '%s': %s", reader->file, strerror(errno));
+    }
+#endif
+
+    reader->main_buf.start = reader->main_buf.cur = reader->str;
+    reader->main_buf.end   = reader->str + reader->len;
+
+    xh_common_reader_init(reader, input, encoding, buf_size);
+}
+
+static void
+xh_mmaped_file_reader_destroy(xh_reader_t *reader)
+{
+    xh_common_reader_destroy(reader);
+
+    if (reader->fd == -1) return;
+
+#ifdef WIN32
+    xh_log_debug1("unmap view of file %s", reader->file);
+    UnmapViewOfFile(reader->str);
+    xh_log_debug1("close handle of file %s", reader->file);
+    CloseHandle(reader->fm);
+#else
+    xh_log_debug1("munmap file %s", reader->file);
+    if (munmap(reader->str, reader->len) == -1) {
+        rb_raise(xh_parse_error_class, "Can't munmap file '%s': %s", reader->file, strerror(errno));
+    }
+#endif
+
+    xh_log_debug1("close file %s", reader->file);
+    if (close(reader->fd) == -1) {
+        rb_raise(xh_parse_error_class, "Can't close file '%s': %s", reader->file, strerror(errno));
+    }
+}
+#else
+static void
+xh_file_reader_init(xh_reader_t *reader, VALUE input, xh_char_t *encoding, size_t buf_size)
+{
+    reader->file = XH_CHAR_CAST SvPV_nolen(input);
+
+    xh_log_debug1("open file: %s", reader->file);
+
+    reader->fd = open((char *) reader->file, O_RDONLY);
+    if (reader->fd == -1) {
+        rb_raise(xh_parse_error_class, "Can't open file '%s': %s", reader->file, strerror(errno));
+    }
+
+    xh_buffer_init(&reader->main_buf, buf_size);
+
+    xh_common_reader_init(reader, input, encoding, buf_size);
+}
+
+static void
+xh_file_reader_destroy(xh_reader_t *reader)
+{
+    xh_common_reader_destroy(reader);
+
+    if (reader->main_buf.start != NULL)
+        free(reader->main_buf.start);
+
+    if (close(reader->fd) == -1) {
+        rb_raise(xh_parse_error_class, "Can't close file '%s': %s", reader->file, strerror(errno));
+    }
+}
+#endif /* XH_HAVE_MMAP */
+
+static size_t
+xh_file_reader_read(xh_reader_t *reader, xh_char_t **buf, xh_char_t *preserve, size_t *off)
+{
+    xh_char_t  *old_buf_addr;
+    size_t      len;
+    xh_buffer_t *main_buf;
+
+    main_buf = &reader->main_buf;
+    *off     = 0;
+
+    xh_log_debug1("read preserve: %p", preserve);
+    if (preserve == NULL) {
+        main_buf->cur = main_buf->start;
+    }
+    else {
+        *off = preserve - main_buf->start;
+        xh_log_debug1("off: %lu", *off);
+        if (*off) {
+            xh_log_debug3("memmove dest: %p src %p size: %lu", main_buf->start, preserve, main_buf->end - preserve);
+            xh_memmove(main_buf->start, preserve, main_buf->end - preserve);
+        }
+        main_buf->cur -= *off;
+        xh_log_debug1("read cur: %p", main_buf->cur);
+    }
+
+    old_buf_addr = main_buf->start;
+
+    xh_buffer_grow50(main_buf);
+
+    if (preserve != NULL && main_buf->start != old_buf_addr) {
+        *off += old_buf_addr - main_buf->start;
+    }
+
+    len = read(reader->fd, main_buf->cur, xh_buffer_avail(main_buf));
+    *buf = main_buf->cur;
+    if (len == (size_t) (-1)) {
+        rb_raise(xh_parse_error_class, "Failed to read file");
+    }
+    main_buf->cur += len;
+
+    return len;
+}
+
+#ifdef XH_HAVE_ENCODER
+static size_t
+xh_file_reader_read_with_encoding(xh_reader_t *reader, xh_char_t **buf, xh_char_t *preserve, size_t *off)
+{
+    xh_char_t   *old_buf_addr;
+    size_t       src_left, dst_left;
+    xh_buffer_t *main_buf, *enc_buf;
+
+    *off     = 0;
+    main_buf = &reader->main_buf;
+    enc_buf  = &reader->enc_buf;
+
+    xh_log_debug4("enc_buf: %p[%.*s] len: %lu", enc_buf->start, enc_buf->cur - enc_buf->start, enc_buf->cur, enc_buf->cur - enc_buf->start);
+
+    xh_log_debug1("preserve data: %p", preserve);
+    if (preserve == NULL) {
+        xh_buffer_seek_top(enc_buf);
+    }
+    else {
+        *off = preserve - enc_buf->start;
+        xh_log_debug1("off: %lu", *off);
+        if (*off) {
+            xh_log_debug3("memmove dest: %p src %p size: %lu", enc_buf->start, preserve, enc_buf->end - preserve);
+            xh_memmove(enc_buf->start, preserve, enc_buf->end - preserve);
+        }
+        enc_buf->cur -= *off;
+    }
+
+    old_buf_addr = enc_buf->start;
+    xh_buffer_grow50(enc_buf);
+
+    if (preserve != NULL && enc_buf->start != old_buf_addr) {
+        *off += old_buf_addr - enc_buf->start;
+    }
+
+    *buf = xh_buffer_pos(enc_buf);
+
+    while (enc_buf->cur < enc_buf->end) {
+        xh_buffer_grow50(main_buf);
+
+        if (reader->fake_read_pos == NULL) {
+            src_left = read(reader->fd, xh_buffer_pos(main_buf), xh_buffer_avail(main_buf));
+        }
+        else {
+            main_buf->cur         = reader->fake_read_pos;
+            src_left              = reader->fake_read_len;
+            reader->fake_read_pos = NULL;
+            reader->fake_read_len = 0;
+        }
+        if (src_left == 0) {
+            if (main_buf->cur == main_buf->end)
+                break;
+            rb_raise(xh_parse_error_class, "Truncate char found");
+        }
+        if (src_left == (size_t) (-1))
+            rb_raise(xh_parse_error_class, "Failed to read file");
+
+        dst_left = xh_buffer_avail(enc_buf);
+
+        xh_log_debug4("main_buf: %.*s src_left: %lu dst_left: %lu", src_left, main_buf->cur, src_left, dst_left);
+
+        xh_encoder_encode_string(reader->encoder, &main_buf->cur, &src_left, &enc_buf->cur, &dst_left);
+
+        xh_log_debug3("enc_buf: %.*s len: %lu", enc_buf->cur - enc_buf->start, enc_buf->start, enc_buf->cur - enc_buf->start);
+
+        switch (reader->encoder->state) {
+            case XH_ENC_TRUNCATED_CHAR_FOUND:
+                if (src_left == 0)
+                    rb_raise(xh_parse_error_class, "Truncated char found but buffer is empty");
+                xh_memmove(main_buf->start, main_buf->cur, src_left);
+                main_buf->cur = main_buf->start + src_left;
+                break;
+            case XH_ENC_BUFFER_OVERFLOW:
+            default:
+                xh_buffer_seek_top(main_buf);
+                goto DONE;
+        }
+    }
+
+DONE:
+    dst_left = enc_buf->cur - *buf;
+    xh_log_debug4("enc_buf: %p[%.*s] len: %lu", enc_buf->start, dst_left, enc_buf->cur, dst_left);
+
+    return dst_left;
+}
+#endif /* XH_HAVE_ENCODER */
+
+static void
+xh_file_reader_switch_encoding(xh_reader_t *reader, xh_char_t *encoding, xh_char_t **buf, size_t *len)
+{
+    xh_common_reader_switch_encoding(reader, encoding, buf, len);
+
+#ifdef XH_HAVE_ENCODER
+    reader->read = reader->encoder == NULL
+        ? xh_file_reader_read
+        : xh_file_reader_read_with_encoding;
+#endif
+}
+
+static void
+xh_ruby_io_reader_init(xh_reader_t *reader, VALUE input, xh_char_t *encoding, size_t buf_size)
+{
+    reader->fd = RFILE(reader->ruby_io)->fptr->fd;
+
+    xh_buffer_init(&reader->main_buf, buf_size);
+
+    xh_common_reader_init(reader, input, encoding, buf_size);
+}
+
+static void
+xh_ruby_io_reader_destroy(xh_reader_t *reader)
+{
+    xh_common_reader_destroy(reader);
+
+    if (reader->main_buf.start != NULL)
+        free(reader->main_buf.start);
+}
+
+void
+xh_reader_init(xh_reader_t *reader, VALUE input, xh_char_t *encoding, size_t buf_size)
+{
+    size_t     len;
+    xh_char_t *str;
+
+    if (RB_TYPE_P(input, T_STRING)) {
+        str = XH_CHAR_CAST RSTRING_PTR(input);
+        len = RSTRING_LEN(input);
+        if (len == 0)
+            rb_raise(xh_parse_error_class, "String is empty");
+
+        /* Parsing string */
+        if (xh_str_is_xml(str)) {
+            reader->type            = XH_READER_STRING_TYPE;
+            reader->init            = xh_string_reader_init;
+            reader->read            = xh_string_reader_read;
+            reader->switch_encoding = xh_string_reader_switch_encoding;
+            reader->destroy         = xh_string_reader_destroy;
+        }
+        /* Parsing file */
+        else {
+#ifdef XH_HAVE_MMAP
+            reader->type            = XH_READER_MMAPED_FILE_TYPE;
+            reader->init            = xh_mmaped_file_reader_init;
+            reader->read            = xh_string_reader_read;
+            reader->switch_encoding = xh_string_reader_switch_encoding;
+            reader->destroy         = xh_mmaped_file_reader_destroy;
+#else
+            reader->type            = XH_READER_FILE_TYPE;
+            reader->init            = xh_file_reader_init;
+            reader->read            = xh_file_reader_read;
+            reader->switch_encoding = xh_file_reader_switch_encoding;
+            reader->destroy         = xh_file_reader_destroy;
+#endif
+        }
+    }
+    else {
+        if (!RB_TYPE_P(input, T_FILE))
+            rb_raise(xh_parse_error_class, "Can't use file handle as a Ruby IO handle");
+
+        /* Ruby IO handle */
+        xh_log_debug0("Ruby IO handle detected");
+        reader->ruby_io         = input;
+        reader->type            = XH_READER_FILE_TYPE;
+        reader->init            = xh_ruby_io_reader_init;
+        reader->read            = xh_file_reader_read;
+        reader->switch_encoding = xh_file_reader_switch_encoding;
+        reader->destroy         = xh_ruby_io_reader_destroy;
+    }
+
+    reader->init(reader, input, encoding, buf_size);
+}
+
+void
+xh_reader_destroy(xh_reader_t *reader)
+{
+    if (reader->destroy != NULL)
+        reader->destroy(reader);
+}

data/ext/fastxml/xh_reader.h

@@ -0,0 +1,43 @@
+#ifndef _XH_READER_H_
+#define _XH_READER_H_
+
+#include "xh_config.h"
+#include "xh_core.h"
+
+typedef enum {
+    XH_READER_STRING_TYPE,
+    XH_READER_FILE_TYPE,
+    XH_READER_MMAPED_FILE_TYPE
+} xh_reader_type_t;
+
+typedef struct _xh_reader_t xh_reader_t;
+struct _xh_reader_t {
+    xh_reader_type_t  type;
+    VALUE             input;
+    xh_char_t        *str;
+    size_t            len;
+    xh_char_t        *file;
+    int               fd;
+    VALUE             ruby_io;
+#ifdef WIN32
+    HANDLE            fm, fh;
+#endif
+#ifdef XH_HAVE_ENCODER
+    xh_encoder_t     *encoder;
+    xh_buffer_t       enc_buf;
+#endif
+    xh_buffer_t       main_buf;
+    xh_ruby_buffer_t  ruby_buf;
+    xh_char_t        *fake_read_pos;
+    size_t            fake_read_len;
+    size_t            buf_size;
+    void              (*init)            (xh_reader_t *reader, VALUE input, xh_char_t *encoding, size_t buf_size);
+    size_t            (*read)            (xh_reader_t *reader, xh_char_t **buf, xh_char_t *preserve, size_t *off);
+    void              (*switch_encoding) (xh_reader_t *reader, xh_char_t *encoding, xh_char_t **buf, size_t *len);
+    void              (*destroy)         (xh_reader_t *reader);
+};
+
+void xh_reader_destroy(xh_reader_t *reader);
+void xh_reader_init(xh_reader_t *reader, VALUE input, xh_char_t *encoding, size_t buf_size);
+
+#endif /* _XH_READER_H_ */