zsv 1.3.0

data/ext/zsv/vendor/zsv-1.3.0/src/zsv_scan_delim.c

@@ -0,0 +1,285 @@
+#ifdef ZSV_SUPPORT_PULL_PARSER
+
+#define zsv_internal_save_reg(x) scanner->pull.regs->delim.x = x
+#define zsv_internal_save_regs(loc)                                                                                    \
+  do {                                                                                                                 \
+    scanner->pull.regs->delim.location = loc;                                                                          \
+    scanner->pull.buff = buff;                                                                                         \
+    scanner->pull.bytes_read = bytes_read;                                                                             \
+    zsv_internal_save_reg(i);                                                                                          \
+    zsv_internal_save_reg(bytes_chunk_end);                                                                            \
+    zsv_internal_save_reg(bytes_read);                                                                                 \
+    zsv_internal_save_reg(delimiter);                                                                                  \
+    zsv_internal_save_reg(c);                                                                                          \
+    zsv_internal_save_reg(skip_next_delim);                                                                            \
+    zsv_internal_save_reg(quote);                                                                                      \
+    zsv_internal_save_reg(mask_total_offset);                                                                          \
+    zsv_internal_save_reg(mask);                                                                                       \
+    zsv_internal_save_reg(mask_last_start);                                                                            \
+  } while (0)
+
+#define zsv_internal_restore_reg(x) x = scanner->pull.regs->delim.x
+#define zsv_internal_restore_regs()                                                                                    \
+  do {                                                                                                                 \
+    buff = scanner->pull.buff;                                                                                         \
+    bytes_read = scanner->pull.bytes_read;                                                                             \
+    zsv_internal_restore_reg(i);                                                                                       \
+    zsv_internal_restore_reg(bytes_chunk_end);                                                                         \
+    zsv_internal_restore_reg(bytes_read);                                                                              \
+    zsv_internal_restore_reg(delimiter);                                                                               \
+    zsv_internal_restore_reg(c);                                                                                       \
+    zsv_internal_restore_reg(skip_next_delim);                                                                         \
+    zsv_internal_restore_reg(quote);                                                                                   \
+    zsv_internal_restore_reg(mask_total_offset);                                                                       \
+    zsv_internal_restore_reg(mask);                                                                                    \
+    zsv_internal_restore_reg(mask_last_start);                                                                         \
+    memset(&v.dl, scanner->opts.delimiter, sizeof(zsv_uc_vector));                                                     \
+    memset(&v.nl, '\n', sizeof(zsv_uc_vector));                                                                        \
+    memset(&v.cr, '\r', sizeof(zsv_uc_vector));                                                                        \
+    memset(&v.qt, scanner->opts.no_quotes > 0 ? 0 : '"', sizeof(v.qt));                                                \
+  } while (0)
+#endif
+
+static enum zsv_status ZSV_SCAN_DELIM(struct zsv_scanner *scanner, unsigned char *buff, size_t bytes_read) {
+  struct {
+    zsv_uc_vector dl;
+    zsv_uc_vector nl;
+    zsv_uc_vector cr;
+    zsv_uc_vector qt;
+  } v;
+
+  size_t i;
+  size_t bytes_chunk_end;
+  char delimiter;
+  unsigned char c;
+  char skip_next_delim;
+  int quote;
+  size_t mask_total_offset;
+  zsv_mask_t mask;
+  int mask_last_start;
+
+#ifdef ZSV_SUPPORT_PULL_PARSER
+  if (scanner->pull.regs->delim.location) {
+    zsv_internal_restore_regs();
+    if (scanner->pull.regs->delim.location == 1)
+      goto zsv_cell_and_row_dl_1;
+    goto zsv_cell_and_row_dl_2;
+  }
+#endif
+  bytes_read += scanner->partial_row_length;
+  i = scanner->partial_row_length;
+  skip_next_delim = 0;
+  bytes_chunk_end = bytes_read >= sizeof(zsv_uc_vector) ? bytes_read - sizeof(zsv_uc_vector) + 1 : 0;
+  delimiter = scanner->opts.delimiter;
+  scanner->partial_row_length = 0;
+
+  // to do: move into one-time execution code?
+  // (but, will also locate away from function stack)
+  quote = scanner->opts.no_quotes > 0 ? -1 : '"';  // ascii code 34
+  memset(&v.dl, delimiter, sizeof(zsv_uc_vector)); // ascii code 44
+  memset(&v.nl, '\n', sizeof(zsv_uc_vector));      // ascii code 10
+  memset(&v.cr, '\r', sizeof(zsv_uc_vector));      // ascii code 13
+  memset(&v.qt, scanner->opts.no_quotes > 0 ? 0 : '"', sizeof(v.qt));
+
+  if (scanner->quoted & ZSV_PARSER_QUOTE_PENDING) {
+    // if we're here, then the last chunk we read ended with a lone quote char inside
+    // a quoted cell, and we are waiting to find out whether it is followed by
+    // another dbl-quote e.g. if the end of the last chunk is |, we had:
+    //    ...,"hel"|"o"
+    //    ...,"hel"|,...
+    //    ...,"hel"|p,...
+    scanner->quoted -= ZSV_PARSER_QUOTE_PENDING;
+    if (buff[i] != quote) {
+      scanner->quoted |= ZSV_PARSER_QUOTE_CLOSED;
+      scanner->quoted &= ~ZSV_PARSER_QUOTE_UNCLOSED; // scanner->quoted -= ZSV_PARSER_QUOTE_UNCLOSED;
+      scanner->quote_close_position = i - scanner->cell_start - 1;
+    } else {
+      scanner->quoted |= ZSV_PARSER_QUOTE_NEEDED;
+      scanner->quoted |= ZSV_PARSER_QUOTE_EMBEDDED;
+      i++;
+    }
+  }
+
+#define scanner_last (i ? buff[i - 1] : scanner->last)
+
+  mask_total_offset = 0;
+  mask = 0;
+  scanner->buffer_end = bytes_read;
+  for (; i < bytes_read; i++) {
+    if (UNLIKELY(mask == 0)) {
+      mask_last_start = i;
+      if (VERY_LIKELY(i < bytes_chunk_end)) {
+        // keep going until we get a delim or we are at the eof
+        mask_total_offset = vec_delims(buff + i, bytes_read - i, &v.dl, &v.nl, &v.cr, &v.qt, &mask);
+        if (LIKELY(mask_total_offset != 0)) {
+          i += mask_total_offset;
+          if (VERY_UNLIKELY(mask == 0 && i == bytes_read))
+            break; // vector processing ended on exactly our buffer end
+        }
+      } else if (skip_next_delim) {
+        skip_next_delim = 0;
+        continue;
+      }
+    }
+    if (VERY_LIKELY(mask)) {
+      size_t next_offset = NEXT_BIT(mask);
+      i = mask_last_start + next_offset - 1;
+      mask = clear_lowest_bit(mask);
+      if (VERY_UNLIKELY(skip_next_delim)) {
+        skip_next_delim = 0;
+        continue;
+      }
+    }
+
+    // to do: consolidate csv and tsv/scanner->delimiter parsers
+    c = buff[i];
+    if (LIKELY(c == delimiter)) { // case ',':
+      if ((scanner->quoted & ZSV_PARSER_QUOTE_UNCLOSED) == 0) {
+        scanner->scanned_length = i;
+        cell_dl(scanner, buff + scanner->cell_start, i - scanner->cell_start);
+        scanner->cell_start = i + 1;
+        c = 0;
+        continue; // this char is not part of the cell content
+      } else
+        // we are inside an open quote, which is needed to escape this char
+        scanner->quoted |= ZSV_PARSER_QUOTE_NEEDED;
+    } else if (UNLIKELY(c == '\r')) {
+#ifndef ZSV_NO_ONLY_CRLF
+      if (VERY_UNLIKELY(scanner->opts.only_crlf_rowend)) {
+        if (scanner->quoted & ZSV_PARSER_QUOTE_PENDING_LF)
+          // if we already had a lone \r in this cell,
+          // flip the flag to ZSV_PARSER_QUOTE_NEEDED
+          scanner->quoted |= ZSV_PARSER_QUOTE_NEEDED;
+        else
+          // otherwise this is the first \r in this cell,
+          // so set ZSV_PARSER_QUOTE_PENDING_LF, which
+          // will be removed if the next char is LF
+          scanner->quoted |= ZSV_PARSER_QUOTE_PENDING_LF;
+        continue;
+      }
+#endif
+      if ((scanner->quoted & ZSV_PARSER_QUOTE_UNCLOSED) == 0) {
+        scanner->scanned_length = i;
+        enum zsv_status stat = cell_and_row_dl(scanner, buff + scanner->cell_start, i - scanner->cell_start);
+        if (VERY_UNLIKELY(stat))
+          return stat;
+#ifdef ZSV_SUPPORT_PULL_PARSER
+        if (scanner->pull.now) {
+          scanner->pull.now = 0;
+          scanner->row.used = scanner->pull.row_used;
+          zsv_internal_save_regs(1);
+          return zsv_status_row;
+        }
+      zsv_cell_and_row_dl_1:
+        scanner->row.used = 0;
+        scanner->pull.regs->delim.location = 0;
+#endif
+        scanner->cell_start = i + 1;
+        scanner->row_start = i + 1;
+        scanner->data_row_count++;
+        continue; // this char is not part of the cell content
+      } else
+        // we are inside an open quote, which is needed to escape this char
+        scanner->quoted |= ZSV_PARSER_QUOTE_NEEDED;
+    } else if (UNLIKELY(c == '\n')) {
+      if ((scanner->quoted & ZSV_PARSER_QUOTE_UNCLOSED) == 0) {
+        int is_crlf = (scanner_last == '\r');
+
+        // Handle logic for when we should SKIP this char (not a row end)
+#ifndef ZSV_NO_ONLY_CRLF
+        if (VERY_UNLIKELY(scanner->opts.only_crlf_rowend)) {
+          if (!is_crlf) {
+            scanner->quoted |= ZSV_PARSER_QUOTE_NEEDED;
+            continue; // only-crlf mode: ignore lone \n
+          } else
+            // remove ZSV_PARSER_QUOTE_PENDING_LF if we have it
+            scanner->quoted &= ~ZSV_PARSER_QUOTE_PENDING_LF;
+        } else
+#endif
+        {
+          if (is_crlf) {
+            // Standard mode: ignore \n because \r already handled the row end
+            scanner->cell_start = i + 1;
+            scanner->row_start = i + 1;
+            continue;
+          }
+        }
+
+        // If we reached here, this is a row end
+        scanner->scanned_length = i;
+
+        // Calculate cell length. In only-crlf mode, we must exclude the preceding \r
+        size_t cell_len = i - scanner->cell_start;
+#ifndef ZSV_NO_ONLY_CRLF
+        if (VERY_UNLIKELY(scanner->opts.only_crlf_rowend))
+          cell_len--;
+#endif
+        enum zsv_status stat = cell_and_row_dl(scanner, buff + scanner->cell_start, cell_len);
+        if (VERY_UNLIKELY(stat))
+          return stat;
+#ifdef ZSV_SUPPORT_PULL_PARSER
+        if (scanner->pull.now) {
+          scanner->pull.now = 0;
+          scanner->row.used = scanner->pull.row_used;
+          zsv_internal_save_regs(2);
+          return zsv_status_row;
+        }
+      zsv_cell_and_row_dl_2:
+        scanner->row.used = 0;
+        scanner->pull.regs->delim.location = 0;
+#endif
+        scanner->cell_start = i + 1;
+        scanner->row_start = i + 1;
+        scanner->data_row_count++;
+        continue; // this char is not part of the cell content
+      } else
+        // we are inside an open quote, which is needed to escape this char
+        scanner->quoted |= ZSV_PARSER_QUOTE_NEEDED;
+    } else if (LIKELY(c == quote)) {
+      if (i == scanner->cell_start && !scanner->buffer_exceeded) {
+        scanner->quoted = ZSV_PARSER_QUOTE_UNCLOSED;
+        scanner->quote_close_position = 0;
+        c = 0;
+      } else if (scanner->quoted & ZSV_PARSER_QUOTE_UNCLOSED) {
+        // the cell started with a quote that is not yet closed
+        if (VERY_LIKELY(i + 1 < bytes_read)) {
+          if (LIKELY(buff[i + 1] != quote)) {
+            // buff[i] is the closing quote (not an escaped quote)
+            scanner->quoted |= ZSV_PARSER_QUOTE_CLOSED;
+            scanner->quoted -= ZSV_PARSER_QUOTE_UNCLOSED;
+
+            // keep track of closing quote position to handle the edge case
+            // where content follows the closing quote e.g. cell content is:
+            //   "this-cell"-did-not-need-quotes
+            if (LIKELY(scanner->quote_close_position == 0))
+              scanner->quote_close_position = i - scanner->cell_start;
+          } else {
+            // next char is also '"'
+            // e.g. cell content is: "this "" is a dbl quote"
+            //            cursor is here => ^
+            // include in cell content and don't further process
+            scanner->quoted |= ZSV_PARSER_QUOTE_NEEDED;
+            scanner->quoted |= ZSV_PARSER_QUOTE_EMBEDDED;
+            skip_next_delim = 1;
+          }
+        } else // we are at the end of this input chunk
+          scanner->quoted |= ZSV_PARSER_QUOTE_PENDING;
+      } else {
+        // cell_length > 0 and cell did not start w quote, so
+        // we have a quote in middle of an unquoted cell
+        // process as a normal char
+        scanner->quoted |= ZSV_PARSER_QUOTE_EMBEDDED;
+        scanner->quote_close_position = scanner->quoted & ZSV_PARSER_QUOTE_CLOSED ? scanner->quote_close_position : 0;
+      }
+    }
+  }
+  scanner->scanned_length = i;
+
+  // save bytes_read-- we will need to shift any remaining partial row
+  // before we read next from our input. however, we intentionally refrain
+  // from doing this until the next parse_more() call, so that the entirety
+  // of all rows parsed thus far are still available until that next call
+  scanner->old_bytes_read = bytes_read;
+
+  return zsv_status_ok;
+}

data/ext/zsv/vendor/zsv-1.3.0/src/zsv_scan_fixed.c

@@ -0,0 +1,88 @@
+static inline char row_fx(struct zsv_scanner *scanner, unsigned char *buff, size_t row_start, size_t row_end) {
+  size_t cell_start = row_start;
+  size_t row_length = row_end - row_start;
+  for (unsigned i = 0; i < scanner->fixed.count; i++) {
+    size_t cell_end = row_start + (scanner->fixed.offsets[i] > row_length ? row_length : scanner->fixed.offsets[i]);
+    size_t cell_length = cell_end - cell_start;
+    unsigned char *s = buff + cell_start;
+    if (UNLIKELY(scanner->opts.cell_handler != NULL))
+      scanner->opts.cell_handler(scanner->opts.ctx, s, cell_length);
+    struct zsv_cell c = {s, cell_length, 1, 0};
+    scanner->row.cells[scanner->row.used++] = c;
+
+    cell_start = cell_end;
+  }
+  if (VERY_LIKELY(scanner->opts.row_handler != NULL))
+    scanner->opts.row_handler(scanner->opts.ctx);
+  scanner->row.used = 0;
+  return scanner->abort;
+}
+
+static enum zsv_status zsv_scan_fixed(struct zsv_scanner *scanner, unsigned char *buff, size_t bytes_read) {
+  bytes_read += scanner->partial_row_length;
+  unsigned char c;
+  size_t bytes_chunk_end = bytes_read >= sizeof(zsv_uc_vector) ? bytes_read - sizeof(zsv_uc_vector) + 1 : 0;
+
+  scanner->partial_row_length = 0;
+
+  // dl_v and qt_v are unused, we just leave them to reuse vec_delims()
+  zsv_uc_vector dl_v;
+  memset(&dl_v, 0, sizeof(zsv_uc_vector));
+  zsv_uc_vector nl_v;
+  memset(&nl_v, '\n', sizeof(zsv_uc_vector));
+  zsv_uc_vector cr_v;
+  memset(&cr_v, '\r', sizeof(zsv_uc_vector));
+  zsv_uc_vector qt_v;
+  memset(&qt_v, 0, sizeof(zsv_uc_vector));
+  size_t mask_total_offset = 0;
+  zsv_mask_t mask = 0;
+  int mask_last_start = 0;
+
+  scanner->buffer_end = bytes_read;
+  for (size_t i = scanner->partial_row_length;; i++) {
+    if (UNLIKELY(mask == 0)) {
+      mask_last_start = i;
+      if (LIKELY(i < bytes_chunk_end)) {
+        // keep going until we get a delim or we are at the eof
+        mask_total_offset = vec_delims(buff + i, bytes_read - i, &dl_v, &nl_v, &cr_v, &qt_v, &mask);
+        if (mask_total_offset)
+          i += mask_total_offset;
+      } else { // we only have a few bytes left, so manually parse
+        for (unsigned i2 = i; i2 < bytes_read; i2++)
+          if (strchr("\n\r", buff[i2]))
+            mask += 1 << (i2 - i);
+      }
+      if (UNLIKELY(mask == 0))
+        break;
+    }
+
+    size_t next_offset = __builtin_ffs(mask);
+    i = mask_last_start + next_offset - 1;
+    mask = clear_lowest_bit(mask);
+
+    c = buff[i];
+    if (LIKELY(c == '\n')) {
+      if (scanner_last == '\r') { // ignore; we are outside a cell and last char was rowend
+        scanner->row_start = i + 1;
+      } else {
+        // this is a row end
+        scanner->scanned_length = i;
+        if (VERY_UNLIKELY(row_fx(scanner, buff, scanner->row_start, i)))
+          return zsv_status_cancelled; // abort
+        scanner->row_start = i + 1;
+      }
+    } else if (UNLIKELY(c == '\r')) {
+      scanner->scanned_length = i;
+      if (VERY_UNLIKELY(row_fx(scanner, buff, scanner->row_start, i)))
+        return zsv_status_cancelled;
+      scanner->row_start = i + 1;
+    }
+  }
+
+  // save bytes_read-- we will need to shift any remaining partial row
+  // before we read next from our input. however, we intentionally refrain
+  // from doing this until the next parse_more() call, so that the entirety
+  // of all rows parsed thus far are still available until that next call
+  scanner->old_bytes_read = bytes_read;
+  return zsv_status_ok;
+}

data/ext/zsv/vendor/zsv-1.3.0/src/zsv_strencode.c

@@ -0,0 +1,51 @@
+/*
+ * zsv_strencode(): standalone file to allow zsv utilities that use this
+ * to be used on a standalone basis without the zsv parser
+ *
+ * This file is part of zsv/lib, distributed under the license defined at
+ * https://opensource.org/licenses/MIT
+ */
+
+#include <zsv/utils/utf8.h>
+#include <zsv/utils/compiler.h>
+
+/**
+ * Ensure valid UTF8 encoding by, if needed, replacing malformed bytes
+ */
+ZSV_EXPORT
+size_t zsv_strencode(unsigned char *s, size_t n, unsigned char replace,
+                     int (*malformed_handler)(void *, const unsigned char *s, size_t n, size_t offset),
+                     void *handler_ctx) {
+  size_t new_len = 0;
+  int clen;
+  for (size_t i2 = 0; i2 < n; i2 += (size_t)clen) {
+    clen = ZSV_UTF8_CHARLEN(s[i2]);
+    if (LIKELY(clen == 1))
+      s[new_len++] = s[i2];
+    else if (UNLIKELY(clen < 0) || UNLIKELY(i2 + clen > n)) {
+      if (malformed_handler)
+        malformed_handler(handler_ctx, s, n, new_len);
+      if (replace)
+        s[new_len++] = replace;
+      clen = 1;
+    } else { /* might be valid multi-byte utf8; check */
+      unsigned char valid_n;
+      for (valid_n = 1; valid_n < clen; valid_n++)
+        if (!ZSV_UTF8_SUBSEQUENT_CHAR_OK(s[i2 + valid_n]))
+          break;
+      if (valid_n == clen) { /* valid_n utf8; copy it */
+        memmove(s + new_len, s + i2, clen);
+        new_len += clen;
+      } else { /* invalid; valid_n smaller than expected */
+        if (malformed_handler)
+          malformed_handler(handler_ctx, s, n, new_len);
+        if (replace) {
+          memset(s + new_len, replace, valid_n);
+          new_len += valid_n;
+        }
+        clen = valid_n;
+      }
+    }
+  }
+  return new_len; // new length
+}