zsv 1.3.0

data/ext/zsv/vendor/zsv-1.3.0/src/zsv_internal.c

@@ -0,0 +1,731 @@
+/*
+ * Copyright (C) 2021 Tai Chi Minh Ralph Eastwood (self), Matt Wong (Guarnerix Inc dba Liquidaty)
+ * All rights reserved
+ *
+ * This file is part of zsv/lib, distributed under the license defined at
+ * https://opensource.org/licenses/MIT
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <ctype.h>
+#include <stdint.h> // uint32_t
+
+#ifdef ZSV_EXTRAS
+#include <time.h>
+#endif
+
+#include <zsv/utils/utf8.h>
+#include <zsv/utils/compiler.h>
+#include <zsv/utils/string.h>
+
+#if !defined(__AVX2__) // -mavx2 compiler flag not present
+#define ZSV_NO_AVX
+#define zsv_mask_t uint16_t
+#define VECTOR_BYTES 16
+#define NEXT_BIT __builtin_ffs
+#if defined(__AVX__)
+#include <emmintrin.h>
+#define zsv_mask_t uint16_t
+#define VECTOR_BYTES 16
+#define NEXT_BIT __builtin_ffs
+#define movemask_pseudo(x) _mm_movemask_epi8((__m128i)x)
+#endif
+#elif defined(HAVE_AVX512)
+#ifndef __AVX512BW__
+#error AVX512 requested, but __AVX512BW__ macro not defined
+#else
+#include <immintrin.h>
+#define VECTOR_BYTES 64
+#define zsv_mask_t uint64_t
+#define movemask_pseudo(x) _mm512_movepi8_mask((__m512i)x)
+#define NEXT_BIT __builtin_ffsl
+#endif
+#elif defined(__AVX2__) // have avx2, not avx512
+#include <immintrin.h>
+#define VECTOR_BYTES 32
+#define zsv_mask_t uint32_t
+#define movemask_pseudo(x) _mm256_movemask_epi8((__m256i)x)
+#define NEXT_BIT __builtin_ffs
+#else
+#define ZSV_NO_AVX
+#define zsv_mask_t uint16_t
+#define VECTOR_BYTES 16
+#define NEXT_BIT __builtin_ffs
+#endif
+
+typedef unsigned char zsv_uc_vector __attribute__((vector_size(VECTOR_BYTES)));
+
+struct zsv_row {
+  size_t used, allocated, overflow;
+  struct zsv_cell *cells;
+};
+
+struct collate_header {
+  struct {
+    unsigned char *buff;
+    size_t used;
+  } buff;
+  size_t *lengths; // length PLUS 1 of each cell
+  size_t column_count;
+};
+
+struct zsv_scan_delim_regs {
+  size_t i;
+  size_t bytes_chunk_end;
+  size_t bytes_read;
+  char delimiter;
+  unsigned char c;
+  char skip_next_delim;
+  int quote;
+  size_t mask_total_offset;
+  zsv_mask_t mask;
+  int mask_last_start;
+  unsigned char location;
+};
+
+struct zsv_scan_fixed_regs {
+  char xx; // to do
+};
+
+#ifdef ZSV_EXTRAS
+#include <inttypes.h>
+#include <sqlite3.h>
+
+struct zsv_overwrite {
+  struct zsv_overwrite_data odata;
+  void *ctx;
+  enum zsv_status (*next)(void *ctx, struct zsv_overwrite_data *odata);
+  enum zsv_status (*close)(void *ctx);
+};
+#endif
+
+struct zsv_scanner {
+  char last;
+  struct {
+    unsigned char *buff; // provided by caller
+    size_t size;         // provided by caller
+  } buff;
+
+  size_t cell_start;
+  unsigned char quoted : 7; // bitfield of ZSV_PARSER_QUOTE_XXX flags
+  unsigned char buffer_exceeded : 1;
+
+  unsigned char waiting_for_end : 1;
+  unsigned char checked_bom : 1;
+  unsigned char free_buff : 1;
+  unsigned char finished : 1;
+  unsigned char had_bom : 1;
+  unsigned char abort : 1;
+  unsigned char have_cell : 1;
+  unsigned char started : 1;
+
+  size_t quote_close_position;
+  struct zsv_opts opts;
+
+  size_t row_start;
+  struct zsv_row row;
+
+  size_t scanned_length;
+  size_t cum_scanned_length;
+  size_t partial_row_length;
+
+  size_t (*read)(void *buff, size_t n, size_t size, void *in);
+  void *in;
+
+  size_t (*filter)(void *ctx, unsigned char *buff, size_t bytes_read);
+  void *filter_ctx;
+
+  size_t buffer_end;
+  size_t old_bytes_read; // only non-zero if we must shift upon next parse_more()
+
+  const char *insert_string;
+
+  size_t empty_header_rows;
+
+  struct zsv_opts opts_orig;
+
+#define ZSV_MODE_DELIM 0
+#define ZSV_MODE_FIXED 1
+#define ZSV_MODE_DELIM_PULL 2
+  unsigned char mode;
+  struct {
+    unsigned *offsets; // 0-based position of each cell end. offset[0] = end of first cell
+    unsigned count;    // number of offsets
+  } fixed;
+
+  struct collate_header *collate_header;
+  size_t data_row_count; /* 0 = in header row; 1 = first data row */
+  struct zsv_cell (*get_cell)(zsv_parser parser, size_t ix);
+
+  struct {
+    union {
+      struct zsv_scan_delim_regs delim;
+      struct zsv_scan_fixed_regs fixed;
+    } *regs;
+    enum zsv_status stat; // last status
+    unsigned char *buff;
+    size_t bytes_read;
+    size_t row_used;
+    unsigned char now;
+  } pull;
+
+  int (*errprintf)(void *ctx, const char *format, ...);
+  void *errf;
+  int (*errclose)(void *ctx);
+
+#ifdef ZSV_EXTRAS
+  struct {
+    size_t cum_row_count; /* total number of rows read */
+    time_t last_time;     /* last time from which to check seconds_interval */
+    size_t max_rows;      /* max rows to read, including header row(s) */
+  } progress;
+  struct zsv_overwrite overwrite;
+#endif
+};
+
+void collate_header_destroy(struct collate_header **chp) {
+  if (*chp) {
+    struct collate_header *ch = *chp;
+    free(ch->buff.buff);
+    free(ch->lengths);
+    free(ch);
+    *chp = NULL;
+  }
+}
+
+/* collate_header_append(): return err */
+static int collate_header_append(struct zsv_scanner *scanner, struct collate_header **chp) {
+  if (!*chp) {
+    if ((*chp = calloc(1, sizeof(struct collate_header))))
+      (*chp)->lengths = calloc(scanner->row.allocated, sizeof(*(*chp)->lengths));
+    if (!(*chp) || !(*chp)->lengths) {
+      free(*chp);
+      scanner->errprintf(scanner->errf, "Out of memory!\n");
+      return -1;
+    }
+  }
+  struct collate_header *ch = *chp;
+  size_t this_row_size = 0;
+  size_t column_count = zsv_cell_count(scanner);
+  for (size_t i = 0, j = column_count; i < j; i++) {
+    struct zsv_cell c = zsv_get_cell_1(scanner, i);
+    if (c.len)
+      this_row_size += c.len + 1; // +1: terminating null or delim
+  }
+  size_t new_row_size = ch->buff.used + this_row_size;
+  unsigned char *new_row = realloc(ch->buff.buff, new_row_size);
+  if (!new_row) {
+    scanner->errprintf(scanner->errf, "Out of memory!\n");
+    return -1;
+  }
+
+  // now: splice the new row into the old row, starting with the last cell
+  // e.g. prior row = A1.B1.C1.
+  //      this row =  A2.B2.C2.
+  //      new_row =   A1.B1.C1..........
+  // starting with last cell in this row, move the old data, then splice new:
+  //      new_row =   A1.B1.C1.......C2.
+  //      new_row =   A1.B1.C1....C1 C2.
+  //      new_row =   A1.B1.C1.B2.C1 C2.
+  //      new_row =   A1.B1.B1 B2.C1 C2.
+  //      new_row =   A1.A2.B1 B2.C1 C2.
+  //      new_row =   A1 A2.B1 B2.C1 C2.
+
+  size_t new_row_end = ch->buff.used + this_row_size;
+  size_t old_row_end = ch->buff.used;
+  ch->buff.used += this_row_size;
+  ch->buff.buff = new_row;
+  for (size_t i = column_count; i > 0; i--) {
+    struct zsv_cell c = zsv_get_cell_1(scanner, i - 1);
+    // copy new row's cell value to end
+    if (c.len) {
+      memcpy(new_row + new_row_end - c.len - 1, c.str, c.len);
+      new_row[new_row_end - 1] = ' ';
+      new_row_end = new_row_end - c.len - 1;
+    }
+
+    // move prior cell value
+    size_t old_cell_len = ch->lengths[i - 1]; // old_cell_len includes delim
+    if (old_cell_len) {
+      // need memmove, not memcpy
+      memmove(new_row + new_row_end - old_cell_len, new_row + old_row_end - old_cell_len, old_cell_len);
+      old_row_end -= old_cell_len;
+      new_row_end -= old_cell_len;
+    }
+    if (c.len)
+      ch->lengths[i - 1] += c.len + 1;
+  }
+  if (column_count > ch->column_count)
+    ch->column_count = column_count;
+  return 0;
+}
+
+__attribute__((always_inline)) static inline void zsv_clear_cell(struct zsv_scanner *scanner) {
+  scanner->quoted = 0;
+}
+
+// always_inline has a noticeable impact. do not remove without benchmarking!
+__attribute__((always_inline)) static inline void cell_dl(struct zsv_scanner *scanner, unsigned char *s, size_t n) {
+  // handle quoting
+  if (VERY_LIKELY(!scanner->buffer_exceeded)) {
+    if (UNLIKELY(scanner->quoted > 0)) {
+      if (LIKELY(scanner->quote_close_position + 1 == n)) {
+        if (LIKELY((scanner->quoted & ZSV_PARSER_QUOTE_EMBEDDED) == 0)) {
+          // this is the easy and usual case: no embedded double-quotes
+          // just remove surrounding quotes from content
+          s++;
+          n -= 2;
+        } else { // embedded dbl-quotes to remove
+          s++;
+          n--;
+          // remove dbl-quotes. TO DO: consider adding option to skip this
+          for (size_t i = 0; i + 1 < n; i++) {
+            if (s[i] == '"' && s[i + 1] == '"') {
+              if (n > i + 2)
+                memmove(s + i + 1, s + i + 2, n - i - 2);
+              n--;
+            }
+          }
+          n--;
+        }
+      } else {
+        if (scanner->quote_close_position) {
+          // the first char was a quote, and we have content after the closing quote
+          // the solution below is a generalized on that will work
+          // for the easy and usual case, but by handling separately
+          // we avoid the memmove in the easy / usual case
+          memmove(s + 1, s, scanner->quote_close_position);
+          s += 2;
+          n -= 2;
+          if (UNLIKELY((scanner->quoted & ZSV_PARSER_QUOTE_EMBEDDED) != 0)) {
+            // remove dbl-quotes
+            for (size_t i = 0; i + 1 < n; i++) {
+              if (s[i] == '"' && s[i + 1] == '"') {
+                if (n > i + 2)
+                  memmove(s + i + 1, s + i + 2, n - i - 2);
+                n--;
+              }
+            }
+          }
+        }
+      }
+    } else if (UNLIKELY(scanner->opts.delimiter != ',')) {
+      if (memchr(s, ',', n))
+        scanner->quoted = ZSV_PARSER_QUOTE_NEEDED;
+    }
+    // end quote handling
+
+    if (scanner->opts.malformed_utf8_replace) {
+      if (scanner->opts.malformed_utf8_replace < 0)
+        n = zsv_strencode(s, n, 0, NULL, NULL);
+      else
+        n = zsv_strencode(s, n, scanner->opts.malformed_utf8_replace, NULL, NULL);
+    }
+  }
+  if (UNLIKELY(scanner->opts.cell_handler != NULL))
+    scanner->opts.cell_handler(scanner->opts.ctx, s, n);
+  if (VERY_LIKELY(scanner->row.used < scanner->row.allocated)) {
+    struct zsv_row *row = &scanner->row;
+    struct zsv_cell c = {s, n, scanner->opts.no_quotes ? 1 : scanner->quoted, 0};
+    row->cells[row->used++] = c;
+  } else
+    scanner->row.overflow++;
+  scanner->have_cell = 1;
+
+  zsv_clear_cell(scanner);
+}
+
+__attribute__((always_inline)) static inline enum zsv_status row_dl(struct zsv_scanner *scanner) {
+  if (VERY_UNLIKELY(scanner->row.overflow)) {
+    scanner->errprintf(scanner->errf, "Warning: number of columns (%zu) exceeds row max (%zu)\n",
+                       scanner->row.allocated + scanner->row.overflow, scanner->row.allocated);
+    scanner->row.overflow = 0;
+  }
+  if (VERY_LIKELY(scanner->opts.row_handler != NULL)) // TO DO: disallow row_handler to be null; if null, set to dummy
+    scanner->opts.row_handler(scanner->opts.ctx);
+    // Note: scanner->data_row_count will be incremented AFTER this call
+    //       in order to accommodate pull parsing, in which case incrementing here
+    //       would be too early
+#ifdef ZSV_EXTRAS
+  scanner->progress.cum_row_count++;
+  if (VERY_UNLIKELY(scanner->opts.progress.rows_interval &&
+                    scanner->progress.cum_row_count % scanner->opts.progress.rows_interval == 0)) {
+    char ok;
+    if (!scanner->opts.progress.seconds_interval)
+      ok = 1;
+    else {
+      // using timer_create() would be better, but is not currently supported on
+      // all platforms, so the fallback is to poll
+      time_t now = time(NULL);
+      if (now > scanner->progress.last_time &&
+          (unsigned int)(now - scanner->progress.last_time) >= scanner->opts.progress.seconds_interval) {
+        ok = 1;
+        scanner->progress.last_time = now;
+      } else
+        ok = 0;
+    }
+    if (ok && scanner->opts.progress.callback)
+      scanner->abort = scanner->opts.progress.callback(scanner->opts.progress.ctx, scanner->progress.cum_row_count);
+#ifndef NDEBUG
+    if (scanner->abort)
+      scanner->errprintf(scanner->errf, "ZSV parsing aborted at %zu\n", scanner->progress.cum_row_count);
+#endif
+  }
+  if (VERY_UNLIKELY(scanner->progress.max_rows > 0)) {
+    if (VERY_UNLIKELY(scanner->progress.cum_row_count == scanner->progress.max_rows)) {
+      scanner->abort = 1;
+      scanner->row.used = 0;
+      return zsv_status_max_rows_read;
+    }
+  }
+
+#endif
+  if (VERY_UNLIKELY(scanner->abort))
+    return zsv_status_cancelled;
+  scanner->have_cell = 0;
+  scanner->row.used = 0;
+  return zsv_status_ok;
+}
+
+__attribute__((always_inline)) static inline enum zsv_status cell_and_row_dl(struct zsv_scanner *scanner,
+                                                                             unsigned char *s, size_t n) {
+  cell_dl(scanner, s, n);
+  return row_dl(scanner);
+}
+
+#ifndef movemask_pseudo
+/*
+  provide our own pseudo-movemask, which sets the 1 bit for each corresponding
+  non-zero value in the vector (as opposed to real movemask which sets the bit
+  only for each corresponding non-zero highest-bit value in the vector)
+*/
+
+#if defined(__EMSCRIPTEN__) && defined(__SSE2__)
+#include <wasm_simd128.h>
+#define movemask_pseudo(x) wasm_i8x16_bitmask(x)
+
+#elif defined(__ARM_NEON) || defined(__ARM_NEON__)
+#include <arm_neon.h>
+static inline zsv_mask_t movemask_pseudo(zsv_uc_vector v) {
+  // see https://stackoverflow.com/questions/11870910/
+  static const uint8_t
+    __attribute__((aligned(16))) _powers[16] = {1, 2, 4, 8, 16, 32, 64, 128, 1, 2, 4, 8, 16, 32, 64, 128};
+  uint8x16_t mm_powers = vld1q_u8(_powers);
+
+  // compute the mask from the input
+  uint64x2_t imask = vpaddlq_u32(vpaddlq_u16(vpaddlq_u8(vandq_u8(v, mm_powers))));
+
+  // Get the resulting bytes
+  uint16_t mask;
+  vst1q_lane_u8((uint8_t *)&mask + 0, (uint8x16_t)imask, 0);
+  vst1q_lane_u8((uint8_t *)&mask + 1, (uint8x16_t)imask, 8);
+  return mask;
+}
+
+#elif defined(__SSE2__)
+
+typedef char zsv_c_vector __attribute__((vector_size(VECTOR_BYTES)));
+#define movemask_pseudo(x) __builtin_ia32_pmovmskb128((zsv_c_vector)x)
+
+#else
+
+// slow path
+#if defined(__EMSCRIPTEN__)
+#warning                                                                                                               \
+  "Compiling with emscripten, without using SIMD. To use SIMD, compile with -msse2 -msimd128 -experimental-wasm-simd and -I/path/to/emsdk/upstream/lib/clang/16.0.0/include"
+#endif
+
+static inline zsv_mask_t movemask_pseudo(zsv_uc_vector v) {
+  zsv_mask_t mask = 0, tmp = 1;
+  for (size_t i = 0; i < sizeof(zsv_uc_vector); i++) {
+    mask |= (v[i] ? tmp : 0);
+    tmp <<= 1;
+  }
+
+  return mask;
+}
+
+#endif // __EMSCRIPTEN__
+#endif // ndef movemask_pseudo
+
+#include "vector_delim.c"
+
+#ifdef ZSV_SUPPORT_PULL_PARSER
+#undef ZSV_SUPPORT_PULL_PARSER
+#endif
+#define ZSV_SCAN_DELIM zsv_scan_delim
+#include "zsv_scan_delim.c"
+#undef ZSV_SCAN_DELIM
+#undef scanner_last
+
+#define ZSV_SUPPORT_PULL_PARSER 1
+#define ZSV_SCAN_DELIM zsv_scan_delim_pull
+#include "zsv_scan_delim.c"
+
+#include "zsv_scan_fixed.c"
+
+static enum zsv_status zsv_scan(struct zsv_scanner *scanner, unsigned char *buff, size_t bytes_read) {
+  switch (scanner->mode) {
+  case ZSV_MODE_FIXED:
+    return zsv_scan_fixed(scanner, buff, bytes_read);
+  case ZSV_MODE_DELIM_PULL:
+    // return zsv_status_row or zsv_status_ok (next call to parse_more)
+    return zsv_scan_delim_pull(scanner, buff, bytes_read);
+  default:
+    return zsv_scan_delim(scanner, buff, bytes_read);
+  }
+}
+
+#define ZSV_BOM "\xef\xbb\xbf"
+
+// optional: set a filter function to filter data before it is processed
+// function should return the number of bytes to process. this may be smaller
+// but may not be larger than the original number of bytes, and any data modification
+// must be done in-place to *buff
+enum zsv_status zsv_set_scan_filter(struct zsv_scanner *scanner,
+                                    size_t (*filter)(void *ctx, unsigned char *buff, size_t bytes_read), void *ctx) {
+  scanner->filter = filter;
+  scanner->filter_ctx = ctx;
+  return zsv_status_ok;
+}
+
+static void apply_callbacks(struct zsv_scanner *scanner) {
+  if (UNLIKELY(scanner->opts.cell_handler != NULL)) {
+    // call the user-provided cell() callback on each cell
+    unsigned char saved_quoted = scanner->quoted;
+    for (size_t i = 0, j = zsv_cell_count(scanner); i < j; i++) {
+      struct zsv_cell c = zsv_get_cell_1(scanner, i);
+      scanner->quoted = c.quoted;
+      scanner->opts.cell_handler(scanner->opts.ctx, c.str, c.len);
+    }
+    scanner->quoted = saved_quoted;
+  }
+  // call the user-provided row() callback
+  if (VERY_LIKELY(scanner->opts.row_handler != NULL))
+    scanner->opts.row_handler(scanner->opts.ctx);
+}
+
+static void set_callbacks(struct zsv_scanner *scanner);
+
+static char zsv_internal_row_is_blank(zsv_parser parser) {
+  for (unsigned int i = 0; i < parser->row.used; i++)
+    if (parser->row.cells[i].len)
+      return 0;
+  return 1;
+}
+
+static void skip_to_first_row_w_data(void *ctx) {
+  struct zsv_scanner *scanner = ctx;
+  if (LIKELY(zsv_internal_row_is_blank(scanner) == 0)) {
+    scanner->opts.keep_empty_header_rows = 1;
+    if (scanner->empty_header_rows) {
+      scanner->errprintf(scanner->errf, "Warning: skipped %zu empty header rows; suggest using:\n  --skip-head %zu\n",
+                         scanner->empty_header_rows, scanner->empty_header_rows + scanner->opts_orig.rows_to_ignore);
+    }
+    set_callbacks(scanner);
+    apply_callbacks(scanner);
+  } else // entire row was empty
+    scanner->empty_header_rows++;
+}
+
+static void ignore_header_rows(void *ctx) {
+  struct zsv_scanner *scanner = ctx;
+  if (scanner->opts.rows_to_ignore)
+    scanner->opts.rows_to_ignore--;
+  if (!scanner->opts.rows_to_ignore)
+    set_callbacks(scanner);
+}
+
+static void collate_header_row(void *ctx) {
+  struct zsv_scanner *scanner = ctx;
+  if (scanner->opts.header_span) {
+    --scanner->opts.header_span;
+
+    // save this row
+
+    // first, make sure this row has at least as many cells as the largest prior row
+    if (scanner->collate_header) {
+      for (size_t i = zsv_cell_count(scanner); i < scanner->row.allocated && i < scanner->collate_header->column_count;
+           i++)
+        memset(&scanner->row.cells[i], 0, sizeof(scanner->row.cells[i]));
+      scanner->row.used = scanner->collate_header->column_count;
+    }
+
+    if (collate_header_append(scanner, &scanner->collate_header))
+      scanner->abort = 1;
+  }
+
+  if (!scanner->opts.header_span) {
+    // finished with header; combine all rows into a single row
+    set_callbacks(scanner);
+    if (scanner->collate_header) {
+      size_t offset = 0;
+      for (size_t i = 0; i < scanner->collate_header->column_count; i++) {
+        size_t len_plus1 = scanner->collate_header->lengths[i];
+        scanner->row.cells[i].str = scanner->collate_header->buff.buff + offset;
+        if (len_plus1) {
+          scanner->row.cells[i].len = len_plus1 - 1;
+          scanner->row.cells[i].quoted = 1;
+        } else
+          scanner->row.cells[i].len = 0;
+        offset += len_plus1;
+      }
+    }
+
+    apply_callbacks(scanner);
+    if (scanner->mode != ZSV_MODE_DELIM_PULL)
+      collate_header_destroy(&scanner->collate_header);
+  }
+}
+
+static void set_callbacks(struct zsv_scanner *scanner) {
+  if (scanner->opts.rows_to_ignore) {
+    scanner->opts.row_handler = ignore_header_rows;
+    scanner->opts.cell_handler = NULL;
+    scanner->opts.ctx = scanner;
+  } else if (scanner->mode != ZSV_MODE_FIXED && !scanner->opts.keep_empty_header_rows) {
+    scanner->opts.row_handler = skip_to_first_row_w_data;
+    scanner->opts.cell_handler = NULL;
+    scanner->opts.ctx = scanner;
+  } else if (scanner->opts.header_span > 1) {
+    scanner->opts.row_handler = collate_header_row;
+    scanner->opts.cell_handler = NULL;
+    scanner->opts.ctx = scanner;
+  } else {
+#ifdef ZSV_EXTRAS
+    if (scanner->overwrite.odata.have)
+      scanner->get_cell = zsv_get_cell_with_overwrite;
+    else
+#endif
+      scanner->get_cell = zsv_get_cell_1;
+    scanner->data_row_count = 0;
+    scanner->opts.row_handler = scanner->opts_orig.row_handler;
+    scanner->opts.cell_handler = scanner->opts_orig.cell_handler;
+    scanner->opts.ctx = scanner->opts_orig.ctx;
+  }
+}
+
+static void zsv_throwaway_row(void *ctx) {
+  struct zsv_scanner *scanner = ctx;
+  if (scanner->opts.overflow_row_handler != NULL) {
+    if (zsv_cell_count(scanner) > 1 || zsv_get_cell_1(scanner, 0).len > 0)
+      scanner->opts.overflow_row_handler(ctx);
+  }
+  scanner->buffer_exceeded = 0;
+  set_callbacks(ctx);
+}
+
+#ifdef ZSV_EXTRAS
+
+static int zsv_have_overwrite(zsv_parser parser, size_t row_ix, size_t col_ix) {
+  struct zsv_overwrite *overwrite = &parser->overwrite;
+  while (overwrite->odata.have && overwrite->odata.row_ix < row_ix)
+    overwrite->next(overwrite->ctx, &overwrite->odata);
+  while (overwrite->odata.have && overwrite->odata.row_ix == row_ix && overwrite->odata.col_ix < col_ix)
+    overwrite->next(overwrite->ctx, &overwrite->odata);
+  if (!overwrite->odata.have)
+    parser->get_cell = zsv_get_cell_1;
+  return overwrite->odata.have && overwrite->odata.row_ix == row_ix && overwrite->odata.col_ix == col_ix;
+}
+
+static struct zsv_cell zsv_get_cell_with_overwrite(zsv_parser parser, size_t col_ix) {
+  if (VERY_LIKELY(col_ix < parser->row.used)) {
+    size_t row_ix = parser->data_row_count;
+    if (!zsv_have_overwrite(parser, row_ix, col_ix))
+      return parser->row.cells[col_ix];
+
+    struct zsv_cell c = parser->overwrite.odata.val;
+    c.overwritten = 1;
+    return c;
+  }
+  struct zsv_cell c = {0, 0, 0, 0};
+  return c;
+}
+#endif
+
+static int zsv_scanner_init(struct zsv_scanner *scanner, struct zsv_opts *opts) {
+  size_t need_buff_size = 0;
+  scanner->errprintf = opts->errprintf ? opts->errprintf : zsv_generic_fprintf;
+  scanner->errf = opts->errf ? opts->errf : stderr;
+  scanner->errclose = opts->errclose;
+  if (opts->malformed_utf8_replace == ZSV_MALFORMED_UTF8_DO_NOT_REPLACE)
+    opts->malformed_utf8_replace = 0;
+  if (opts->buffsize < opts->max_row_size * 2)
+    need_buff_size = opts->max_row_size * 2;
+  opts->delimiter = opts->delimiter ? opts->delimiter : ',';
+  if (opts->delimiter == '\n' || opts->delimiter == '\r' || opts->delimiter == '"') {
+    scanner->errprintf(scanner->errf, "warning: ignoring illegal delimiter\n");
+    opts->delimiter = ',';
+  }
+
+  if (opts->insert_header_row)
+    scanner->insert_string = opts->insert_header_row;
+
+  if (need_buff_size < ZSV_MIN_SCANNER_BUFFSIZE)
+    need_buff_size = ZSV_MIN_SCANNER_BUFFSIZE;
+  if (opts->buffsize < need_buff_size) {
+    if (opts->buffsize > 0) {
+      if (need_buff_size == ZSV_MIN_SCANNER_BUFFSIZE)
+        scanner->errprintf(scanner->errf, "Increasing --buff-size to minimum %zu\n", need_buff_size);
+      else
+        scanner->errprintf(scanner->errf, "Increasing --buff-size to %zu to accommmodate max-row-size of %u\n",
+                           need_buff_size, opts->max_row_size);
+    }
+    opts->buffsize = need_buff_size;
+  }
+  if (opts->buffsize == 0)
+    opts->buffsize = ZSV_DEFAULT_SCANNER_BUFFSIZE;
+  else if (opts->buffsize < ZSV_MIN_SCANNER_BUFFSIZE)
+    opts->buffsize = ZSV_MIN_SCANNER_BUFFSIZE;
+
+  scanner->in = opts->stream;
+  if (!opts->read) {
+    scanner->read = (zsv_generic_read)fread;
+    if (!opts->stream)
+      scanner->in = stdin;
+  } else {
+    scanner->read = opts->read;
+    scanner->in = opts->stream;
+  }
+  scanner->buff.buff = opts->buff;
+  scanner->buff.size = opts->buffsize;
+
+  if (opts->buffsize && !opts->buff) {
+    scanner->buff.buff = malloc(opts->buffsize);
+    scanner->free_buff = 1;
+  }
+
+#ifdef ZSV_EXTRAS
+  if (opts->max_rows)
+    scanner->progress.max_rows = opts->max_rows;
+#endif
+  if (scanner->buff.buff) {
+    scanner->opts = *opts;
+    scanner->opts_orig = *opts;
+    if (!scanner->opts.max_columns)
+      scanner->opts.max_columns = 1024;
+    set_callbacks(scanner);
+    if ((scanner->row.allocated = scanner->opts.max_columns) &&
+        (scanner->row.cells = calloc(scanner->row.allocated, sizeof(*scanner->row.cells)))) {
+#ifdef ZSV_EXTRAS
+      // initialize overwrites
+      if (scanner->opts.overwrite.open && !scanner->opts.overwrite.cancel) {
+        if (scanner->opts.overwrite.open(scanner->opts.overwrite.ctx) == zsv_status_ok) {
+          scanner->overwrite.odata.have = 1;
+          scanner->overwrite.next = scanner->opts.overwrite.next;
+          scanner->overwrite.close = scanner->opts.overwrite.close;
+          scanner->overwrite.ctx = scanner->opts.overwrite.ctx;
+          // load the first overwrite
+          scanner->overwrite.next(scanner->overwrite.ctx, &scanner->overwrite.odata);
+          return 0;
+        }
+        return 1;
+      }
+#endif
+      return 0;
+    }
+  }
+  return 1;
+}