zsv 1.3.0

data/ext/zsv/vendor/zsv-1.3.0/app/utils/file-mem.c

@@ -0,0 +1,180 @@
+#include <string.h>
+#include <errno.h>
+
+// TO DO: make this a standalone repo
+
+// The structure defining our dual-storage stream
+struct zsv_memfile {
+  size_t size;          // size of buffer
+  size_t used;          // Actual bytes written to memory
+  char *tmp_fn;         // name of temp file
+  FILE *tmp_f;          // temp FILE *
+  size_t total_written; // Total bytes written (memory + disk)
+  size_t read_offset;   // Current read position for the entire stream
+  bool write_mode;      // Flag to prevent writing after rewind
+  char buffer_start;    // start of buffer
+};
+typedef struct zsv_memfile zsv_memfile;
+
+// --- API Implementation ---
+/**
+ * Equivalent to fopen: Allocates and initializes the stream.
+ */
+zsv_memfile *zsv_memfile_open(size_t buffersize) {
+  zsv_memfile *zfm = malloc(sizeof(zsv_memfile) + buffersize);
+  if (!zfm)
+    return NULL;
+  memset(zfm, 0, sizeof(zsv_memfile));
+  zfm->size = buffersize;
+  zfm->write_mode = true;
+  return zfm;
+}
+
+size_t zsv_memfile_write(const void *data, size_t sz, size_t n, zsv_memfile *zfm) {
+  if (!zfm || !zfm->write_mode) {
+    errno = EPERM; // Operation not permitted
+    return 0;
+  }
+
+  size_t nbytes = sz * n;
+  if (nbytes == 0)
+    return 0;
+
+  const char *data_ptr = (const char *)data;
+  size_t remaining_bytes = nbytes;
+  size_t written_total = 0;
+
+  // 1. Write to Memory (until full)
+  if (zfm->used < zfm->size) {
+    size_t mem_space_avail = zfm->size - zfm->used;
+    size_t write_to_mem = (remaining_bytes < mem_space_avail) ? remaining_bytes : mem_space_avail;
+
+    memcpy(&zfm->buffer_start + zfm->used, data_ptr, write_to_mem);
+
+    zfm->used += write_to_mem;
+    zfm->total_written += write_to_mem;
+    data_ptr += write_to_mem;
+    remaining_bytes -= write_to_mem;
+    written_total += write_to_mem;
+
+    // If memory is now full, transition to disk if data remains
+    if (zfm->used == zfm->size && remaining_bytes > 0) {
+      // Allocate the temporary disk file. We use tmpfile() for simplicity and security.
+      zfm->tmp_f = zsv_tmpfile("zfm_", &zfm->tmp_fn, "wb+");
+      if (!zfm->tmp_f) {
+        perror("Failed to create temporary file");
+        return written_total; // Return what was successfully written to memory
+      }
+    }
+  }
+
+  // 2. Write to Disk (for overflow data)
+  if (remaining_bytes > 0 && zfm->tmp_f) {
+    size_t written_to_disk = fwrite(data_ptr, 1, remaining_bytes, zfm->tmp_f);
+
+    zfm->total_written += written_to_disk;
+    written_total += written_to_disk;
+  }
+
+  return written_total;
+}
+
+/**
+ * Equivalent to freopen (simplified): Switches stream to read-only mode and resets read pointer.
+ * Assumes no further writing will occur.
+ */
+int zsv_memfile_rewind(zsv_memfile *zfm) {
+  if (!zfm)
+    return -1;
+
+  // Transition to read-only mode
+  zfm->write_mode = false;
+
+  // Reset the overall read pointer to the start of the combined stream
+  zfm->read_offset = 0;
+
+  // Reset the disk file pointer if it exists, essential for the read logic.
+  if (zfm->tmp_f) {
+    rewind(zfm->tmp_f);
+  }
+  return 0;
+}
+
+/**
+ * Equivalent to fread: Reads data seamlessly from memory and then disk.
+ * This function enforces the seamless abstraction layer.
+ */
+size_t zsv_memfile_read(void *buffer, size_t size, size_t nitems, zsv_memfile *zfm) {
+  if (!zfm || zfm->write_mode) {
+    // Must call zsv_memfile_rewind() before reading
+    errno = EPERM;
+    return 0;
+  }
+
+  size_t nbytes = size * nitems;
+  char *buffer_ptr = (char *)buffer;
+  size_t remaining_bytes = nbytes;
+  size_t read_total = 0;
+
+  // Total available bytes to read across the entire stream
+  size_t available_bytes = zfm->total_written - zfm->read_offset;
+  if (available_bytes == 0)
+    return 0; // EOF
+
+  // Limit read request to available data
+  if (remaining_bytes > available_bytes) {
+    remaining_bytes = available_bytes;
+  }
+
+  // 1. Read from Memory
+  if (zfm->read_offset < zfm->size) {
+    // Calculate the starting position within the memory buffer
+    size_t mem_start = zfm->read_offset;
+
+    // Calculate how much memory data is left to read
+    size_t mem_data_left = zfm->used - mem_start;
+
+    // Calculate how much to read from memory in this call
+    size_t read_from_mem = (remaining_bytes < mem_data_left) ? remaining_bytes : mem_data_left;
+
+    if (read_from_mem > 0) {
+      memcpy(buffer_ptr, &zfm->buffer_start + mem_start, read_from_mem);
+
+      zfm->read_offset += read_from_mem;
+      buffer_ptr += read_from_mem;
+      remaining_bytes -= read_from_mem;
+      read_total += read_from_mem;
+    }
+  }
+
+  // 2. Read from Disk (if necessary)
+  if (remaining_bytes > 0 && zfm->tmp_f) {
+    // Note: The disk file pointer was already managed by zsv_memfile_rewind or is positioned correctly
+    // relative to the read_offset shift from step 1.
+    size_t read_from_disk = fread(buffer_ptr, 1, remaining_bytes, zfm->tmp_f);
+
+    zfm->read_offset += read_from_disk;
+    read_total += read_from_disk;
+  }
+
+  return read_total;
+}
+
+/**
+ * Equivalent to fclose: Cleans up resources.
+ */
+void zsv_memfile_close(zsv_memfile *zfm) {
+  if (!zfm)
+    return;
+
+  // Close and implicitly delete the temporary file if it was opened
+  if (zfm->tmp_f) {
+    fclose(zfm->tmp_f);
+    unlink(zfm->tmp_fn);
+  }
+  free(zfm->tmp_fn);
+  zfm->tmp_fn = NULL;
+
+  // Free the main structure
+  free(zfm);
+}

data/ext/zsv/vendor/zsv-1.3.0/app/utils/file.c

@@ -0,0 +1,256 @@
+/*
+ * Copyright (C) 2021 Liquidaty and the zsv/lib contributors
+ * All rights reserved
+ *
+ * This file is part of zsv/lib, distributed under the license defined at
+ * https://opensource.org/licenses/MIT
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <unistd.h> // for close()
+#include <fcntl.h>  // open
+
+#include <zsv/utils/dirs.h>
+#include <zsv/utils/os.h>
+#include <zsv/utils/file.h>
+
+/**
+ * Get a temp file name. The returned value, if any, will have been allocated
+ * on the heap, and the caller should `free()`
+ *
+ * @param prefix string with which the resulting file name will be prefixed
+ */
+#if defined(_WIN32) || defined(WIN32) || defined(WIN)
+#include <windows.h>
+
+char *zsv_get_temp_filename(const char *prefix) {
+  TCHAR lpTempPathBuffer[MAX_PATH];
+  DWORD dwRetVal = GetTempPath(MAX_PATH,          // length of the buffer
+                               lpTempPathBuffer); // buffer for path
+  if (!(dwRetVal > 0 && dwRetVal < MAX_PATH))
+    zsv_perror("GetTempPath");
+  else {
+    char szTempFileName[MAX_PATH];
+    UINT uRetVal = GetTempFileName(lpTempPathBuffer, // directory for tmp files
+                                   TEXT(prefix),     // temp file name prefix
+                                   0,                // create unique name
+                                   szTempFileName);  // buffer for name
+    if (uRetVal > 0)
+      return strdup(szTempFileName);
+    zsv_perror(lpTempPathBuffer);
+  }
+
+  return NULL;
+}
+#else
+
+char *zsv_get_temp_filename(const char *prefix) {
+  char *s = NULL;
+  char *tmpdir = getenv("TMPDIR");
+  if (!tmpdir)
+    tmpdir = ".";
+  asprintf(&s, "%s/%s_XXXXXXXX", tmpdir, prefix);
+  if (!s) {
+    const char *msg = strerror(errno);
+    fprintf(stderr, "%s%c%s: %s\n", tmpdir, FILESLASH, prefix, msg ? msg : "Unknown error");
+  } else {
+    int fd = mkstemp(s);
+    if (fd > 0) {
+      close(fd);
+      return s;
+    }
+    free(s);
+  }
+  return NULL;
+}
+
+#endif
+
+/**
+ *  Replacement for tmpfile().
+ *  Returns filename; file must be manually removed after fclose
+ *
+ *  @param mode optional mode passed to fopen(); if NULL, defaults to "wb"
+ */
+FILE *zsv_tmpfile(const char *prefix, char **filename, const char *mode) {
+  char *fn = zsv_get_temp_filename(prefix);
+  if (fn) {
+    FILE *f = fopen(fn, mode == NULL ? "wb" : mode);
+    if (f) {
+      *filename = fn;
+      return f;
+    }
+  }
+  int e = errno;
+  free(fn);
+  errno = e;
+  return NULL;
+}
+
+/**
+ * Temporarily redirect a FILE * (e.g. stdout / stderr) to a temp file
+ * temp_filename and bak are set as return values
+ * caller must free temp_filename
+ *
+ * @param old_fd file descriptor of file to dupe e.g. fileno(stdout);
+ * @return fd needed to pass on to zsv_redirect_file_from_temp
+ */
+#if defined(_WIN32) || defined(__FreeBSD__)
+#include <sys/stat.h> // S_IRUSR S_IWUSR
+#endif
+
+int zsv_redirect_file_to_temp(FILE *f, const char *tempfile_prefix, char **temp_filename) {
+  int new_fd;
+  int old_fd = fileno(f);
+  fflush(f);
+  int bak = dup(old_fd);
+  *temp_filename = zsv_get_temp_filename(tempfile_prefix);
+
+  new_fd = open(*temp_filename, O_WRONLY | O_CREAT, S_IRUSR | S_IWUSR);
+
+  dup2(new_fd, old_fd);
+  close(new_fd);
+  return bak;
+}
+
+/**
+ * Restore a FILE * that was redirected by zsv_redirect_file_to_temp()
+ */
+void zsv_redirect_file_from_temp(FILE *f, int bak, int old_fd) {
+  fflush(f);
+  dup2(bak, old_fd);
+  close(bak);
+}
+
+#if defined(_WIN32) || defined(WIN32) || defined(WIN)
+int zsv_file_exists(const char *filename) {
+  DWORD attributes = GetFileAttributes(filename);
+  return (attributes != INVALID_FILE_ATTRIBUTES && !(attributes & FILE_ATTRIBUTE_DIRECTORY));
+}
+#else
+#include <sys/stat.h> // S_IRUSR S_IWUSR
+
+int zsv_file_exists(const char *filename) {
+  struct stat buffer;
+  if (stat(filename, &buffer) == 0) {
+    char is_dir = buffer.st_mode & S_IFDIR ? 1 : 0;
+    if (!is_dir)
+      return 1;
+  }
+  return 0;
+}
+#endif
+
+/**
+ * Copy a file, given source and destination paths
+ * On error, output error message and return non-zero
+ */
+int zsv_copy_file(const char *src, const char *dest) {
+  // create one or more directories if needed
+  if (zsv_mkdirs(dest, 1)) {
+    fprintf(stderr, "Unable to create directories needed for %s\n", dest);
+    return -1;
+  }
+
+  // copy the file
+  int err = 0;
+  FILE *fsrc = zsv_fopen(src, "rb");
+  if (!fsrc)
+    err = errno ? errno : -1, perror(src);
+  else {
+    FILE *fdest = zsv_fopen(dest, "wb");
+    if (!fdest)
+      err = errno ? errno : -1, perror(dest);
+    else {
+      err = zsv_copy_file_ptr(fsrc, fdest);
+      if (err)
+        perror(dest);
+      fclose(fdest);
+    }
+    fclose(fsrc);
+  }
+  return err;
+}
+
+/**
+ * Copy a file-like, given source and destination handles
+ * and read/write functions
+ * Return error number per errno.h
+ */
+int zsv_copy_filelike_ptr(
+  FILE *src, size_t (*freadx)(void *restrict ptr, size_t size, size_t nitems, void *restrict stream), FILE *dest,
+  size_t (*fwritex)(const void *restrict ptr, size_t size, size_t nitems, void *restrict stream)) {
+  int err = 0;
+  char buffer[4096 * 16];
+  size_t bytes_read;
+  while ((bytes_read = freadx(buffer, 1, sizeof(buffer), src)) > 0) {
+    if (fwritex(buffer, 1, bytes_read, dest) != bytes_read) {
+      err = errno ? errno : -1;
+      break;
+    }
+  }
+  return err;
+}
+
+/**
+ * Copy a file, given source and destination FILE pointers
+ * Return error number per errno.h
+ */
+int zsv_copy_file_ptr(FILE *src, FILE *dest) {
+  return zsv_copy_filelike_ptr(
+    src, (size_t(*)(void *restrict ptr, size_t size, size_t nitems, void *restrict stream))fread, dest,
+    (size_t(*)(const void *restrict ptr, size_t size, size_t nitems, void *restrict stream))fwrite);
+}
+
+size_t zsv_dir_len_basename(const char *filepath, const char **basename) {
+  for (size_t len = strlen(filepath); len; len--) {
+    if (filepath[len - 1] == '/' || filepath[len - 1] == '\\') {
+      *basename = filepath + len;
+      return len - 1;
+    }
+  }
+
+  *basename = filepath;
+  return 0;
+}
+
+int zsv_file_readable(const char *filename, int *err, FILE **f_out) {
+  FILE *f;
+  int rc;
+  if (err)
+    *err = 0;
+  // to do: use fstat()
+  if ((f = zsv_fopen(filename, "rb")) == NULL) {
+    rc = 0;
+    if (err)
+      *err = errno;
+    else
+      perror(filename);
+  } else {
+    rc = 1;
+    if (f_out)
+      *f_out = f;
+    else
+      fclose(f);
+  }
+  return rc;
+}
+
+/**
+ * Function that is the same as `fwrite()`, but can be used as a callback
+ * argument to `zsv_set_scan_filter()`
+ */
+size_t zsv_filter_write(void *FILEp, unsigned char *buff, size_t bytes_read) {
+  fwrite(buff, 1, bytes_read, (FILE *)FILEp);
+  return bytes_read;
+}
+
+int zsv_no_printf(void *_ctx, const char *_format, ...) {
+  // do nothing!
+  return 0;
+}
+
+#include "file-mem.c"

data/ext/zsv/vendor/zsv-1.3.0/app/utils/index.c

@@ -0,0 +1,197 @@
+#include <assert.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <zsv.h>
+#include <zsv/utils/prop.h>
+#include "index.h"
+
+struct zsv_index *zsv_index_new(void) {
+  struct zsv_index *ix = calloc(1, sizeof(*ix));
+
+  if (!ix)
+    return ix;
+
+  const size_t init_cap = 512;
+  ix->first = calloc(1, sizeof(*ix->first) + init_cap * sizeof(ix->first->u64s[0]));
+  ix->first->capacity = init_cap;
+
+  return ix;
+}
+
+void zsv_index_delete(struct zsv_index *ix) {
+  if (ix) {
+    struct zsv_index_array *arr = ix->first;
+
+    while (arr) {
+      struct zsv_index_array *a = arr;
+      arr = arr->next;
+      free(a);
+    }
+
+    free(ix);
+  }
+}
+
+enum zsv_index_status zsv_index_add_row(struct zsv_index *ix, uint64_t line_end) {
+  struct zsv_index_array *arr = ix->first;
+  size_t len = arr->len, cap = arr->capacity;
+
+  if (!ix->header_line_end) {
+    ix->header_line_end = line_end;
+    return zsv_index_status_ok;
+  }
+
+  ix->row_count_local++;
+
+  if ((ix->row_count_local & (ZSV_INDEX_ROW_N - 1)) != 0)
+    return zsv_index_status_ok;
+
+  while (len >= cap) {
+    assert(len == cap);
+
+    if (!arr->next) {
+      len = 0;
+      cap *= 2;
+      arr->next = calloc(1, sizeof(*arr) + cap * sizeof(arr->u64s[0]));
+      arr = arr->next;
+      if (!arr)
+        return zsv_index_status_memory;
+      arr->capacity = cap;
+    } else {
+      arr = arr->next;
+      len = arr->len;
+      cap = arr->capacity;
+    }
+  }
+
+  arr->u64s[len] = line_end;
+  arr->len++;
+
+  return zsv_index_status_ok;
+}
+
+void zsv_index_commit_rows(struct zsv_index *ix) {
+  ix->row_count = ix->row_count_local;
+}
+
+enum zsv_index_status zsv_index_row_end_offset(const struct zsv_index *ix, uint64_t row, uint64_t *offset_out,
+                                               uint64_t *remaining_rows_out) {
+  assert(ix->row_count <= ix->row_count_local);
+
+  if (row > ix->row_count)
+    return zsv_index_status_error;
+
+  if (row < ZSV_INDEX_ROW_N) {
+    *offset_out = ix->header_line_end;
+    *remaining_rows_out = row;
+
+    return zsv_index_status_ok;
+  }
+
+  const size_t i = (row >> ZSV_INDEX_ROW_SHIFT) - 1;
+  struct zsv_index_array *arr = ix->first;
+  size_t lens = 0;
+
+  while (i >= lens + arr->len) {
+    assert(arr->next);
+
+    lens += arr->len;
+    arr = arr->next;
+  }
+
+  *offset_out = (long)arr->u64s[i - lens];
+  *remaining_rows_out = row & (ZSV_INDEX_ROW_N - 1);
+
+  return zsv_index_status_ok;
+}
+
+struct seek_row_ctx {
+  uint64_t remaining_rows;
+  zsv_parser parser;
+};
+
+static void seek_row_handler(void *ctx) {
+  struct seek_row_ctx *c = ctx;
+
+  c->remaining_rows--;
+  if (c->remaining_rows > 0)
+    return;
+
+  zsv_abort(c->parser);
+}
+
+static enum zsv_index_status seek_and_check_newline(long *offset, struct zsv_opts *opts) {
+  char new_line[2];
+  zsv_generic_read read = (zsv_generic_read)fread;
+  zsv_generic_seek seek = (zsv_generic_seek)fseek;
+  FILE *stream = opts->stream;
+
+  if (opts->seek)
+    seek = opts->seek;
+
+  if (opts->read)
+    read = opts->read;
+
+  if (seek(stream, *offset, SEEK_SET))
+    return zsv_index_status_error;
+
+  size_t nmemb = read(new_line, 1, 2, stream);
+
+  if (nmemb < 1)
+    return zsv_index_status_error;
+
+  if (new_line[0] == '\n') {
+    *offset += 1;
+  } else if (new_line[0] == '\r') {
+    *offset += 1;
+
+    if (new_line[1] == '\n')
+      *offset += 1;
+  } else {
+    return zsv_index_status_error;
+  }
+
+  if (seek(stream, *offset, SEEK_SET))
+    return zsv_index_status_error;
+
+  return zsv_index_status_ok;
+}
+
+enum zsv_index_status zsv_index_seek_row(const struct zsv_index *ix, struct zsv_opts *opts, uint64_t row) {
+  uint64_t offset;
+  uint64_t remaining_rows;
+  enum zsv_index_status zist = zsv_index_row_end_offset(ix, row, &offset, &remaining_rows);
+
+  if (zist != zsv_index_status_ok)
+    return zist;
+
+  if ((zist = seek_and_check_newline((long *)&offset, opts)) != zsv_index_status_ok)
+    return zist;
+
+  if (!remaining_rows)
+    return zsv_index_status_ok;
+
+  struct seek_row_ctx ctx = {
+    .remaining_rows = remaining_rows,
+  };
+  struct zsv_opts o;
+  memcpy(&o, opts, sizeof(o));
+  o.ctx = &ctx;
+  o.row_handler = seek_row_handler;
+  zsv_parser parser = zsv_new(&o);
+  ctx.parser = parser;
+
+  enum zsv_status zst;
+  while ((zst = zsv_parse_more(parser)) == zsv_status_ok)
+    ;
+
+  if (zst != zsv_status_cancelled)
+    return zsv_index_status_error;
+
+  offset += zsv_cum_scanned_length(parser);
+
+  zsv_delete(parser);
+
+  return seek_and_check_newline((long *)&offset, opts);
+}

data/ext/zsv/vendor/zsv-1.3.0/app/utils/index.h

@@ -0,0 +1,49 @@
+#ifndef ZSV_UTILS_INDEX_H
+#define ZSV_UTILS_INDEX_H
+
+#include <stdint.h>
+#include <stddef.h>
+
+#include "zsv/common.h"
+
+// Decides the number of rows we skip when storing the line end
+// 1 << 10 = 1024 means that we store every 1024th line end
+#define ZSV_INDEX_ROW_SHIFT 10
+#define ZSV_INDEX_ROW_N (1 << ZSV_INDEX_ROW_SHIFT)
+
+enum zsv_index_status {
+  zsv_index_status_ok = 0,
+  zsv_index_status_memory,
+  zsv_index_status_error,
+  zsv_index_status_utf8,
+};
+
+// An array of uint64_t. Needs to be reallocated to extend the capacity.
+// Reallocation can be avoided by adding new arrays instead.
+struct zsv_index_array {
+  size_t capacity;
+  size_t len;
+  struct zsv_index_array *next;
+  uint64_t u64s[];
+};
+
+struct zsv_index {
+  uint64_t header_line_end;
+  // Reading and writing should be protected with a lock
+  uint64_t row_count;
+  // Should only be updated by the thread building the index
+  uint64_t row_count_local;
+
+  // array containing the offsets of every ZSV_INDEX_ROW_N line end
+  struct zsv_index_array *first;
+};
+
+struct zsv_index *zsv_index_new(void);
+void zsv_index_delete(struct zsv_index *ix);
+enum zsv_index_status zsv_index_add_row(struct zsv_index *ix, uint64_t line_end);
+void zsv_index_commit_rows(struct zsv_index *ix);
+enum zsv_index_status zsv_index_row_end_offset(const struct zsv_index *ix, uint64_t row, uint64_t *offset_out,
+                                               uint64_t *remaining_rows_out);
+enum zsv_index_status zsv_index_seek_row(const struct zsv_index *ix, struct zsv_opts *opts, uint64_t row);
+
+#endif