zsv 1.3.0 → 1.4.3

data/ext/zsv/vendor/zsv-1.3.0/app/rm.c

@@ -1,131 +0,0 @@
-/* Copyright (C) 2022 Guarnerix Inc dba Liquidaty - All Rights Reserved
- * Unauthorized copying of this file, via any medium is strictly prohibited
- * Proprietary and confidential
- * Written by Matt Wong <matt@guarnerix.com>
- */
-
-/*
- * remove a given file and its cache
- */
-
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <unistd.h> // unlink()
-#include <errno.h>
-
-#define ZSV_COMMAND_NO_OPTIONS
-#define ZSV_COMMAND rm
-#include "zsv_command.h"
-
-#include <zsv/utils/dirs.h>
-#include <zsv/utils/cache.h>
-
-/**
- * TO DO: add --orphaned option to remove all orphaned caches
- */
-const char *zsv_rm_usage_msg[] = {
-  APPNAME ": remove a file and its related cache",
-  "",
-  "Usage: " APPNAME " [options] <filepath>",
-  "",
-  "Options:",
-  "  -v,--verbose   : verbose output",
-#ifndef NO_STDIN
-  "  -f,--force     : do not prompt for confirmation",
-#endif
-  "  -k,--keep      : do not remove related cache",
-  "  -C,--cache     : only remove related cache (not the file)",
-  NULL,
-};
-
-static int zsv_rm_usage(FILE *target) {
-  for (size_t j = 0; zsv_rm_usage_msg[j]; j++)
-    fprintf(target, "%s\n", zsv_rm_usage_msg[j]);
-  return target == stdout ? 0 : 1;
-}
-
-int ZSV_MAIN_NO_OPTIONS_FUNC(ZSV_COMMAND)(int argc, const char *argv[]) {
-  int err = 0;
-  if (argc > 1 && (!strcmp(argv[1], "-h") || !strcmp(argv[1], "--help")))
-    err = zsv_rm_usage(stdout);
-  else if (argc < 2)
-    err = zsv_rm_usage(stderr);
-  else {
-    const char *filepath = NULL;
-    char force = 0;
-#ifdef NO_STDIN
-    force = 1;
-#endif
-    char remove_cache = 1;
-    char remove_file = 1;
-    char verbose = 0;
-    for (int i = 1; !err && i < argc; i++) {
-      const char *arg = argv[i];
-      if (*arg == '-') {
-        if (!strcmp(arg, "-v") || !strcmp(arg, "--verbose"))
-          verbose = 1;
-        else if (!strcmp(arg, "-f") || !strcmp(arg, "--force"))
-          force = 1;
-        else if (!strcmp(arg, "-k") || !strcmp(arg, "--keep"))
-          remove_cache = 0;
-        else if (!strcmp(arg, "-C") || !strcmp(arg, "--cache"))
-          remove_file = 0;
-        else
-          err = zsv_printerr(1, "Unrecognized option: %s", arg);
-      } else if (!filepath)
-        filepath = arg;
-      else
-        err = zsv_printerr(1, "Unrecognized option: %s", arg);
-    }
-
-    if (!err && !filepath)
-      err = zsv_rm_usage(stderr);
-    else if (remove_file == 0 && remove_cache == 0)
-      err = fprintf(stderr, "Nothing to remove\n");
-
-    if (!err) {
-      char ok = 1;
-#ifndef NO_STDIN
-      if (!force) {
-        ok = 0;
-        if (!remove_file)
-          printf("Are you sure you want to remove the entire cache for the file %s?\n", filepath);
-        else
-          printf("Are you sure you want to remove the file %s%s?\n", filepath,
-                 remove_cache ? " and all of its cache contents" : "");
-        char buff[64];
-        if (fscanf(stdin, "%60s", buff) == 1 && strchr("Yy", buff[0]))
-          ok = 1;
-      }
-#endif
-      if (ok) {
-        if (remove_file) {
-          if (verbose)
-            fprintf(stderr, "Removing %s", filepath);
-          err = unlink(filepath);
-          if (err) {
-            if (err == ENOENT && force)
-              err = 0;
-            else
-              perror(filepath);
-          }
-        }
-        if (!err) {
-          unsigned char *cache_dir = zsv_cache_path((const unsigned char *)filepath, NULL, 0);
-          if (!cache_dir)
-            err = zsv_printerr(ENOMEM, "Out of memory!");
-          else if (zsv_dir_exists((const char *)cache_dir)) {
-            err = zsv_remove_dir_recursive(cache_dir);
-            if (verbose) {
-              if (!err)
-                fprintf(stderr, "Removed cache %s", cache_dir);
-            }
-          }
-          free(cache_dir);
-        }
-      }
-    }
-  }
-  return err;
-}

data/ext/zsv/vendor/zsv-1.3.0/app/select/fixed.c

@@ -1,130 +0,0 @@
-/**
- * Get a list of ending positions for each column name based on the ending position of each column name
- * where the first row is of the below form (dash = whitespace):
- * ----COLUMN1----COLUMN2-----COLUMN3----
- *
- * Approach:
- * - read the first [256k] of data [to do: alternatively, read only the first line]
- * - merge all read lines into a single line where line[i] = 'x' for each position i at which
- *   a non-white char appeared in any scanned line
- * - from the merged line, find each instance of white followed by not-white,
- *   but ignore the first instance of it
- */
-static enum zsv_status auto_detect_fixed_column_sizes(struct fixed *fixed, struct zsv_opts *opts, unsigned char *buff,
-                                                      size_t buffsize, size_t *buff_used, char verbose) {
-  size_t max_lines = fixed->max_lines ? fixed->max_lines : 9999999999;
-  enum zsv_status stat = zsv_status_ok;
-
-  fixed->count = 0;
-  char *line = calloc(buffsize, sizeof(*buff));
-  if (!line) {
-    stat = zsv_status_memory;
-    goto auto_detect_fixed_column_sizes_exit;
-  }
-  memset(line, ' ', buffsize);
-
-  *buff_used = fread(buff, 1, buffsize, opts->stream);
-  if (!*buff_used) {
-    stat = zsv_status_no_more_input;
-    goto auto_detect_fixed_column_sizes_exit;
-  }
-
-  size_t lines_read = 0;
-  size_t line_end = 0;
-  size_t line_cursor = 0;
-  char first = 1;
-  char was_space = 1;
-  char was_line_end = 0;
-  for (size_t i = 0; i < *buff_used && (!line_end || max_lines == 0 || lines_read < max_lines);
-       i++, line_cursor = was_line_end ? 0 : line_cursor + 1) {
-    was_line_end = 0;
-    // TO DO: support multi-byte unicode chars?
-    switch (buff[i]) {
-    case '\n':
-    case '\r':
-      if (line_cursor > line_end)
-        line_end = line_cursor;
-      was_line_end = 1;
-      was_space = 1;
-      lines_read++;
-      break;
-    case '\t':
-    case '\v':
-    case '\f':
-    case ' ':
-      was_space = 1;
-      break;
-    default:
-      line[line_cursor] = 'x';
-      if (was_space) {
-        if (!line_end) { // only count columns for the first line
-          if (first)
-            first = 0;
-          else
-            fixed->count++;
-        }
-      }
-      was_space = 0;
-    }
-  }
-  if (!first)
-    fixed->count++;
-
-  if (!line_end) {
-    stat = zsv_status_error;
-    goto auto_detect_fixed_column_sizes_exit;
-  }
-
-  if (verbose)
-    fprintf(stderr, "Calculating %zu columns from line:\n%.*s\n", fixed->count, (int)line_end, line);
-
-  // allocate offsets
-  free(fixed->offsets);
-  fixed->offsets = NULL; // unnecessary line to silence codeQL false positive
-  fixed->offsets = calloc(fixed->count, sizeof(*fixed->offsets));
-  if (!fixed->offsets) {
-    stat = zsv_status_memory;
-    goto auto_detect_fixed_column_sizes_exit;
-  }
-
-  // now we have our merged line, so calculate the sizes
-  // do the loop again, but assign values this time
-  int count = 0;
-  was_space = 1;
-  first = 1;
-  if (verbose)
-    fprintf(stderr, "Running --fixed ");
-  size_t i;
-  for (i = 0; i < line_end; i++) {
-    if (line[i] == 'x') {
-      if (was_space) {
-        if (first)
-          first = 0;
-        else {
-          if (verbose)
-            fprintf(stderr, "%s%zu", count ? "," : "", i);
-          fixed->offsets[count++] = i;
-        }
-      }
-      was_space = 0;
-    } else
-      was_space = 1;
-  }
-  if (!first) {
-    if (verbose)
-      fprintf(stderr, "%s%zu", count ? "," : "", i);
-    if (i)
-      fixed->offsets[count++] = i;
-    fixed->count = count;
-  }
-  if (verbose)
-    fprintf(stderr, "\n");
-
-  // add a buffer to the last column in case subsequent lines are longer than what we scanned
-  if (fixed->count)
-    fixed->offsets[fixed->count - 1] += 50;
-
-auto_detect_fixed_column_sizes_exit:
-  free(line);
-  return stat;
-}

data/ext/zsv/vendor/zsv-1.3.0/app/select/internal.h

@@ -1,118 +0,0 @@
-#include <sys/types.h> // Required for off_t
-
-#ifndef ZSV_NO_PARALLEL
-#include "parallel.h"
-#endif
-
-#define ZSV_SELECT_MAX_COLS_DEFAULT 1024
-#define ZSV_SELECT_MAX_COLS_DEFAULT_S "1024"
-
-struct zsv_select_search_str {
-  struct zsv_select_search_str *next;
-  const char *value;
-  size_t len;
-};
-
-struct zsv_select_uint_list {
-  struct zsv_select_uint_list *next;
-  unsigned int value;
-};
-
-struct fixed {
-  size_t *offsets;
-  size_t count;
-  size_t max_lines; // max lines to use to calculate offsets
-  char autodetect;
-};
-
-struct zsv_select_data {
-  const char *input_path;
-  unsigned int current_column_ix;
-  size_t data_row_count;
-
-  struct zsv_opts *opts;
-  zsv_parser parser;
-  unsigned int errcount;
-
-  unsigned int output_col_index; // num of cols printed in current row
-
-  // output columns:
-  const char **col_argv;
-  int col_argc;
-  char *cols_to_print; // better: bitfield
-
-  struct {
-    unsigned int ix; // index of the input column to be output
-    struct {         // merge data: only used with --merge
-      struct zsv_select_uint_list *indexes, **last_index;
-    } merge;
-  } *out2in; // array of .output_cols_count length; out2in[x] = y where x = output ix, y = input info
-
-  unsigned int output_cols_count; // total count of output columns
-
-#define MAX_EXCLUSIONS 1024
-  const unsigned char *exclusions[MAX_EXCLUSIONS];
-  unsigned int exclusion_count;
-
-  unsigned int header_name_count;
-  unsigned char **header_names;
-
-  const char *prepend_header; // --prepend-header
-
-  char header_finished;
-
-  char embedded_lineend;
-
-  double sample_pct;
-
-  unsigned sample_every_n;
-
-  size_t data_rows_limit;
-  size_t skip_data_rows;
-
-  struct zsv_select_search_str *search_strings;
-#ifdef HAVE_PCRE2_8
-  struct zsv_select_regex *search_regexs;
-#endif
-
-  zsv_csv_writer csv_writer;
-
-  size_t overflow_size;
-
-  struct fixed fixed;
-
-#ifndef ZSV_NO_PARALLEL
-  unsigned num_chunks;
-  off_t end_offset_limit;                  // Byte offset where the current parser instance should stop
-  off_t next_row_start;                    // Actual byte offset of the last row that was processed
-  struct zsv_parallel_data *parallel_data; // Pointer to the thread management structure
-#endif
-
-  // FLAGS
-  unsigned char whitespace_clean_flags;
-
-  unsigned char print_all_cols : 1;
-  unsigned char use_header_indexes : 1;
-  unsigned char no_trim_whitespace : 1;
-  unsigned char cancelled : 1;
-  unsigned char skip_this_row : 1;
-  unsigned char verbose : 1;
-  unsigned char clean_white : 1;
-  unsigned char prepend_line_number : 1;
-
-  unsigned char any_clean : 1;
-#define ZSV_SELECT_DISTINCT_MERGE 2
-  unsigned char distinct : 2; // 1 = ignore subsequent cols, ZSV_SELECT_DISTINCT_MERGE = merge subsequent cols (first
-                              // non-null value)
-  unsigned char unescape : 1;
-  unsigned char no_header : 1;       // --no-header
-  unsigned char run_in_parallel : 1; // Flag if parallel mode is active
-  unsigned char _ : 2;               // Reduced padding by 1 bit due to addition of run_in_parallel
-};
-
-enum zsv_select_column_index_selection_type {
-  zsv_select_column_index_selection_type_none = 0,
-  zsv_select_column_index_selection_type_single,
-  zsv_select_column_index_selection_type_range,
-  zsv_select_column_index_selection_type_lower_bounded
-};

data/ext/zsv/vendor/zsv-1.3.0/app/select/parallel.c

@@ -1,45 +0,0 @@
-#include <errno.h>
-
-static struct zsv_parallel_data *zsv_parallel_data_new(unsigned num_chunks) {
-  struct zsv_parallel_data *pdata = calloc(1, sizeof(*pdata));
-  if (pdata) {
-    pdata->threads = calloc(num_chunks, sizeof(*pdata->threads));
-    pdata->chunk_data = calloc(num_chunks, sizeof(*pdata->chunk_data));
-    pdata->num_chunks = num_chunks;
-    if (pdata->threads && pdata->chunk_data)
-      return pdata;
-    zsv_parallel_data_delete(pdata);
-  }
-  // if we got here, we had a memory allocation failure
-  errno = ENOMEM;
-  return NULL;
-}
-
-static void zsv_chunk_data_clear_output(struct zsv_chunk_data *c) {
-  if (c) {
-#ifdef __linux__
-    if (c->tmp_output_filename) {
-      unlink(c->tmp_output_filename);
-      free(c->tmp_output_filename);
-      c->tmp_output_filename = NULL;
-    }
-#else
-    if (c->tmp_f) {
-      zsv_memfile_close(c->tmp_f);
-      c->tmp_f = NULL;
-    }
-#endif
-  }
-}
-
-static void zsv_parallel_data_delete(struct zsv_parallel_data *pdata) {
-  if (pdata) {
-    for (unsigned int i = 0; i < pdata->num_chunks; i++) {
-      if (pdata->chunk_data)
-        zsv_chunk_data_clear_output(&pdata->chunk_data[i]);
-    }
-    free(pdata->threads);
-    free(pdata->chunk_data);
-    free(pdata);
-  }
-}

data/ext/zsv/vendor/zsv-1.3.0/app/select/parallel.h

@@ -1,41 +0,0 @@
-#ifndef ZSV_SELECT_PARALLEL_H
-#define ZSV_SELECT_PARALLEL_H
-
-#include <sys/types.h> // Required for off_t
-#include <pthread.h>   // Required for pthread_t
-
-/**
- * @brief Data structure passed to each worker thread (Chunk 2, 3, 4)
- * Uses in-memory output buffer to avoid I/O lock contention.
- */
-struct zsv_chunk_data {
-#ifdef __linux__
-  char *tmp_output_filename; // use temp file because we can use zero-copy file concatenation
-#else
-  void *tmp_f; // use zsv_memfile * because we cannot do zero-copy file concatenation
-#endif
-  off_t start_offset;
-  off_t end_offset; // Stop processing when current offset exceeds this
-  off_t actual_next_row_start;
-  struct zsv_opts *opts; // Configuration options (read-only)
-  enum zsv_status status;
-  int id;
-  unsigned char skip : 1;
-  unsigned char _ : 7;
-};
-
-/**
- * @brief Structure to manage thread handles and chunk data for parallel processing.
- */
-struct zsv_parallel_data {
-  struct zsv_select_data *main_data;
-  unsigned num_chunks;
-  pthread_t *threads;                // array of N (num_chunks) pointers
-  struct zsv_chunk_data *chunk_data; // array of N chunk datas
-};
-
-static struct zsv_parallel_data *zsv_parallel_data_new(unsigned num_chunks);
-static void zsv_chunk_data_clear_output(struct zsv_chunk_data *c);
-static void zsv_parallel_data_delete(struct zsv_parallel_data *pdata);
-
-#endif

data/ext/zsv/vendor/zsv-1.3.0/app/select/processing.c

@@ -1,107 +0,0 @@
-#ifndef NDEBUG
-__attribute__((always_inline)) static inline
-#endif
-  unsigned char *
-  zsv_select_cell_clean(struct zsv_select_data *data, unsigned char *utf8_value, char *quoted, size_t *lenp) {
-
-  size_t len = *lenp;
-  // to do: option to replace or warn non-printable chars 0 - 31:
-  // vectorized scan
-  // replace or warn if found
-  if (UNLIKELY(data->unescape)) {
-    size_t new_len = zsv_strunescape_backslash(utf8_value, len);
-    if (new_len != len) {
-      *quoted = 1;
-      len = new_len;
-    }
-  }
-
-  if (UNLIKELY(!data->no_trim_whitespace))
-    utf8_value = (unsigned char *)zsv_strtrim(utf8_value, &len);
-
-  if (UNLIKELY(data->clean_white))
-    len = zsv_strwhite(utf8_value, len, data->whitespace_clean_flags); // to do: zsv_clean
-
-  if (UNLIKELY(data->embedded_lineend && *quoted)) {
-    unsigned char *tmp;
-    const char *to_replace[] = {"\r\n", "\r", "\n"};
-    for (int i = 0; i < 3; i++) {
-      while ((tmp = memmem(utf8_value, len, to_replace[i], strlen(to_replace[i])))) {
-        if (strlen(to_replace[i]) == 1)
-          *tmp = data->embedded_lineend;
-        else {
-          size_t right_len = utf8_value + len - tmp;
-          memmove(tmp + 1, tmp + 2, right_len - 2);
-          *tmp = data->embedded_lineend;
-          len--;
-        }
-      }
-    }
-    if (data->no_trim_whitespace)
-      utf8_value = (unsigned char *)zsv_strtrim(utf8_value, &len);
-  }
-  *lenp = len;
-  return utf8_value;
-}
-
-static inline char zsv_select_row_search_hit(struct zsv_select_data *data) {
-  if (!data->search_strings
-#ifdef HAVE_PCRE2_8
-      && !data->search_regexs
-#endif
-  )
-    return 1;
-
-  char have_overwrite = 0;
-  unsigned int j = zsv_cell_count(data->parser);
-  // Convert all bytes between cells to NUL so we can accurately search the entire row in one goe
-  unsigned char *start = NULL;
-  unsigned char *end = NULL;
-  for (unsigned int i = 0; i < j; i++) {
-    struct zsv_cell cell = zsv_get_cell(data->parser, i);
-    if (cell.overwritten)
-      have_overwrite = 1;
-    if (i == 0)
-      start = cell.str;
-    if (UNLIKELY(data->any_clean != 0))
-      cell.str = zsv_select_cell_clean(data, cell.str, &cell.quoted, &cell.len);
-    if (end) {
-      while (end < cell.str) {
-        *end = '\0';
-        end++;
-      }
-    }
-    end = cell.str + cell.len;
-  }
-
-  if (have_overwrite) {
-    for (unsigned int i = 0; i < j; i++) {
-      struct zsv_cell cell = zsv_get_cell(data->parser, i);
-      if (cell.len) {
-        start = cell.str;
-        end = cell.str + cell.len;
-        for (struct zsv_select_search_str *ss = data->search_strings; ss; ss = ss->next)
-          if (ss->value && *ss->value && end > start && memmem(start, end - start, ss->value, ss->len))
-            return 1;
-#ifdef HAVE_PCRE2_8
-        for (struct zsv_select_regex *rs = data->search_regexs; rs; rs = rs->next)
-          if (rs->regex && zsv_pcre2_8_match(rs->regex, start, end - start))
-            return 1;
-#endif
-      }
-    }
-  } else {
-    if (end > start) {
-      for (struct zsv_select_search_str *ss = data->search_strings; ss; ss = ss->next)
-        if (ss->value && *ss->value && end > start && memmem(start, end - start, ss->value, ss->len))
-          return 1;
-
-#ifdef HAVE_PCRE2_8
-      for (struct zsv_select_regex *rs = data->search_regexs; rs; rs = rs->next)
-        if (rs->regex && zsv_pcre2_8_match(rs->regex, start, end - start))
-          return 1;
-#endif
-    }
-  }
-  return 0;
-}

data/ext/zsv/vendor/zsv-1.3.0/app/select/rand.c

@@ -1,20 +0,0 @@
-// demo_random_bw_1_and_100(): this is a poor random number generator. you probably
-// will want to use a better one
-static double demo_random_bw_1_and_100(void) {
-#ifdef HAVE_ARC4RANDOM_UNIFORM
-  return (long double)(arc4random_uniform(1000000)) / 10000;
-#else
-  double max = 100.0;
-  unsigned int n;
-#ifdef HAVE_RAND_S
-  unsigned int tries = 0;
-  while (rand_s(&n) && tries++ < 10)
-    ;
-  return (double)n / ((double)UINT_MAX + 1) * max;
-#else
-  unsigned int umax = ~0;
-  n = rand();
-  return (double)n / ((double)(umax) + 1) * max;
-#endif
-#endif
-}

data/ext/zsv/vendor/zsv-1.3.0/app/select/regex.c

@@ -1,61 +0,0 @@
-#ifdef HAVE_PCRE2_8
-
-#include "../utils/pcre2-8/pcre2-8.h"
-
-struct zsv_select_regex {
-  struct zsv_select_regex *next;
-  const char *pattern;
-  regex_handle_t *regex;
-  // unsigned char has_anchors:1;
-  unsigned char _ : 7;
-};
-
-static void zsv_select_regexs_delete(struct zsv_select_regex *rs) {
-  for (struct zsv_select_regex *next; rs; rs = next) {
-    next = rs->next;
-    zsv_pcre2_8_delete(rs->regex);
-    free(rs);
-  }
-}
-
-static void zsv_select_add_regex(struct zsv_select_data *data, const char *pattern) {
-  if (pattern && *pattern) {
-    struct zsv_select_regex *sr = calloc(1, sizeof(*sr));
-    sr->pattern = pattern;
-    sr->regex = zsv_pcre2_8_new(pattern, 0);
-    if (sr->regex) {
-      sr->next = data->search_regexs;
-      data->search_regexs = sr;
-    } else
-      free(sr->regex);
-  }
-}
-
-/**
- * @brief create a full, independent duplicate of the regex list
- */
-static struct zsv_select_regex *zsv_select_regexs_dup(struct zsv_select_regex *src) {
-  struct zsv_select_regex *head = NULL;
-  struct zsv_select_regex **tail = &head;
-
-  while (src) {
-    struct zsv_select_regex *new_node = calloc(1, sizeof(*new_node));
-    if (!new_node) {
-      zsv_select_regexs_delete(head);
-      return NULL;
-    }
-
-    // Copy the pattern pointer (safe, strings are static/const)
-    new_node->pattern = src->pattern;
-
-    // FULL RECOMPILE: Call the standard new() function.
-    // This creates a fresh pcre2_code AND a fresh match_data buffer.
-    new_node->regex = zsv_pcre2_8_new(src->pattern, 0);
-
-    *tail = new_node;
-    tail = &new_node->next;
-    src = src->next;
-  }
-  return head;
-}
-#endif

data/ext/zsv/vendor/zsv-1.3.0/app/select/search.c

@@ -1,14 +0,0 @@
-static void zsv_select_search_str_delete(struct zsv_select_search_str *ss) {
-  for (struct zsv_select_search_str *next; ss; ss = next) {
-    next = ss->next;
-    free(ss);
-  }
-}
-
-static void zsv_select_add_search(struct zsv_select_data *data, const char *value) {
-  struct zsv_select_search_str *ss = calloc(1, sizeof(*ss));
-  ss->value = value;
-  ss->len = value ? strlen(value) : 0;
-  ss->next = data->search_strings;
-  data->search_strings = ss;
-}