zsv 1.3.0

data/ext/zsv/vendor/zsv-1.3.0/app/rm.c

@@ -0,0 +1,131 @@
+/* Copyright (C) 2022 Guarnerix Inc dba Liquidaty - All Rights Reserved
+ * Unauthorized copying of this file, via any medium is strictly prohibited
+ * Proprietary and confidential
+ * Written by Matt Wong <matt@guarnerix.com>
+ */
+
+/*
+ * remove a given file and its cache
+ */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h> // unlink()
+#include <errno.h>
+
+#define ZSV_COMMAND_NO_OPTIONS
+#define ZSV_COMMAND rm
+#include "zsv_command.h"
+
+#include <zsv/utils/dirs.h>
+#include <zsv/utils/cache.h>
+
+/**
+ * TO DO: add --orphaned option to remove all orphaned caches
+ */
+const char *zsv_rm_usage_msg[] = {
+  APPNAME ": remove a file and its related cache",
+  "",
+  "Usage: " APPNAME " [options] <filepath>",
+  "",
+  "Options:",
+  "  -v,--verbose   : verbose output",
+#ifndef NO_STDIN
+  "  -f,--force     : do not prompt for confirmation",
+#endif
+  "  -k,--keep      : do not remove related cache",
+  "  -C,--cache     : only remove related cache (not the file)",
+  NULL,
+};
+
+static int zsv_rm_usage(FILE *target) {
+  for (size_t j = 0; zsv_rm_usage_msg[j]; j++)
+    fprintf(target, "%s\n", zsv_rm_usage_msg[j]);
+  return target == stdout ? 0 : 1;
+}
+
+int ZSV_MAIN_NO_OPTIONS_FUNC(ZSV_COMMAND)(int argc, const char *argv[]) {
+  int err = 0;
+  if (argc > 1 && (!strcmp(argv[1], "-h") || !strcmp(argv[1], "--help")))
+    err = zsv_rm_usage(stdout);
+  else if (argc < 2)
+    err = zsv_rm_usage(stderr);
+  else {
+    const char *filepath = NULL;
+    char force = 0;
+#ifdef NO_STDIN
+    force = 1;
+#endif
+    char remove_cache = 1;
+    char remove_file = 1;
+    char verbose = 0;
+    for (int i = 1; !err && i < argc; i++) {
+      const char *arg = argv[i];
+      if (*arg == '-') {
+        if (!strcmp(arg, "-v") || !strcmp(arg, "--verbose"))
+          verbose = 1;
+        else if (!strcmp(arg, "-f") || !strcmp(arg, "--force"))
+          force = 1;
+        else if (!strcmp(arg, "-k") || !strcmp(arg, "--keep"))
+          remove_cache = 0;
+        else if (!strcmp(arg, "-C") || !strcmp(arg, "--cache"))
+          remove_file = 0;
+        else
+          err = zsv_printerr(1, "Unrecognized option: %s", arg);
+      } else if (!filepath)
+        filepath = arg;
+      else
+        err = zsv_printerr(1, "Unrecognized option: %s", arg);
+    }
+
+    if (!err && !filepath)
+      err = zsv_rm_usage(stderr);
+    else if (remove_file == 0 && remove_cache == 0)
+      err = fprintf(stderr, "Nothing to remove\n");
+
+    if (!err) {
+      char ok = 1;
+#ifndef NO_STDIN
+      if (!force) {
+        ok = 0;
+        if (!remove_file)
+          printf("Are you sure you want to remove the entire cache for the file %s?\n", filepath);
+        else
+          printf("Are you sure you want to remove the file %s%s?\n", filepath,
+                 remove_cache ? " and all of its cache contents" : "");
+        char buff[64];
+        if (fscanf(stdin, "%60s", buff) == 1 && strchr("Yy", buff[0]))
+          ok = 1;
+      }
+#endif
+      if (ok) {
+        if (remove_file) {
+          if (verbose)
+            fprintf(stderr, "Removing %s", filepath);
+          err = unlink(filepath);
+          if (err) {
+            if (err == ENOENT && force)
+              err = 0;
+            else
+              perror(filepath);
+          }
+        }
+        if (!err) {
+          unsigned char *cache_dir = zsv_cache_path((const unsigned char *)filepath, NULL, 0);
+          if (!cache_dir)
+            err = zsv_printerr(ENOMEM, "Out of memory!");
+          else if (zsv_dir_exists((const char *)cache_dir)) {
+            err = zsv_remove_dir_recursive(cache_dir);
+            if (verbose) {
+              if (!err)
+                fprintf(stderr, "Removed cache %s", cache_dir);
+            }
+          }
+          free(cache_dir);
+        }
+      }
+    }
+  }
+  return err;
+}

data/ext/zsv/vendor/zsv-1.3.0/app/select/fixed.c

@@ -0,0 +1,130 @@
+/**
+ * Get a list of ending positions for each column name based on the ending position of each column name
+ * where the first row is of the below form (dash = whitespace):
+ * ----COLUMN1----COLUMN2-----COLUMN3----
+ *
+ * Approach:
+ * - read the first [256k] of data [to do: alternatively, read only the first line]
+ * - merge all read lines into a single line where line[i] = 'x' for each position i at which
+ *   a non-white char appeared in any scanned line
+ * - from the merged line, find each instance of white followed by not-white,
+ *   but ignore the first instance of it
+ */
+static enum zsv_status auto_detect_fixed_column_sizes(struct fixed *fixed, struct zsv_opts *opts, unsigned char *buff,
+                                                      size_t buffsize, size_t *buff_used, char verbose) {
+  size_t max_lines = fixed->max_lines ? fixed->max_lines : 9999999999;
+  enum zsv_status stat = zsv_status_ok;
+
+  fixed->count = 0;
+  char *line = calloc(buffsize, sizeof(*buff));
+  if (!line) {
+    stat = zsv_status_memory;
+    goto auto_detect_fixed_column_sizes_exit;
+  }
+  memset(line, ' ', buffsize);
+
+  *buff_used = fread(buff, 1, buffsize, opts->stream);
+  if (!*buff_used) {
+    stat = zsv_status_no_more_input;
+    goto auto_detect_fixed_column_sizes_exit;
+  }
+
+  size_t lines_read = 0;
+  size_t line_end = 0;
+  size_t line_cursor = 0;
+  char first = 1;
+  char was_space = 1;
+  char was_line_end = 0;
+  for (size_t i = 0; i < *buff_used && (!line_end || max_lines == 0 || lines_read < max_lines);
+       i++, line_cursor = was_line_end ? 0 : line_cursor + 1) {
+    was_line_end = 0;
+    // TO DO: support multi-byte unicode chars?
+    switch (buff[i]) {
+    case '\n':
+    case '\r':
+      if (line_cursor > line_end)
+        line_end = line_cursor;
+      was_line_end = 1;
+      was_space = 1;
+      lines_read++;
+      break;
+    case '\t':
+    case '\v':
+    case '\f':
+    case ' ':
+      was_space = 1;
+      break;
+    default:
+      line[line_cursor] = 'x';
+      if (was_space) {
+        if (!line_end) { // only count columns for the first line
+          if (first)
+            first = 0;
+          else
+            fixed->count++;
+        }
+      }
+      was_space = 0;
+    }
+  }
+  if (!first)
+    fixed->count++;
+
+  if (!line_end) {
+    stat = zsv_status_error;
+    goto auto_detect_fixed_column_sizes_exit;
+  }
+
+  if (verbose)
+    fprintf(stderr, "Calculating %zu columns from line:\n%.*s\n", fixed->count, (int)line_end, line);
+
+  // allocate offsets
+  free(fixed->offsets);
+  fixed->offsets = NULL; // unnecessary line to silence codeQL false positive
+  fixed->offsets = calloc(fixed->count, sizeof(*fixed->offsets));
+  if (!fixed->offsets) {
+    stat = zsv_status_memory;
+    goto auto_detect_fixed_column_sizes_exit;
+  }
+
+  // now we have our merged line, so calculate the sizes
+  // do the loop again, but assign values this time
+  int count = 0;
+  was_space = 1;
+  first = 1;
+  if (verbose)
+    fprintf(stderr, "Running --fixed ");
+  size_t i;
+  for (i = 0; i < line_end; i++) {
+    if (line[i] == 'x') {
+      if (was_space) {
+        if (first)
+          first = 0;
+        else {
+          if (verbose)
+            fprintf(stderr, "%s%zu", count ? "," : "", i);
+          fixed->offsets[count++] = i;
+        }
+      }
+      was_space = 0;
+    } else
+      was_space = 1;
+  }
+  if (!first) {
+    if (verbose)
+      fprintf(stderr, "%s%zu", count ? "," : "", i);
+    if (i)
+      fixed->offsets[count++] = i;
+    fixed->count = count;
+  }
+  if (verbose)
+    fprintf(stderr, "\n");
+
+  // add a buffer to the last column in case subsequent lines are longer than what we scanned
+  if (fixed->count)
+    fixed->offsets[fixed->count - 1] += 50;
+
+auto_detect_fixed_column_sizes_exit:
+  free(line);
+  return stat;
+}

data/ext/zsv/vendor/zsv-1.3.0/app/select/internal.h

@@ -0,0 +1,118 @@
+#include <sys/types.h> // Required for off_t
+
+#ifndef ZSV_NO_PARALLEL
+#include "parallel.h"
+#endif
+
+#define ZSV_SELECT_MAX_COLS_DEFAULT 1024
+#define ZSV_SELECT_MAX_COLS_DEFAULT_S "1024"
+
+struct zsv_select_search_str {
+  struct zsv_select_search_str *next;
+  const char *value;
+  size_t len;
+};
+
+struct zsv_select_uint_list {
+  struct zsv_select_uint_list *next;
+  unsigned int value;
+};
+
+struct fixed {
+  size_t *offsets;
+  size_t count;
+  size_t max_lines; // max lines to use to calculate offsets
+  char autodetect;
+};
+
+struct zsv_select_data {
+  const char *input_path;
+  unsigned int current_column_ix;
+  size_t data_row_count;
+
+  struct zsv_opts *opts;
+  zsv_parser parser;
+  unsigned int errcount;
+
+  unsigned int output_col_index; // num of cols printed in current row
+
+  // output columns:
+  const char **col_argv;
+  int col_argc;
+  char *cols_to_print; // better: bitfield
+
+  struct {
+    unsigned int ix; // index of the input column to be output
+    struct {         // merge data: only used with --merge
+      struct zsv_select_uint_list *indexes, **last_index;
+    } merge;
+  } *out2in; // array of .output_cols_count length; out2in[x] = y where x = output ix, y = input info
+
+  unsigned int output_cols_count; // total count of output columns
+
+#define MAX_EXCLUSIONS 1024
+  const unsigned char *exclusions[MAX_EXCLUSIONS];
+  unsigned int exclusion_count;
+
+  unsigned int header_name_count;
+  unsigned char **header_names;
+
+  const char *prepend_header; // --prepend-header
+
+  char header_finished;
+
+  char embedded_lineend;
+
+  double sample_pct;
+
+  unsigned sample_every_n;
+
+  size_t data_rows_limit;
+  size_t skip_data_rows;
+
+  struct zsv_select_search_str *search_strings;
+#ifdef HAVE_PCRE2_8
+  struct zsv_select_regex *search_regexs;
+#endif
+
+  zsv_csv_writer csv_writer;
+
+  size_t overflow_size;
+
+  struct fixed fixed;
+
+#ifndef ZSV_NO_PARALLEL
+  unsigned num_chunks;
+  off_t end_offset_limit;                  // Byte offset where the current parser instance should stop
+  off_t next_row_start;                    // Actual byte offset of the last row that was processed
+  struct zsv_parallel_data *parallel_data; // Pointer to the thread management structure
+#endif
+
+  // FLAGS
+  unsigned char whitespace_clean_flags;
+
+  unsigned char print_all_cols : 1;
+  unsigned char use_header_indexes : 1;
+  unsigned char no_trim_whitespace : 1;
+  unsigned char cancelled : 1;
+  unsigned char skip_this_row : 1;
+  unsigned char verbose : 1;
+  unsigned char clean_white : 1;
+  unsigned char prepend_line_number : 1;
+
+  unsigned char any_clean : 1;
+#define ZSV_SELECT_DISTINCT_MERGE 2
+  unsigned char distinct : 2; // 1 = ignore subsequent cols, ZSV_SELECT_DISTINCT_MERGE = merge subsequent cols (first
+                              // non-null value)
+  unsigned char unescape : 1;
+  unsigned char no_header : 1;       // --no-header
+  unsigned char run_in_parallel : 1; // Flag if parallel mode is active
+  unsigned char _ : 2;               // Reduced padding by 1 bit due to addition of run_in_parallel
+};
+
+enum zsv_select_column_index_selection_type {
+  zsv_select_column_index_selection_type_none = 0,
+  zsv_select_column_index_selection_type_single,
+  zsv_select_column_index_selection_type_range,
+  zsv_select_column_index_selection_type_lower_bounded
+};

data/ext/zsv/vendor/zsv-1.3.0/app/select/parallel.c

@@ -0,0 +1,45 @@
+#include <errno.h>
+
+static struct zsv_parallel_data *zsv_parallel_data_new(unsigned num_chunks) {
+  struct zsv_parallel_data *pdata = calloc(1, sizeof(*pdata));
+  if (pdata) {
+    pdata->threads = calloc(num_chunks, sizeof(*pdata->threads));
+    pdata->chunk_data = calloc(num_chunks, sizeof(*pdata->chunk_data));
+    pdata->num_chunks = num_chunks;
+    if (pdata->threads && pdata->chunk_data)
+      return pdata;
+    zsv_parallel_data_delete(pdata);
+  }
+  // if we got here, we had a memory allocation failure
+  errno = ENOMEM;
+  return NULL;
+}
+
+static void zsv_chunk_data_clear_output(struct zsv_chunk_data *c) {
+  if (c) {
+#ifdef __linux__
+    if (c->tmp_output_filename) {
+      unlink(c->tmp_output_filename);
+      free(c->tmp_output_filename);
+      c->tmp_output_filename = NULL;
+    }
+#else
+    if (c->tmp_f) {
+      zsv_memfile_close(c->tmp_f);
+      c->tmp_f = NULL;
+    }
+#endif
+  }
+}
+
+static void zsv_parallel_data_delete(struct zsv_parallel_data *pdata) {
+  if (pdata) {
+    for (unsigned int i = 0; i < pdata->num_chunks; i++) {
+      if (pdata->chunk_data)
+        zsv_chunk_data_clear_output(&pdata->chunk_data[i]);
+    }
+    free(pdata->threads);
+    free(pdata->chunk_data);
+    free(pdata);
+  }
+}

data/ext/zsv/vendor/zsv-1.3.0/app/select/parallel.h

@@ -0,0 +1,41 @@
+#ifndef ZSV_SELECT_PARALLEL_H
+#define ZSV_SELECT_PARALLEL_H
+
+#include <sys/types.h> // Required for off_t
+#include <pthread.h>   // Required for pthread_t
+
+/**
+ * @brief Data structure passed to each worker thread (Chunk 2, 3, 4)
+ * Uses in-memory output buffer to avoid I/O lock contention.
+ */
+struct zsv_chunk_data {
+#ifdef __linux__
+  char *tmp_output_filename; // use temp file because we can use zero-copy file concatenation
+#else
+  void *tmp_f; // use zsv_memfile * because we cannot do zero-copy file concatenation
+#endif
+  off_t start_offset;
+  off_t end_offset; // Stop processing when current offset exceeds this
+  off_t actual_next_row_start;
+  struct zsv_opts *opts; // Configuration options (read-only)
+  enum zsv_status status;
+  int id;
+  unsigned char skip : 1;
+  unsigned char _ : 7;
+};
+
+/**
+ * @brief Structure to manage thread handles and chunk data for parallel processing.
+ */
+struct zsv_parallel_data {
+  struct zsv_select_data *main_data;
+  unsigned num_chunks;
+  pthread_t *threads;                // array of N (num_chunks) pointers
+  struct zsv_chunk_data *chunk_data; // array of N chunk datas
+};
+
+static struct zsv_parallel_data *zsv_parallel_data_new(unsigned num_chunks);
+static void zsv_chunk_data_clear_output(struct zsv_chunk_data *c);
+static void zsv_parallel_data_delete(struct zsv_parallel_data *pdata);
+
+#endif

data/ext/zsv/vendor/zsv-1.3.0/app/select/processing.c

@@ -0,0 +1,107 @@
+#ifndef NDEBUG
+__attribute__((always_inline)) static inline
+#endif
+  unsigned char *
+  zsv_select_cell_clean(struct zsv_select_data *data, unsigned char *utf8_value, char *quoted, size_t *lenp) {
+
+  size_t len = *lenp;
+  // to do: option to replace or warn non-printable chars 0 - 31:
+  // vectorized scan
+  // replace or warn if found
+  if (UNLIKELY(data->unescape)) {
+    size_t new_len = zsv_strunescape_backslash(utf8_value, len);
+    if (new_len != len) {
+      *quoted = 1;
+      len = new_len;
+    }
+  }
+
+  if (UNLIKELY(!data->no_trim_whitespace))
+    utf8_value = (unsigned char *)zsv_strtrim(utf8_value, &len);
+
+  if (UNLIKELY(data->clean_white))
+    len = zsv_strwhite(utf8_value, len, data->whitespace_clean_flags); // to do: zsv_clean
+
+  if (UNLIKELY(data->embedded_lineend && *quoted)) {
+    unsigned char *tmp;
+    const char *to_replace[] = {"\r\n", "\r", "\n"};
+    for (int i = 0; i < 3; i++) {
+      while ((tmp = memmem(utf8_value, len, to_replace[i], strlen(to_replace[i])))) {
+        if (strlen(to_replace[i]) == 1)
+          *tmp = data->embedded_lineend;
+        else {
+          size_t right_len = utf8_value + len - tmp;
+          memmove(tmp + 1, tmp + 2, right_len - 2);
+          *tmp = data->embedded_lineend;
+          len--;
+        }
+      }
+    }
+    if (data->no_trim_whitespace)
+      utf8_value = (unsigned char *)zsv_strtrim(utf8_value, &len);
+  }
+  *lenp = len;
+  return utf8_value;
+}
+
+static inline char zsv_select_row_search_hit(struct zsv_select_data *data) {
+  if (!data->search_strings
+#ifdef HAVE_PCRE2_8
+      && !data->search_regexs
+#endif
+  )
+    return 1;
+
+  char have_overwrite = 0;
+  unsigned int j = zsv_cell_count(data->parser);
+  // Convert all bytes between cells to NUL so we can accurately search the entire row in one goe
+  unsigned char *start = NULL;
+  unsigned char *end = NULL;
+  for (unsigned int i = 0; i < j; i++) {
+    struct zsv_cell cell = zsv_get_cell(data->parser, i);
+    if (cell.overwritten)
+      have_overwrite = 1;
+    if (i == 0)
+      start = cell.str;
+    if (UNLIKELY(data->any_clean != 0))
+      cell.str = zsv_select_cell_clean(data, cell.str, &cell.quoted, &cell.len);
+    if (end) {
+      while (end < cell.str) {
+        *end = '\0';
+        end++;
+      }
+    }
+    end = cell.str + cell.len;
+  }
+
+  if (have_overwrite) {
+    for (unsigned int i = 0; i < j; i++) {
+      struct zsv_cell cell = zsv_get_cell(data->parser, i);
+      if (cell.len) {
+        start = cell.str;
+        end = cell.str + cell.len;
+        for (struct zsv_select_search_str *ss = data->search_strings; ss; ss = ss->next)
+          if (ss->value && *ss->value && end > start && memmem(start, end - start, ss->value, ss->len))
+            return 1;
+#ifdef HAVE_PCRE2_8
+        for (struct zsv_select_regex *rs = data->search_regexs; rs; rs = rs->next)
+          if (rs->regex && zsv_pcre2_8_match(rs->regex, start, end - start))
+            return 1;
+#endif
+      }
+    }
+  } else {
+    if (end > start) {
+      for (struct zsv_select_search_str *ss = data->search_strings; ss; ss = ss->next)
+        if (ss->value && *ss->value && end > start && memmem(start, end - start, ss->value, ss->len))
+          return 1;
+
+#ifdef HAVE_PCRE2_8
+      for (struct zsv_select_regex *rs = data->search_regexs; rs; rs = rs->next)
+        if (rs->regex && zsv_pcre2_8_match(rs->regex, start, end - start))
+          return 1;
+#endif
+    }
+  }
+  return 0;
+}

data/ext/zsv/vendor/zsv-1.3.0/app/select/rand.c

@@ -0,0 +1,20 @@
+// demo_random_bw_1_and_100(): this is a poor random number generator. you probably
+// will want to use a better one
+static double demo_random_bw_1_and_100(void) {
+#ifdef HAVE_ARC4RANDOM_UNIFORM
+  return (long double)(arc4random_uniform(1000000)) / 10000;
+#else
+  double max = 100.0;
+  unsigned int n;
+#ifdef HAVE_RAND_S
+  unsigned int tries = 0;
+  while (rand_s(&n) && tries++ < 10)
+    ;
+  return (double)n / ((double)UINT_MAX + 1) * max;
+#else
+  unsigned int umax = ~0;
+  n = rand();
+  return (double)n / ((double)(umax) + 1) * max;
+#endif
+#endif
+}

data/ext/zsv/vendor/zsv-1.3.0/app/select/regex.c

@@ -0,0 +1,61 @@
+#ifdef HAVE_PCRE2_8
+
+#include "../utils/pcre2-8/pcre2-8.h"
+
+struct zsv_select_regex {
+  struct zsv_select_regex *next;
+  const char *pattern;
+  regex_handle_t *regex;
+  // unsigned char has_anchors:1;
+  unsigned char _ : 7;
+};
+
+static void zsv_select_regexs_delete(struct zsv_select_regex *rs) {
+  for (struct zsv_select_regex *next; rs; rs = next) {
+    next = rs->next;
+    zsv_pcre2_8_delete(rs->regex);
+    free(rs);
+  }
+}
+
+static void zsv_select_add_regex(struct zsv_select_data *data, const char *pattern) {
+  if (pattern && *pattern) {
+    struct zsv_select_regex *sr = calloc(1, sizeof(*sr));
+    sr->pattern = pattern;
+    sr->regex = zsv_pcre2_8_new(pattern, 0);
+    if (sr->regex) {
+      sr->next = data->search_regexs;
+      data->search_regexs = sr;
+    } else
+      free(sr->regex);
+  }
+}
+
+/**
+ * @brief create a full, independent duplicate of the regex list
+ */
+static struct zsv_select_regex *zsv_select_regexs_dup(struct zsv_select_regex *src) {
+  struct zsv_select_regex *head = NULL;
+  struct zsv_select_regex **tail = &head;
+
+  while (src) {
+    struct zsv_select_regex *new_node = calloc(1, sizeof(*new_node));
+    if (!new_node) {
+      zsv_select_regexs_delete(head);
+      return NULL;
+    }
+
+    // Copy the pattern pointer (safe, strings are static/const)
+    new_node->pattern = src->pattern;
+
+    // FULL RECOMPILE: Call the standard new() function.
+    // This creates a fresh pcre2_code AND a fresh match_data buffer.
+    new_node->regex = zsv_pcre2_8_new(src->pattern, 0);
+
+    *tail = new_node;
+    tail = &new_node->next;
+    src = src->next;
+  }
+  return head;
+}
+#endif

data/ext/zsv/vendor/zsv-1.3.0/app/select/search.c

@@ -0,0 +1,14 @@
+static void zsv_select_search_str_delete(struct zsv_select_search_str *ss) {
+  for (struct zsv_select_search_str *next; ss; ss = next) {
+    next = ss->next;
+    free(ss);
+  }
+}
+
+static void zsv_select_add_search(struct zsv_select_data *data, const char *value) {
+  struct zsv_select_search_str *ss = calloc(1, sizeof(*ss));
+  ss->value = value;
+  ss->len = value ? strlen(value) : 0;
+  ss->next = data->search_strings;
+  data->search_strings = ss;
+}