zsv 1.3.0

data/ext/zsv/vendor/zsv-1.3.0/app/builtin/help.c

@@ -0,0 +1,123 @@
+/*
+ * Copyright (C) 2021 Liquidaty and the zsv/lib contributors
+ * All rights reserved
+ *
+ * This file is part of zsv/lib, distributed under the license defined at
+ * https://opensource.org/licenses/MIT
+ */
+
+#include <stdio.h>
+
+static int main_help(int argc, const char *argv[]) {
+  (void)(argc);
+  (void)(argv);
+
+  FILE *f = stdout;
+  static const char *usage[] = {
+    "zsv: streaming csv processor",
+    "",
+    "Usage:",
+    "  zsv version: display version info (and if applicable, extension info)",
+#ifndef __EMSCRIPTEN__
+    "  zsv (un)register [<extension_id>]    : (un)register an extension",
+    "      Registration info is saved in zsv.ini located in a directory determined as:",
+    "        ZSV_CONFIG_DIR environment variable value, if set",
+#if defined(_WIN32)
+    "        LOCALAPPDATA environment variable value, if set",
+    "        otherwise, C:\\temp",
+#else
+    "        otherwise, " PREFIX "/etc",
+#endif
+#endif
+    "  zsv help [<command>]",
+    "  zsv <command> <options> <arguments>  : run a command on data (see below for details)",
+#ifndef __EMSCRIPTEN__
+    "  zsv <id>-<cmd> <options> <arguments> : invoke command 'cmd' of extension 'id'",
+    "  zsv license [<extension_id>]",
+#endif
+    "  zsv thirdparty                       : view third-party licenses & acknowledgements",
+    "",
+    "Options common to all commands except `prop`, `rm` and `jq`:",
+#ifdef ZSV_EXTRAS
+    "  -L,--limit-rows <n>      : limit processing to the given number of rows (including any header row(s))",
+#endif
+    "  -c,--max-column-count <n>: set the maximum number of columns parsed per row. defaults to 1024",
+    "  -r,--max-row-size <n>    : set the minimum supported maximum row size. defaults to 64k",
+    "  -B,--buff-size <n>       : set internal buffer size. defaults to 256k",
+    "  -t,--tab-delim           : set column delimiter to tab",
+    "  -O,--other-delim <char>  : set column delimiter to specified character",
+    "  -q,--no-quote            : turn off quote handling",
+    "  -R,--skip-head <n>       : skip specified number of initial rows",
+    "  -d,--header-row-span <n> : apply header depth (rowspan) of n",
+    "  -u,--malformed-utf8-replacement <string>: replacement string (can be empty) in case of malformed UTF8 input",
+    "       (default for \"desc\" command is '?')",
+    "  -S,--keep-blank-headers  : disable default behavior of ignoring leading blank rows",
+    "  -0,--header-row <header> : insert the provided CSV as the first row (in position 0)",
+    "                             e.g. --header-row 'col1,col2,\"my col 3\"'",
+#ifndef ZSV_NO_ONLY_CRLF
+    "  --only-crlf              : only treat CRLF as row delimiter",
+    "                             CR or LF alone are treated as normal chars that do not require quotes",
+#endif
+#ifdef ZSV_EXTRAS
+    "  -1,--apply-overwrites    : automatically apply overwrites saved via `overwrite` command",
+#endif
+    "  -v,--verbose             : verbose output",
+    "",
+    "Commands that parse CSV or other tabular data:",
+    "  echo     : write tabular input to stdout with optional cell overwrites",
+    "  check    : check for anolomolies (column counts, utf8 encoding etc)",
+    "  count    : print the number of rows",
+    "  select   : extract rows/columns by name or position and perform other basic and 'cleanup' operations",
+    "  desc     : describe each column",
+    "  sql      : run ad-hoc SQL on one or more CSV files",
+    "  pretty   : pretty print for console display",
+    "  serialize: convert into 3-column format (id, column name, cell value)",
+    "  flatten  : flatten a table consisting of N groups of data, each with 1 or",
+    "             more rows in the table, into a table of N rows",
+    "  2json    : convert CSV or sqlite3 db table to json",
+    "  2tsv     : convert to tab-delimited text",
+    "  stack    : stack tables vertically, aligning columns with common names",
+    "  paste    : horizontally paste two tables together: given inputs X, Y, ... of N rows",
+    "  compare  : compare two or more tables and output differences",
+    "  overwrite: save, modify or apply overwrites",
+    "",
+    "Other commands:",
+    "  2db      : convert json to sqlite3 db",
+    "  prop     : save parsing options associated with a file that are subsequently",
+    "             applied by default when processing that file",
+    "  rm       : remove a file and its related cache",
+    "  mv       : rename (move) a file and/or its related cache",
+#ifdef USE_JQ
+    "  jq       : run a jq filter on json input",
+#endif
+    NULL,
+  };
+
+  for (size_t i = 0; usage[i]; i++)
+    fprintf(f, "%s\n", usage[i]);
+
+#ifndef __EMSCRIPTEN__
+  char printed_init = 0;
+  struct cli_config config;
+  if (!config_init(&config, 1, 1, 0)) {
+    for (struct zsv_ext *ext = config.extensions; ext; ext = ext->next) {
+      if (ext->inited == zsv_init_ok) {
+        if (!printed_init) {
+          printed_init = 1;
+          fprintf(f, "\nExtended commands:\n");
+        } else
+          fprintf(f, "\n");
+        if (ext->help)
+          fprintf(f, "  Extension '%s': %s\n", ext->id, ext->help);
+        for (struct zsv_ext_command *cmd = ext->commands; cmd; cmd = cmd->next)
+          fprintf(f, "    %s-%s%s%s\n", ext->id, cmd->id, cmd->help ? ": " : "", cmd->help ? cmd->help : "");
+      }
+    }
+    config_free(&config);
+  }
+  if (!printed_init)
+    fprintf(f, "\n(No extended commands)\n");
+#endif
+
+  return 0;
+}

data/ext/zsv/vendor/zsv-1.3.0/app/builtin/license.c

@@ -0,0 +1,39 @@
+/*
+ * Copyright (C) 2021 Liquidaty and the zsv/lib contributors
+ * All rights reserved
+ *
+ * This file is part of zsv/lib, distributed under the license defined at
+ * https://opensource.org/licenses/MIT
+ */
+
+static int main_license(int argc, const char *argv[]) {
+  (void)(argc);
+  (void)(argv);
+
+  fprintf(stderr, "Note: for third-party licenses & acknowledgements, run `zsv thirdparty`\n");
+  printf("\n====================================================\n");
+  printf("ZSV/lib license");
+  printf(" ");
+  printf("\n====================================================\n");
+
+  fwrite(zsv_license_text_MIT, 1, strlen(zsv_license_text_MIT), stdout);
+
+  struct cli_config config;
+  if (!config_init(&config, 0, 1, 0)) {
+    for (struct zsv_ext *ext = config.extensions; ext; ext = ext->next) {
+      printf("\n====================================================\n");
+      printf("License for extension '%s'", ext->id);
+      printf("\n====================================================\n");
+      if (ext->license && *ext->license) {
+        size_t len = strlen(ext->license);
+        fwrite(ext->license, 1, len, stdout);
+        if (ext->license[len - 1] != '\n')
+          printf("\n");
+      } else
+        printf("Unknown\n");
+    }
+  }
+  config_free(&config);
+
+  return 0;
+}

data/ext/zsv/vendor/zsv-1.3.0/app/builtin/register.c

@@ -0,0 +1,104 @@
+/*
+ * Copyright (C) 2021 Liquidaty and the zsv/lib contributors
+ * All rights reserved
+ *
+ * This file is part of zsv/lib, distributed under the license defined at
+ * https://opensource.org/licenses/MIT
+ */
+
+/**
+ * Check an extension's implementation for completeness and print any errors or warnings
+ * This function does not yet impose any requirements on any extension
+ *
+ * @param ext the extension to check
+ * @return error
+ */
+static int check_extension(struct zsv_ext *ext) {
+  /* to do:
+   * check that each extension command supports `zsv my-cmd --help`;
+   *   redirect, examine & restore stdin/out/err
+   */
+  if (!ext)
+    return 1;
+
+  ext_init(ext);
+  if (ext->inited != zsv_init_ok)
+    return 1;
+  return 0;
+}
+
+static int register_help(char do_register) {
+  static const char *register_help = "zsv register: register an extension\n\n"
+                                     "usage: zsv register [-g,--global] <extension_id>";
+  static const char *unregister_help = "zsv unregister: unregister an extension\n\n"
+                                       "usage: zsv unregister [-g,--global] <extension_id>";
+  printf("%s\n", do_register ? register_help : unregister_help);
+  return 0;
+}
+
+static int main_register_aux(int argc, const char *argv[]) {
+  int err = 0;
+  struct cli_config config;
+  char do_register = *argv[0] == 'r';
+  const char *extension_id = NULL;
+  char global = 0;
+  for (int i = 1; i < argc; i++) {
+    const char *arg = argv[i];
+    if (!strcmp(arg, "--help") || !strcmp(arg, "-h"))
+      return register_help(do_register);
+    if (!strcmp(arg, "-g") || !strcmp(arg, "--global"))
+      global = 1;
+    else {
+      if (extension_id != NULL)
+        return register_help(do_register);
+      extension_id = arg;
+    }
+  }
+
+  if (!extension_id)
+    fprintf(stderr, "No extension id provided\n"), err = 1;
+  else if (strlen(extension_id) < ZSV_EXTENSION_ID_MIN_LEN || strlen(extension_id) > ZSV_EXTENSION_ID_MAX_LEN)
+    fprintf(stderr, "Extension id must be 1 to 8 bytes\n"), err = 1;
+  else if (config_init_2(&config, !do_register, 1, 1, global, !global))
+    config_free(&config); // unable to init config
+  else {
+    struct zsv_ext *found;
+    if ((found = find_extension(&config, extension_id))) {
+      // found this extension registered
+      if (do_register) {
+        fprintf(stderr, "Extension %s already registered\n", extension_id), err = 1;
+        check_extension(found);
+      }
+      // unregister
+      else if (!(err = remove_extension(&config.extensions, found)))
+        if (!(err = config_save(&config)))
+          fprintf(stderr, "Extension %s unregistered\n", extension_id);
+    } else {
+      // this extension has not been registered
+      if (!do_register)
+        fprintf(stderr, "Extension %s was not already registered\n", extension_id), err = 1;
+
+      // register
+      else {
+        // confirm we can successfully load dll, then register
+        // add_extension() adds this extension to the front of the list
+        if (!(err = add_extension(extension_id, &config.extensions, 1, 1))) {
+          if (!check_extension(config.extensions)) {
+            if (!(err = config_save(&config)))
+              fprintf(stderr, "Extension %s registered\n", extension_id);
+          }
+        }
+      }
+    }
+    config_free(&config);
+  }
+  return err;
+}
+
+static int main_register(int argc, const char *argv[]) {
+  return main_register_aux(argc, argv);
+}
+
+static int main_unregister(int argc, const char *argv[]) {
+  return main_register_aux(argc, argv);
+}

data/ext/zsv/vendor/zsv-1.3.0/app/builtin/thirdparty.c

@@ -0,0 +1,41 @@
+/*
+ * Copyright (C) 2021 Liquidaty and the zsv/lib contributors
+ * All rights reserved
+ *
+ * This file is part of zsv/lib, distributed under the license defined at
+ * https://opensource.org/licenses/MIT
+ */
+
+static void print_str_array(const char *name, const char *name2, const char **ss) {
+  printf("\n\n==========================\n%s%s\n==========================\n", name, name2 ? name2 : "?");
+  if (!ss)
+    printf("No third-party information\n");
+  for (int i = 0; ss && ss[i]; i++) {
+    const char *s = ss[i];
+    fwrite(s, 1, strlen(s), stdout);
+  }
+}
+
+static int main_thirdparty(int argc, const char *argv[]) {
+  (void)(argc);
+  (void)(argv);
+
+  printf("Third-party licenses and acknowledgements");
+  print_str_array("ZSV/lib third-party dependencies", "", zsv_thirdparty);
+
+#ifndef __EMSCRIPTEN__
+  struct cli_config config;
+  const char *ss[2];
+  ss[1] = NULL;
+  if (!config_init(&config, 0, 1, 0)) {
+    for (struct zsv_ext *ext = config.extensions; ext; ext = ext->next) {
+      if (ext->thirdparty) {
+        ss[0] = ext->thirdparty;
+        print_str_array("Extension: ", (const char *)ext->id, ss);
+      }
+    }
+  }
+#endif
+  printf("\n");
+  return 0;
+}

data/ext/zsv/vendor/zsv-1.3.0/app/builtin/unregister.c

@@ -0,0 +1 @@
+/* intentionally left blank. see register.c */

data/ext/zsv/vendor/zsv-1.3.0/app/builtin/version.c

@@ -0,0 +1,14 @@
+/*
+ * Copyright (C) 2021 Liquidaty and the zsv/lib contributors
+ * All rights reserved
+ *
+ * This file is part of zsv/lib, distributed under the license defined at
+ * https://opensource.org/licenses/MIT
+ */
+
+static int main_version(int argc, const char *argv[]) {
+  (void)(argc);
+  (void)(argv);
+  printf("zsv version %s (lib %s)\n", VERSION, zsv_lib_version());
+  return 0;
+}

data/ext/zsv/vendor/zsv-1.3.0/app/check/simdutf_wrapper.h

@@ -0,0 +1,19 @@
+/* simdutf_wrapper.h - C-callable wrapper around simdutf */
+
+#ifndef SIMDUTF_WRAPPER_H
+#define SIMDUTF_WRAPPER_H
+
+#include <stddef.h> /* for size_t */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Returns 1 if valid UTF-8, 0 otherwise */
+int simdutf_is_valid_utf8(const char *buf, size_t len);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* SIMDUTF_WRAPPER_H */

data/ext/zsv/vendor/zsv-1.3.0/app/check/utf8.c

@@ -0,0 +1,116 @@
+#include <stddef.h>
+#include <stdint.h>
+#include <stdbool.h>
+
+bool utf8_is_valid(const uint8_t *s, size_t len);
+
+/////
+static inline bool is_cont(uint8_t b) {
+  // Continuation byte: 10xxxxxx
+  return (b & 0xC0) == 0x80;
+}
+
+bool utf8_is_valid(const uint8_t *s, size_t len) {
+  size_t i = 0;
+
+  while (i < len) {
+    uint8_t c = s[i];
+
+    // ASCII fast path
+    if (c < 0x80) {
+      i++;
+      continue;
+    }
+
+    // Reject continuation bytes as leading bytes, and C0/C1 overlongs
+    if (c < 0xC2) {
+      return false;
+    }
+
+    // 2-byte sequence: 110xxxxx 10xxxxxx
+    if (c < 0xE0) {
+      if (i + 1 >= len)
+        return false;
+      uint8_t c1 = s[i + 1];
+      if (!is_cont(c1))
+        return false;
+      i += 2;
+      continue;
+    }
+
+    // 3-byte sequence: 1110xxxx 10xxxxxx 10xxxxxx
+    if (c < 0xF0) {
+      if (i + 2 >= len)
+        return false;
+      uint8_t c1 = s[i + 1];
+      uint8_t c2 = s[i + 2];
+
+      // Special rules to avoid overlongs and surrogates:
+      //   E0 A0–BF 80–BF
+      //   E1–EC 80–BF 80–BF
+      //   ED 80–9F 80–BF (avoid D800–DFFF)
+      //   EE–EF 80–BF 80–BF
+      if (!is_cont(c2))
+        return false;
+
+      switch (c) {
+      case 0xE0:
+        if (c1 < 0xA0 || c1 > 0xBF)
+          return false;
+        break;
+      case 0xED:
+        if (c1 < 0x80 || c1 > 0x9F)
+          return false;
+        break;
+      default:
+        if (!is_cont(c1))
+          return false;
+        break;
+      }
+
+      i += 3;
+      continue;
+    }
+
+    // 4-byte sequence: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
+    // Valid range is up to U+10FFFF:
+    //   F0 90–BF 80–BF 80–BF
+    //   F1–F3 80–BF 80–BF 80–BF
+    //   F4 80–8F 80–BF 80–BF
+    if (c < 0xF5) {
+      if (i + 3 >= len)
+        return false;
+      uint8_t c1 = s[i + 1];
+      uint8_t c2 = s[i + 2];
+      uint8_t c3 = s[i + 3];
+
+      if (!is_cont(c2) || !is_cont(c3))
+        return false;
+
+      switch (c) {
+      case 0xF0:
+        // Avoid overlongs: codepoints < 0x10000
+        if (c1 < 0x90 || c1 > 0xBF)
+          return false;
+        break;
+      case 0xF4:
+        // Avoid values > U+10FFFF
+        if (c1 < 0x80 || c1 > 0x8F)
+          return false;
+        break;
+      default:
+        if (!is_cont(c1))
+          return false;
+        break;
+      }
+
+      i += 4;
+      continue;
+    }
+
+    // 0xF5–0xFF are invalid in UTF-8
+    return false;
+  }
+
+  return true;
+}

data/ext/zsv/vendor/zsv-1.3.0/app/check.c

@@ -0,0 +1,194 @@
+/*
+ * Copyright (C) 2025 Liquidaty and zsv contributors. All rights reserved.
+ *
+ * This file is part of zsv/lib, distributed under the MIT license as defined at
+ * https://opensource.org/licenses/MIT
+ */
+
+#include <stdio.h>
+#include <errno.h>
+
+#define ZSV_COMMAND check
+#include "zsv_command.h"
+
+#define _GNU_SOURCE 1
+#include <string.h>
+#include <zsv/utils/arg.h>
+#include <zsv/utils/err.h>
+
+#ifdef USE_SIMDUTF
+#include "check/simdutf_wrapper.h"
+#define USAGE_APPNAME APPNAME " (simdutf8)"
+#else
+#include "check/utf8.c"
+#define USAGE_APPNAME APPNAME
+#endif
+
+struct zsv_check_data {
+  FILE *in;
+  FILE *out;
+  const char *input_path;
+  zsv_parser parser;
+  size_t row_ix;
+  size_t column_count;
+  int err;
+  unsigned char display_row : 1;
+  unsigned char check_utf8 : 1;
+  unsigned char _ : 6;
+};
+
+#ifdef USE_SIMDUTF
+#define UTF8VALIDATOR simdutf_is_valid_utf8
+#else
+#define UTF8VALIDATOR utf8_is_valid
+#endif
+
+static void zsv_check_row(void *ctx) {
+  struct zsv_check_data *data = ctx;
+  size_t column_count = zsv_cell_count(data->parser);
+  unsigned const char *row_start = NULL;
+  size_t row_len;
+  if (column_count != data->column_count) {
+    fprintf(data->out, "Row %zu column count (%zu) differs from header (%zu)", data->row_ix, column_count,
+            data->column_count);
+    data->err = 1;
+    if (data->display_row && column_count > 0) {
+      row_start = zsv_get_cell(data->parser, 0).str;
+      struct zsv_cell last_cell = zsv_get_cell(data->parser, column_count - 1);
+      row_len = (last_cell.str + last_cell.len - row_start);
+      fprintf(data->out, ": %.*s", (int)row_len, row_start);
+    }
+    fprintf(data->out, "\n");
+  }
+  if (data->check_utf8) {
+    if (!row_start) {
+      row_start = zsv_get_cell(data->parser, 0).str;
+      struct zsv_cell last_cell = zsv_get_cell(data->parser, column_count - 1);
+      row_len = (last_cell.str + last_cell.len - row_start);
+    }
+    if (row_len > 0 && !UTF8VALIDATOR(row_start, row_len)) {
+      data->err = 1;
+      fprintf(data->out, "Row %zu invalid utf8", data->row_ix);
+      if (data->display_row)
+        fprintf(data->out, ": %.*s", (int)row_len, row_start);
+      fprintf(data->out, "\n");
+    }
+  }
+  data->row_ix++;
+}
+
+static void zsv_check_header(void *ctx) {
+  struct zsv_check_data *data = ctx;
+  data->column_count = zsv_cell_count(data->parser);
+  data->row_ix++;
+  zsv_set_row_handler(data->parser, zsv_check_row);
+}
+
+static int zsv_check_usage(void) {
+  const char *zsv_check_usage_msg[] = {
+    USAGE_APPNAME ": check input for anomalies",
+    "",
+    "Usage: " APPNAME " <filename>",
+    "",
+    "Options:",
+    "  -o,--output <path> : output to specified file path",
+    "  --display-row      : display the row contents with any reported issue",
+    //    "  --utf8             : check for invalid utf8",
+    // "  --all              : run all checks",
+    "",
+    "If no check options are provided, all checks are run",
+    NULL,
+  };
+  for (size_t i = 0; zsv_check_usage_msg[i]; i++)
+    fprintf(stdout, "%s\n", zsv_check_usage_msg[i]);
+  return 1;
+}
+
+static void zsv_check_cleanup(struct zsv_check_data *data) {
+  if (data->in && data->in != stdin)
+    fclose(data->in);
+  if (data->out && data->out != stdout)
+    fclose(data->out);
+}
+
+int ZSV_MAIN_FUNC(ZSV_COMMAND)(int argc, const char *argv[], struct zsv_opts *optsp,
+                               struct zsv_prop_handler *custom_prop_handler) {
+  if (argc < 1 || (argc > 1 && (!strcmp(argv[1], "-h") || !strcmp(argv[1], "--help")))) {
+    zsv_check_usage();
+    return 0;
+  }
+  struct zsv_opts opts = *optsp;
+  struct zsv_check_data data = {0};
+  int err = 0;
+  for (int arg_i = 1; !err && arg_i < argc; arg_i++) {
+    const char *arg = argv[arg_i];
+    if (!strcmp(arg, "--display-row"))
+      data.display_row = 1;
+    // else if (!strcmp(arg, "--utf8"))
+    // data.check_utf8 = 1;
+    else if (!strcmp(arg, "-o") || !strcmp(arg, "--output")) {
+      if (data.out)
+        err = zsv_printerr(1, "Output specified more than once");
+      else {
+        const char *fn = zsv_next_arg(++arg_i, argc, argv, &err);
+        if (!(fn && *fn))
+          err = zsv_printerr(1, "%s requires a filename value", arg);
+        else if (!(data.out = fopen(fn, "wb"))) {
+          err = errno;
+          perror(fn);
+        }
+      }
+    } else if (!data.in) {
+#ifndef NO_STDIN
+      if (!strcmp(arg, "-"))
+        data.in = stdin;
+#endif
+      if (!data.in) {
+        if (!(data.in = fopen(arg, "rb"))) {
+          err = 1;
+          perror(arg);
+        } else
+          data.input_path = arg;
+      }
+    } else {
+      fprintf(stderr, "Unrecognized option: %s\n", arg);
+      err = 1;
+    }
+  }
+
+  if (!err && !data.in) {
+#ifndef NO_STDIN
+    data.in = stdin;
+#else
+    fprintf(stderr, "Please specify an input file\n");
+    err = 1;
+#endif
+  }
+
+  if (err) {
+    zsv_check_cleanup(&data);
+    return 1;
+  }
+
+  data.check_utf8 = 1;
+  if (!data.out)
+    data.out = stdout;
+  opts.row_handler = zsv_check_header;
+  opts.stream = data.in;
+  opts.ctx = &data;
+  if (zsv_new_with_properties(&opts, custom_prop_handler, data.input_path, &data.parser) != zsv_status_ok)
+    err = 1;
+  else {
+    // process the input data
+    zsv_handle_ctrl_c_signal();
+    enum zsv_status status;
+    while (!zsv_signal_interrupted && (status = zsv_parse_more(data.parser)) == zsv_status_ok)
+      ;
+    zsv_finish(data.parser);
+    zsv_delete(data.parser);
+  }
+  zsv_check_cleanup(&data);
+  if (!err)
+    err = data.err;
+  return err;
+}