zsv 1.3.0

data/ext/zsv/vendor/zsv-1.3.0/include/zsv/api.h

@@ -0,0 +1,336 @@
+/*
+ * Copyright (C) 2021 Tai Chi Minh Ralph Eastwood, Matt Wong and Guarnerix dba Liquidaty
+ * All rights reserved
+ *
+ * This file is part of zsv/lib, distributed under the license defined at
+ * https://opensource.org/licenses/MIT
+ */
+
+#ifndef ZSV_API_H
+#define ZSV_API_H
+
+#define ZSV_ROW_MAX_SIZE_DEFAULT 65536
+#define ZSV_ROW_MAX_SIZE_DEFAULT_S "64k"
+
+#define ZSV_MAX_COLS_DEFAULT 1024
+
+#define ZSV_ROW_MAX_SIZE_MIN 1024
+#define ZSV_ROW_MAX_SIZE_MIN_S "1024"
+
+#define ZSV_MIN_SCANNER_BUFFSIZE 4096
+#define ZSV_DEFAULT_SCANNER_BUFFSIZE (1 << 18) // 256k
+
+#include "zsv_export.h"
+/*****************************************************************************
+ * libzsv API
+ *
+ * Functions provided by the zsv library are described herein. This document is
+ * organized into three sections:
+ * - required functions. any time libzsv is used and any input at all is parsed,
+ *   each of these functions should be used (usually, exactly once)
+ * - minimal access functions. these functions are generally necessary to use
+ *   libzsv for any non-trivial task
+ * - other functions
+ ******************************************************************************/
+
+/******************************************************************************
+ * Required functions:
+ * - zsv_new(): allocate a parser
+ * - zsv_parse_more(): parse some data
+ * - zsv_finish(): tie up loose ends
+ * - zsv_delete(): dispose the parser
+ ******************************************************************************/
+
+ZSV_EXPORT
+int zsv_peek(zsv_parser);
+
+/**
+ * Create a zsv parser. Typically, passed options will at least include a
+ * a `row_handler()` callback. Many, but not all, options can be subsequently
+ * set or modified after calling `zsv_new()`
+ *
+ * @param options see `struct zsv_opts` in common.h
+ * @returns zsv parser handle
+ */
+ZSV_EXPORT
+zsv_parser zsv_new(struct zsv_opts *opts);
+
+/**
+ * Parse the next chunk of data from the input stream:
+ * - Immediately after a cell (column) delimiter is parsed, the configured
+ *   `cell_handler()` callback, if any, is called
+ * - Immediately after a row delimiter is parsed, the configured
+ *   `row_handler()` callback, if any, is called.
+ *
+ * @param parser
+ * @returns zsv_status_ok if more data remains to be parsed,
+ *          zsv_status_no_more_input if the stream's EOF has been reached,
+ *          or other zsv status code in the event of error or cancellation
+ */
+ZSV_EXPORT enum zsv_status zsv_parse_more(zsv_parser parser);
+
+/**
+ * Finish any remaining processing, after all input has been read
+ */
+ZSV_EXPORT enum zsv_status zsv_finish(zsv_parser);
+
+/**
+ * Dispose of a parser that was created with `zsv_new()` or `zsv_new_with_properties()`
+ */
+ZSV_EXPORT enum zsv_status zsv_delete(zsv_parser);
+
+/******************************************************************************
+ * minimal access functions:
+ * - zsv_cell_count(): get the number of cells in the row
+ * - zsv_get_cell(): retrieve a cell value
+ ******************************************************************************/
+
+/**
+ * Get the number of cells in the row that was just parsed. This function
+ * is typically called from within your `row_handler()` callback. In the event
+ * that the last row did not contain a single cell delimiter, returns 1
+ *
+ * @param parser
+ * @returns number, >= 1, of cells in the row that was just parsed
+ */
+ZSV_EXPORT
+size_t zsv_cell_count(zsv_parser parser);
+
+/**
+ * Get the contents of a cell in the row that was just parsed. See `struct zsv_cell`
+ * in common.h for further details
+ *
+ * @param parser
+ * @param index zero-based index of the cell to fetch
+ * @return `zsv_cell` structure with the bytes and length of this cell value
+ *
+ * An example of a `row_handler()` loop to print each cell in a row might be:
+ * ```
+ *   size_t cell_count = zsv_cell_count(parser);
+ *   for(size_t i = 0; i < cell_count; i++) {
+ *     struct zsv_cell c = zsv_get_cell(parser, i);
+ *     printf("%.*s", c.len, (const char *)c.str);
+ *   }
+ * ```
+ */
+struct zsv_cell zsv_get_cell(zsv_parser parser, size_t index);
+
+/**
+ * `zsv_get_cell_len()` is not needed in most cases, but may be useful in
+ * restrictive cases such as when calling from Javascript into wasm
+ *
+ * @param  parser
+ * @param  ix     0-based index of the cell to copy
+ * @return length of cell contents (0 if cell is empty)
+ */
+ZSV_EXPORT
+size_t zsv_get_cell_len(zsv_parser parser, size_t ix);
+
+/**
+ * get a pointer to the cell contents (not NULL-terminated)
+ * @param parser
+ * @param ix     0-based index of the cell to copy. Caller must ensure validity
+ * @return pointer to the cell contents, or NULL if contents are empty
+ */
+ZSV_EXPORT
+unsigned char *zsv_get_cell_str(zsv_parser parser, size_t ix);
+
+/******************************************************************************
+ * other functions
+ ******************************************************************************/
+
+/**
+ * Get the library version
+ */
+ZSV_EXPORT
+const char *zsv_lib_version(void);
+
+/**
+ * Change a parser's row handler. This function may be called at any time
+ * during the parsing process to change the row handler that is called each
+ * time a row is parsed
+ *
+ * @param parser
+ * @param row_handler new callback value
+ */
+ZSV_EXPORT void zsv_set_row_handler(zsv_parser, void (*row_handler)(void *ctx));
+
+/**
+ * Check if the row we just parsed consisted entirely of blank data
+ *
+ * @param parser
+ * @return non-zero if blank, 0 if non-blank
+ */
+ZSV_EXPORT
+char zsv_row_is_blank(zsv_parser parser);
+
+/**
+ * Set the context pointer that is passed to our callbacks
+ * @param parser
+ * @param ctx new context pointer value
+ */
+ZSV_EXPORT
+void zsv_set_context(zsv_parser parser, void *ctx);
+
+/**
+ * Set the read function that is invoked by `zsv_parse_more()` to fetch more data.
+ * If not explicitly set, defaults to fread
+ *
+ * @param parser
+ * @param read_function
+ * @param stream        value that is passed to read_function when it is called
+ */
+ZSV_EXPORT
+void zsv_set_read(zsv_parser parser, size_t (*read_func)(void *restrict, size_t n, size_t size, void *restrict));
+
+/**
+ * Set the input stream our parser reads from. If not explicity set, defaults to
+ * stdin. This can be used to read multiple inputs as a single combined input
+ * by calling `zsv_set_input()` after `zsv_parse_more()` returns
+ * `zsv_status_no_more_input`
+ */
+ZSV_EXPORT
+void zsv_set_input(zsv_parser, void *in);
+
+/**
+ * Insert a filter to process or modify, before parsing, the next chunk of raw
+ * bytes read from the input stream. For example, to save a copy of the raw
+ * input to a file, `zsv_set_scan_filter()` could be called with
+ * `zsv_filter_write` passed as the filter argument, and the target FILE *
+ * passed as the context pointer.
+ *
+ * @param parser
+ * @param filter callback that is called on each chunk that is read from the
+ *               input stream, before the chunk is parsed. The callback may
+ *               modify the contents of the buffer so long as its return value
+ *               does not exceed the bufflen it was passed
+ */
+ZSV_EXPORT enum zsv_status zsv_set_scan_filter(zsv_parser parser,
+                                               size_t (*filter)(void *ctx, unsigned char *buff, size_t bufflen),
+                                               void *ctx);
+
+/**
+ * Set parsing mode to fixed-width. Once set to fixed mode, a parser may not be
+ *   set back to CSV mode
+ * @return status code
+ * @param parser parser handle
+ * @param count number of elements in offsets
+ * @param offsets array of cell-end offsets. offsets[0] should be the length of the first cell
+ */
+ZSV_EXPORT enum zsv_status zsv_set_fixed_offsets(zsv_parser parser, size_t count, size_t *offsets);
+
+/**
+ * Parse a buffer of bytes. This function is usually not needed, but
+ * can be used to parse in a push instead of pull manner
+ *
+ * @param parser
+ * @param buff   the input buffer to parse. This buffer may not overlap with
+ *               the parser buffer!
+ * @param len    length of the input to parse
+ */
+ZSV_EXPORT enum zsv_status zsv_parse_bytes(zsv_parser parser, const unsigned char *restrict buff, size_t len);
+
+/**
+ * Get a text description of a status code
+ */
+ZSV_EXPORT
+const unsigned char *zsv_parse_status_desc(enum zsv_status status);
+
+/**
+ * Abort parsing. After this function is called, no further
+ * `row_handler()` or `cell_handler()` calls will be made, and parse functions
+ * will return zsv_status_cancelled
+ */
+ZSV_EXPORT
+void zsv_abort(zsv_parser);
+
+/**
+ * @return number of bytes scanned from the last zsv_parse_more() invocation
+ */
+ZSV_EXPORT size_t zsv_scanned_length(zsv_parser);
+
+/**
+ * @return cumulative number of bytes scanned across all requests by this parser
+ */
+ZSV_EXPORT size_t zsv_cum_scanned_length(zsv_parser parser);
+
+/**
+ * @return number of raw bytes scanned from the beginning of the row
+ * to the end of this row. Subtract from zsv_cum_scanned_length() to get the
+ * position of the beginning of the row
+ */
+ZSV_EXPORT size_t zsv_row_length_raw_bytes(zsv_parser parser);
+
+/**
+ * Check the quoted status of the last cell that was read. This function is only
+ * applicable when called from within a cell_handler() callback. Furthermore, this
+ * function is generally only useful when the cell value will subsequent be
+ * output in CSV format
+ *
+ * @returns 0 if value will *not* need to be quoted when output as CSV, or
+ *          1 if it might need to be quoted
+ */
+ZSV_EXPORT
+char zsv_quoted(zsv_parser parser);
+
+/**
+ * Create a zsv_opts structure and return its handle
+ *
+ * This is only necessary in environments where structures cannot be directly
+ * instantiated such as web assembly. Otherwise, you should avoid this function
+ * and just create a `struct zsv_opts` on the stack
+ *
+ * Each argument to `zsv_opts_new()` corresponds to the same-named `struct zsv_opts` element
+ * See common.h for details
+ */
+ZSV_EXPORT struct zsv_opts *zsv_opts_new(void (*row_handler)(void *ctx),
+                                         void (*cell_handler)(void *ctx, unsigned char *utf8_value, size_t len),
+                                         void *ctx, zsv_generic_read read, void *stream, unsigned char *buff,
+                                         size_t buffsize, unsigned max_columns, unsigned max_row_size, char delimiter,
+                                         char no_quotes
+#ifdef ZSV_EXTRAS
+                                         ,
+                                         size_t max_rows
+#endif
+);
+
+/**
+ * Destroy an option structure that was created by zsv_opts_new()
+ */
+ZSV_EXPORT void zsv_opts_delete(struct zsv_opts *);
+
+/******************************************************************************
+ * Pull parsing functions
+ ******************************************************************************/
+
+/**
+ * To use pull parsing, do not use row or cell handlers, handler context
+ * or zsv_parse_more(). Instead, call zsv_next_row() until a non-ok result
+ * @param  parser parser handle
+ * @return zsv_status_ok on success, other status code on error
+ */
+ZSV_EXPORT
+enum zsv_status zsv_next_row(zsv_parser parser);
+
+/******************************************************************************
+ * Miscellaneous functions used by the parser that may have standalone utility
+ ******************************************************************************/
+
+/**
+ * Force a string to conform to UTF8 encoding. Replaces any non-conforming utf8
+ * with the specified char, or removes from the string (and shortens the string)
+ * if replace = 0
+ * @param  s        input string. invalid UTF8 bytes will be overwritten
+ * @param  n        length (in bytes) of input
+ * @param  replace  the character to replace any malformed UTF8 bytes with, or 0
+ *                  to remove and shorten the result
+ * @param  callback optional callback invoked upon scanning malformed UTF8
+ * @param  ctx      context pointer passed to callback
+ * @return          length of the valid string
+ */
+ZSV_EXPORT
+size_t zsv_strencode(unsigned char *s, size_t n, unsigned char replace,
+                     int (*malformed_handler)(void *, const unsigned char *s, size_t n, size_t offset),
+                     void *handler_ctx);
+
+#endif

data/ext/zsv/vendor/zsv-1.3.0/include/zsv/common.h

@@ -0,0 +1,361 @@
+/*
+ * Copyright (C) 2021 Tai Chi Minh Ralph Eastwood, Matt Wong and Guarnerix dba Liquidaty
+ * All rights reserved
+ *
+ * This file is part of zsv/lib, distributed under the license defined at
+ * https://opensource.org/licenses/MIT
+ */
+
+#ifndef ZSV_COMMON_H
+#define ZSV_COMMON_H
+
+#ifdef __cplusplus
+#define ZSV_BEGIN_DECL extern "C" {
+#define ZSV_END_DECL }
+#else
+#define ZSV_BEGIN_DECL
+#define ZSV_END_DECL /* empty */
+#endif
+
+enum zsv_status {
+  zsv_status_ok = 0,
+  zsv_status_cancelled,
+  zsv_status_no_more_input,
+  zsv_status_invalid_option,
+  zsv_status_memory,
+  zsv_status_error,
+  zsv_status_row,
+  zsv_status_done = 100
+#ifdef ZSV_EXTRAS
+    ,
+  zsv_status_max_rows_read = 999
+#endif
+};
+
+/**
+ * `zsv_parser` is the type of a zsv parser handle
+ */
+typedef struct zsv_scanner *zsv_parser;
+
+/**
+ * Structure returned by `zsv_get_cell()` for fetching a parsed CSV cell value
+ */
+struct zsv_cell {
+  /**
+   * address of cell contents (not null-terminated)
+   */
+  unsigned char *str;
+
+  /**
+   * length of cell contents
+   */
+  size_t len;
+
+  /**
+   * bitfield values for `quoted` flags
+   */
+#define ZSV_PARSER_QUOTE_NONE 0        /* content does not need to be quoted */
+#define ZSV_PARSER_QUOTE_UNCLOSED 1    /* only used internally by parser */
+#define ZSV_PARSER_QUOTE_CLOSED 2      /* value was quoted */
+#define ZSV_PARSER_QUOTE_NEEDED 4      /* value contains delimiter or dbl-quote */
+#define ZSV_PARSER_QUOTE_EMBEDDED 8    /* value contains dbl-quote */
+#define ZSV_PARSER_QUOTE_PENDING 16    /* only used internally by parser */
+#define ZSV_PARSER_QUOTE_PENDING_LF 32 /* only used internally by parser */
+  /**
+   * quoted flags enable additional efficiency, in particular when input data will
+   * be output as text (csv, json etc), by indicating whether the cell contents may
+   * require special handling. For example, if the caller will output the cell value as
+   * CSV and quoted == 0, the caller need not scan the cell contents to check if
+   * quoting or escaping will be required
+   */
+  char quoted;
+  unsigned char overwritten : 1;
+};
+
+typedef size_t (*zsv_generic_write)(const void *restrict, size_t, size_t, void *restrict);
+typedef size_t (*zsv_generic_read)(void *restrict, size_t n, size_t size, void *restrict);
+typedef int (*zsv_generic_seek)(void *, long, int);
+
+#define zsv_generic_fprintf (int (*)(void *, const char *, ...)) fprintf;
+#define zsv_generic_fclose (int (*)(void *)) fclose;
+
+#ifdef ZSV_EXTRAS
+/**
+ * progress callback function signature
+ * @param context pointer set in parser opts.progress.ctx
+ * @param cumulative_row_count number of input rows read so far
+ * @return zero to continue processing, non-zero to cancel parse
+ */
+typedef int (*zsv_progress_callback)(void *ctx, size_t cumulative_row_count);
+
+/**
+ * completed callback function signature
+ * @param context pointer set in parser opts.progress.ctx
+ * @param exit code
+ */
+typedef void (*zsv_completed_callback)(void *ctx, int code);
+
+/**
+ * Data can be "overwritten" on-the-fly by providing custom callbacks
+ * data from the calling code is passed to the zsv library
+ * via the `zsv_overwrite_data` structure
+ */
+struct zsv_overwrite_data {
+  size_t row_ix; // 0-based
+  size_t col_ix; // 0-based
+  size_t timestamp;
+  struct zsv_cell val;
+  struct zsv_cell author;
+  struct zsv_cell old_value;
+  char have; // 1 = we have unprocessed overwrites
+};
+
+struct zsv_opt_overwrite {
+  void *ctx;
+  enum zsv_status (*open)(void *ctx);
+  enum zsv_status (*next)(void *ctx, struct zsv_overwrite_data *odata);
+  enum zsv_status (*close)(void *ctx);
+  char cancel; // explicitly cancel application of overwrites
+};
+
+#endif
+
+struct zsv_opts {
+  /**
+   * Callback that is called for each row that is parsed. In most use cases,
+   * this is where most of the code logic resides
+   */
+  void (*row_handler)(void *ctx);
+
+  /**
+   * Callback that is called immediately after a cell is parsed.
+   * The most common usage pattern is to omit the cell handler, and just loop
+   * through each cell in the `row_handler`. But if you prefer, you can use
+   * only a cell handler, or both a cell handler and a row handler.
+   */
+  void (*cell_handler)(void *ctx, unsigned char *utf8_value, size_t len);
+
+  /**
+   * If a row was too long to fit in the allocated memory, then
+   * if `overflow_row_handler` is set, it will be called after the remaining
+   * portion of the row is parsed. For example, if a row consists of
+   * XXXXX,YYYYY and there is only enough memory to hold XXXX
+   * then `row_handler()` will be called after parsing `XXXX,` and
+   * `overflow_row_handler()` called twice, after parsing `X,YY` and `YYY`,
+   * before the subsequent row is parsed
+   *
+   * Note: we considered adding a callback that would be called before the initial
+   * `row_handler()` call, in the event that a subsequent `overflow_row_handler()`
+   * call was anticipated. Because this scenario occurs so infrequently, we
+   * decided to keep it simple with a single callback. But we may reconsider
+   * if there is demand for that (or another) alternative approach
+   */
+  void (*overflow_row_handler)(void *ctx);
+
+  /**
+   * The context that is passed to each of our handlers
+   */
+  void *ctx;
+
+  /**
+   * Caller can specify its own read function for fetching data to be parsed
+   * If not specified, the default value is `fread()`
+   */
+  zsv_generic_read read;
+
+  /**
+   * Caller can specify its own seek function for setting the file position
+   * with zsv_index_seek. If not specified, the default value is `fseek()`
+   */
+  zsv_generic_seek seek;
+
+  /**
+   * Caller can specify its own stream that is passed to the read function
+   * If not specified, the default value is stdin
+   */
+  void *stream;
+
+  /**
+   * Caller can specify its own buffer for the parser to use of at least
+   * ZSV_MIN_SCANNER_BUFFSIZE (4096) in size. If not provided, an internal
+   * buffer is allocated
+   */
+  unsigned char *buff;
+
+  /**
+   * If caller specifies its own buffer, this should be its size
+   * Otherwise, this can be specified as the size of the internal buffer that
+   * will be created, subject to increase if/as appropriate if max_row_size
+   * is specified. Defaults to 256k
+   *
+   * cli option: -B,--buff-size
+   */
+  size_t buffsize;
+
+  /**
+   * Maximum number of columns to parse. defaults to 1024
+   *
+   * cli option: -c,--max-column-count
+   */
+  unsigned max_columns;
+
+  /**
+   * maximum row size can be used as an alternative way to specify the
+   * internal buffer size, which will be at least as large as max_row_size * 2
+   *
+   * cli option: -r,--max-row-size
+   */
+  unsigned max_row_size;
+
+  /**
+   * delimiter: typically a comma or tab
+   * can be any char other than newline, form feed or quote
+   * defaults to comma
+   *
+   * cli option: -t,--tab-delim or -O,--other-delim <delim>
+   */
+  char delimiter;
+
+  /**
+   * no_quotes: if > 0, this flag indicates that the parser should treat double-quotes
+   * just like any ordinary character
+   * defaults to 0
+   *
+   * cli option: -q,--no-quote
+   */
+  char no_quotes;
+
+#ifndef ZSV_NO_ONLY_CRLF
+  /**
+   * only_crlf_rowend: if non-zero, *only* accept CRLF as row end
+   *
+   * cli option: --only-crlf
+   */
+  char only_crlf_rowend;
+#endif
+  /**
+   * flag to print more verbose messages to the console
+   * cli option: -v,--verbose
+   */
+  char verbose;
+
+  /**
+   * if the actual data does not have a header row with column names, the caller
+   * should provide one (in CSV format) which will be treated as if it was the
+   * first row of data
+   *
+   * cli option: -0,--header-row
+   */
+  const char *insert_header_row;
+
+  /**
+   * number of rows that the header row spans. If 0 or 1, header is assumed to span 1 row
+   * otherwise, set to number > 1 to span multiple rows
+   */
+  unsigned int header_span;
+
+  /**
+   * number of rows to ignore before the initial row is processed
+   */
+  unsigned int rows_to_ignore;
+
+  /**
+   * by default, zsv ignores empty header rows; the number of ignored rows
+   * can be fetched via `zsv_get_empty_header_rows()`). To disable this
+   * behavior, set `keep_empty_header_rows` to 1
+   */
+  unsigned char keep_empty_header_rows;
+
+  /**
+   * by default, zsv ignores malformed UTF8; set malformed_utf8_replace to
+   * a value between 1 and 127 to replace malformed UTF8 with that single
+   * char, to ZSV_MALFORMED_UTF8_REMOVE to remove
+   * or ZSV_MALFORMED_UTF8_DO_NOT_REPLACE to explicitly leave untouched
+   */
+#define ZSV_MALFORMED_UTF8_DO_NOT_REPLACE -2
+#define ZSV_MALFORMED_UTF8_REMOVE -1
+  char malformed_utf8_replace;
+
+  /**
+   * `overrides` is a bitfield that indicates what ZSV options, if any, were
+   * specifically set in the command invocation and is used to ensure
+   * that option values set in the command invocation take priority over
+   * default values, or values saved in related property values such as
+   * .zsv/data/<filename>/props.json
+   *
+   * For example, if a file has a saved header row span of 2, but the
+   * command-line arguments explicitly included `--header-row-span 3`,
+   * then setting header_span to 3 and setting overrides.header_row_span
+   * ensures that the value of 3 is used
+   */
+  struct {
+    unsigned char header_row_span : 1;
+    unsigned char skip_head : 1;
+    unsigned char max_column_count : 1;
+    unsigned char malformed_utf8_replacement : 1;
+    unsigned char _ : 4;
+  } option_overrides;
+
+  int (*errprintf)(void *ctx, const char *format, ...);
+  void *errf;
+  int (*errclose)(void *ctx);
+
+#ifdef ZSV_EXTRAS
+  struct {
+    /**
+     * min number of rows between progress callback calls
+     */
+    size_t rows_interval;
+
+    /**
+     * min number of seconds b/w callback calls
+     */
+    unsigned int seconds_interval;
+
+    /**
+     * Progress callback, called periodically to provide progress updates
+     */
+    zsv_progress_callback callback;
+
+    /**
+     * Context passed to the callback, when the callback is invoked
+     */
+    void *ctx;
+  } progress;
+
+  struct {
+    /**
+     * Optional callback. If set, it is called by zsv_finish()
+     */
+    zsv_completed_callback callback;
+    /**
+     * Context passed to the callback, when the callback is invoked
+     */
+    void *ctx;
+  } completed;
+
+  /**
+   * maximum number of rows to parse (including any header rows)
+   */
+  size_t max_rows;
+
+  /**
+   * If non-zero, automatically apply overwrites located in
+   * /path/to/.zsv/data/my-data.csv/overwrite.sqlite3 for a given
+   * input /path/to/my-data.csv
+   *
+   * This flag is only used by zsv_new_with_properties()
+   * if using zsv_new(), this flag is ignored (use the `overwrite` structure instead)
+   */
+  char overwrite_auto;
+
+  /**
+   * Optional cell-level values that overwrite data returned to the caller by the API
+   * Use when not using overwrite_auto together with zsv_new_with_properties()
+   */
+  struct zsv_opt_overwrite overwrite;
+
+#endif /* ZSV_EXTRAS */
+};
+
+#endif