simdjson 0.2.0 → 0.3.0

data/vendor/simdjson/tools/minify.cpp

@@ -1,21 +1,79 @@
 #include <iostream>
+#if (!(_MSC_VER) && !(__MINGW32__) && !(__MINGW64__))
+#include <dirent.h>
+#endif
+#include <unistd.h>
 
-#include "simdjson/jsonioutil.h"
-#include "simdjson/jsonminifier.h"
+#include "simdjson.h"
+
+SIMDJSON_PUSH_DISABLE_ALL_WARNINGS
+#ifndef __cpp_exceptions
+#define CXXOPTS_NO_EXCEPTIONS
+#endif
+#include "cxxopts.hpp"
+SIMDJSON_POP_DISABLE_WARNINGS
+
+cxxopts::Options options("minify", "Runs the parser against the given json files in a loop, measuring speed and other statistics.");
+
+void usage(std::string message) {
+  std::cerr << message << std::endl;
+  std::cerr << options.help() << std::endl;
+}
 
 int main(int argc, char *argv[]) {
-  if (argc != 2) {
-    std::cerr << "Usage: " << argv[0] << " <jsonfile>\n";
-    exit(1);
+#ifdef __cpp_exceptions
+  try {
+#endif
+  std::stringstream ss;
+  ss << "Parser implementation (by default, detects the most advanced implementation supported on the host machine)."  << std::endl;
+  ss << "Available parser implementations:" << std::endl;
+  for (auto impl : simdjson::available_implementations) {
+    ss << "-a " << std::left << std::setw(9) << impl->name() << " - Use the " << impl->description() << " parser implementation." << std::endl;
   }
+  options.add_options()
+    ("a,arch", ss.str(), cxxopts::value<std::string>())
+    ("f,file", "File name.", cxxopts::value<std::string>())
+    ("h,help", "Print usage.")
+  ;
+
+  options.parse_positional({"file"});
+  auto result = options.parse(argc, argv);
+
+  if(result.count("help")) {
+    usage("");
+    return EXIT_SUCCESS;
+  }
+
+  if(!result.count("file")) {
+    usage("No filename specified.");
+    return EXIT_FAILURE;
+  }
+  if(result.count("arch")) {
+    const simdjson::implementation *impl = simdjson::available_implementations[result["arch"].as<std::string>().c_str()];
+    if(!impl) {
+      usage("Unsupported implementation.");
+      return EXIT_FAILURE;
+    }
+    simdjson::active_implementation = impl;
+  }
+
+  std::string filename = result["file"].as<std::string>();
+
   simdjson::padded_string p;
-  std::string filename = argv[argc - 1];
-  try {
-    simdjson::get_corpus(filename).swap(p);
-  } catch (const std::exception &) {
-    std::cout << "Could not load the file " << filename << std::endl;
+  auto error = simdjson::padded_string::load(filename).get(p);
+  if (error) {
+    std::cerr << "Could not load the file " << filename << std::endl;
+    return EXIT_FAILURE;
+  }
+  simdjson::padded_string copy(p.length());
+  size_t copy_len;
+  error = simdjson::active_implementation->minify((const uint8_t*)p.data(), p.length(), (uint8_t*)copy.data(), copy_len);
+  if (error) { std::cerr << error << std::endl; return 1; }
+  printf("%s", copy.data());
+#ifdef __cpp_exceptions
+  } catch (const cxxopts::OptionException& e) {
+    std::cout << "error parsing options: " << e.what() << std::endl;
     return EXIT_FAILURE;
   }
-  simdjson::json_minify(p, p.data());
-  printf("%s", p.data());
+#endif
 }

data/vendor/simdjson/tools/release.py

@@ -7,7 +7,13 @@ import re
 import subprocess
 import io
 import os
+import fileinput
+if sys.version_info < (3, 0):
+    sys.stdout.write("Sorry, requires Python 3.x or better\n")
+    sys.exit(1)
 
+def colored(r, g, b, text):
+    return "\033[38;2;{};{};{}m{} \033[38;2;255;255;255m".format(r, g, b, text)
 
 def extractnumbers(s):
     return tuple(map(int,re.findall("(\d+)\.(\d+)\.(\d+)",str(s))[0]))
@@ -22,17 +28,13 @@ pipe = subprocess.Popen(["git", "rev-parse", "--abbrev-ref", "HEAD"], stdout=sub
 branchresult = pipe.communicate()[0].decode().strip()
 
 if(branchresult != "master"):
-    print("release on master, you are on '"+branchresult+"'")
-    sys.exit(-1)
-
+    print(colored(255, 0, 0, "We recommend that you release on master, you are on '"+branchresult+"'"))
 
 ret = subprocess.call(["git", "remote", "update"])
 
 if(ret != 0):
     sys.exit(ret)
 
-
-
 pipe = subprocess.Popen(["git", "log", "HEAD..", "--oneline"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
 uptodateresult = pipe.communicate()[0].decode().strip()
 
@@ -42,7 +44,7 @@ if(len(uptodateresult) != 0):
 
 pipe = subprocess.Popen(["git", "rev-parse", "--show-toplevel"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
 maindir = pipe.communicate()[0].decode().strip()
-
+scriptlocation = os.path.dirname(os.path.abspath(__file__))
 
 print("repository: "+maindir)
 
@@ -76,50 +78,120 @@ elif (newversion[1] !=  currentv[1]):
 else :
     assert newversion[2] ==  currentv[2] + 1
 
+atleastminor= (currentv[0] != newversion[0]) or (currentv[1] != newversion[1])
 
+if(atleastminor):
+    print(colored(0, 255, 0, "This is more than a revision."))
+    releasefile = maindir + os.sep + "RELEASES.md"
+    releasedata = open(releasefile).read()
+    pattern = re.compile("#\s+\d+\.\d+")
+    m = pattern.search(releasedata)
+    if(m == None):
+        print(colored(255, 0, 0, "You are preparing a new minor release and you have not yet updated RELEASES.md."))
+        sys.exit(-1)
 
 versionfilerel = os.sep + "include" + os.sep + "simdjson" + os.sep + "simdjson_version.h"
 versionfile = maindir + versionfilerel
 
 with open(versionfile, 'w') as file:
-    file.write("// "+versionfilerel+" automatically generated by release.py,\n")
+    file.write("// /include/simdjson/simdjson_version.h automatically generated by release.py,\n")
     file.write("// do not change by hand\n")
-    file.write("#ifndef SIMDJSON_INCLUDE_SIMDJSON_VERSION\n")
-    file.write("#define SIMDJSON_INCLUDE_SIMDJSON_VERSION\n")
+    file.write("#ifndef SIMDJSON_SIMDJSON_VERSION_H\n")
+    file.write("#define SIMDJSON_SIMDJSON_VERSION_H\n")
+    file.write("\n")
+    file.write("/** The version of simdjson being used (major.minor.revision) */\n")
     file.write("#define SIMDJSON_VERSION "+toversionstring(*newversion)+"\n")
+    file.write("\n")
     file.write("namespace simdjson {\n")
     file.write("enum {\n")
+    file.write("  /**\n")
+    file.write("   * The major version (MAJOR.minor.revision) of simdjson being used.\n")
+    file.write("   */\n")
     file.write("  SIMDJSON_VERSION_MAJOR = "+str(newversion[0])+",\n")
+    file.write("  /**\n")
+    file.write("   * The minor version (major.MINOR.revision) of simdjson being used.\n")
+    file.write("   */\n")
     file.write("  SIMDJSON_VERSION_MINOR = "+str(newversion[1])+",\n")
+    file.write("  /**\n")
+    file.write("   * The revision (major.minor.REVISION) of simdjson being used.\n")
+    file.write("   */\n")
     file.write("  SIMDJSON_VERSION_REVISION = "+str(newversion[2])+"\n")
     file.write("};\n")
-    file.write("}\n")
-    file.write("#endif // SIMDJSON_INCLUDE_SIMDJSON_VERSION\n")
-
+    file.write("} // namespace simdjson\n")
+    file.write("\n")
+    file.write("#endif // SIMDJSON_SIMDJSON_VERSION_H\n")
 
 print(versionfile + " modified")
 
-import fileinput
-import re
-
 newmajorversionstring = str(newversion[0])
 mewminorversionstring = str(newversion[1])
 newrevversionstring = str(newversion[2])
 newversionstring = str(newversion[0]) + "." + str(newversion[1]) + "." + str(newversion[2])
 cmakefile = maindir + os.sep + "CMakeLists.txt"
+
+sonumber = None
+pattern = re.compile("set\(SIMDJSON_LIB_SOVERSION \"(\d+)\" CACHE STRING \"simdjson library soversion\"\)")
+with open (cmakefile, 'rt') as myfile:
+    for line in myfile:
+        m = pattern.search(line)
+        if m != None:
+            sonumber = int(m.group(1))
+            break
+print("so library number "+str(sonumber))
+
+if(atleastminor):
+    print("Given that we have a minor revision, it seems necessary to bump the so library number")
+    print("See https://github.com/simdjson/simdjson/issues/661")
+    sonumber += 1
+
 for line in fileinput.input(cmakefile, inplace=1, backup='.bak'):
-    line = re.sub('SIMDJSON_LIB_VERSION "\d+\.\d+\.\d+','SIMDJSON_LIB_VERSION "'+newversionstring, line.rstrip())
-    line = re.sub('SIMDJSON_LIB_SOVERSION "\d+','SIMDJSON_LIB_SOVERSION "'+newmajorversionstring, line)
+    line = re.sub('SIMDJSON_SEMANTIC_VERSION "\d+\.\d+\.\d+','SIMDJSON_SEMANTIC_VERSION "'+newversionstring, line.rstrip())
+    line = re.sub('SIMDJSON_LIB_VERSION "\d+','SIMDJSON_LIB_VERSION "'+str(sonumber), line)
     line = re.sub('set\(PROJECT_VERSION_MAJOR \d+','set(PROJECT_VERSION_MAJOR '+newmajorversionstring, line)
     line = re.sub('set\(PROJECT_VERSION_MINOR \d+','set(PROJECT_VERSION_MINOR '+mewminorversionstring, line)
     line = re.sub('set\(PROJECT_VERSION_PATCH \d+','set(PROJECT_VERSION_PATCH '+newrevversionstring, line)
+    line = re.sub('set\(SIMDJSON_LIB_SOVERSION \"\d+\"','set(SIMDJSON_LIB_SOVERSION \"'+str(sonumber)+'\"', line)
     print(line)
 
-
 print("modified "+cmakefile+", a backup was made")
 
-scriptlocation = os.path.dirname(os.path.abspath(__file__))
 
+doxyfile = maindir + os.sep + "Doxyfile"
+for line in fileinput.input(doxyfile, inplace=1, backup='.bak'):
+    line = re.sub('PROJECT_NUMBER         = "\d+\.\d+\.\d+','PROJECT_NUMBER         = "'+newversionstring, line.rstrip())
+    print(line)
+print("modified "+doxyfile+", a backup was made")
+
+
+
+cp = subprocess.run(["bash", "amalgamate.sh"], stdout=subprocess.DEVNULL, cwd=maindir+ os.sep + "singleheader")  # doesn't capture output
+if(cp.returncode != 0):
+    print("Failed to run amalgamate")
+
+cp = subprocess.run(["doxygen"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, cwd=maindir)  # doesn't capture output
+if(cp.returncode != 0):
+    print("Failed to run doxygen")
+
+#ipe = subprocess.Popen(["doxygen"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=maindir)
+#doxygenresult = pipe.communicate()[0].decode().strip()
 
-print("Please run the tests before issuing a release, do make test && make amalgamate \n")
+pattern = re.compile("https://simdjson.org/api/(\d+\.\d+\.\d+)/index.html")
+readmefile = maindir + os.sep + "README.md"
+readmedata = open(readmefile).read()
+m = pattern.search(readmedata)
+if m == None:
+    print(colored(255, 0, 0, 'I cannot find a link to the API documentation in your README?????'))
+else: 
+    detectedreadme = m.group(1)
+    print("found a link to your API documentation in the README file: "+detectedreadme+" ("+toversionstring(*newversion)+")")
+    if(atleastminor):
+       if(detectedreadme != toversionstring(*newversion)):
+           print(colored(255, 0, 0, "Consider updating the readme link to "+toversionstring(*newversion)))
+
+
+
+print("Please run the tests before issuing a release. \n")
 print("to issue release, enter \n git commit -a && git push  &&  git tag -a v"+toversionstring(*newversion)+" -m \"version "+toversionstring(*newversion)+"\" &&  git push --tags \n")
+
+
+

data/vendor/simdjson/windows/CMakeLists.txt

@@ -0,0 +1,6 @@
+add_library(simdjson-windows-headers INTERFACE)
+if(MSVC OR MINGW)
+  target_include_directories(simdjson-windows-headers INTERFACE .)
+  # getopt.h triggers bogus CRT_SECURE warnings. If you include them, you need this.
+  target_compile_definitions(simdjson-windows-headers INTERFACE _CRT_SECURE_NO_WARNINGS)
+endif()

data/vendor/simdjson/windows/dirent_portable.h

@@ -1,1043 +1,10 @@
-// only use under Visual Studio and only for jsoncheck.cpp
-/*
- * This file was originally: "dirent for Visual C++" from: http://softagalleria.net/dirent.php (version 1.20.1)
- * However I've modified it to <dirent_portable.h> by adding:
- *
- * a fallback to <dirent.h> if _WIN32 is not defined
- * two missing methods: scandir(...) and alphasort(...)
- * and some other minor modifications (see below)
- *
- *
- * Original license from http://softagalleria.net/dirent.php
- *
- *========================================================================
- *
- * dirent.h - dirent API for Microsoft Visual Studio
- *
- * Copyright (C) 2006-2012 Toni Ronkko
- *
- * Permission is hereby granted, free of charge, to any person obtaining
- * a copy of this software and associated documentation files (the
- * ``Software''), to deal in the Software without restriction, including
- * without limitation the rights to use, copy, modify, merge, publish,
- * distribute, sublicense, and/or sell copies of the Software, and to
- * permit persons to whom the Software is furnished to do so, subject to
- * the following conditions:
- *
- * The above copyright notice and this permission notice shall be included
- * in all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS
- * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
- * IN NO EVENT SHALL TONI RONKKO BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * $Id: dirent.h,v 1.20 2014/03/19 17:52:23 tronkko Exp $
- *
- * =========================================================================
- * Added:
- * -> some undefs to prevent possible compiler warnings
- * -> the scandir(...) and alphasort(...) methods
- * -> the optional DIRENT_USE_ASCII_SHORT_PATHS_ON_WINDOWS definition (needed for browsing with short ASCII paths instead of long UTF8 paths).
- *    WARNING: in my tests the usage of the long UTF8 paths is not fully functional (patches are welcome)
- * All these additions have been made to made <dirent_portable.h> usage for Windows consistent
- * with what I get using <direct.h> under my Ubuntu Linux OS.
- * =========================================================================
- *
- * The code of the scandir(...) method come from the musl library (http://www.musl-libc.org/)
- * (MIT licensed, Copyright © 2005-2014 Rich Felker, et al.).
- *
- * The code of the alphasort(...) method and of all the other minor modifications is in the public domain.
- *
- */
-
-#if (!defined(_WIN32) && !defined(_WIN64))
-#	include <dirent.h>
-#else // #if (!defined(_WIN32) && !defined(_WIN64))
-
-#ifndef DIRENT_H
-#define DIRENT_H
-
-/*
- * Define architecture flags so we don't need to include windows.h.
- * Avoiding windows.h makes it simpler to use windows sockets in conjunction
- * with dirent.h.
- */
-#if !defined(_68K_) && !defined(_MPPC_) && !defined(_X86_) && !defined(_IA64_) && !defined(_AMD64_) && defined(_M_IX86)
-#   define _X86_
-#endif
-#if !defined(_68K_) && !defined(_MPPC_) && !defined(_X86_) && !defined(_IA64_) && !defined(_AMD64_) && defined(_M_AMD64)
-#define _AMD64_
-#endif
-#include <stringapiset.h>
-#include <stdio.h>
-#include <stdarg.h>
-#include <windef.h>     // MAX_PATH is defined here,
-#include <winbase.h>
-#include <wchar.h>
-#include <string.h>
-#include <stdlib.h>
-#include <malloc.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <errno.h>
-#if (!defined(SIZE_MAX) && !defined(INT_MAX))
-#include <limits.h> // INT_MAX
-#endif //(!defined(SIZE_MAX) && !defined(INT_MAX))
-
-
-//#define DIRENT_USE_ASCII_SHORT_PATHS_ON_WINDOWS  // set it globally, not just here]
-
-/* Indicates that d_type field is available in dirent structure */
-#define _DIRENT_HAVE_D_TYPE
-
-/* Indicates that d_namlen field is available in dirent structure */
-#define _DIRENT_HAVE_D_NAMLEN
-
-/* Entries missing from MSVC 6.0 */
-#if !defined(FILE_ATTRIBUTE_DEVICE)
-#   define FILE_ATTRIBUTE_DEVICE 0x40
-#endif
-
-/* File type and permission flags for stat() */
-#if !defined(S_IFMT)
-#   define S_IFMT   _S_IFMT                     /* File type mask */
-#endif
-#if !defined(S_IFDIR)
-#   define S_IFDIR  _S_IFDIR                    /* Directory */
-#endif
-#if !defined(S_IFCHR)
-#   define S_IFCHR  _S_IFCHR                    /* Character device */
-#endif
-#if !defined(S_IFFIFO)
-#   define S_IFFIFO _S_IFFIFO                   /* Pipe */
-#endif
-#if !defined(S_IFREG)
-#   define S_IFREG  _S_IFREG                    /* Regular file */
-#endif
-#if !defined(S_IREAD)
-#   define S_IREAD  _S_IREAD                    /* Read permission */
-#endif
-#if !defined(S_IWRITE)
-#   define S_IWRITE _S_IWRITE                   /* Write permission */
-#endif
-#if !defined(S_IEXEC)
-#   define S_IEXEC  _S_IEXEC                    /* Execute permission */
-#endif
-#if !defined(S_IFIFO)
-#   define S_IFIFO _S_IFIFO                     /* Pipe */
-#endif
-#if !defined(S_IFBLK)
-#   define S_IFBLK   0                          /* Block device */
-#endif
-#if !defined(S_IFLNK)
-#   define S_IFLNK   0                          /* Link */
-#endif
-#if !defined(S_IFSOCK)
-#   define S_IFSOCK  0                          /* Socket */
-#endif
-
-#if defined(_MSC_VER)
-#   define S_IRUSR  S_IREAD                     /* Read user */
-#   define S_IWUSR  S_IWRITE                    /* Write user */
-#   define S_IXUSR  0                           /* Execute user */
-#   define S_IRGRP  0                           /* Read group */
-#   define S_IWGRP  0                           /* Write group */
-#   define S_IXGRP  0                           /* Execute group */
-#   define S_IROTH  0                           /* Read others */
-#   define S_IWOTH  0                           /* Write others */
-#   define S_IXOTH  0                           /* Execute others */
-#endif
-
-/* Maximum length of file name */
-#ifndef DIRENT_MAX_PATH
-#   ifndef MAX_PATH
-#       define MAX_PATH PATH_MAX    // it should be in <limits.h> AFAIK
-#   endif //MAX_PATH
-#   ifndef DIRENT_USE_ASCII_SHORT_PATHS_ON_WINDOWS    // utf8 strings can have up to 4 bytes per char
-#       define DIRENT_MAX_PATH (MAX_PATH*4)
-#   else //DIRENT_USE_ASCII_SHORT_PATHS_ON_WINDOWS
-#       define DIRENT_MAX_PATH (MAX_PATH)
-#   endif //DIRENT_USE_ASCII_SHORT_PATHS_ON_WINDOWS
-#endif //DIRENT_MAX_PATH
-
-/* File type flags for d_type */
-#define DT_UNKNOWN  0
-#define DT_REG      S_IFREG
-#define DT_DIR      S_IFDIR
-#define DT_FIFO     S_IFIFO
-#define DT_SOCK     S_IFSOCK
-#define DT_CHR      S_IFCHR
-#define DT_BLK      S_IFBLK
-#define DT_LNK      S_IFLNK
-
-/* Macros for converting between st_mode and d_type */
-#define IFTODT(mode) ((mode) & S_IFMT)
-#define DTTOIF(type) (type)
-
-/*
- * File type macros.  Note that block devices, sockets and links cannot be
- * distinguished on Windows and the macros S_ISBLK, S_ISSOCK and S_ISLNK are
- * only defined for compatibility.  These macros should always return false
- * on Windows.
- */
-
-// Added some undefs to prevent possible compiler warnings
-#undef S_ISFIFO
-#undef S_ISDIR
-#undef S_ISREG
-#undef S_ISLNK
-#undef S_ISSOCK
-#undef S_ISSOCK
-#undef S_ISCHR
-#undef S_ISBLK
-
-#define	S_ISFIFO(mode) (((mode) & S_IFMT) == S_IFIFO)
-#define	S_ISDIR(mode)  (((mode) & S_IFMT) == S_IFDIR)
-#define	S_ISREG(mode)  (((mode) & S_IFMT) == S_IFREG)
-#define	S_ISLNK(mode)  (((mode) & S_IFMT) == S_IFLNK)
-#define	S_ISSOCK(mode) (((mode) & S_IFMT) == S_IFSOCK)
-#define	S_ISCHR(mode)  (((mode) & S_IFMT) == S_IFCHR)
-#define	S_ISBLK(mode)  (((mode) & S_IFMT) == S_IFBLK)
-
-/* Return the exact length of d_namlen without zero terminator */
-#define _D_EXACT_NAMLEN(p) ((p)->d_namlen)
-
-/* Return number of bytes needed to store d_namlen */
-#define _D_ALLOC_NAMLEN(p) (DIRENT_MAX_PATH)
-
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-
-/* Wide-character version */
-struct _wdirent {
-    long d_ino;                                 /* Always zero */
-    unsigned short d_reclen;                    /* Structure size */
-    size_t d_namlen;                            /* Length of name without \0 */
-    int d_type;                                 /* File type */
-    wchar_t d_name[DIRENT_MAX_PATH];                   /* File name */
-};
-typedef struct _wdirent _wdirent;
-
-struct _WDIR {
-    struct _wdirent ent;                        /* Current directory entry */
-    WIN32_FIND_DATAW data;                      /* Private file data */
-    int cached;                                 /* True if data is valid */
-    HANDLE handle;                              /* Win32 search handle */
-    wchar_t *patt;                              /* Initial directory name */
-};
-typedef struct _WDIR _WDIR;
-
-static _WDIR *_wopendir (const wchar_t *dirname);
-static struct _wdirent *_wreaddir (_WDIR *dirp);
-static int _wclosedir (_WDIR *dirp);
-static void _wrewinddir (_WDIR* dirp);
-
-
-/* For compatibility with Symbian */
-#define wdirent _wdirent
-#define WDIR _WDIR
-#define wopendir _wopendir
-#define wreaddir _wreaddir
-#define wclosedir _wclosedir
-#define wrewinddir _wrewinddir
-
-
-/* Multi-byte character versions */
-struct dirent {
-    long d_ino;                                 /* Always zero */
-    unsigned short d_reclen;                    /* Structure size */
-    size_t d_namlen;                            /* Length of name without \0 */
-    int d_type;                                 /* File type */
-    char d_name[DIRENT_MAX_PATH];               /* File name */
-};
-typedef struct dirent dirent;
-
-struct DIR {
-    struct dirent ent;
-    struct _WDIR *wdirp;
-};
-typedef struct DIR DIR;
-
-static DIR *opendir (const char *dirname);
-static struct dirent *readdir (DIR *dirp);
-static int closedir (DIR *dirp);
-static void rewinddir (DIR* dirp);
-
-
-/* Internal utility functions */
-static WIN32_FIND_DATAW *dirent_first (_WDIR *dirp);
-static WIN32_FIND_DATAW *dirent_next (_WDIR *dirp);
-
-static int dirent_mbstowcs_s(
-    size_t *pReturnValue,
-    wchar_t *wcstr,
-    size_t sizeInWords,
-    const char *mbstr,
-    size_t count);
-
-static int dirent_wcstombs_s(
-    size_t *pReturnValue,
-    char *mbstr,
-    size_t sizeInBytes,
-    const wchar_t *wcstr,
-    size_t count);
-
-static void dirent_set_errno (int error);
-
-/*
- * Open directory stream DIRNAME for read and return a pointer to the
- * internal working area that is used to retrieve individual directory
- * entries.
- */
-static _WDIR*
-_wopendir(
-    const wchar_t *dirname)
-{
-    _WDIR *dirp = NULL;
-    int error;
-
-    /* Must have directory name */
-    if (dirname == NULL  ||  dirname[0] == '\0') {
-        dirent_set_errno (ENOENT);
-        return NULL;
-    }
-
-    /* Allocate new _WDIR structure */
-    dirp = (_WDIR*) malloc (sizeof (struct _WDIR));
-    if (dirp != NULL) {
-        DWORD n;
-
-        /* Reset _WDIR structure */
-        dirp->handle = INVALID_HANDLE_VALUE;
-        dirp->patt = NULL;
-        dirp->cached = 0;
-
-        /* Compute the length of full path plus zero terminator */
-        n = GetFullPathNameW (dirname, 0, NULL, NULL);
-
-        /* Allocate room for absolute directory name and search pattern */
-        dirp->patt = (wchar_t*) malloc (sizeof (wchar_t) * n + 16);
-        if (dirp->patt) {
-
-            /*
-             * Convert relative directory name to an absolute one.  This
-             * allows rewinddir() to function correctly even when current
-             * working directory is changed between opendir() and rewinddir().
-             */
-            n = GetFullPathNameW (dirname, n, dirp->patt, NULL);
-            if (n > 0) {
-                wchar_t *p;
-
-                /* Append search pattern \* to the directory name */
-                p = dirp->patt + n;
-                if (dirp->patt < p) {
-                    switch (p[-1]) {
-                    case '\\':
-                    case '/':
-                    case ':':
-                        /* Directory ends in path separator, e.g. c:\temp\ */
-                        /*NOP*/;
-                        break;
-
-                    default:
-                        /* Directory name doesn't end in path separator */
-                        *p++ = '\\';
-                    }
-                }
-                *p++ = '*';
-                *p = '\0';
-
-                /* Open directory stream and retrieve the first entry */
-                if (dirent_first (dirp)) {
-                    /* Directory stream opened successfully */
-                    error = 0;
-                } else {
-                    /* Cannot retrieve first entry */
-                    error = 1;
-                    dirent_set_errno (ENOENT);
-                }
-
-            } else {
-                /* Cannot retrieve full path name */
-                dirent_set_errno (ENOENT);
-                error = 1;
-            }
-
-        } else {
-            /* Cannot allocate memory for search pattern */
-            error = 1;
-        }
-
-    } else {
-        /* Cannot allocate _WDIR structure */
-        error = 1;
-    }
-
-    /* Clean up in case of error */
-    if (error  &&  dirp) {
-        _wclosedir (dirp);
-        dirp = NULL;
-    }
-
-    return dirp;
-}
-
-/*
- * Read next directory entry.  The directory entry is returned in dirent
- * structure in the d_name field.  Individual directory entries returned by
- * this function include regular files, sub-directories, pseudo-directories
- * "." and ".." as well as volume labels, hidden files and system files.
- */
-static struct _wdirent*
-_wreaddir(
-    _WDIR *dirp)
-{
-    WIN32_FIND_DATAW *datap;
-    struct _wdirent *entp;
-
-    /* Read next directory entry */
-    datap = dirent_next (dirp);
-    if (datap) {
-        size_t n;
-        DWORD attr;
-
-        /* Pointer to directory entry to return */
-        entp = &dirp->ent;
-
-        /*
-         * Copy file name as wide-character string.  If the file name is too
-         * long to fit in to the destination buffer, then truncate file name
-         * to DIRENT_MAX_PATH characters and zero-terminate the buffer.
-         */
-        n = 0;
-        while (n + 1 < DIRENT_MAX_PATH  &&  datap->cFileName[n] != 0) {
-            entp->d_name[n] = datap->cFileName[n];
-            n++;
-        }
-        dirp->ent.d_name[n] = 0;
-
-        /* Length of file name excluding zero terminator */
-        entp->d_namlen = n;
-
-        /* File type */
-        attr = datap->dwFileAttributes;
-        if ((attr & FILE_ATTRIBUTE_DEVICE) != 0) {
-            entp->d_type = DT_CHR;
-        } else if ((attr & FILE_ATTRIBUTE_DIRECTORY) != 0) {
-            entp->d_type = DT_DIR;
-        } else {
-            entp->d_type = DT_REG;
-        }
-
-        /* Reset dummy fields */
-        entp->d_ino = 0;
-        entp->d_reclen = sizeof (struct _wdirent);
-
-    } else {
-
-        /* Last directory entry read */
-        entp = NULL;
-
-    }
-
-    return entp;
-}
-
-/*
- * Close directory stream opened by opendir() function.  This invalidates the
- * DIR structure as well as any directory entry read previously by
- * _wreaddir().
- */
-static int
-_wclosedir(
-    _WDIR *dirp)
-{
-    int ok;
-    if (dirp) {
-
-        /* Release search handle */
-        if (dirp->handle != INVALID_HANDLE_VALUE) {
-            FindClose (dirp->handle);
-            dirp->handle = INVALID_HANDLE_VALUE;
-        }
-
-        /* Release search pattern */
-        if (dirp->patt) {
-            free (dirp->patt);
-            dirp->patt = NULL;
-        }
-
-        /* Release directory structure */
-        free (dirp);
-        ok = /*success*/0;
-
-    } else {
-        /* Invalid directory stream */
-        dirent_set_errno (EBADF);
-        ok = /*failure*/-1;
-    }
-    return ok;
-}
-
-/*
- * Rewind directory stream such that _wreaddir() returns the very first
- * file name again.
- */
-static void
-_wrewinddir(
-    _WDIR* dirp)
-{
-    if (dirp) {
-        /* Release existing search handle */
-        if (dirp->handle != INVALID_HANDLE_VALUE) {
-            FindClose (dirp->handle);
-        }
-
-        /* Open new search handle */
-        dirent_first (dirp);
-    }
-}
-
-/* Get first directory entry (internal) */
-static WIN32_FIND_DATAW*
-dirent_first(
-    _WDIR *dirp)
-{
-    WIN32_FIND_DATAW *datap;
-
-    /* Open directory and retrieve the first entry */
-    dirp->handle = FindFirstFileW (dirp->patt, &dirp->data);
-    if (dirp->handle != INVALID_HANDLE_VALUE) {
-
-        /* a directory entry is now waiting in memory */
-        datap = &dirp->data;
-        dirp->cached = 1;
-
-    } else {
-
-        /* Failed to re-open directory: no directory entry in memory */
-        dirp->cached = 0;
-        datap = NULL;
-
-    }
-    return datap;
-}
-
-/* Get next directory entry (internal) */
-static WIN32_FIND_DATAW*
-dirent_next(
-    _WDIR *dirp)
-{
-    WIN32_FIND_DATAW *p;
-
-    /* Get next directory entry */
-    if (dirp->cached != 0) {
-
-        /* A valid directory entry already in memory */
-        p = &dirp->data;
-        dirp->cached = 0;
-
-    } else if (dirp->handle != INVALID_HANDLE_VALUE) {
-
-        /* Get the next directory entry from stream */
-        if (FindNextFileW (dirp->handle, &dirp->data) != FALSE) {
-            /* Got a file */
-            p = &dirp->data;
-        } else {
-            /* The very last entry has been processed or an error occured */
-            FindClose (dirp->handle);
-            dirp->handle = INVALID_HANDLE_VALUE;
-            p = NULL;
-        }
-
-    } else {
-
-        /* End of directory stream reached */
-        p = NULL;
-
-    }
-
-    return p;
-}
-
-/*
- * Open directory stream using plain old C-string.
- */
-static DIR*
-opendir(
-    const char *dirname)
-{
-    struct DIR *dirp;
-    int error;
-
-    /* Must have directory name */
-    if (dirname == NULL  ||  dirname[0] == '\0') {
-        dirent_set_errno (ENOENT);
-        return NULL;
-    }
-
-    /* Allocate memory for DIR structure */
-    dirp = (DIR*) malloc (sizeof (struct DIR));
-    if (dirp) {
-        wchar_t wname[DIRENT_MAX_PATH];
-        size_t n;
-
-        /* Convert directory name to wide-character string */
-        error = dirent_mbstowcs_s (&n, wname, DIRENT_MAX_PATH, dirname, DIRENT_MAX_PATH);
-        if (!error) {
-
-            /* Open directory stream using wide-character name */
-            dirp->wdirp = _wopendir (wname);
-            if (dirp->wdirp) {
-                /* Directory stream opened */
-                error = 0;
-            } else {
-                /* Failed to open directory stream */
-                error = 1;
-            }
-
-        } else {
-            /*
-             * Cannot convert file name to wide-character string.  This
-             * occurs if the string contains invalid multi-byte sequences or
-             * the output buffer is too small to contain the resulting
-             * string.
-             */
-            error = 1;
-        }
-
-    } else {
-        /* Cannot allocate DIR structure */
-        error = 1;
-    }
-
-    /* Clean up in case of error */
-    if (error  &&  dirp) {
-        free (dirp);
-        dirp = NULL;
-    }
-
-    return dirp;
-}
-
-/*
- * Read next directory entry.
- *
- * When working with text consoles, please note that file names returned by
- * readdir() are represented in the default ANSI code page while any output to
- * console is typically formatted on another code page.  Thus, non-ASCII
- * characters in file names will not usually display correctly on console.  The
- * problem can be fixed in two ways: (1) change the character set of console
- * to 1252 using chcp utility and use Lucida Console font, or (2) use
- * _cprintf function when writing to console.  The _cprinf() will re-encode
- * ANSI strings to the console code page so many non-ASCII characters will
- * display correcly.
- */
-static struct dirent*
-readdir(
-    DIR *dirp)
-{
-    WIN32_FIND_DATAW *datap;
-    struct dirent *entp;
-
-    /* Read next directory entry */
-    datap = dirent_next (dirp->wdirp);
-    if (datap) {
-        size_t n;
-        int error;
-
-        /* Attempt to convert file name to multi-byte string */
-        error = dirent_wcstombs_s(
-            &n, dirp->ent.d_name, DIRENT_MAX_PATH, datap->cFileName, DIRENT_MAX_PATH);
-
-        /*
-         * If the file name cannot be represented by a multi-byte string,
-         * then attempt to use old 8+3 file name.  This allows traditional
-         * Unix-code to access some file names despite of unicode
-         * characters, although file names may seem unfamiliar to the user.
-         *
-         * Be ware that the code below cannot come up with a short file
-         * name unless the file system provides one.  At least
-         * VirtualBox shared folders fail to do this.
-         */
-        if (error  &&  datap->cAlternateFileName[0] != '\0') {
-            error = dirent_wcstombs_s(
-                &n, dirp->ent.d_name, DIRENT_MAX_PATH,
-                datap->cAlternateFileName, DIRENT_MAX_PATH);
-        }
-
-        if (!error) {
-            DWORD attr;
-
-            /* Initialize directory entry for return */
-            entp = &dirp->ent;
-
-            /* Length of file name excluding zero terminator */
-            entp->d_namlen = n - 1;
-
-            /* File attributes */
-            attr = datap->dwFileAttributes;
-            if ((attr & FILE_ATTRIBUTE_DEVICE) != 0) {
-                entp->d_type = DT_CHR;
-            } else if ((attr & FILE_ATTRIBUTE_DIRECTORY) != 0) {
-                entp->d_type = DT_DIR;
-            } else {
-                entp->d_type = DT_REG;
-            }
-
-            /* Reset dummy fields */
-            entp->d_ino = 0;
-            entp->d_reclen = sizeof (struct dirent);
-
-        } else {
-            /*
-             * Cannot convert file name to multi-byte string so construct
-             * an errornous directory entry and return that.  Note that
-             * we cannot return NULL as that would stop the processing
-             * of directory entries completely.
-             */
-            entp = &dirp->ent;
-            entp->d_name[0] = '?';
-            entp->d_name[1] = '\0';
-            entp->d_namlen = 1;
-            entp->d_type = DT_UNKNOWN;
-            entp->d_ino = 0;
-            entp->d_reclen = 0;
-        }
-
-    } else {
-        /* No more directory entries */
-        entp = NULL;
-    }
-
-    return entp;
-}
-
-/*
- * Close directory stream.
- */
-static int
-closedir(
-    DIR *dirp)
-{
-    int ok;
-    if (dirp) {
-
-        /* Close wide-character directory stream */
-        ok = _wclosedir (dirp->wdirp);
-        dirp->wdirp = NULL;
-
-        /* Release multi-byte character version */
-        free (dirp);
-
-    } else {
-
-        /* Invalid directory stream */
-        dirent_set_errno (EBADF);
-        ok = /*failure*/-1;
-
-    }
-    return ok;
-}
-
-/*
- * Rewind directory stream to beginning.
- */
-static void
-rewinddir(
-    DIR* dirp)
-{
-    /* Rewind wide-character string directory stream */
-    _wrewinddir (dirp->wdirp);
-}
-
-/* Convert multi-byte string to wide character string */
-static int
-dirent_mbstowcs_s(
-    size_t *pReturnValue,
-    wchar_t *wcstr,
-    size_t sizeInWords,
-    const char *mbstr,
-    size_t count)
-{
-    int error;
-#ifndef DIRENT_USE_ASCII_SHORT_PATHS_ON_WINDOWS
-    // we don't use "count" at all: we assume mstr is zero terminated:
-    size_t n = (size_t) MultiByteToWideChar (CP_UTF8, 0, mbstr, -1, wcstr, 0);//sizeInWords);
-    if (n==0) {
-        error = 1;
-        if (sizeInWords>0) wcstr[0]=L'\0';
-        if (pReturnValue) *pReturnValue = 0;
-    }
-    else if (n<=sizeInWords) {
-        error = MultiByteToWideChar (CP_UTF8, 0, mbstr, -1, wcstr, n) == 0 ? 1 : 0;
-        if (pReturnValue) *pReturnValue = n;
-    }
-    else {
-        // Buffer too low:
-        if (sizeInWords>0)  {
-            if (sizeInWords>1) MultiByteToWideChar (CP_UTF8, 0, mbstr, -1, wcstr, sizeInWords-1);
-            wcstr[sizeInWords-1] = L'\0';
-        }
-        if (pReturnValue) *pReturnValue = sizeInWords;
-        error = 1;
-    }
-
-    /*
-    if (!wcstr  ||  n < count) {
-        // Zero-terminate output buffer
-        if (wcstr  &&  sizeInWords) {
-            if (n >= sizeInWords) {
-                n = sizeInWords - 1;
-            }
-            wcstr[n] = 0;
-        }
-
-        // Length of resuting multi-byte string WITH zero terminator
-        if (pReturnValue) {
-            *pReturnValue = n + 1;
-        }
-
-        // Success
-        error = 0;
-
-    } else {
-
-        // Could not convert string
-        error = 1;
-
-    }
-    */
-#else //DIRENT_USE_ASCII_SHORT_PATHS_ON_WINDOWS
-#if defined(_MSC_VER)  &&  _MSC_VER >= 1400
-
-    /* Microsoft Visual Studio 2005 or later */
-    error = mbstowcs_s (pReturnValue, wcstr, sizeInWords, mbstr, count);
+#ifndef SIMDJSON_DIRENT_PORTABLE_INC_
+#define SIMDJSON_DIRENT_PORTABLE_INC_
 
+#if (!defined(_WIN32) && !defined(_WIN64) && !(__MINGW32__) && !(__MINGW64__))
+#include <dirent.h>
 #else
-
-    /* Older Visual Studio or non-Microsoft compiler */
-    size_t n;
-
-    /* Convert to wide-character string (or count characters) */
-     n = mbstowcs (wcstr, mbstr, sizeInWords);
-    if (!wcstr  ||  n < count) {
-
-        /* Zero-terminate output buffer */
-        if (wcstr  &&  sizeInWords) {
-            if (n >= sizeInWords) {
-                n = sizeInWords - 1;
-            }
-            wcstr[n] = 0;
-        }
-
-        /* Length of resuting multi-byte string WITH zero terminator */
-        if (pReturnValue) {
-            *pReturnValue = n + 1;
-        }
-
-        /* Success */
-        error = 0;
-
-    } else {
-
-        /* Could not convert string */
-        error = 1;
-
-    }
-
-#endif
-#endif //DIRENT_USE_ASCII_SHORT_PATHS_ON_WINDOWS
-
-    return error;
-}
-
-/* Convert wide-character string to multi-byte string */
-static int
-dirent_wcstombs_s(
-    size_t *pReturnValue,
-    char *mbstr,
-    size_t sizeInBytes, /* max size of mbstr */
-    const wchar_t *wcstr,
-    size_t count)
-{
-    int error;
-
-#ifndef DIRENT_USE_ASCII_SHORT_PATHS_ON_WINDOWS
-    // we don't use "count" at all: we assume wcstr is zero terminated:
-    size_t n = (size_t) WideCharToMultiByte (CP_UTF8, 0, wcstr, -1, mbstr, 0,NULL,NULL);//sizeInBytes, NULL, NULL);
-    if (n==0) {
-        error = 1;
-        if (sizeInBytes>0) mbstr[0]='\0';
-        if (pReturnValue) *pReturnValue = 0;
-    }
-    else if (n<=sizeInBytes) {
-        error = WideCharToMultiByte (CP_UTF8, 0, wcstr, -1, mbstr, n, NULL, NULL) == 0 ? 1 : 0;
-        if (pReturnValue) *pReturnValue = n;
-    }
-    else {
-        // Buffer too low:
-        if (sizeInBytes>0)  {
-            if (sizeInBytes>1) WideCharToMultiByte (CP_UTF8, 0, wcstr, -1, mbstr, sizeInBytes-1, NULL, NULL);
-            mbstr[sizeInBytes-1] = '\0';
-        }
-        if (pReturnValue) *pReturnValue = sizeInBytes;
-        error = 1;
-    }
-    /*
-    if (!mbstr  ||  n < count) {
-
-        // Zero-terminate output buffer
-        if (mbstr  &&  sizeInBytes) {
-            if (n >= sizeInBytes) {
-                n = sizeInBytes - 1;
-            }
-            mbstr[n] = '\0';
-        }
-
-        // Lenght of resulting multi-bytes string WITH zero-terminator
-        if (pReturnValue) {
-            *pReturnValue = n + 1;
-        }
-
-
-        // Success
-        error = 0;
-
-    } else {
-
-        // Cannot convert string
-        error = 1;
-
-    }
-    */
-#else //DIRENT_USE_ASCII_SHORT_PATHS_ON_WINDOWS
-#if defined(_MSC_VER)  &&  _MSC_VER >= 1400
-
-    /* Microsoft Visual Studio 2005 or later */
-    error = wcstombs_s (pReturnValue, mbstr, sizeInBytes, wcstr, count);
-
-#else
-
-    /* Older Visual Studio or non-Microsoft compiler */
-    size_t n;
-
-    /* Convert to multi-byte string (or count the number of bytes needed) */
-    n = wcstombs (mbstr, wcstr, sizeInBytes);
-    if (!mbstr  ||  n < count) {
-
-        /* Zero-terminate output buffer */
-        if (mbstr  &&  sizeInBytes) {
-            if (n >= sizeInBytes) {
-                n = sizeInBytes - 1;
-            }
-            mbstr[n] = '\0';
-        }
-
-        /* Lenght of resulting multi-bytes string WITH zero-terminator */
-        if (pReturnValue) {
-            *pReturnValue = n + 1;
-        }
-
-        /* Success */
-        error = 0;
-
-    } else {
-
-        /* Cannot convert string */
-        error = 1;
-
-    }
-
-#endif
-#endif //DIRENT_USE_ASCII_SHORT_PATHS_ON_WINDOWS
-    return error;
-}
-
-/* Set errno variable */
-static void
-dirent_set_errno(
-    int error)
-{
-#if defined(_MSC_VER)  &&  _MSC_VER >= 1400
-
-    /* Microsoft Visual Studio 2005 and later */
-    _set_errno (error);
-
-#else
-
-    /* Non-Microsoft compiler or older Microsoft compiler */
-    errno = error;
-
-#endif
-}
-
-
-// The code of this single method comes from the musl library
-// (MIT licensed, Copyright © 2005-2014 Rich Felker, et al.)
-#   ifndef SIZE_MAX                 // Can't we just set it to INT_MAX or something like that ?
-#       define SIZE_MAX_WAS_NOT_DEFINED
-#       define SIZE_MAX INT_MAX     // should be in limits.h AFAIK
-#   endif //SIZE_MAX
-inline static int scandir(const char *path, struct dirent ***res,
-    int (*sel)(const struct dirent *),
-    int (*cmp)(const struct dirent **, const struct dirent **))
-{
-    DIR *d = opendir(path);
-    struct dirent *de, **names=0, **tmp;
-    size_t cnt=0, len=0;
-    int old_errno = errno;
-
-    if (!d) return -1;
-
-    while ((errno=0), (de = readdir(d))) {
-        if (sel && !sel(de)) continue;
-        if (cnt >= len) {
-            len = 2*len+1;
-            if (len > SIZE_MAX/sizeof *names) break;
-            tmp = (dirent**)realloc(names, len * sizeof *names);
-            if (!tmp) break;
-            names = tmp;
-        }
-        names[cnt] = (dirent*)malloc(de->d_reclen);
-        if (!names[cnt]) break;
-        memcpy(names[cnt++], de, de->d_reclen);
-    }
-
-    closedir(d);
-
-    if (errno) {
-        if (names) while (cnt-->0) free(names[cnt]);
-        free(names);
-        return -1;
-    }
-    errno = old_errno;
-
-    if (cmp) qsort(names, cnt, sizeof *names, (int (*)(const void *, const void *))cmp);
-    *res = names;
-    return cnt;
-}
-#ifdef SIZE_MAX_WAS_NOT_DEFINED
-#   undef SIZE_MAX
-#   undef SIZE_MAX_WAS_NOT_DEFINED
-#endif //SIZE_MAX_WAS_NOT_DEFINED
-
-// alphasort: Function to compare two `struct dirent's alphabetically.
-inline static int alphasort (const struct dirent **e1,const struct dirent **e2)    {
-    return strcmp((*e1)->d_name,(*e2)->d_name);
-}
-
-
-#ifdef __cplusplus
-}
+#include "toni_ronnko_dirent.h"
 #endif
-#endif /*DIRENT_H*/
 
-#endif //#if (!defined(_WIN32) && !defined(_WIN64))
+#endif // SIMDJSON_DIRENT_PORTABLE_INC_