@port-labs/jq-node-bindings 0.0.13 → 0.0.15-dev1

package/deps/jq/modules/oniguruma/src/make_unicode_fold_data.py

@@ -1,9 +1,10 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
+# make_unicode_fold_data.py
+# Copyright (c) 2016-2022  K.Kosako
 
 import sys
 import re
-import codecs
 
 SOURCE_FILE = 'CaseFolding.txt'
 GPERF_UNFOLD_KEY_FILE = 'unicode_unfold_key.gperf'
@@ -15,9 +16,9 @@ DataName = 'OnigUnicodeFolds'
 ENCODING = 'utf-8'
 
 LINE_REG = re.compile("([0-9A-F]{1,6}); (.); ([0-9A-F]{1,6})(?: ([0-9A-F]{1,6}))?(?: ([0-9A-F]{1,6}))?;(?:\s*#\s*)(.*)")
-VERSION_REG  = re.compile("#.*-(\d\.\d\.\d)\.txt")
+VERSION_REG  = re.compile("#.*-(\d+)\.(\d+)\.(\d+)\.txt")
 
-VERSION_INFO = None
+VERSION_INFO = [-1, -1, -1]
 
 FOLDS = {}
 TURKISH_FOLDS = {}
@@ -27,6 +28,35 @@ UNFOLDS = {}
 TURKISH_UNFOLDS = {}
 LOCALE_UNFOLDS  = {}
 
+COPYRIGHT = '''
+/*-
+ * Copyright (c) 2017-2022  K.Kosako
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+'''.strip()
+
+
 class Entry:
     def __init__(self, fold):
         self.fold = fold
@@ -54,19 +84,25 @@ def form3bytes(x):
     x2 = (x>>16) & 0xff
     return "\\x%02x\\x%02x\\x%02x" % (x2, x1, x0)
 
+def enc_len(code, encode):
+    u = unichr(code)
+    s = u.encode(encode)
+    return len(s)
+
 def check_version_info(s):
-    global VERSION_INFO
-    if VERSION_INFO is None:
-        m = VERSION_REG.match(s)
-        if m is not None:
-            VERSION_INFO = m.group(1)
+  m = VERSION_REG.match(s)
+  if m is not None:
+    VERSION_INFO[0] = int(m.group(1))
+    VERSION_INFO[1] = int(m.group(2))
+    VERSION_INFO[2] = int(m.group(3))
 
 def parse_line(s):
     if len(s) == 0:
-        return False
+      return False
     if s[0] == '#':
+      if VERSION_INFO[0] < 0:
         check_version_info(s)
-        return False
+      return False
 
     m = LINE_REG.match(s)
     if m is None:
@@ -229,27 +265,31 @@ def output_macros(f, name):
 
 def output_fold_source(f, out_comment):
     print >> f, "/* This file was generated by make_unicode_fold_data.py. */"
+    print >> f, COPYRIGHT
+    print >> f, "\n"
     print >> f, '#include "regenc.h"'
     print >> f, ''
-    if VERSION_INFO is not None:
-        print "#define CASEFOLD_VERSION  %s" % re.sub(r'[\.-]', '_', VERSION_INFO)
-        print ''
+    if VERSION_INFO[0] < 0:
+      raise RuntimeError("Version is not found")
+
+    print "#define UNICODE_CASEFOLD_VERSION  %02d%02d%02d" % (VERSION_INFO[0], VERSION_INFO[1], VERSION_INFO[2])
+    print ''
     #output_macros(f, DataName)
     print >> f, ''
     #output_typedef(f)
     output_fold_data(f, DataName, out_comment)
 
+
 def output_gperf_unfold_key(f):
-    head = """\
-%{
-/* This gperf source file was generated by make_unicode_fold_data.py */
-#include <string.h>
-#include "regenc.h"
+    head = "%{\n/* This gperf source file was generated by make_unicode_fold_data.py */\n\n" + COPYRIGHT + """\
+
+#include "regint.h"
 %}
+
 struct ByUnfoldKey {
   OnigCodePoint code;
-  int   index;
-  int   fold_len;
+  short int   index;
+  short int   fold_len;
 };
 %%
 """
@@ -264,13 +304,12 @@ struct ByUnfoldKey {
     print >> f, '%%'
 
 def output_gperf_fold_key(f, key_len):
-    head = """\
-%{
-/* This gperf source file was generated by make_unicode_fold_data.py */
-#include <string.h>
-#include "regenc.h"
+    head = "%{\n/* This gperf source file was generated by make_unicode_fold_data.py */\n\n" + COPYRIGHT + """\
+
+#include "regint.h"
 %}
-int
+
+short int
 %%
 """
     f.write(head)
@@ -293,6 +332,138 @@ def output_gperf_source():
        with open(GPERF_FOLD_KEY_FILES[i-1], 'w') as f:
            output_gperf_fold_key(f, i)
 
+def unfolds_byte_length_check(encode):
+    l = UNFOLDS.items()
+    sl = sorted(l, key=lambda (k,e):(e.fold_len, e.index))
+    for unfold, e in sl:
+        key_len = enc_len(unfold, encode)
+        fold_len = sum(map(lambda c: enc_len(c, encode), e.fold))
+        if key_len > fold_len:
+            sfolds = ' '.join(map(lambda c: "0x%06x" % c, e.fold))
+            s = "%s byte length: %d > %d: 0x%06x => %s" % (encode, key_len, fold_len, unfold, sfolds)
+            print >> sys.stderr, s
+
+def double_fold_check():
+    l = UNFOLDS.items()
+    sl = sorted(l, key=lambda (k,e):(e.fold_len, e.index))
+    for unfold, e in sl:
+        for f in e.fold:
+            #print >> sys.stderr, ("check 0x%06x" % f)
+            e2 = UNFOLDS.get(f)
+            if e2 is not None:
+                s = "double folds: 0x%06x => %s, 0x%06x => %s" % (unfold, e.fold, f, e2.fold)
+                print >> sys.stderr, s
+
+def unfold_is_multi_code_folds_head_check():
+    l = UNFOLDS.items()
+    l2 = filter(lambda (k,e):e.fold_len == 2, l)
+    l3 = filter(lambda (k,e):e.fold_len == 3, l)
+    sl = sorted(l, key=lambda (k,e):(e.fold_len, e.index))
+    for unfold, _ in sl:
+        for k, e in l2:
+            if e.fold[0] == unfold:
+                s = "unfold 0x%06x is multi-code fold head in %s" % (unfold, e.fold)
+                print >> sys.stderr, s
+        for k, e in l3:
+            if e.fold[0] == unfold:
+                s = "unfold 0x%06x is multi-code fold head in %s" % (unfold, e.fold)
+                print >> sys.stderr, s
+
+def make_one_folds(l):
+    h = {}
+    for unfold, e in l:
+        if e.fold_len != 1:
+            continue
+        fold = e.fold[0]
+        unfolds = h.get(fold)
+        if unfolds is None:
+            unfolds = [unfold]
+            h[fold] = unfolds
+        else:
+            unfolds.append(unfold)
+
+    return h
+
+def make_foldn_heads(l, fold_len, one_folds):
+    h = {}
+    for unfold, e in l:
+        if e.fold_len != fold_len:
+            continue
+        unfolds = one_folds.get(e.fold[0])
+        h[e.fold[0]] = (e, unfolds)
+
+    return h
+
+def fold2_expansion_num(e, one_folds):
+    n = len(e.unfolds)
+    n0 = 1
+    u0 = one_folds.get(e.fold[0])
+    if u0 is not None:
+        n0 += len(u0)
+    n1 = 1
+    u1 = one_folds.get(e.fold[1])
+    if u1 is not None:
+        n1 += len(u1)
+    n += (n0 * n1)
+    return n
+
+def fold3_expansion_num(e, one_folds):
+    n = len(e.unfolds)
+    n0 = 1
+    u0 = one_folds.get(e.fold[0])
+    if u0 is not None:
+        n0 += len(u0)
+    n1 = 1
+    u1 = one_folds.get(e.fold[1])
+    if u1 is not None:
+        n1 += len(u1)
+    n2 = 1
+    u2 = one_folds.get(e.fold[2])
+    if u2 is not None:
+        n2 += len(u2)
+    n += (n0 * n1 * n2)
+    return n
+
+def get_all_folds_expansion_num(x, one_folds, fold2_heads, fold3_heads):
+    e = UNFOLDS[x]
+    n = 0
+    if e.fold_len == 1:
+        n1 = len(e.unfolds) + 1 # +1: fold
+        fx = e.fold[0]
+        r = fold2_heads.get(fx)
+        n2 = n3 = 0
+        if r is not None:
+            e2, _ = r
+            n2 = fold2_expansion_num(e2, one_folds)
+        r = fold3_heads.get(fx)
+        if r is not None:
+            e3, _ = r
+            n3 = fold3_expansion_num(e3, one_folds)
+        n = max(n1, n2, n3)
+    elif e.fold_len == 2:
+        n = fold2_expansion_num(e, one_folds)
+    elif e.fold_len == 3:
+        n = fold3_expansion_num(e, one_folds)
+    else:
+        raise RuntimeError("Invalid fold_len %d" % (e.fold_len))
+
+    return n
+
+def get_all_folds_expansion_max_num():
+    l = UNFOLDS.items()
+    one_folds = make_one_folds(l)
+    fold2_heads = make_foldn_heads(l, 2, one_folds)
+    fold3_heads = make_foldn_heads(l, 3, one_folds)
+    sl = sorted(l, key=lambda (k,e):(e.fold_len, e.index))
+    nmax = 0
+    max_unfold = None
+    for unfold, e in sl:
+        n = get_all_folds_expansion_num(unfold, one_folds, fold2_heads, fold3_heads)
+        if nmax < n:
+            nmax = n
+            max_unfold = unfold
+
+    return (nmax, max_unfold)
 
 ## main ##
 with open(SOURCE_FILE, 'r') as f:
@@ -304,3 +475,12 @@ out_comment = True
 output_fold_source(sys.stdout, out_comment)
 
 output_gperf_source()
+
+#unfolds_byte_length_check('utf-8')
+#unfolds_byte_length_check('utf-16')
+double_fold_check()
+unfold_is_multi_code_folds_head_check()
+
+#max_num, max_code = get_all_folds_expansion_max_num()
+#max_num -= 1  # remove self
+#print >> sys.stderr, "max expansion: 0x%06x: %d" % (max_code, max_num)

package/deps/jq/modules/oniguruma/src/make_unicode_property.sh

@@ -1,18 +1,27 @@
 #!/bin/sh
 
+GPERF=gperf
+SED=sed
+
 NAME=unicode_property_data
-TMP=gperf.tmp
-#GPERF_OPT='-P -Q prop_name_pool -C -c -t -j1 -L ANSI-C --ignore-case'
-GPERF_OPT='-T -C -c -t -j1 -L ANSI-C --ignore-case'
+TMP1=gperf1.tmp
+TMP2=gperf2.tmp
+TMP=
+
+GPERF_OPT='-T -C -c -t -j1 -L ANSI-C --ignore-case --pic -Q unicode_prop_name_pool'
+POOL_CAST='s/\(int *\)\(size_t *\)&\(\(struct +unicode_prop_name_pool_t *\* *\) *0\)->unicode_prop_name_pool_str([^,]+)/pool_offset(\1)/g'
+ADD_STATIC='s/(const +struct +PoolPropertyNameCtype +\*)/static \1/'
+ADD_CAST='s/unsigned +int +hval *= *len/unsigned int hval = (unsigned int )len/'
+
+python2 make_unicode_property_data.py > ${NAME}.gperf
+python2 make_unicode_property_data.py -posix > ${NAME}_posix.gperf
 
-./make_unicode_property_data.py > ${NAME}.gperf
-./make_unicode_property_data.py -posix > ${NAME}_posix.gperf
+${GPERF} ${GPERF_OPT} -N unicode_lookup_property_name --output-file ${TMP1} ${NAME}.gperf
+cat ${TMP1} | ${SED} -e 's/^#line.*$//g' | ${SED} -E "${POOL_CAST}" | ${SED} -E "${ADD_STATIC}" | ${SED} -E "${ADD_CAST}" > ${NAME}.c
 
-gperf ${GPERF_OPT} -N unicode_lookup_property_name --output-file ${TMP} ${NAME}.gperf
-sed -e 's/^#line.*$//g' ${TMP} > ${NAME}.c
-gperf ${GPERF_OPT} -N unicode_lookup_property_name --output-file ${TMP} ${NAME}_posix.gperf
-sed -e 's/^#line.*$//g' ${TMP} > ${NAME}_posix.c
+${GPERF} ${GPERF_OPT} -N unicode_lookup_property_name --output-file ${TMP2} ${NAME}_posix.gperf
+cat ${TMP2} | ${SED} -e 's/^#line.*$//g' | ${SED} -E "${POOL_CAST}" | ${SED} -E "${ADD_STATIC}" > ${NAME}_posix.c
 
-rm -f ${NAME}.gperf ${NAME}_posix.gperf ${TMP}
+rm -f ${NAME}.gperf ${NAME}_posix.gperf ${TMP1} ${TMP2}
 
 exit 0