tree-sitter 0.0.1 → 0.1.0

data/ext/tree-sitter/tree-sitter/externals/gyp/test/win/win-tool/copies_readonly_files.gyp

@@ -0,0 +1,29 @@
+{
+  'targets': [
+    {
+      'target_name': 'foo',
+      'type': 'none',
+      'copies': [
+        {
+          'destination': '<(PRODUCT_DIR)/dest',
+          'files': [
+            'read-only-file',
+          ],
+        },
+      ],
+    },  # target: foo
+
+    {
+      'target_name': 'bar',
+      'type': 'none',
+      'copies': [
+        {
+          'destination': '<(PRODUCT_DIR)/dest',
+          'files': [
+            'subdir/',
+          ],
+        },
+      ],
+    },  # target: bar
+  ],
+}

data/ext/tree-sitter/tree-sitter/externals/gyp/test/win/win-tool/gyptest-win-tool-handles-readonly-files.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2014 Google Inc. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""
+Make sure overwriting read-only files works as expected (via win-tool).
+"""
+
+import TestGyp
+
+import filecmp
+import os
+import stat
+import sys
+
+if sys.platform == 'win32':
+  test = TestGyp.TestGyp(formats=['ninja'])
+
+  # First, create the source files.
+  os.makedirs('subdir')
+  read_only_files = ['read-only-file', 'subdir/A', 'subdir/B', 'subdir/C']
+  for f in read_only_files:
+    test.write(f, 'source_contents')
+    test.chmod(f, stat.S_IREAD)
+    if os.access(f, os.W_OK):
+      test.fail_test()
+
+  # Second, create the read-only destination files. Note that we are creating
+  # them where the ninja and win-tool will try to copy them to, in order to test
+  # that copies overwrite the files.
+  os.makedirs(test.built_file_path('dest/subdir'))
+  for f in read_only_files:
+    f = os.path.join('dest', f)
+    test.write(test.built_file_path(f), 'SHOULD BE OVERWRITTEN')
+    test.chmod(test.built_file_path(f), stat.S_IREAD)
+    # Ensure not writable.
+    if os.access(test.built_file_path(f), os.W_OK):
+      test.fail_test()
+
+  test.run_gyp('copies_readonly_files.gyp')
+  test.build('copies_readonly_files.gyp')
+
+  # Check the destination files were overwritten by ninja.
+  for f in read_only_files:
+    f = os.path.join('dest', f)
+    test.must_contain(test.built_file_path(f), 'source_contents')
+
+  # This will fail if the files are not the same mode or contents.
+  for f in read_only_files:
+    if not filecmp.cmp(f, test.built_file_path(os.path.join('dest', f))):
+      test.fail_test()
+
+  test.pass_test()

data/ext/tree-sitter/tree-sitter/externals/gyp/tools/emacs/gyp.el

@@ -15,14 +15,36 @@
                  "recent emacsen), not from the older and less maintained "
                  "python-mode.el")))
 
-(defadvice python-calculate-indentation (after ami-outdent-closing-parens
-                                               activate)
+(defadvice python-indent-calculate-levels (after gyp-outdent-closing-parens
+                                                 activate)
   "De-indent closing parens, braces, and brackets in gyp-mode."
-  (if (and (eq major-mode 'gyp-mode)
-           (string-match "^ *[])}][],)}]* *$"
-                         (buffer-substring-no-properties
-                          (line-beginning-position) (line-end-position))))
-      (setq ad-return-value (- ad-return-value 2))))
+  (when (and (eq major-mode 'gyp-mode)
+             (string-match "^ *[])}][],)}]* *$"
+                           (buffer-substring-no-properties
+                            (line-beginning-position) (line-end-position))))
+    (setf (first python-indent-levels)
+          (- (first python-indent-levels) python-continuation-offset))))
+
+(defadvice python-indent-guess-indent-offset (around
+                                              gyp-indent-guess-indent-offset
+                                              activate)
+  "Guess correct indent offset in gyp-mode."
+  (or (and (not (eq major-mode 'gyp-mode))
+           ad-do-it)
+      (save-excursion
+        (save-restriction
+          (widen)
+          (goto-char (point-min))
+          ;; Find first line ending with an opening brace that is not a comment.
+          (or (and (re-search-forward "\\(^[[{]$\\|^.*[^#].*[[{]$\\)")
+                   (forward-line)
+                   (/= (current-indentation) 0)
+                   (set (make-local-variable 'python-indent-offset)
+                        (current-indentation))
+                   (set (make-local-variable 'python-continuation-offset)
+                        (current-indentation)))
+              (message "Can't guess gyp indent offset, using default: %s"
+                       python-continuation-offset))))))
 
 (define-derived-mode gyp-mode python-mode "Gyp"
   "Major mode for editing .gyp files. See http://code.google.com/p/gyp/"
@@ -35,9 +57,10 @@
 
 (defun gyp-set-indentation ()
   "Hook function to configure python indentation to suit gyp mode."
-  (setq python-continuation-offset 2
-        python-indent 2
-        python-guess-indent nil))
+  (set (make-local-variable 'python-indent-offset) 2)
+  (set (make-local-variable 'python-continuation-offset) 2)
+  (set (make-local-variable 'python-indent-guess-indent-offset) t)
+  (python-indent-guess-indent-offset))
 
 (add-hook 'gyp-mode-hook 'gyp-set-indentation)
 
@@ -218,11 +241,11 @@
     ;; Top-level keywords
     (list (concat "['\"]\\("
               (regexp-opt (list "action" "action_name" "actions" "cflags"
-                                "conditions" "configurations" "copies" "defines"
-                                "dependencies" "destination"
+                                "cflags_cc" "conditions" "configurations"
+                                "copies" "defines" "dependencies" "destination"
                                 "direct_dependent_settings"
                                 "export_dependent_settings" "extension" "files"
-                                "include_dirs" "includes" "inputs" "libraries"
+                                "include_dirs" "includes" "inputs" "ldflags" "libraries"
                                 "link_settings" "mac_bundle" "message"
                                 "msvs_external_rule" "outputs" "product_name"
                                 "process_outputs_as_sources" "rules" "rule_name"

data/ext/tree-sitter/tree-sitter/externals/gyp/tools/pretty_sln.py

@@ -38,12 +38,13 @@ def ParseSolution(solution_file):
 
   # Regular expressions that matches the SLN format.
   # The first line of a project definition.
-  begin_project = re.compile(('^Project\("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942'
-                              '}"\) = "(.*)", "(.*)", "(.*)"$'))
+  begin_project = re.compile(r'^Project\("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942'
+                             r'}"\) = "(.*)", "(.*)", "(.*)"$')
   # The last line of a project definition.
   end_project = re.compile('^EndProject$')
   # The first line of a dependency list.
-  begin_dep = re.compile('ProjectSection\(ProjectDependencies\) = postProject$')
+  begin_dep = re.compile(
+      r'ProjectSection\(ProjectDependencies\) = postProject$')
   # The last line of a dependency list.
   end_dep = re.compile('EndProjectSection$')
   # A line describing a dependency.

data/ext/tree-sitter/tree-sitter/externals/utf8proc/CMakeLists.txt

@@ -6,9 +6,11 @@ disallow_intree_builds()
 
 project (utf8proc C)
 
-# Be sure to also update these in Makefile!
+# This is the ABI version number, which may differ from the
+# API version number (defined in utf8proc.h).
+# Be sure to also update these in Makefile and MANIFEST!
 set(SO_MAJOR 2)
-set(SO_MINOR 0)
+set(SO_MINOR 1)
 set(SO_PATCH 0)
 
 add_definitions (

data/ext/tree-sitter/tree-sitter/externals/utf8proc/MANIFEST

@@ -2,6 +2,6 @@ include/
 include/utf8proc.h
 lib/
 lib/libutf8proc.a
-lib/libutf8proc.so -> libutf8proc.so.2.0.0
-lib/libutf8proc.so.2 -> libutf8proc.so.2.0.0
-lib/libutf8proc.so.2.0.0
+lib/libutf8proc.so -> libutf8proc.so.2.1.0
+lib/libutf8proc.so.2 -> libutf8proc.so.2.1.0
+lib/libutf8proc.so.2.1.0

data/ext/tree-sitter/tree-sitter/externals/utf8proc/Makefile

@@ -11,16 +11,17 @@ FIND=find
 CFLAGS ?= -O2
 PICFLAG = -fPIC
 C99FLAG = -std=c99
-WCFLAGS = -Wall -Wmissing-prototypes -pedantic
+WCFLAGS = -Wall -pedantic
 UCFLAGS = $(CFLAGS) $(PICFLAG) $(C99FLAG) $(WCFLAGS) -DUTF8PROC_EXPORTS
 
 # shared-library version MAJOR.MINOR.PATCH ... this may be *different*
 # from the utf8proc version number because it indicates ABI compatibility,
 # not API compatibility: MAJOR should be incremented whenever *binary*
-# compatibility is broken, even if the API is backward-compatible
-# Be sure to also update these in MANIFEST and CMakeLists.txt!
+# compatibility is broken, even if the API is backward-compatible.
+# The API version number is defined in utf8proc.h.
+# Be sure to also update these ABI versions in MANIFEST and CMakeLists.txt!
 MAJOR=2
-MINOR=0
+MINOR=1
 PATCH=0
 
 OS := $(shell uname)
@@ -48,7 +49,7 @@ clean:
 ifneq ($(OS),Darwin)
 	rm -f libutf8proc.so.$(MAJOR)
 endif
-	rm -f test/tests.o test/normtest test/graphemetest test/printproperty test/charwidth test/valid test/iterate test/case
+	rm -f test/tests.o test/normtest test/graphemetest test/printproperty test/charwidth test/valid test/iterate test/case test/custom
 	rm -rf MANIFEST.new tmp
 	$(MAKE) -C bench clean
 	$(MAKE) -C data clean
@@ -135,7 +136,10 @@ test/iterate: test/iterate.c test/tests.o utf8proc.o utf8proc.h test/tests.h
 test/case: test/case.c test/tests.o utf8proc.o utf8proc.h test/tests.h
 	$(CC) $(UCFLAGS) test/case.c test/tests.o utf8proc.o -o $@
 
-check: test/normtest data/NormalizationTest.txt test/graphemetest data/GraphemeBreakTest.txt test/printproperty test/case test/charwidth test/valid test/iterate bench/bench.c bench/util.c bench/util.h utf8proc.o
+test/custom: test/custom.c test/tests.o utf8proc.o utf8proc.h test/tests.h
+	$(CC) $(UCFLAGS) test/custom.c test/tests.o utf8proc.o -o $@
+
+check: test/normtest data/NormalizationTest.txt test/graphemetest data/GraphemeBreakTest.txt test/printproperty test/case test/custom test/charwidth test/valid test/iterate bench/bench.c bench/util.c bench/util.h utf8proc.o
 	$(MAKE) -C bench
 	test/normtest data/NormalizationTest.txt
 	test/graphemetest data/GraphemeBreakTest.txt
@@ -143,3 +147,4 @@ check: test/normtest data/NormalizationTest.txt test/graphemetest data/GraphemeB
 	test/valid
 	test/iterate
 	test/case
+	test/custom

data/ext/tree-sitter/tree-sitter/externals/utf8proc/NEWS.md

@@ -1,5 +1,62 @@
 # utf8proc release history #
 
+## Version 2.1 ##
+
+2016-12-16:
+
+- New functions `utf8proc_map_custom` and `utf8proc_decompose_custom`
+  to allow user-supplied transformations of codepoints, in conjunction
+  with other transformations ([#89]).
+
+- New function `utf8proc_normalize_utf32` to apply normalizations
+  directly to UTF-32 data (not just UTF-8) ([#88]).
+
+- Fixed stack overflow that could occur due to incorrect definition
+  of `UINT16_MAX` with some compilers ([#84]).
+
+- Fixed conflict with `stdbool.h` in Visual Studio ([#90]).
+
+- Updated font metrics to use Unifont 9.0.04.
+
+## Version 2.0.2 ##
+
+2016-07-27:
+
+- Move `-Wmissing-prototypes` warning flag from `Makefile` to `.travis.yml`
+  since MSVC does not understand this flag and it is occasionally useful to
+  build using MSVC through the `Makefile` ([#79]).
+
+- Use a different variable name for a nested loop in `bench/bench.c`, and
+  declare it in a C89 way rather than inside the `for` to avoid "error:
+  'for' loop initial declarations are only allowed in C99 mode" ([#80]).
+
+## Version 2.0.1 ##
+
+2016-07-13:
+
+- Bug fix in `utf8proc_grapheme_break_stateful` ([#77]).
+
+- Tests now use versioned Unicode files, so they will no longer
+  break when a new version of Unicode is released ([#78]).
+
+## Version 2.0 ##
+
+2016-07-13:
+
+- Updated for Unicode 9.0 ([#70]).
+
+- New `utf8proc_grapheme_break_stateful` to handle the complicated
+  grapheme-breaking rules in Unicode 9.  The old `utf8proc_grapheme_break`
+  is still provided, but may incorrectly identify grapheme breaks
+  in some Unicode-9 sequences.
+
+- Smaller Unicode tables ([#62], [#68]).  This required changes
+  in the `utf8proc_property_t` structure, which breaks backward
+  compatibility if you access this `struct` directly.  The
+  functions in the API remain backward-compatible, however.
+
+- Buffer overrun fix ([#66]).
+
 ## Version 1.3.1 ##
 
 2015-11-02:
@@ -232,3 +289,15 @@ Release of version 1.0.1
 [#51]: https://github.com/JuliaLang/utf8proc/issues/51
 [#55]: https://github.com/JuliaLang/utf8proc/issues/55
 [#58]: https://github.com/JuliaLang/utf8proc/issues/58
+[#62]: https://github.com/JuliaLang/utf8proc/issues/62
+[#66]: https://github.com/JuliaLang/utf8proc/issues/66
+[#68]: https://github.com/JuliaLang/utf8proc/issues/68
+[#70]: https://github.com/JuliaLang/utf8proc/issues/70
+[#77]: https://github.com/JuliaLang/utf8proc/issues/77
+[#78]: https://github.com/JuliaLang/utf8proc/issues/78
+[#79]: https://github.com/JuliaLang/utf8proc/issues/79
+[#80]: https://github.com/JuliaLang/utf8proc/issues/80
+[#84]: https://github.com/JuliaLang/utf8proc/pull/84
+[#88]: https://github.com/JuliaLang/utf8proc/pull/88
+[#89]: https://github.com/JuliaLang/utf8proc/pull/89
+[#90]: https://github.com/JuliaLang/utf8proc/issues/90

data/ext/tree-sitter/tree-sitter/externals/utf8proc/README.md

@@ -1,5 +1,7 @@
 # utf8proc
-[![Build Status](https://travis-ci.org/JuliaLang/utf8proc.png)](https://travis-ci.org/JuliaLang/utf8proc)
+[![Travis CI Status](https://travis-ci.org/JuliaLang/utf8proc.png)](https://travis-ci.org/JuliaLang/utf8proc)
+[![AppVeyor Status](https://ci.appveyor.com/api/projects/status/aou20lfkyhj8xbwq/branch/master?svg=true)](https://ci.appveyor.com/project/tkelman/utf8proc/branch/master)
+
 
 [utf8proc](http://julialang.org/utf8proc/) is a small, clean C
 library that provides Unicode normalization, case-folding, and other
@@ -38,7 +40,7 @@ The C library is found in this directory after successful compilation
 and is named `libutf8proc.a` (for the static library) and
 `libutf8proc.so` (for the dynamic library).
 
-The Unicode version supported is 8.0.0.
+The Unicode version supported is 9.0.0.
 
 For Unicode normalizations, the following options are used:
 

data/ext/tree-sitter/tree-sitter/externals/utf8proc/bench/bench.c

@@ -7,7 +7,7 @@
 
 int main(int argc, char **argv)
 {
-	 int i;
+	 int i, j;
 	 int options = 0;
 	 
 	 for (i = 1; i < argc; ++i) {
@@ -44,7 +44,7 @@ int main(int argc, char **argv)
 		  }
 		  uint8_t *dest;
 		  mytime start = gettime();
-		  for (int i = 0; i < 100; ++i) {
+		  for (j = 0; j < 100; ++j) {
 			   utf8proc_map(src, len, &dest, options);
 			   free(dest);
 		  }

data/ext/tree-sitter/tree-sitter/externals/utf8proc/data/Makefile

@@ -20,7 +20,7 @@ utf8proc_data.c.new: data_generator.rb UnicodeData.txt GraphemeBreakProperty.txt
 	$(RUBY) data_generator.rb < UnicodeData.txt > $@
 
 # GNU Unifont version for font metric calculations:
-UNIFONT_VERSION=8.0.01
+UNIFONT_VERSION=9.0.04
 
 unifont.ttf:
 	$(CURL) $(CURLFLAGS) -o $@ $(URLCACHE)https://mirrors.kernel.org/gnu/unifont/unifont-$(UNIFONT_VERSION)/unifont-$(UNIFONT_VERSION).ttf
@@ -34,29 +34,32 @@ unifont_upper.ttf:
 CharWidths.txt: charwidths.jl unifont.sfd unifont_upper.sfd EastAsianWidth.txt
 	$(JULIA) charwidths.jl > $@
 
+# Unicode data version
+UNICODE_VERSION=9.0.0
+
 UnicodeData.txt:
-	$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/UNIDATA/UnicodeData.txt
+	$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/UnicodeData.txt
 
 EastAsianWidth.txt:
-	$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/UNIDATA/EastAsianWidth.txt
+	$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/EastAsianWidth.txt
 
 GraphemeBreakProperty.txt:
-	$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/UCD/latest/ucd/auxiliary/GraphemeBreakProperty.txt
+	$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/auxiliary/GraphemeBreakProperty.txt
 
 DerivedCoreProperties.txt:
-	$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/UNIDATA/DerivedCoreProperties.txt
+	$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/DerivedCoreProperties.txt
 
 CompositionExclusions.txt:
-	$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/UNIDATA/CompositionExclusions.txt
+	$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/CompositionExclusions.txt
 
 CaseFolding.txt:
-	$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/UNIDATA/CaseFolding.txt
+	$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/CaseFolding.txt
 
 NormalizationTest.txt:
-	$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/UNIDATA/NormalizationTest.txt
+	$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/NormalizationTest.txt
 
 GraphemeBreakTest.txt:
-	$(CURL) $(CURLFLAGS) $(URLCACHE)http://www.unicode.org/Public/UCD/latest/ucd/auxiliary/GraphemeBreakTest.txt | $(PERL) -pe 's,÷,/,g;s,×,+,g' > $@
+	$(CURL) $(CURLFLAGS) $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/auxiliary/GraphemeBreakTest.txt | $(PERL) -pe 's,÷,/,g;s,×,+,g' > $@
 
 clean:
 	rm -f UnicodeData.txt EastAsianWidth.txt GraphemeBreakProperty.txt DerivedCoreProperties.txt CompositionExclusions.txt CaseFolding.txt NormalizationTest.txt GraphemeBreakTest.txt CharWidths.txt unifont*.ttf unifont*.sfd

data/ext/tree-sitter/tree-sitter/externals/utf8proc/data/data_generator.rb

@@ -115,22 +115,52 @@ def str2c(string, prefix)
   return "0" if string.nil?
   return "UTF8PROC_#{prefix}_#{string.upcase}"
 end
-def ary2c(array)
-  return "UINT16_MAX" if array.nil?
-  unless $int_array_indicies[array]
+def pushary(array)
+  idx = $int_array_indicies[array]
+  unless idx
     $int_array_indicies[array] = $int_array.length
+    idx = $int_array.length
     array.each { |entry| $int_array << entry }
-    $int_array << -1
   end
-  raise "Array index out of bound" if $int_array_indicies[array] >= 65535
-  return "#{$int_array_indicies[array]}"
+  return idx
+end
+def cpary2utf16encoded(array)
+  return array.flat_map { |cp|
+      if (cp <= 0xFFFF)
+        raise "utf-16 code: #{cp}" if cp & 0b1111100000000000 == 0b1101100000000000
+        cp
+      else
+        temp = cp - 0x10000
+        [(temp >> 10) | 0b1101100000000000, (temp & 0b0000001111111111) | 0b1101110000000000]
+      end
+    }
+end
+def cpary2c(array)
+  return "UINT16_MAX" if array.nil? || array.length == 0
+  lencode = array.length - 1 #no sequence has len 0, so we encode len 1 as 0, len 2 as 1, ... 
+  array = cpary2utf16encoded(array)
+  if lencode >= 7 #we have only 3 bits for the length (which is already cutting it close. might need to change it to 2 bits in future Unicode versions)
+    array = [lencode] + array 
+    lencode = 7
+  end  
+  idx = pushary(array) 
+  raise "Array index out of bound" if idx > 0x1FFF
+  return "#{idx | (lencode << 13)}"
+end
+def singlecpmap(cp)
+  return "UINT16_MAX" if cp == nil
+  idx = pushary(cpary2utf16encoded([cp]))
+  raise "Array index out of bound" if idx > 0xFFFF
+  return "#{idx}"
 end
 
 class UnicodeChar
   attr_accessor :code, :name, :category, :combining_class, :bidi_class,
                 :decomp_type, :decomp_mapping,
                 :bidi_mirrored,
-                :uppercase_mapping, :lowercase_mapping, :titlecase_mapping
+                :uppercase_mapping, :lowercase_mapping, :titlecase_mapping,
+                #caches:
+                :c_entry_index, :c_decomp_mapping, :c_case_folding
   def initialize(line)
     raise "Could not parse input." unless line =~ /^
       ([0-9A-F]+);        # code
@@ -165,25 +195,23 @@ class UnicodeChar
   def case_folding
     $case_folding[code]
   end
-  def c_entry(comb1_indicies, comb2_indicies)
+  def c_entry(comb_indicies)
     "  " <<
     "{#{str2c category, 'CATEGORY'}, #{combining_class}, " <<
     "#{str2c bidi_class, 'BIDI_CLASS'}, " <<
     "#{str2c decomp_type, 'DECOMP_TYPE'}, " <<
-    "#{ary2c decomp_mapping}, " <<
-    "#{ary2c case_folding}, " <<
-    "#{uppercase_mapping or -1}, " <<
-    "#{lowercase_mapping or -1}, " <<
-    "#{titlecase_mapping or -1}, " <<
-    "#{comb1_indicies[code] ?
-       (comb1_indicies[code]*comb2_indicies.keys.length) : -1
-      }, #{comb2_indicies[code] or -1}, " <<
+    "#{c_decomp_mapping}, " <<
+    "#{c_case_folding}, " <<
+    "#{singlecpmap uppercase_mapping }, " <<
+    "#{singlecpmap lowercase_mapping }, " <<
+    "#{singlecpmap titlecase_mapping }, " <<
+    "#{comb_indicies[code] ? comb_indicies[code]: 'UINT16_MAX'}, " <<
     "#{bidi_mirrored}, " <<
     "#{$exclusions.include?(code) or $excl_version.include?(code)}, " <<
     "#{$ignorable.include?(code)}, " <<
     "#{%W[Zl Zp Cc Cf].include?(category) and not [0x200C, 0x200D].include?(category)}, " <<
-    "#{$grapheme_boundclass[code]}, " <<
-    "#{$charwidth[code]}},\n"
+    "#{$charwidth[code]}, 0, " <<
+    "#{$grapheme_boundclass[code]}},\n"
   end
 end
 
@@ -215,6 +243,8 @@ end
 
 comb1st_indicies = {}
 comb2nd_indicies = {}
+comb2nd_indicies_sorted_keys = []
+comb2nd_indicies_nonbasic = {}
 comb_array = []
 
 chars.each do |char|
@@ -222,27 +252,69 @@ chars.each do |char|
       char.decomp_mapping.length == 2 and !char_hash[char.decomp_mapping[0]].nil? and
       char_hash[char.decomp_mapping[0]].combining_class == 0 and
       not $exclusions.include?(char.code)
-    unless comb1st_indicies[char.decomp_mapping[0]]
-      comb1st_indicies[char.decomp_mapping[0]] = comb1st_indicies.keys.length
+
+    dm0 = char.decomp_mapping[0]
+    dm1 = char.decomp_mapping[1]
+    unless comb1st_indicies[dm0]
+      comb1st_indicies[dm0] = comb1st_indicies.keys.length
+    end
+    unless comb2nd_indicies[dm1]
+      comb2nd_indicies_sorted_keys << dm1
+      comb2nd_indicies[dm1] = comb2nd_indicies.keys.length 
     end
-    unless comb2nd_indicies[char.decomp_mapping[1]]
-      comb2nd_indicies[char.decomp_mapping[1]] = comb2nd_indicies.keys.length
+    comb_array[comb1st_indicies[dm0]] ||= []
+    raise "Duplicate canonical mapping: #{char.code} #{dm0} #{dm1}" if comb_array[comb1st_indicies[dm0]][comb2nd_indicies[dm1]]
+    comb_array[comb1st_indicies[dm0]][comb2nd_indicies[dm1]] = char.code
+    
+    comb2nd_indicies_nonbasic[dm1] = true if char.code > 0xFFFF
+  end
+  char.c_decomp_mapping = cpary2c(char.decomp_mapping)
+  char.c_case_folding = cpary2c(char.case_folding)
+end 
+
+comb_indicies = {}
+cumoffset = 0
+comb1st_indicies_lastoffsets = []
+comb1st_indicies_firstoffsets = []
+comb1st_indicies.each do |dm0, index|
+  first = nil
+  last = nil
+  offset = 0
+  comb2nd_indicies_sorted_keys.each_with_index do |dm1, b|
+    if comb_array[index][b] 
+      first = offset unless first
+      last = offset
+      last += 1 if comb2nd_indicies_nonbasic[dm1]
     end
-    comb_array[comb1st_indicies[char.decomp_mapping[0]]] ||= []
-    raise "Duplicate canonical mapping" if
-      comb_array[comb1st_indicies[char.decomp_mapping[0]]][
-      comb2nd_indicies[char.decomp_mapping[1]]]
-    comb_array[comb1st_indicies[char.decomp_mapping[0]]][
-      comb2nd_indicies[char.decomp_mapping[1]]] = char.code
+    offset += 1
+    offset += 1 if comb2nd_indicies_nonbasic[dm1]
+  end
+  comb1st_indicies_firstoffsets[index] = first
+  comb1st_indicies_lastoffsets[index] = last
+  raise "double index" if comb_indicies[dm0]
+  comb_indicies[dm0] = cumoffset
+  cumoffset += last - first + 1 + 2
+end
+
+offset = 0
+comb2nd_indicies_sorted_keys.each do |dm1|
+  raise "double index" if comb_indicies[dm1]
+  comb_indicies[dm1] = 0x8000 | (comb2nd_indicies[dm1] + offset)
+  raise "too large comb index" if  comb2nd_indicies[dm1] + offset > 0x4000
+  if comb2nd_indicies_nonbasic[dm1]
+    comb_indicies[dm1] = comb_indicies[dm1] | 0x4000
+    offset += 1
   end
 end
 
 properties_indicies = {}
 properties = []
 chars.each do |char|
-  c_entry = char.c_entry(comb1st_indicies, comb2nd_indicies)
-  unless properties_indicies[c_entry]
+  c_entry = char.c_entry(comb_indicies)
+  char.c_entry_index = properties_indicies[c_entry]
+  unless char.c_entry_index
     properties_indicies[c_entry] = properties.length
+    char.c_entry_index = properties.length
     properties << c_entry
   end
 end
@@ -254,8 +326,7 @@ for code in 0...0x110000
   stage2_entry = []
   for code2 in code...(code+0x100)
     if char_hash[code2]
-      stage2_entry << (properties_indicies[char_hash[code2].c_entry(
-        comb1st_indicies, comb2nd_indicies)] + 1)
+      stage2_entry << (char_hash[code2].c_entry_index + 1)
     else
       stage2_entry << 0
     end
@@ -269,7 +340,7 @@ for code in 0...0x110000
   end
 end
 
-$stdout << "const utf8proc_int32_t utf8proc_sequences[] = {\n  "
+$stdout << "const utf8proc_uint16_t utf8proc_sequences[] = {\n  "
 i = 0
 $int_array.each do |entry|
   i += 1
@@ -306,23 +377,35 @@ end
 $stdout << "};\n\n"
 
 $stdout << "const utf8proc_property_t utf8proc_properties[] = {\n"
-$stdout << "  {0, 0, 0, 0, UINT16_MAX, UINT16_MAX, -1, -1, -1, -1, -1, false,false,false,false, UTF8PROC_BOUNDCLASS_OTHER, 0},\n"
+$stdout << "  {0, 0, 0, 0, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,  false,false,false,false, 0, 0, UTF8PROC_BOUNDCLASS_OTHER},\n"
 properties.each { |line|
   $stdout << line
 }
 $stdout << "};\n\n"
 
-$stdout << "const utf8proc_int32_t utf8proc_combinations[] = {\n  "
+
+
+$stdout << "const utf8proc_uint16_t utf8proc_combinations[] = {\n  "
 i = 0
-comb1st_indicies.keys.sort.each_index do |a|
-  comb2nd_indicies.keys.sort.each_index do |b|
-    i += 1
-    if i == 8
-      i = 0
-      $stdout << "\n  "
+comb1st_indicies.keys.each_index do |a|
+  offset = 0
+  $stdout << comb1st_indicies_firstoffsets[a] << ", " << comb1st_indicies_lastoffsets[a] << ", "
+  comb2nd_indicies_sorted_keys.each_with_index do |dm1, b|
+    break if offset > comb1st_indicies_lastoffsets[a] 
+    if offset >= comb1st_indicies_firstoffsets[a]
+      i += 1
+      if i == 8
+        i = 0
+        $stdout << "\n  "
+      end
+      v = comb_array[a][b] ? comb_array[a][b] : 0
+      $stdout << (( v & 0xFFFF0000 ) >> 16) << ", " if comb2nd_indicies_nonbasic[dm1]
+      $stdout << (v & 0xFFFF) << ", "
     end
-    $stdout << ( comb_array[a][b] or -1 ) << ", "
+    offset += 1
+    offset += 1 if comb2nd_indicies_nonbasic[dm1]    
   end
+  $stdout  << "\n"
 end
 $stdout << "};\n\n"