ffi-hydrogen 0.1.0

data/vendor/stringencoders/make-ci.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+set -x
+set -e
+
+INDENT=clang-format-3.8 ./indent.sh
+git status --porcelain | wc -l
+
+autoreconf --install --force
+automake --add-missing >/dev/null 2>&1
+./configure
+make
+make check
+

data/vendor/stringencoders/makerelease.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+FILE=stringencoders-v3.11.0
+rm -rf ${FILE}
+git clone --depth=1 git@github.com:client9/stringencoders.git ${FILE}
+(cd ${FILE} && ./bootstrap.sh)
+find $FILE -name ".git" | xargs rm -rf
+
+rm -f ${FILE}/Doxyfile
+rm -f ${FILE}/makerelease.sh
+rm -rf ${FILE}/doxy
+
+tar -czvf ${FILE}.tar.gz ${FILE}
+rm -rf ${FILE}
+echo "DONE"
+ls -lh ${FILE}.tar.gz

data/vendor/stringencoders/python/b85.py

@@ -0,0 +1,176 @@
+#!/usr/bin/env python
+
+#
+# This is the MIT License
+# http://www.opensource.org/licenses/mit-license.php
+#
+# Copyright (c) 2008 Nick Galbreath
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+gsCharToInt = [
+    256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256,
+    256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256,
+    256, 256, 256, 256, 256, 256, 256, 256, 256,   0, 256,   1,
+    2,   3, 256,   4,   5,   6,   7,   8, 256,   9,  10,  11,
+    12,  13,  14,  15,  16,  17,  18,  19,  20,  21,  22, 256,
+    23,  24,  25,  26,  27,  28,  29,  30,  31,  32,  33,  34,
+    35,  36,  37,  38,  39,  40,  41,  42,  43,  44,  45,  46,
+    47,  48,  49,  50,  51,  52,  53,  54, 256,  55,  56,  57,
+    58,  59,  60,  61,  62,  63,  64,  65,  66,  67,  68,  69,
+    70,  71,  72,  73,  74,  75,  76,  77,  78,  79,  80,  81,
+    82,  83,  84, 256, 256, 256, 256, 256, 256, 256, 256, 256,
+    256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256,
+    256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256,
+    256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256,
+    256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256,
+    256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256,
+    256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256,
+    256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256,
+    256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256,
+    256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256,
+    256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256,
+    256, 256, 256, 256]
+
+gsIntToChar = [
+    '!',  '#',  '$',  '%', '\'',  '(',  ')',  '*',  '+',  '-',
+    '.',  '/',  '0',  '1',  '2',  '3',  '4',  '5',  '6',  '7',
+    '8',  '9',  ':',  '<',  '=',  '>',  '?',  '@',  'A',  'B',
+    'C',  'D',  'E',  'F',  'G',  'H',  'I',  'J',  'K',  'L',
+    'M',  'N',  'O',  'P',  'Q',  'R',  'S',  'T',  'U',  'V',
+    'W',  'X',  'Y',  'Z',  '[',  ']',  '^',  '_',  '`',  'a',
+    'b',  'c',  'd',  'e',  'f',  'g',  'h',  'i',  'j',  'k',
+    'l',  'm',  'n',  'o',  'p',  'q',  'r',  's',  't',  'u',
+    'v',  'w',  'x',  'y',  'z'
+    ]
+
+from struct import unpack, pack
+# covert 4 characters into 5
+def b85_encode(s):
+    parts = []
+    numchunks = len(s) // 4
+    format = '!' + str(numchunks) + 'I'
+    for x in  unpack(format, s):
+        # network order (big endian), 32-bit unsigned integer
+        # note: x86 is little endian
+        parts.append(gsIntToChar[x // 52200625])
+        parts.append(gsIntToChar[(x // 614125) % 85])
+        parts.append(gsIntToChar[(x // 7225) % 85])
+        parts.append(gsIntToChar[(x // 85) % 85])
+        parts.append(gsIntToChar[x % 85])
+    return ''.join(parts)
+
+#
+# also does not use the 'struct' module which may be desirable
+# to some
+#
+def b85_encode2(s):
+    parts = []
+    for i in xrange(0,len(s), 4):
+        chunk = s[i:i+4]
+        x = ord(chunk[3]) + 256*(ord(chunk[2]) + 256*(ord(chunk[1]) + 256*ord(chunk[0])))
+
+        # network order (big endian), 32-bit unsigned integer
+        # note: x86 is little endian
+        parts.append(gsIntToChar[x // 52200625])
+        parts.append(gsIntToChar[(x // 614125) % 85])
+        parts.append(gsIntToChar[(x // 7225) % 85])
+        parts.append(gsIntToChar[(x // 85) % 85])
+        parts.append(gsIntToChar[x % 85])
+    return ''.join(parts)
+
+# convert 5 characters to 4
+def b85_decode(s):
+    parts = []
+    for i in xrange(0, len(s), 5):
+        bsum = 0;
+        for j in xrange(0,5):
+            val = gsCharToInt[ord(s[i+j])]
+            bsum = 85*bsum + val
+        tmp = pack('!I', bsum)
+        parts.append(tmp)
+        #parts += tmp
+        #parts += unpack('cccc', tmp)
+    return ''.join(parts)
+
+# convert 5 characters to 4
+def b85_decode2(s):
+    parts = []
+    for i in xrange(0, len(s), 5):
+        bsum = 0;
+        for j in xrange(0,5):
+            val = gsCharToInt[ord(s[i+j])]
+            bsum = 85*bsum + val
+        parts.append(chr((bsum >> 24) & 0xff))
+        parts.append(chr((bsum >> 16) & 0xff))
+        parts.append(chr((bsum >> 8) & 0xff))
+        parts.append(chr(bsum & 0xff))
+
+    return ''.join(parts)
+
+import unittest
+class B85Test(unittest.TestCase):
+
+    def testDecode1(self):
+        s = b85_decode('!!!!#')
+        self.assertEquals(4, len(s))
+        self.assertEquals(0, ord(s[0]))
+        self.assertEquals(0, ord(s[1]))
+        self.assertEquals(0, ord(s[2]))
+        self.assertEquals(1, ord(s[3]))
+
+        e = b85_encode(s)
+        self.assertEquals('!!!!#', e)
+
+
+if __name__ == '__main__':
+    from time import clock
+    #unittest.main()
+
+    s = '!!!!#' * 10
+
+    t0 = clock()
+    for i in xrange(1000000):
+        b85_decode(s)
+    t1 = clock()
+    print "decode v1",  t1-t0
+
+    t0 = clock()
+    for i in xrange(1000000):
+       b85_decode2(s)
+    t1 = clock()
+    print "decode v2",  t1-t0
+
+
+    s = b85_decode('!!!!#' * 10)
+
+    t0 = clock()
+    for i in xrange(1000000):
+       b85_encode(s)
+    t1 = clock()
+    print "encode v1",  t1-t0
+
+    t0 = clock()
+    for i in xrange(1000000):
+        b85_encode2(s)
+    t1 = clock()
+    print "encode v2",  t1-t0
+
+

data/vendor/stringencoders/src/Makefile.am

@@ -0,0 +1,134 @@
+
+include_HEADERS = \
+	extern_c_begin.h extern_c_end.h \
+	modp_b16.h modp_b64.h modp_b64w.h modp_b64r.h \
+	modp_b85.h modp_burl.h modp_bjavascript.h \
+	modp_numtoa.h modp_ascii.h modp_b2.h \
+	modp_qsiter.h modp_xml.h modp_html.h modp_json.h
+
+lib_LTLIBRARIES = libmodpbase64.la
+libmodpbase64_la_SOURCES = \
+	modp_b2.c modp_b2.h modp_b2_data.h \
+	modp_b16.c modp_b16.h modp_b16_data.h \
+	modp_b64.c modp_b64.h modp_b64_data.h \
+	modp_b64w.c modp_b64w.h modp_b64w_data.h \
+	modp_b64r.c modp_b64r.h modp_b64r_data.h \
+	modp_b85.h modp_b85.c modp_b85_data.h \
+	modp_burl.h modp_burl.c modp_burl_data.h \
+	modp_bjavascript.h modp_bjavascript.c modp_bjavascript_data.h \
+	modp_numtoa.h modp_numtoa.c \
+	modp_qsiter.h modp_qsiter.c \
+	modp_xml.h modp_xml.c \
+	modp_ascii.h modp_ascii.c modp_ascii_data.h \
+	modp_utf8.h modp_utf8.c \
+	modp_html.h modp_html.c \
+	modp_json.h modp_json.c 
+
+pkgconfigdir = $(libdir)/pkgconfig
+pkgconfig_DATA = stringencoders.pc
+
+#libmodpbase64_la_DEPENDENCIES = \
+#	modp_b2_data.h modp_b2_gen \
+#	modp_b16_data.h modp_b16_gen \
+#	modp_b64_data.h modp_b64w_data.h modp_b64r_data.h modp_b64_gen \
+#	modp_b85_data.h modp_b85_gen \
+#	modp_ascii_data.h modp_ascii_gen
+
+CLEANFILES = *.log \
+	*~ src/*~ test/*~ \
+	*.tmp src/*.tmp test/*.tmp \
+	doxy/html/* \
+	doxy/*.tmp \
+	m4 \
+	*.loT \
+	*.ll *.ll.out \
+	*.gcda *.gcno *.gcov stringencoders.info \
+	modp_b16_data.h modp_b64_data.h modp_b85_data.h \
+	modp_b64w_data.h modp_b64w.c test/modp_b64w_test.c \
+	modp_b64r_data.h modp_b64r.c test/modp_b64r_test.c \
+	modp_ascii_data.h modp_b2_data.h \
+	modp_xml_test modp_qsiter_test modp_html_test modp_json_test \
+	cxx_test
+
+modp_b2.c: modp_b2.h modp_b2_data.h
+
+modp_b16.c: modp_b16.h modp_b16_data.h
+
+modp_b64.c: modp_b64.h modp_b64_data.h
+
+modp_b64r.c: modp_b64.c modp_b64r.h modp_b64r_data.h
+	perl -p -i -e 's/b64/b64r/g' < modp_b64.c > modp_b64r.c
+
+modp_b64w.c: modp_b64.c modp_b64w.h modp_b64w_data.h
+	perl -p -i -e 's/b64/b64w/g' < modp_b64.c > modp_b64w.c
+
+modp_b85.c: modp_b85.h modp_b85_data.h
+
+modp_burl.c: modp_burl.h modp_burl_data.h
+
+modp_bjavascript.c: modp_bjavascript.h modp_bjavascript_data.h
+
+modp_json.c: modp_json.h modp_json_data.h
+
+modp_ascii.c: modp_ascii.h modp_ascii_data.h
+
+modp_qsiter.c: modp_qsiter.h
+
+modp_xml.c: modp_xml.h
+
+modp_b2_data.h: modp_b2_gen
+	./modp_b2_gen > modp_b2_data.h
+
+modp_b16_data.h: modp_b16_gen
+	./modp_b16_gen > modp_b16_data.h
+
+modp_b64_data.h: modp_b64_gen
+	./modp_b64_gen > modp_b64_data.h
+
+modp_ascii_data.h: modp_ascii_gen
+	./modp_ascii_gen > modp_ascii_data.h
+
+#
+# Recall B64WCHARS by default is "-_."
+#
+modp_b64w_data.h: modp_b64_gen
+	./modp_b64_gen $(B64WCHARS) > modp_b64w_data.h
+
+#
+# RFC 4648 alphabet is -_=
+# http://www.ietf.org/rfc/rfc4648.txt
+modp_b64r_data.h: modp_b64_gen
+	./modp_b64_gen "-_=" > modp_b64r_data.h
+
+modp_b85_data.h: modp_b85_gen
+	./modp_b85_gen > modp_b85_data.h
+
+modp_burl_data.h: modp_burl_gen
+	./modp_burl_gen > modp_burl_data.h
+
+modp_bjavascript_data.h: modp_bjavascript_gen
+	./modp_bjavascript_gen > modp_bjavascript_data.h
+
+modp_json_data.h: modp_json_gen.py
+	./modp_json_gen.py
+
+noinst_PROGRAMS = \
+	modp_b2_gen modp_b16_gen modp_b64_gen modp_b85_gen \
+	modp_burl_gen modp_ascii_gen modp_bjavascript_gen
+
+modp_b2_gen_SOURCES = arraytoc.c modp_b2_gen.c
+
+modp_b16_gen_SOURCES = arraytoc.c modp_b16_gen.c
+
+modp_b64_gen_SOURCES = arraytoc.c modp_b64_gen.c
+
+modp_b85_gen_SOURCES = arraytoc.c modp_b85_gen.c
+
+modp_burl_gen_SOURCES = arraytoc.c modp_burl_gen.c
+
+modp_ascii_gen_SOURCES = arraytoc.c modp_ascii_gen.c
+
+modp_bjavascript_gen_SOURCES = arraytoc.c modp_bjavascript_gen.c
+
+
+

data/vendor/stringencoders/src/arraytoc.c

@@ -0,0 +1,85 @@
+#include "arraytoc.h"
+#include <stdio.h>
+
+/* dump uint32_t as hex digits */
+void uint32_array_to_c_hex(const uint32_t* ary, size_t sz, const char* name)
+{
+    size_t i = 0;
+
+    printf("static const uint32_t %s[%d] = {\n    ", name, (int)sz);
+    for (;;) {
+        printf("0x%08x", ary[i]);
+        ++i;
+        if (i == sz)
+            break;
+        if (i % 6 == 0) {
+            printf(",\n    ");
+        } else {
+            printf(", ");
+        }
+    }
+    printf("\n};\n");
+}
+
+/**
+ * prints char array as a c program snippet
+ */
+void char_array_to_c(const char* ary, size_t sz, const char* name)
+{
+    uint8_t tmp;
+    size_t i = 0;
+
+    printf("static const uint8_t %s[%d] = {\n    ", name, (int)sz);
+
+    for (;;) {
+        if (ary[i] == 0) {
+            printf("'\\0'");
+        } else if (ary[i] == '\n') {
+            printf("'\\n'");
+        } else if (ary[i] == '\t') {
+            printf("'\\t'");
+        } else if (ary[i] == '\r') {
+            printf("'\\r'");
+        } else if (ary[i] == '\'') {
+            printf("'\\''");
+        } else if (ary[i] == '\\') {
+            printf("'\\\\'");
+        } else if (ary[i] < 32 || ary[i] > 126) {
+            tmp = (uint8_t)ary[i];
+            printf("0x%02x", tmp);
+        } else {
+            printf("'%c'", (char)ary[i]);
+        }
+        ++i;
+        if (i == sz)
+            break;
+        if (i % 10 == 0) {
+            printf(",\n    ");
+        } else {
+            printf(", ");
+        }
+    }
+    printf("\n};\n");
+}
+
+/**
+ * prints an uint array as a c program snippet
+ */
+void uint32_array_to_c(const uint32_t* ary, size_t sz, const char* name)
+{
+    size_t i = 0;
+
+    printf("static const uint32_t %s[%d] = {\n    ", name, (int)sz);
+    for (;;) {
+        printf("%u", ary[i]);
+        ++i;
+        if (i == sz)
+            break;
+        if (i % 12 == 0) {
+            printf(",\n    ");
+        } else {
+            printf(", ");
+        }
+    }
+    printf("\n};\n");
+}

data/vendor/stringencoders/src/arraytoc.h

@@ -0,0 +1,43 @@
+#ifndef COM_MODP_STRINGENCODERS_ARRAYTOC
+#define COM_MODP_STRINGENCODERS_ARRAYTOC
+
+#include "modp_stdint.h"
+
+#ifdef __cplusplus
+#define BEGIN_C extern "C" {
+#define END_C }
+#else
+#define BEGIN_C
+#define END_C
+#endif
+
+BEGIN_C
+
+/** \brief output a uint32_t array into source code
+ *
+ *
+ * \param[in] ary the input array
+ * \param[in] size number of elements in array
+ * \param[in] name the name of the struct for the source code
+ *
+ */
+void uint32_array_to_c(const uint32_t* ary, size_t size, const char* name);
+
+/** \brief output an uint32_t array into source code as hex values
+ *
+ * \param[in] ary the input array
+ * \param[in] size number of elements in array
+ * \param[in] name the name of the struct for source code
+ *
+ */
+void uint32_array_to_c_hex(const uint32_t* ary, size_t size, const char* name);
+
+/** \brief output a char array into source code
+ *
+ * \param[in] ary the input array
+ * \param[in] size number of elements in array
+ * \param[in] name the name of the struct for source code
+ */
+void char_array_to_c(const char* ary, size_t size, const char* name);
+
+#endif

data/vendor/stringencoders/src/extern_c_begin.h

@@ -0,0 +1,3 @@
+#ifdef __cplusplus
+extern "C" {
+#endif

data/vendor/stringencoders/src/extern_c_end.h

@@ -0,0 +1,3 @@
+#ifdef __cplusplus
+}
+#endif

data/vendor/stringencoders/src/html_named_entities_generator.py

@@ -0,0 +1,203 @@
+#!/usr/bin/env python
+
+
+class Trie(object):
+    def __init__(self):
+        self.struct = [
+            [ ]
+        ]
+
+    def dump(self):
+        count = 0
+        for i in self.struct:
+            count += len(i)
+            print i
+        print len(self.struct)
+        print count
+
+    def longest_match(self, s):
+        """
+        sample code
+        """
+        leaf = self.struct[0]
+        matches = []
+        for ch in s:
+            newleaf = None
+            for node in leaf:
+                if ch == node['value']:
+                    found = True
+                    if node['userdata'] is not None:
+                        matches.append(node['userdata'])
+                    if node['next'] == 0:
+                        return matches
+                    newleaf = self.struct[node['next']]
+                    break
+            if newleaf:
+                leaf = newleaf
+            else:
+                break
+        return matches
+
+    def compile(self):
+        """
+        The current data struct is a a ragged array
+        0 - [ ]
+        1 - [   ]
+        2 - [      ]
+        3 - [  ]
+        i.e. an array of vectors which are ugly to
+        statically allocate in C
+        http://stackoverflow.com/questions/1083658/jagged-array-in-c
+
+        value, more, next, [userdata or 0]
+        """
+        offsets = [ 0 ]
+        count = 0
+        for leaf in self.struct:
+            count += len(leaf)
+            offsets.append(count)
+        print """
+#ifndef _MODP_HTML_NAMED_ENTITIES_H
+#define _MODP_HTML_NAMED_ENTITIES_H
+#include <string.h>
+
+typedef struct trienode {
+    char value;    /* 1 byte -- char value - we deal only in pure ASCII */
+    char more;     /* 1 byte -- should be read next value in array (boundary marker) */
+    short next;    /* 2 byte -- position of next leaf */
+    int codepoint; /* 4 byte -- codepoint or 0 if none */
+} trienode_t;
+
+static const trienode_t entities[];
+static int html_named_entity_decode(const char* s, size_t len, size_t* consumed);
+
+static int html_named_entity_decode(const char* s, size_t len, size_t* consumed)
+{
+   int codepoint = 0;
+   size_t pos = 0;
+   int i = 0;
+
+   *consumed = 0;
+
+   while (pos < len) {
+        if (s[pos] == entities[i].value) {
+           if (entities[i].codepoint) {
+               codepoint = entities[i].codepoint;
+               *consumed = pos + 1;
+           }
+           if (entities[i].next != 0) {
+               i = entities[i].next;
+               pos += 1;
+               continue;
+           } else {
+               break;
+           }
+        }
+        if (entities[i].more == 0) {
+           break;
+        }
+        ++i;
+   }
+   return codepoint;
+}
+/*
+#include <stdio.h>
+int main(int argc, char* argv[]) {
+    size_t consumed;
+    int codepoint;
+    codepoint = html_named_entity_decode(argv[1], strlen(argv[1]), &consumed);
+    printf("cp = %d, consumed = %d\\n", codepoint, consumed);
+    return 0;
+}
+*/
+static const trienode_t entities[] = {
+"""
+        for leaf in self.struct:
+            nodelen = 0
+            for node in leaf:
+                nodelen += 1
+                if node['full'] != None:
+                    comment = ' /* &' + node['full'] + ' */'
+                else:
+                    comment = ''
+
+                print "{{ {0}, {1}, {2}, {3} }},{4}".format(ord(node['value']),
+                                                            int(nodelen != len(leaf)),
+                                                            offsets[node['next']],
+                                                            node['userdata'],
+                                                            comment
+                                                        )
+        print "};"
+        print "#endif"
+
+    def add(self, s, userdata):
+        idx = 0
+        leaf = self.struct[0]
+        count = 0
+        for ch in s[0:-1]:
+            #print "CHAR " + i
+            key = None
+            for node in leaf:
+                if ch == node['value']:
+                    key = node
+                    break
+            if key is None:
+                #print "key is none"
+                newleaf = [ ]
+                self.struct.append(newleaf)
+                idx = len(self.struct) -1
+                leaf.append( {
+                    "full": None,
+                    "value": ch,
+                    "userdata": 0,
+                    "next": idx
+                } )
+                leaf = newleaf
+            elif key['next'] != 0:
+                # just follow
+                #print "got key index of " + str(key[1])
+                leaf = self.struct[key['next']]
+            else:
+                newleaf = [ ]
+                self.struct.append(newleaf)
+                idx = len(self.struct) -1
+                key['next'] = idx
+                leaf = newleaf
+
+        key = None
+        ch = s[-1]
+        for node in leaf:
+            if ch == node['value']:
+                key = node
+                break
+        if key is None:
+            leaf.append( {
+                'full': s,
+                'userdata': userdata,
+                'next': 0,
+                'value': s[-1]
+            })
+        else:
+            key['full'] = s
+            key['userdata'] = userdata
+
+
+
+import json
+with open('entities.json', 'r') as fd:
+    obj = json.load(fd)
+
+t = Trie()
+total = 0
+for name, data in obj.iteritems():
+    total += len(name) -1
+    t.add(name[1:], data['codepoints'][0])
+
+#for i in ["fo", "fa", "foo"]:
+#    t.add(i)
+
+#t.dump()
+#print t.longest_match("amp;")
+
+t.compile()
+