u 0.5.0

data/ext/encoding/character/utf-8/extconf.rb

@@ -0,0 +1,67 @@
+require 'mkmf'
+
+def try_compiler_option(opt, &block)
+  checking_for "#{opt} option to compiler" do
+    $CFLAGS += " #{opt}" if try_compile '', opt, &block
+  end
+end
+
+try_compiler_option '-std=c99'
+try_compiler_option '-finline-functions'
+try_compiler_option '-fno-common'
+try_compiler_option '-Wall'
+try_compiler_option '-Waggregate-return'
+try_compiler_option '-Wcast-align'
+try_compiler_option '-Wextra'
+try_compiler_option '-Wformat=2'
+try_compiler_option '-Winit-self'
+try_compiler_option '-Winline'
+try_compiler_option '-Wmissing-declarations'
+try_compiler_option '-Wmissing-format-attribute'
+try_compiler_option '-Wmissing-include-dirs'
+try_compiler_option '-Wmissing-noreturn'
+try_compiler_option '-Wmissing-prototypes'
+try_compiler_option '-Wnested-externs'
+try_compiler_option '-Wold-style-definition'
+try_compiler_option '-Wpacked'
+try_compiler_option '-Wp,-D_FORTIFY_SOURCE=2'
+try_compiler_option '-Wpointer-arith'
+try_compiler_option '-Wsign-compare'
+try_compiler_option '-Wstrict-aliasing=2'
+try_compiler_option '-Wswitch-default'
+try_compiler_option '-Wswitch-enum'
+try_compiler_option '-Wundef'
+try_compiler_option '-Wunsafe-loop-optimizations'
+try_compiler_option '-Wwrite-strings'
+
+checking_for 'GNUC visibility attribute' do
+  $defs.push('-DHAVE_GNUC_VISIBILITY') if try_compile <<EOC, '-Werror'
+void f_hidden(void);
+void __attribute__((visibility("hidden")))
+f_hidden(void)
+{
+}
+int main(void)
+{
+  f_hidden();
+  return 0;
+}
+EOC
+end
+
+have_header 'assert.h'
+have_header 'limits.h'
+have_header 'locale.h'
+have_header 'stdbool.h'
+have_header 'stddef.h'
+have_header 'stdint.h'
+have_header 'stdio.h'
+have_header 'stdlib.h'
+have_header 'string.h'
+have_header 'sys/types.h'
+have_header 'wchar.h'
+
+$INSTALLFILES ||= []
+$INSTALLFILES << ['unicode.h', '$(RUBYARCHDIR)', 'lib']
+
+create_makefile 'encoding/character/utf-8/utf8'

data/ext/encoding/character/utf-8/private.c

@@ -0,0 +1,62 @@
+/*
+ * contents: Private functions used by the UTF-8 character-encoding library.
+ *
+ * Copyright © 2007 Nikolai Weibull <now@bitwi.se>
+ */
+
+#include <ruby.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+#include "unicode.h"
+
+#include "private.h"
+
+/* Lookup C in the sorted TABLE using binary search.  TABLE consists of N
+ * entries, where each entry is SIZEOF_ENTRY bytes in size and the first
+ * component is a unichar of size SIZEOF_CHAR.  If C is found in TABLE, its
+ * index is stored in INDEX and true is returned.  Otherwise, false is returned
+ * and INDEX is left untouched. */
+bool
+binary_search_unicode_table(const void *table, size_t n, size_t sizeof_entry, size_t sizeof_char, unichar c, int *index)
+{
+#define ENTRY(index) ((unichar)(*(unichar *)((const char *)table + ((index) * sizeof_entry))) & char_mask)
+
+	int begin = 0;
+        int end = n - 1;
+        int middle;
+
+        /* This is ugly, but not all tables use unichars as their lookup
+         * character.  The casefold table, for example, uses uint16_t-sized
+         * characters.  To only get the interesting part of our table entry
+         * we’ll have to mask the retrieved value. */
+        int char_mask = (1 << (8 * sizeof_char)) - 1;
+
+        /* Drop out early if we know for certain that C can’t be in the
+         * decomposition table. */
+        if (c < ENTRY(0) || c > ENTRY(end))
+                return false;
+
+        while (begin <= end) {
+                middle = binary_search_middle_of(begin, end);
+
+                unichar probe = ENTRY(middle);
+                if (c < probe)
+                        end = middle - 1;
+                else if (c > probe)
+                        begin = middle + 1;
+                else
+                        break;
+        }
+
+        if (begin > end)
+                return false;
+
+        *index = middle;
+
+        return true;
+
+#undef ENTRY
+}

data/ext/encoding/character/utf-8/private.h

@@ -0,0 +1,51 @@
+/*
+ * contents: Private Unicode related information.
+ *
+ * Copyright (C) 2004 Nikolai Weibull <source@pcppopper.org>
+ */
+
+#ifndef PRIVATE_H
+#define PRIVATE_H
+
+#define NUL '\0'
+#define lengthof(ary)   (sizeof(ary) / sizeof((ary)[0]))
+
+#if defined(HAVE_GNUC_VISIBILITY)
+#  define HIDDEN   \
+        __attribute__((visibility("hidden")))
+#else
+#  define HIDDEN
+#endif
+
+#if defined(__GNUC__)
+#  define UNUSED(u)   \
+        u __attribute__((__unused__))
+#else
+#  define UNUSED(u)   \
+        u
+#endif
+
+#define binary_search_middle_of(begin, end)     \
+        (((unsigned)((begin) + (end))) >> 1)
+
+#define unicode_table_lookup(table, c, index)    \
+        binary_search_unicode_table(table, lengthof(table), sizeof((table)[0]), sizeof((table)[0].ch), c, index)
+
+bool binary_search_unicode_table(const void *table, size_t n, size_t sizeof_entry, size_t sizeof_char, unichar c, int *index) HIDDEN;
+
+#define SPLIT_UNICODE_TABLE_LOOKUP_PAGE(data, part, page, c)  \
+        ((part[page] >= UNICODE_MAX_TABLE_INDEX) \
+         ? (part[page] - UNICODE_MAX_TABLE_INDEX) \
+         : (data[part[page]][(c) & 0xff]))
+
+#define SPLIT_UNICODE_TABLE_LOOKUP(data, part1, part2, c, fallback)    \
+        (((c) <= UNICODE_LAST_CHAR_PART1) \
+         ? SPLIT_UNICODE_TABLE_LOOKUP_PAGE(data, part1, (c) >> 8, c) \
+         : (((c) >= UNICODE_FIRST_CHAR_PART2 && (c) <= UNICODE_LAST_CHAR) \
+            ? SPLIT_UNICODE_TABLE_LOOKUP_PAGE(data, part2, ((c) - UNICODE_FIRST_CHAR_PART2) >> 8, c) \
+            : (fallback)))
+
+unichar *_utf_normalize_wc(const char *str, size_t max_len, bool use_len,
+                           NormalizeMode mode) HIDDEN;
+
+#endif /* PRIVATE_H */

data/ext/encoding/character/utf-8/properties.c

@@ -0,0 +1,1056 @@
+/*
+ * contents: Unicode character properties.
+ *
+ * Copyright (C) 2004 Nikolai Weibull <source@pcppopper.org>
+ */
+
+#include <ruby.h>
+#include <assert.h>
+#include <locale.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+#include "unicode.h"
+#include "private.h"
+#include "data/character-tables.h"
+
+
+#define COMBINING_DOT_ABOVE                     ((unichar)0x0307)
+#define LATIN_CAPITAL_LETTER_I_WITH_DOT_ABOVE   ((unichar)0x0130)
+#define COMBINING_GREEK_YPOGEGRAMMENI           ((unichar)0x0345)
+#define GREEK_CAPITAL_LETTER_IOTA               ((unichar)0x0399)
+#define LATIN_SMALL_LETTER_I                    ((unichar)0x0069)
+#define LATIN_SMALL_LETTER_DOTLESS_I            ((unichar)0x0131)
+#define LATIN_CAPITAL_LETTER_I_WITH_GRAVE       ((unichar)0x00cc)
+#define LATIN_CAPITAL_LETTER_I_WITH_ACUTE       ((unichar)0x00cd)
+#define LATIN_CAPITAL_LETTER_I_WITH_TILDE       ((unichar)0x0128)
+#define LATIN_CAPITAL_LETTER_I_WITH_OGONEK      ((unichar)0x012e)
+#define COMBINING_GRAVE_ACCENT                  ((unichar)0x0300)
+#define COMBINING_ACUTE_ACCENT                  ((unichar)0x0301)
+#define COMBINING_TILDE                         ((unichar)0x0303)
+#define GREEK_CAPITAL_LETTER_SIGMA              ((unichar)0x03a3)
+#define GREEK_SMALL_LETTER_SIGMA                ((unichar)0x03c3)
+#define GREEK_SMALL_LETTER_FINAL_SIGMA          ((unichar)0x03c2)
+
+#define OFFSET_IF(buf, len)    (((buf) != NULL) ? (buf) + (len) : NULL)
+
+/* {{{1
+ * Macros for accessing the Unicode character attribute table.
+ *
+ * TODO: Turn these macros into full-fledged functions, as this is rather silly
+ * when we have ‹inline› in C99.
+ */
+#define ATTR_TABLE(page) \
+	(((page) <= UNICODE_LAST_PAGE_PART1) \
+	 ? attr_table_part1[page] \
+	 : attr_table_part2[(page) - 0xe00])
+
+#define ATTTABLE(page, char) \
+	((ATTR_TABLE(page) == UNICODE_MAX_TABLE_INDEX) \
+	 ? 0 : (attr_data[ATTR_TABLE(page)][char]))
+
+
+/* {{{1
+ * Internal function used for figuring out the type of a given character.
+ */
+static inline int
+s_type(unichar c)
+{
+	const int16_t *table;
+	unsigned int page;
+
+	if (c <= UNICODE_LAST_CHAR_PART1) {
+		page = c >> 8;
+		table = type_table_part1;
+	} else if (c >= UNICODE_FIRST_CHAR_PART2 && c <= UNICODE_LAST_CHAR) {
+		page = (c - UNICODE_FIRST_CHAR_PART2) >> 8;
+		table = type_table_part2;
+	} else {
+		return UNICODE_UNASSIGNED;
+	}
+
+	if (table[page] >= UNICODE_MAX_TABLE_INDEX)
+		return table[page] - UNICODE_MAX_TABLE_INDEX;
+	else
+		return type_data[table[page]][c & 0xff];
+}
+
+
+/* {{{1
+ * Bit-fiddling macros for testing the class of a type.
+ */
+#define IS(type, class) (((unsigned int)1 << (type)) & (class))
+#define OR(type, rest)  (((unsigned int)1 << (type)) | (rest))
+
+
+/* {{{1
+ * Internal function used to check if the given type represents a digit type.
+ */
+static inline bool
+s_isdigit(int type)
+{
+        return IS(type,
+                  OR(UNICODE_DECIMAL_NUMBER,
+                     OR(UNICODE_LETTER_NUMBER,
+                        OR(UNICODE_OTHER_NUMBER, 0))));
+}
+
+
+/* {{{1
+ * Internal function used to check if the given type represents an alphabetic
+ * type.
+ */
+static inline bool
+s_isalpha(int type)
+{
+        return IS(type,
+                  OR(UNICODE_LOWERCASE_LETTER,
+                     OR(UNICODE_UPPERCASE_LETTER,
+                        OR(UNICODE_TITLECASE_LETTER,
+                           OR(UNICODE_MODIFIER_LETTER,
+                              OR(UNICODE_OTHER_LETTER, 0))))));
+}
+
+
+/* {{{1
+ * Internal function used to check if the given type represents a mark type.
+ */
+static inline bool
+s_ismark(int type)
+{
+        return IS(type,
+                  OR(UNICODE_NON_SPACING_MARK,
+                     OR(UNICODE_COMBINING_MARK,
+                        OR(UNICODE_ENCLOSING_MARK, 0))));
+}
+
+
+/* {{{1
+ * Determine whether ‘c’ is an alphanumeric, such as A, B, C, 0, 1, or 2.
+ */
+bool
+unichar_isalnum(unichar c)
+{
+	int type = s_type(c);
+
+	return s_isdigit(type) || s_isalpha(type);
+}
+
+
+/* {{{1
+ * Determine whether ‘c’ is an alphabetic (i.e. a letter), such as A, B, or C.
+ */
+bool
+unichar_isalpha(unichar c)
+{
+	return s_isalpha(s_type(c));
+}
+
+
+/* {{{1
+ * Determine whether ‘c’ is a control character, such as ‹NUL›.
+ */
+bool
+unichar_iscntrl(unichar c)
+{
+	return s_type(c) == UNICODE_CONTROL;
+}
+
+
+/* {{{1
+ * Determine whether ‘c’ is a digit, such as 0, 1, or 2.
+ */
+bool
+unichar_isdigit(unichar c)
+{
+	return s_type(c) == UNICODE_DECIMAL_NUMBER;
+}
+
+
+/* {{{1
+ * Determine whether ‘c’ is printable and not a space or control character such
+ * as tab or <NUL›, such as A, B, or C.
+ */
+bool
+unichar_isgraph(unichar c)
+{
+        return !IS(s_type(c),
+                   OR(UNICODE_CONTROL,
+                      OR(UNICODE_FORMAT,
+                         OR(UNICODE_UNASSIGNED,
+                            OR(UNICODE_PRIVATE_USE,
+                               OR(UNICODE_SURROGATE,
+                                  OR(UNICODE_SPACE_SEPARATOR, 0)))))));
+}
+
+
+/* {{{1
+ * Determine whether ‘c’ is a lowercase letter, such as a, b, or c.
+ */
+bool
+unichar_islower(unichar c)
+{
+	return s_type(c) == UNICODE_LOWERCASE_LETTER;
+}
+
+
+/* {{{1
+ * Determine whether ‘c’ is printable, which works the same as
+ * unichar_isgraph(), except that space characters are also printable.
+ */
+bool
+unichar_isprint(unichar c)
+{
+        return !IS(s_type(c),
+                   OR(UNICODE_CONTROL,
+                      OR(UNICODE_FORMAT,
+                         OR(UNICODE_UNASSIGNED,
+                            OR(UNICODE_PRIVATE_USE,
+                               OR(UNICODE_SURROGATE, 0))))));
+}
+
+
+/* {{{1
+ * Determine whether ‘c’ is some form of punctuation or other symbol.
+ */
+bool
+unichar_ispunct(unichar c)
+{
+        return IS(s_type(c),
+                  OR(UNICODE_CONNECT_PUNCTUATION,
+                     OR(UNICODE_DASH_PUNCTUATION,
+                        OR(UNICODE_OPEN_PUNCTUATION,
+                           OR(UNICODE_CLOSE_PUNCTUATION,
+                              OR(UNICODE_INITIAL_PUNCTUATION,
+                                 OR(UNICODE_FINAL_PUNCTUATION,
+                                    OR(UNICODE_OTHER_PUNCTUATION,
+                                       OR(UNICODE_MODIFIER_SYMBOL,
+                                          OR(UNICODE_MATH_SYMBOL,
+                                             OR(UNICODE_CURRENCY_SYMBOL,
+                                                OR(UNICODE_OTHER_SYMBOL, 0)))))))))))) ? true : false;
+}
+
+
+/* {{{1
+ * Determine whether ‘c’ is some form of whitespace, such as space, tab or a
+ * line separator (newline, carriage return, etc.).
+ */
+bool
+unichar_isspace(unichar c)
+{
+	switch (c) {
+	case '\t':
+	case '\n':
+	case '\r':
+	case '\f':
+		return true;
+	default:
+                return IS(s_type(c),
+                          OR(UNICODE_SPACE_SEPARATOR,
+                             OR(UNICODE_LINE_SEPARATOR,
+                                OR(UNICODE_PARAGRAPH_SEPARATOR, 0)))) ? true : false;
+	}
+}
+
+
+/* {{{1
+ * Determine whether ‘c’ is an uppeercase letter, such as A, B, or C
+ */
+bool
+unichar_isupper(unichar c)
+{
+	return s_type(c) == UNICODE_UPPERCASE_LETTER;
+}
+
+
+/* {{{1
+ * Determine whether ‘c’ is a titlecase letter, such as the slavic digraph DZ,
+ * which at the beginning of a word is written as Dz, where only the initial D
+ * is capitalized.  (Complicated huh?)
+ */
+bool
+unichar_istitle(unichar c)
+{
+	/* TODO: binary search helpful? */
+	for (size_t i = 0; i < lengthof(title_table); i++)
+		if (title_table[i][0] == c)
+			return true;
+
+	return false;
+}
+
+
+/* {{{1
+ * Determine whether ‘c’ is a new-line.
+ */
+#define UNICHAR_NEXT_LINE               ((unichar)0x0085)
+#define UNICHAR_LINE_SEPARATOR          ((unichar)0x2028)
+#define UNICHAR_PARAGRAPH_SEPARATOR     ((unichar)0x2029)
+
+bool
+unichar_isnewline(unichar c)
+{
+        switch (c) {
+        case '\n': case '\f': case '\r': case UNICHAR_NEXT_LINE:
+        case UNICHAR_LINE_SEPARATOR: case UNICHAR_PARAGRAPH_SEPARATOR:
+                return true;
+        default:
+                return false;
+        }
+}
+
+/* {{{1
+ * Determine whether ‘c’ is a hexadecimal digit, such as 0, 1, ..., 9, a, b,
+ * ..., f, or A, B, ..., F.
+ */
+#define UNICHAR_FULLWIDTH_A     0xff21
+#define UNICHAR_FULLWIDTH_F     0xff26
+#define UNICHAR_FULLWIDTH_a     0xff41
+#define UNICHAR_FULLWIDTH_f     0xff46
+bool
+unichar_isxdigit(unichar c)
+{
+	return ((c >= 'a' && c <= 'f') ||
+		(c >= 'A' && c <= 'F') ||
+                (c >= UNICHAR_FULLWIDTH_a && c <= UNICHAR_FULLWIDTH_f) ||
+                (c >= UNICHAR_FULLWIDTH_A && c <= UNICHAR_FULLWIDTH_F) ||
+                (s_type(c) == UNICODE_DECIMAL_NUMBER));
+//		s_isdigit(s_type(c)));
+}
+
+
+/* {{{1
+ * Determine whether code point ‘c’ has been assigned a code value.
+ */
+bool
+unichar_isassigned(unichar c)
+{
+	return s_type(c) != UNICODE_UNASSIGNED;
+}
+
+
+/* {{{1
+ * Determine whether ‘c’ is a wide character, thus is typically rendered in a
+ * double-width cell on a terminal.
+ */
+bool
+unichar_iswide(unichar c)
+{
+	if (c < 0x1100)
+		return false;
+
+        return (c <= 0x115f || 				/* Hangul Jamo init. consonants */
+                c == 0x2329 || c == 0x232a || 		/* angle brackets */
+                (c >= 0x2e80 && c <= 0xa4cf && 		/* CJK ... Yi */
+                 (c < 0x302a || c > 0x302f) &&
+                 c != 0x303f && c != 0x3099 && c != 0x309a) ||
+                (c >= 0xac00 && c <= 0xd7a3) || 	/* Hangul syllables */
+                (c >= 0xf900 && c <= 0xfaff) || 	/* CJK comp. graphs */
+                (c >= 0xfe30 && c <= 0xfe6f) || 	/* CJK comp. forms */
+                (c >= 0xff00 && c <= 0xff60) || 	/* fullwidth forms */
+                (c >= 0xffe0 && c <= 0xffe6) || 	/*       -"-       */
+                (c >= 0x20000 && c <= 0x2fffd) || 	/* CJK extra stuff */
+                (c >= 0x30000 && c <= 0x3fffd));    	/*       -"-       */
+}
+
+
+/* {{{1
+ * Convert ‘c’ to its uppercase representation (if any).
+ */
+static unichar
+special_case_table_lookup(unichar c)
+{
+        unichar tv = ATTTABLE(c >> 8, c & 0xff);
+
+        if (tv >= UNICODE_SPECIAL_CASE_TABLE_START)
+                tv = utf_char(special_case_table +
+                              tv - UNICODE_SPECIAL_CASE_TABLE_START);
+
+        if (tv == '\0')
+                return c;
+
+        return tv;
+}
+
+static unichar
+titlecase_table_lookup(unichar c, bool want_upper)
+{
+        for (size_t i = 0; i < lengthof(title_table); i++)
+                if (title_table[i][0] == c)
+                        return title_table[i][want_upper ? 1 : 2];
+
+        return c;
+}
+
+unichar
+unichar_toupper(unichar c)
+{
+	int type = s_type(c);
+
+	if (type == UNICODE_LOWERCASE_LETTER)
+                return special_case_table_lookup(c);
+        
+        if (type == UNICODE_TITLECASE_LETTER)
+                return titlecase_table_lookup(c, true);
+
+        return c;
+}
+
+
+/* {{{1
+ * Convert ‘c’ to its lowercase representation (if any).
+ */
+unichar
+unichar_tolower(unichar c)
+{
+	int type = s_type(c);
+
+	if (type == UNICODE_UPPERCASE_LETTER)
+                return special_case_table_lookup(c);
+        
+        if (type == UNICODE_TITLECASE_LETTER)
+                return titlecase_table_lookup(c, false);
+
+        return c;
+}
+
+
+/* {{{1
+ * Convert ‘c’ to its titlecase representation (if any).
+ */
+unichar
+unichar_totitle(unichar c)
+{
+	for (size_t i = 0; i < lengthof(title_table); i++)
+		if (title_table[i][0] == c ||
+                    title_table[i][1] == c ||
+                    title_table[i][2] == c)
+			return title_table[i][0];
+
+        if (s_type(c) == UNICODE_LOWERCASE_LETTER)
+                return unichar_toupper(c);
+
+        return c;
+}
+
+
+/* {{{1
+ * Return the numeric value of ‘c’ if it's a decimal digit, or -1 if not.
+ */
+int
+unichar_digit_value(unichar c)
+{
+	if (s_type(c) == UNICODE_DECIMAL_NUMBER)
+		return ATTTABLE(c >> 8, c & 0xff);
+
+        return -1;
+}
+
+
+/* {{{1
+ * Return the numeric value of ‘c’ if it's a hexadecimal digit, or -1 if not.
+ */
+int
+unichar_xdigit_value(unichar c)
+{
+	if (c >= 'a' && c <= 'f')
+		return c - 'a' + 10;
+	else if (c >= 'A' && c <= 'F')
+		return c - 'A' + 10;
+        else if (c >= UNICHAR_FULLWIDTH_a && c <= UNICHAR_FULLWIDTH_f)
+                return c - UNICHAR_FULLWIDTH_a + 10;
+        else if (c >= UNICHAR_FULLWIDTH_A && c <= UNICHAR_FULLWIDTH_F)
+                return c - UNICHAR_FULLWIDTH_A + 10;
+	else
+		return unichar_digit_value(c);
+}
+
+
+/* {{{1
+ * Determine the Unicode character type of ‘c’.
+ */
+UnicodeType
+unichar_type(unichar c)
+{
+	return s_type(c);
+}
+
+
+/* {{{1
+ * LocaleType: This ‹enum› is used for dealing with different locales for
+ * turning strings into uppercase or lowercase.
+ */
+typedef enum {
+	LOCALE_NORMAL,
+	LOCALE_TURKIC,
+	LOCALE_LITHUANIAN
+} LocaleType;
+
+
+/* {{{1
+ * Retrieve the locale type from the environment (LC_CTYPE).
+ */
+static LocaleType
+get_locale_type(void)
+{
+	const char *locale = setlocale(LC_CTYPE, NULL);
+
+	if ((locale[0] == 'a' && locale[1] == 'z') ||
+	    (locale[0] == 't' && locale[1] == 'r'))
+		return LOCALE_TURKIC;
+
+	if (locale[0] == 'l' && locale[1] == 't')
+		return LOCALE_LITHUANIAN;
+
+        return LOCALE_NORMAL;
+}
+
+
+/* {{{1
+ * Put character marks found in ‘p_inout’ into itself.  If ‘remove_dot’ is
+ * true, remove the dot over an uppercase I for a turkish locale.
+ */
+static size_t
+output_marks(const char **p_inout, char *buf, bool remove_dot)
+{
+	size_t len = 0;
+	const char *p = *p_inout;
+
+	for ( ; *p != '\0'; p = utf_next(p)) {
+		unichar c = utf_char(p);
+
+                if (!s_ismark(s_type(c)))
+                        break;
+
+                if (!remove_dot || c != COMBINING_DOT_ABOVE)
+                        len += unichar_to_utf(c, (buf != NULL) ? buf + len : NULL);
+	}
+
+	*p_inout = p;
+
+	return len;
+}
+
+/* {{{1
+ * Output titlecases where appropriate.
+ */
+static size_t
+output_special_case(char *buf, int offset, int type, bool upper)
+{
+	const char *p = special_case_table + offset;
+
+	if (type != UNICODE_TITLECASE_LETTER)
+		p = utf_next(p);
+
+	if (upper)
+		p += utf_byte_length(p) + 1;
+
+	size_t len = utf_byte_length(p);
+
+	if (buf != NULL)
+		memcpy(buf, p, len);
+
+	return len;
+}
+
+/* {{{1
+ * Do uppercasing of ‘p’ for Lithuanian locales.
+ */
+static size_t
+remove_all_combining_dot_above(unichar c, char *buf)
+{
+        size_t decomp_len;
+        unichar *decomp = unicode_canonical_decomposition(c, &decomp_len);
+
+        size_t len = 0;
+        for (size_t i = 0; i < decomp_len; i++)
+                if (decomp[i] != COMBINING_DOT_ABOVE)
+                        len += unichar_to_utf(unichar_toupper(decomp[i]),
+                                              OFFSET_IF(buf, len));
+
+        free(decomp);
+
+        return len;
+}
+
+static size_t
+real_toupper_lithuanian(const char **p, unichar c, int type, char *buf,
+                        bool *was_i)
+{
+	if (c == 'i') {
+		*was_i = true;
+		return 0;
+	}
+
+	if (*was_i) {
+                size_t len = remove_all_combining_dot_above(c, buf);
+		return len + output_marks(p, OFFSET_IF(buf, len), true);
+	}
+
+	if (!s_ismark(type))
+		*was_i = false;
+
+	return 0;
+}
+
+/* {{{1
+ * Do real upcasing. */
+static inline size_t
+real_do_toupper(unichar c, int type, char *buf)
+{
+	bool upper = (type != UNICODE_LOWERCASE_LETTER);
+	unichar tv = ATTTABLE(c >> 8, c & 0xff);
+
+	if (tv >= UNICODE_SPECIAL_CASE_TABLE_START)
+                return output_special_case(buf,
+                                           tv - UNICODE_SPECIAL_CASE_TABLE_START,
+                                           type, upper);
+
+        /* TODO: this should really use titlecase_table_lookup somehow. */
+        if (type == UNICODE_TITLECASE_LETTER)
+                for (size_t i = 0; i < lengthof(title_table); i++)
+                        if (title_table[i][0] == c)
+                                return unichar_to_utf(title_table[i][1], buf);
+
+        return unichar_to_utf(tv != '\0' ? tv : c, buf); 
+}
+
+/* {{{1
+ * Do real uppercasing of ‘str’.
+ */
+static size_t
+real_toupper_one(const char **p, const char *prev, char *buf,
+                 LocaleType locale_type, bool *was_i)
+{
+        unichar c = utf_char(prev);
+        int type = s_type(c);
+
+        if (locale_type == LOCALE_LITHUANIAN) {
+                size_t len = real_toupper_lithuanian(p, c, type, buf, was_i);
+                if (len > 0)
+                        return len;
+        }
+
+        if (locale_type == LOCALE_TURKIC && c == 'i')
+                return unichar_to_utf(LATIN_CAPITAL_LETTER_I_WITH_DOT_ABOVE,
+                                      buf);
+
+        if (c == COMBINING_GREEK_YPOGEGRAMMENI) {
+                /* Nasty, need to move it after other combining marks...this
+                 * would go away if we normalized first. */
+                /* TODO: don’t we need to make sure we don’t go beyond the end
+                 * of ‘p’? */
+                size_t len = output_marks(p, buf, false);
+                return len + unichar_to_utf(GREEK_CAPITAL_LETTER_IOTA,
+                                            OFFSET_IF(buf, len));
+        }
+        
+        if (IS(type, OR(UNICODE_LOWERCASE_LETTER,
+                        OR(UNICODE_TITLECASE_LETTER, 0))))
+                return real_do_toupper(c, type, buf);
+
+        size_t len = s_utf_skip_lengths[*(const unsigned char *)prev];
+
+        if (buf != NULL)
+                memcpy(buf, prev, len);
+
+        return len;
+}
+
+static size_t
+real_toupper(const char *str, size_t max, bool use_max, char *buf,
+	     LocaleType locale_type)
+{
+	const char *p = str;
+	size_t len = 0;
+	bool p_was_i = false;
+
+	while ((!use_max || p < str + max) && *p != '\0') {
+		const char *prev = p;
+		p = utf_next(p);
+
+                len += real_toupper_one(&p, prev, OFFSET_IF(buf, len),
+                                        locale_type, &p_was_i);
+	}
+
+	return len;
+}
+
+/* {{{1
+ * Wrapper around real_toupper() for dealing with memory allocation and such.
+ */
+static char *
+utf_upcase_impl(const char *str, size_t max, bool use_max)
+{
+	assert(str != NULL);
+
+	LocaleType locale_type = get_locale_type();
+
+	size_t len = real_toupper(str, max, use_max, NULL, locale_type);
+	char *result = ALLOC_N(char, len + 1);
+	real_toupper(str, max, use_max, result, locale_type);
+	result[len] = '\0';
+
+	return result;
+}
+
+
+/* {{{1
+ * Convert all characters in ‘str’ to their uppercase representation if
+ * applicable.  Returns the freshly allocated representation.
+ */
+char *
+utf_upcase(const char *str)
+{
+	return utf_upcase_impl(str, 0, false);
+}
+
+
+/* {{{1
+ * Convert all characters in ‘str’ to their uppercase representation if
+ * applicable.  Returns the freshly allocated representation.  Do this for at
+ * most ‘len˚ bytes from ‘str’.
+ */
+char *
+utf_upcase_n(const char *str, size_t len)
+{
+	return utf_upcase_impl(str, len, true);
+}
+
+
+/* {{{1
+ * Traverse the string checking for characters with combining class == 230
+ * until a base character is found.
+ */ 
+static bool
+has_more_above(const char *str)
+{
+	for (const char *p = str; *p != '\0'; p = utf_next(p)) {
+		int c_class = unichar_combining_class(utf_char(p));
+
+		if (c_class == 230)
+			return true;
+
+		if (c_class == 0)
+			return false;
+	}
+
+	return false;
+}
+
+static inline size_t
+real_do_tolower(unichar c, int type, char *buf)
+{
+	unichar tv = ATTTABLE(c >> 8, c & 0xff);
+
+	if (tv >= UNICODE_SPECIAL_CASE_TABLE_START)
+                return output_special_case(buf,
+                                           tv - UNICODE_SPECIAL_CASE_TABLE_START,
+                                           type, false);
+
+        /* TODO: this should really use titlecase_table_lookup somehow. */
+        if (type == UNICODE_TITLECASE_LETTER)
+                for (size_t i = 0; i < lengthof(title_table); i++)
+                        if (title_table[i][0] == c)
+                                return unichar_to_utf(title_table[i][2], buf);
+
+        return unichar_to_utf(tv != '\0' ? tv : c, buf);
+}
+
+/* {{{1
+ * The real implementation of downcase.
+ */
+static size_t
+tolower_turkic_i(const char **p, char *buf)
+{
+        unichar i = LATIN_SMALL_LETTER_DOTLESS_I;
+
+        if (utf_char(*p) == COMBINING_DOT_ABOVE) {
+                /* TODO: don’t we need to make sure we don’t go beyond the end
+                 * of ‘p’? */
+                *p = utf_next(*p);
+                i = LATIN_SMALL_LETTER_I;
+        } 
+
+        return unichar_to_utf(i, buf);
+}
+
+static size_t
+tolower_lithuianian_i(char *buf, unichar base, unichar combiner)
+{
+        size_t len = unichar_to_utf(base, buf);
+        len += unichar_to_utf(COMBINING_DOT_ABOVE, OFFSET_IF(buf, len));
+        if (combiner != '\0')
+                len += unichar_to_utf(combiner, OFFSET_IF(buf, len));
+
+        return len;
+}
+
+static size_t
+tolower_sigma(const char **p, char *buf, const char *end, bool use_end)
+{
+        unichar sigma = GREEK_SMALL_LETTER_FINAL_SIGMA;
+
+        /* SIGMA maps differently depending on whether it is final or not.  The
+         * following simplified test would fail in the case of combining marks
+         * following the sigma, but I don't think that occurs in real text.
+         * The test here matches that in ICU. */
+        if ((!use_end || *p < end) && **p != '\0' && s_isalpha(s_type(utf_char(*p))))
+                sigma = GREEK_SMALL_LETTER_SIGMA;
+
+        return unichar_to_utf(sigma, buf);
+}
+
+static size_t
+real_tolower_one(const char **p, const char *prev, char *buf,
+                 LocaleType locale_type, const char *end, bool use_end)
+{
+        unichar c = utf_char(prev);
+        int type = s_type(c);
+
+        if (locale_type == LOCALE_TURKIC && c == 'I')
+                return tolower_turkic_i(p, buf);
+
+        /* Introduce an explicit dot above the lowercasing capital I’s
+         * and J’s whenever there are more accents above.
+         * [SpecialCasing.txt] */
+        if (locale_type == LOCALE_LITHUANIAN) {
+                unichar base = LATIN_SMALL_LETTER_I;
+                unichar combiner = '\0';
+
+                switch (c) {
+                case LATIN_CAPITAL_LETTER_I_WITH_GRAVE:
+                        combiner = COMBINING_GRAVE_ACCENT;
+                        break;
+                case LATIN_CAPITAL_LETTER_I_WITH_ACUTE:
+                        combiner = COMBINING_ACUTE_ACCENT;
+                        break;
+                case LATIN_CAPITAL_LETTER_I_WITH_TILDE:
+                        combiner = COMBINING_TILDE;
+                        break;
+                case 'I':
+                case 'J':
+                case LATIN_CAPITAL_LETTER_I_WITH_OGONEK:
+                        if (!has_more_above(*p))
+                                goto no_lithuanian_i_casing;
+
+                        base = unichar_tolower(c);
+                        break;
+                default:
+                        goto no_lithuanian_i_casing;
+                }
+
+                return tolower_lithuianian_i(buf, base, combiner);
+        }
+
+no_lithuanian_i_casing:
+
+        if (c == GREEK_CAPITAL_LETTER_SIGMA)
+                return tolower_sigma(p, buf, end, use_end); 
+        
+        if (IS(type, OR(UNICODE_UPPERCASE_LETTER,
+                        OR(UNICODE_TITLECASE_LETTER, 0))))
+                return real_do_tolower(c, type, buf);
+
+        size_t len = s_utf_skip_lengths[*(const unsigned char *)prev];
+
+        if (buf != NULL)
+                memcpy(buf, prev, len);
+
+        return len;
+}
+
+static size_t
+real_tolower(const char *str, size_t max, bool use_max, char *buf,
+             LocaleType locale_type)
+{
+	const char *p = str;
+        const char *end = str + max;
+	size_t len = 0;
+
+	while ((!use_max || p < end) && *p != '\0') {
+		const char *prev = p;
+		p = utf_next(p);
+
+                len += real_tolower_one(&p, prev, OFFSET_IF(buf, len),
+                                        locale_type, end, use_max);
+	}
+
+	return len;
+}
+
+
+/* {{{1 */
+static char *
+utf_downcase_impl(const char *str, size_t max, bool use_max)
+{
+	assert(str != NULL);
+
+	LocaleType locale_type = get_locale_type();
+
+	size_t len = real_tolower(str, max, use_max, NULL, locale_type);
+	char *result = ALLOC_N(char, len + 1);
+	real_tolower(str, max, use_max, result, locale_type);
+	result[len] = '\0';
+
+	return result;
+}
+
+
+/* {{{1
+ * Convert all characters in ‘str’ to their lowercase representation if
+ * applicable.  Returns the freshly allocated representation.
+ */
+char *
+utf_downcase(const char *str)
+{
+	return utf_downcase_impl(str, 0, false);
+}
+
+
+/* {{{1
+ * Convert all characters in ‘str’ to their lowercase representation if
+ * applicable.  Returns the freshly allocated representation.  Do this for at
+ * most ‘len˚ bytes from ‘str’.
+ */
+char *
+utf_downcase_n(const char *str, size_t len)
+{
+	return utf_downcase_impl(str, len, true);
+}
+
+
+/* {{{1
+ * The real implementation of case folding below.
+ */
+
+static bool
+casefold_table_lookup(unichar c, char *folded, size_t *len)
+{
+        int index;
+
+        if (!unicode_table_lookup(casefold_table, c, &index))
+                return false;
+
+        char const *folded_c = casefold_table[index].data;
+
+        if (folded != NULL)
+                strcpy(folded, folded_c);
+
+        *len += utf_byte_length(folded_c);
+
+        return true;
+}
+
+static char *
+utf_foldcase_impl(const char *str, size_t max, bool use_max)
+{
+	assert(str != NULL);
+
+	char *folded = NULL;
+	size_t len = 0;
+
+again:
+	for (const char *p = str; (!use_max || p < str + max) && *p != '\0'; p = utf_next(p)) {
+		unichar c = utf_char(p);
+
+                if (casefold_table_lookup(c, OFFSET_IF(folded, len), &len))
+                        continue;
+
+		len += unichar_to_utf(unichar_tolower(c), OFFSET_IF(folded, len));
+	}
+
+	if (folded == NULL) {
+		folded = ALLOC_N(char, len + 1);
+		folded[0] = NUL;
+		len = 0;
+		goto again;
+	}
+
+	folded[len] = '\0';
+
+	return folded;
+}
+
+
+/* {{{1
+ * Convert a string into a form that is independent of case.  Return the
+ * freshly allocated representation.
+ */
+char *
+utf_foldcase(const char *str)
+{
+	return utf_foldcase_impl(str, 0, false);
+}
+
+
+/* {{{1
+ * Convert a string into a form that is independent of case.  Return the
+ * freshly allocated representation.  Do this for at most ‘len’ bytes from the
+ * string.
+ */
+char *
+utf_foldcase_n(const char *str, size_t len)
+{
+	return utf_foldcase_impl(str, len, true);
+}
+
+
+/* {{{1
+ * The real implementation of utf_width() and utf_width_n() below.
+ */
+static size_t
+utf_width_impl(const char *str, size_t len, bool use_len)
+{
+	assert(str != NULL);
+
+	size_t width = 0;
+
+	for (const char *p = str; (!use_len || p < str + len) && *p != NUL; p = utf_next(p))
+		width += unichar_iswide(utf_char(p)) ? 2 : 1;
+
+	return width;
+}
+
+
+/* {{{1
+ * Calculate the width in cells of ‘str’.
+ */
+size_t
+utf_width(const char *str)
+{
+	return utf_width_impl(str, 0, false);
+}
+
+
+/* {{{1
+ * Calculate the width in cells of ‘str’, which is of length ‘len’.
+ */
+size_t
+utf_width_n(const char *str, size_t len)
+{
+	return utf_width_impl(str, len, true);
+}
+
+
+/* {{{1
+ * Retrieve the mirrored representation of ‘c’ (if any) and store it in
+ * ‘mirrored’.
+ */
+bool
+unichar_mirror(unichar c, unichar *mirrored)
+{
+        int index;
+
+        if (!unicode_table_lookup(bidi_mirroring_table, c, &index))
+                return false;
+
+        if (mirrored != NULL)
+                *mirrored = bidi_mirroring_table[index].mirrored_ch;
+
+        return true;
+}
+
+
+/* }}}1 */