bibtex_to_scrapbox 0.1.0

data/gems/gems/unicode-0.4.4.4/ext/unicode/wstring.h

@@ -0,0 +1,43 @@
+/*
+ * Simple wide string library
+ * Version 0.1
+ * 1999 by yoshidam
+ */
+
+#ifndef _WSTRING_H
+#define _WSTRING_H
+
+#include "ustring.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define WSTR_INITIAL_STRING_LEN 1024
+#define WSTR_STRING_EXTEND_LEN 1024
+
+typedef struct _WString {
+  int* str;
+  int len;
+  int size;
+} WString;
+
+WString* WStr_alloc(WString* str);
+WString* WStr_allocWithUTF8(WString* s, const char* u);
+WString* WStr_allocWithUTF8L(WString* s, const char* u, int len);
+WString* WStr_enlarge(WString* str, int size);
+void WStr_free(WString* str);
+int WStr_addWChars(WString* s, const int* a, int len);
+int WStr_addWChar(WString* s, int a);
+int WStr_pushWString(WString* s, const WString* add);
+int WStr_addWChar2(WString* s, int a1, int a2);
+int WStr_addWChar3(WString* s, int a1, int a2, int a3);
+UString* WStr_convertIntoUString(WString* wstr, UString* ustr);
+UString* WStr_convertIntoUString2(WString* wstr, int start, int len, UString* ustr);
+void WStr_dump(WString* s);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

data/gems/gems/unicode-0.4.4.4/lib/unicode.rb

@@ -0,0 +1,6 @@
+begin
+  require "unicode/#{RUBY_VERSION[/\d+.\d+/]}/unicode_native"
+rescue LoadError => err
+  raise if err.respond_to?(:path) && !err.path
+  require 'unicode/unicode_native'
+end

data/gems/gems/unicode-0.4.4.4/test/test.rb

@@ -0,0 +1,69 @@
+#! /usr/local/bin/ruby -KU
+# -*- coding: utf-8 -*-
+
+require 'unicode'
+
+## dump Unicode string
+class String
+  def udump
+    ustr = self.unpack("U*")
+    ret = []
+    ustr.each do |e|
+      if e.is_a?(Integer)
+        ret << "U+%04X" % e
+      else
+        ret << e
+      end
+    end
+    ret
+  end
+end
+
+
+print "Canonical decomposition vs compatibility decomposition\n"
+p Unicode::decompose("⑽ o\xef\xac\x83ce").udump
+p Unicode::decompose_compat("⑽ o\xef\xac\x83ce")
+
+print "Canonical equivalent vs Compatibility equivalent\n"
+p Unicode::strcmp("ｶﾞ", "ガ")
+p Unicode::strcmp("ガ", "ｶﾞ")
+p Unicode::strcmp_compat("ｶﾞ", "ガ")
+
+print "Decomposition/composition\n"
+p Unicode::normalize_D([0x63, 0x301, 0x327].pack("U*")).udump
+p Unicode::normalize_D([0x63, 0x327, 0x301].pack("U*")).udump
+p Unicode::normalize_D([0x107, 0x327].pack("U*")).udump
+p Unicode::normalize_D([0xe7, 0x301].pack("U*")).udump
+p Unicode::normalize_C([0x63, 0x301, 0x327].pack("U*")).udump
+p Unicode::normalize_C([0x63, 0x327, 0x301].pack("U*")).udump
+p Unicode::normalize_C([0x107, 0x327].pack("U*")).udump
+p Unicode::normalize_C([0xe7, 0x301].pack("U*")).udump
+
+print "Kana Normalization\n"
+p Unicode::normalize_D("ｶﾞガ").udump
+p Unicode::normalize_C("ｶﾞガ").udump
+p Unicode::normalize_KD("ｶﾞガ").udump
+p Unicode::normalize_KC("ｶﾞガ").udump
+
+print "Hangul\n"
+p "요시담".udump
+p Unicode::normalize_D("요시담").udump
+p Unicode::normalize_C("요시담").udump
+
+print "Composition Exclusion\n"
+print "   ANGSTROM SIGN [U+212B]\n"
+p Unicode::normalize_D([0x212b].pack("U")).udump
+p Unicode::normalize_C([0x212b].pack("U")).udump
+print "   LATIN CAPITAL LETTER A WITH RING ABOVE [U+00C5]\n"
+p Unicode::normalize_D([0x00c5].pack("U")).udump
+p Unicode::normalize_C([0x00c5].pack("U")).udump
+
+print "Case conversion\n"
+p Unicode::normalize_C(Unicode::upcase([0x63, 0x301, 0x327, 0xff41].pack("U*"))).udump
+p Unicode::normalize_C(Unicode::downcase([0x43, 0x301, 0x327, 0xff21].pack("U*"))).udump
+p Unicode::capitalize([0x1f1, 0x41, 0x61, 0xff21].pack("U*")).udump
+
+
+## Local variables:
+## coding: utf-8
+## End:

data/gems/gems/unicode-0.4.4.4/tools/README

@@ -0,0 +1,7 @@
+The bundled unidata.map is created from UnicodeData.txt,
+DerivedNormalizationProps.txt, SpecialCasing.txt and EastAsianWidth.txt
+of Unicode 6.0.
+
+To update unidata.map,
+
+  ruby mkunidata.rb UnicodeData.txt DerivedNormalizationProps.txt SpecialCasing.txt EastAsianWidth.txt  > unidata.map

data/gems/gems/unicode-0.4.4.4/tools/mkunidata.rb

@@ -0,0 +1,293 @@
+#! /usr/local/bin/ruby -KU
+
+#if $KCODE != 'UTF8'
+#  raise "$KCODE must be UTF8"
+#end
+
+HEAD=<<EOS
+/*
+ * UnicodeData
+ * Copyright 1999, 2004, 2010, 2012 by yoshidam
+ *
+ */
+
+#ifndef _UNIDATA_MAP
+#define _UNIDATA_MAP
+
+EOS
+
+HEAD1=<<EOS
+
+enum GeneralCategory {
+  /* Letter */
+  c_Lu = 1, c_Ll, c_Lt, c_LC, c_Lm, c_Lo,
+  /* Mark */
+  c_Mn, c_Mc, c_Me,
+  /* Number */
+  c_Nd, c_Nl, c_No,
+  /* Punctuation */
+  c_Pc, c_Pd, c_Ps, c_Pe, c_Pi, c_Pf, c_Po,
+  /* Symbol */
+  c_Sm, c_Sc, c_Sk, c_So,
+  /* Separator */
+  c_Zs, c_Zl, c_Zp,
+  /* Other */
+  c_Cc, c_Cf, c_Cs, c_Co, c_Cn
+};
+
+const char* const gencat_abbr[] = {
+  "", /* 0 */
+  /* Letter */
+  "Lu", "Ll", "Lt", "LC", "Lm", "Lo",
+  /* Mark */
+  "Mn", "Mc", "Me",
+  /* Number */
+  "Nd", "Nl", "No",
+  /* Punctuation */
+  "Pc", "Pd", "Ps", "Pe", "Pi", "Pf", "Po",
+  /* Symbol */
+  "Sm", "Sc", "Sk", "So",
+  /* Separator */
+  "Zs", "Zl", "Zp",
+  /* Other */
+  "Cc", "Cf", "Cs", "Co", "Cn"
+};
+
+const char* const gencat_long[] = {
+  "",
+  "Uppercase_Letter",
+  "Lowercase_Letter",
+  "Titlecase_Letter",
+  "Cased_Letter",
+  "Modifier_Letter",
+  "Other_Letter",
+  "Nonspacing_Mark",
+  "Spacing_Mark",
+  "Enclosing_Mark",
+  "Decimal_Number",
+  "Letter_Number",
+  "Other_Number",
+  "Connector_Punctuation",
+  "Dash_Punctuation",
+  "Open_Punctuation",
+  "Close_Punctuation",
+  "Initial_Punctuation",
+  "Final_Punctuation",
+  "Other_Punctuation",
+  "Math_Symbol",
+  "Currency_Symbol",
+  "Modifier_Symbol",
+  "Other_Symbol",
+  "Space_Separator",
+  "Line_Separator",
+  "Paragraph_Separator",
+  "Control",
+  "Format",
+  "Surrogate",
+  "Private_Use",
+  "Unassigned"
+};
+
+enum EastAsianWidth {
+  w_N = 1, w_A, w_H, w_W, w_F, w_Na
+};
+
+struct unicode_data {
+  const int code;
+  const char* const canon;
+  const char* const compat;
+  const char* const uppercase;
+  const char* const lowercase;
+  const char* const titlecase;
+  const unsigned char combining_class;
+  const unsigned char exclusion;
+  const unsigned char general_category;
+  const unsigned char east_asian_width;
+};
+
+static const struct unicode_data unidata[] = {
+EOS
+
+TAIL=<<EOS
+};
+
+#endif
+EOS
+
+def hex2str(hex)
+  if hex.nil? || hex == ''
+    return [nil, nil]
+  end
+  canon = ""
+  compat = ""
+  chars = hex.split(" ")
+  if chars[0] =~ /^[0-9A-F]{4,6}$/
+    chars.each do |c|
+      canon << [c.hex].pack("U")
+    end
+    compat = canon
+  elsif chars[0] =~ /^<.+>$/
+    chars.shift
+    chars.each do |c|
+      compat << [c.hex].pack("U")
+    end
+    canon = nil
+  else
+    raise "unknown value: " + hex
+  end
+  [canon, compat]
+end
+
+def hex_or_nil(str)
+  return nil if str.nil? || str == ''
+  ret = ""
+  chars = str.split(" ")
+  chars.each do |c|
+    ret << [c.hex].pack("U")
+  end
+  return ret
+end
+
+def printstr(str)
+  return "NULL" if !str
+  ret = ""
+  str.each_byte do |c|
+    if c >= 32 && c < 127 && c != 34 && c != 92
+      ret << c
+    else
+      ret << format("\\%03o", c)
+    end
+  end
+  return '"' + ret + '"'
+end
+
+if ARGV.length != 4
+  puts "Usage: #{$0} <UnicodeData.txt> <DerivedNormalizationProps.txt> <SpecialCasing.txt> <EastAsianWidth.txt>"
+  exit 0
+end
+
+## scan Composition Exclusions
+exclusion = {}
+open(ARGV[1]) do |f|
+  while l = f.gets
+    next if l =~ /^\#/ || l =~ /^$/
+    next if l !~ /Full_Composition_Exclusion/
+    code, = l.split(/\s/)
+    if code =~ /^[0-9A-F]+$/
+      code = code.hex
+      exclusion[code] = true
+    elsif code =~ /^([0-9A-F]+)\.\.([0-9A-F]+)$/
+#      p [$1, $2]
+      scode = $1.hex
+      ecode = $2.hex
+      for code in scode..ecode
+        exclusion[code] = true
+      end
+    end
+  end
+end
+
+## scan Special Casing
+casing = {}
+open(ARGV[2]) do |f|
+  while l = f.gets
+    l.chomp!
+    next if l =~ /^\#/ || l =~ /^$/
+    l =~ /^(.*)#\s*(.*)$/
+    l = $1
+    comment = $2
+    code,lower,title,upper,cond = l.split(/;\s/)
+    next if cond
+    lower = nil if code == lower
+    title = nil if code == title
+    upper = nil if code == upper
+    code = code.hex
+    casing[code] = [hex_or_nil(lower), hex_or_nil(title), hex_or_nil(upper)]
+  end
+end
+
+## scan UnicodeData
+udata = {}
+range_data = []
+open(ARGV[0]) do |f|
+  while l = f.gets
+    l.chomp!
+    code, charname, gencat, ccclass, bidicat,decomp,
+      dec, digit, num, mirror, uni1_0, comment, upcase,
+      lowcase, titlecase = l.split(";", 15);
+    code = code.hex
+    ccclass = ccclass.to_i
+    canon, compat = hex2str(decomp)
+    upcase = hex_or_nil(upcase)
+    lowcase = hex_or_nil(lowcase)
+    titlecase = hex_or_nil(titlecase)
+    udata[code] = [ccclass, canon, compat, upcase, lowcase, titlecase, gencat]
+    if charname =~ /^<(.*, (First|Last))>$/
+      charname = $1.upcase.gsub(/,? /, '_')
+      range_data << [charname, code]
+    end
+  end
+end
+
+## scan EastAsianWidth
+ea_width = {}
+open(ARGV[3]) do |f|
+  while l = f.gets
+    l.chomp!
+    next if l =~ /^\#/ || l =~ /^$/
+    l =~ /^(.*)\s+#\s*(.*)$/
+    l = $1
+    comment = $2
+    code,width = l.split(/;/)
+    if code =~ /\.\./
+      start_code, end_code = code.split('..')
+      start_code = start_code.hex
+      end_code = end_code.hex
+      (start_code..end_code).each do |code|
+        ea_width[code] = width
+      end
+      next
+    end
+    code = code.hex
+    ea_width[code] = width
+  end
+end
+
+print HEAD
+range_data.each do |charname, code|
+  printf("#define %s\t(0x%04x)\n", charname, code)
+end
+
+print HEAD1
+udata.sort.each do |code, data|
+  ccclass, canon, compat, upcase, lowcase, titlecase, gencat = data
+  ## Exclusions
+  ex = 0
+  if exclusion[code]  ## Script-specifics or Post Composition Version
+    ex = 1
+  elsif canon =~ /^.$/ ## Singltons
+    ex = 2
+  elsif !canon.nil?
+    starter = canon.unpack("U*")[0]
+    if udata[starter][0] != 0 ## Non-stater decompositions
+      ex = 3
+    end
+  end
+  ## Special Casing
+  if casing[code]
+    lowcase = casing[code][0] if casing[code][0]
+    titlecase = casing[code][1] if casing[code][1]
+    upcase = casing[code][2] if casing[code][2]
+  end
+  width = 'N'
+  if ea_width[code]
+    width = ea_width[code]
+  end
+
+  printf("  { 0x%04x, %s, %s, %s, %s, %s, %d, %d, c_%s, w_%s }, \n",
+         code, printstr(canon),
+         printstr(compat), printstr(upcase), printstr(lowcase),
+         printstr(titlecase), ccclass, ex, gencat, width)
+end
+printf("  { -1, NULL, NULL, NULL, NULL, NULL, 0, 0, 0, 0 }\n")
+print TAIL

data/gems/gems/unicode-0.4.4.4/tools/normtest.rb

@@ -0,0 +1,111 @@
+#! /usr/local/bin/ruby -KU
+
+## Conformance test with NormaliztionTest.txt
+## Copyrigth 2010 yoshidam
+
+require 'unicode'
+
+TESTFILE = "NormalizationTest.txt"
+
+def from_hex(str)
+  ret = ""
+  chars = str.split(" ")
+  chars.each do |c|
+    ret << [c.hex].pack("U")
+  end
+  return ret
+end
+
+def to_hex(str)
+  ret = ""
+  str = str.unpack('U*')
+  str.each do |c|
+    ret += sprintf("%04X ", c)
+  end
+  ret
+end
+
+open(TESTFILE) do |f|
+  while l = f.gets
+    next if l =~ /^#/
+    l.chomp
+    if l =~ /^@/
+      puts l
+      next
+    end
+    c1, c2, c3, c4, c5 = l.split(';')
+    code = c1
+    c1 = from_hex(c1)
+    c2 = from_hex(c2)
+    c3 = from_hex(c3)
+    c4 = from_hex(c4)
+    c5 = from_hex(c5)
+    ## NFC TEST
+    if c2 == Unicode.nfc(c1) && c2 == Unicode.nfc(c2) &&
+        c2 == Unicode.nfc(c3) &&
+        c4 == Unicode.nfc(c4) && c4 == Unicode.nfc(c4)
+      ##puts "NFC OK: " + code
+    else
+      puts "NFC NG: " + to_hex(c1)
+      printf("  c2=%s NFC(c1)=%s NFC(c2)=%s NFC(c3)=%s\n",
+             to_hex(c2),
+             to_hex(Unicode.nfc(c1)),
+             to_hex(Unicode.nfc(c2)),
+             to_hex(Unicode.nfc(c3)))
+      printf("  c4=%s NFC(c4)=%s NFC(c5)=%s\n",
+             to_hex(c4),
+             to_hex(Unicode.nfc(c4)),
+             to_hex(Unicode.nfc(c5)))
+    end
+
+    ## NFD TEST
+    if c3 == Unicode.nfd(c1) && c3 == Unicode.nfd(c2) &&
+        c3 == Unicode.nfd(c3) &&
+        c5 == Unicode.nfd(c4) && c5 == Unicode.nfd(c5)
+      ##puts "NFD OK: " + code
+    else
+      puts "NFD NG: " + to_hex(c1)
+      printf("  c3=%s NFD(c1)=%s NFD(c2)=%s NFD(c3)=%s\n",
+             to_hex(c3),
+             to_hex(Unicode.nfd(c1)),
+             to_hex(Unicode.nfd(c2)),
+             to_hex(Unicode.nfd(c3)))
+      printf("  c5=%s NFD(c4)=%s NFD(c5)=%s\n",
+             to_hex(c5),
+             to_hex(Unicode.nfd(c4)),
+             to_hex(Unicode.nfd(c5)))
+    end
+
+    ## NFKC TEST
+    if c4 == Unicode.nfkc(c1) && c4 == Unicode.nfkc(c2) &&
+        c4 == Unicode.nfkc(c3) &&
+        c4 == Unicode.nfkc(c4) && c4 == Unicode.nfkc(c5)
+      ##puts "NFKC OK: " + code
+    else
+      puts "NFKC NG: " + to_hex(c1)
+      printf("  c4=%s NFKC(c1)=%s NFKC(c2)=%s NFKC(c3)=%s NFKC(c4)=%s NFKC(c5)=%s\n",
+             to_hex(c4),
+             to_hex(Unicode.nfkc(c1)),
+             to_hex(Unicode.nfkc(c2)),
+             to_hex(Unicode.nfkc(c3)),
+             to_hex(Unicode.nfkc(c4)),
+             to_hex(Unicode.nfkc(c5)))
+    end
+
+    ## NFKD TEST
+    if c5 == Unicode.nfkd(c1) && c5 == Unicode.nfkd(c2) &&
+        c5 == Unicode.nfkd(c3) &&
+        c5 == Unicode.nfkd(c4) && c5 == Unicode.nfkd(c5)
+      ##puts "NFKD OK: " + code
+    else
+      puts "NFKD NG: " + to_hex(c1)
+      printf("  c5=%s NFKD(c1)=%s NFKD(c2)=%s NFKD(c3)=%s NFKD(c4)=%s NFKD(c5)=%s\n",
+             to_hex(c5),
+             to_hex(Unicode.nfkd(c1)),
+             to_hex(Unicode.nfkd(c2)),
+             to_hex(Unicode.nfkd(c3)),
+             to_hex(Unicode.nfkd(c4)),
+             to_hex(Unicode.nfkd(c5)))
+    end
+  end
+end