tiny_segmenter 0.0.1

data/lib/tiny_segmenter.rb

@@ -0,0 +1,136 @@
+#encoding: utf-8
+
+class TinySegmenter
+  def initialize
+    @chartype = []
+    @BIAS = -332
+    set_score_hashes
+    # Compile regex patterns
+    {
+      "[一二三四五六七八九十百千万億兆]" => "M", # numbers (japanese)
+      "[一-龠々〆ヵヶ]" => "H", # kanji & misc characters
+      "[ぁ-ん]" => "I", # hiragana
+      "[ァ-ヴーｱ-ﾝﾞｰ]" => "K", # katakana
+      "[a-zA-Zａ-ｚＡ-Ｚ]" => "A", # ascii / romaji letters
+      "[0-9０-９]" => "N", # ascii / romaji numbers
+    }.each do |pattern, value|
+      @chartype << [Regexp.compile(pattern), value]
+    end
+  end
+
+  def segment(text)
+    return []  if text.nil? || text.strip.empty?
+    text = text.strip
+    result = []
+    segments = %w[B3 B2 B1]
+    ctypes = %w[O O O]
+    text.split(//).each do |char|
+      char.strip!
+      next  if char.empty?
+      segments << char
+      ctypes << ctype(char)
+    end
+    segments.concat(%w[E1 E2 E3])
+    ctypes.concat(%w[O O O])
+
+    word = segments[3]
+    p1, p2, p3 = %w[U U U]
+    (4..segments.size-4).to_a.each do |i|
+      score = @BIAS
+      words = []
+      chars = []
+      (-3..2).to_a.each do |idx|
+        words << segments[i + idx]
+        chars << ctypes[i + idx]
+      end
+
+      score += sum_scores(p1, p2, p3, *words, *chars)
+      p_new = "O"
+      if score > 0
+        result << word
+        word = ""
+        p_new = "B"
+      end
+      p1, p2, p3 = p2, p3, p_new
+      word = "#{word}#{segments[i]}"
+    end
+    result << word  unless word.empty?
+    result
+  end
+
+  private
+
+  def ctype(str)
+    @chartype.each do |pattern, value|
+      return value  if str.match(pattern)
+    end
+    "O"
+  end
+
+  def sum_scores(p1, p2, p3, w1, w2, w3, w4, w5, w6, c1, c2, c3, c4, c5, c6)
+    score = 0
+    [
+      [@UP1, p1], [@UP2, p2], [@UP3, p3],
+      [@BP1, p1, p2], [@BP2, p2, p3],
+      [@UW1, w1], [@UW2, w2], [@UW3, w3], [@UW4, w4], [@UW5, w5], [@UW6, w6],
+      [@BW1, w2, w3], [@BW2, w3, w4], [@BW3, w4, w5],
+      [@TW1, w1, w2, w3], [@TW2, w2, w3, w4], [@TW3, w3, w4, w5], [@TW4, w4, w5, w6],
+      [@UC1, c1], [@UC2, c2], [@UC3, c3], [@UC4, c4], [@UC5, c5], [@UC6, c6],
+      [@BC1, c2, c3], [@BC2, c3, c4], [@BC3, c4, c5],
+      [@TC1, c1, c2, c3], [@TC2, c2, c3, c4], [@TC3, c3, c4, c5], [@TC4, c4, c5, c6],
+      [@UQ1, p1, c1], [@UQ2, p2, c2], [@UQ3, p3, c3],
+      [@BQ1, p2, c2, c3], [@BQ2, p2, c3, c4], [@BQ3, p3, c2, c3], [@BQ4, p3, c3, c4],
+      [@TQ1, p2, c1, c2, c3], [@TQ2, p2, c2, c3, c4], [@TQ3, p3, c1, c2, c3], [@TQ4, p3, c2, c3, c4],
+    ].each do |score_hash_and_key|
+      score_hash = score_hash_and_key[0]
+      key = score_hash_and_key[1..-1].join("")
+      score += score_hash[key] || 0
+    end
+    score
+  end
+
+  def set_score_hashes
+    @BC1 = {"HH" => 6,"II" => 2461,"KH" => 406,"OH" => -1378}
+    @BC2 = {"AA" => -3267,"AI" => 2744,"AN" => -878,"HH" => -4070,"HM" => -1711,"HN" => 4012,"HO" => 3761,"IA" => 1327,"IH" => -1184,"II" => -1332,"IK" => 1721,"IO" => 5492,"KI" => 3831,"KK" => -8741,"MH" => -3132,"MK" => 3334,"OO" => -2920}
+    @BC3 = {"HH" => 996,"HI" => 626,"HK" => -721,"HN" => -1307,"HO" => -836,"IH" => -301,"KK" => 2762,"MK" => 1079,"MM" => 4034,"OA" => -1652,"OH" => 266}
+    @BP1 = {"BB" => 295,"OB" => 304,"OO" => -125,"UB" => 352}
+    @BP2 = {"BO" => 60,"OO" => -1762}
+    @BQ1 = {"BHH" => 1150,"BHM" => 1521,"BII" => -1158,"BIM" => 886,"BMH" => 1208,"BNH" => 449,"BOH" => -91,"BOO" => -2597,"OHI" => 451,"OIH" => -296,"OKA" => 1851,"OKH" => -1020,"OKK" => 904,"OOO" => 2965}
+    @BQ2 = {"BHH" => 118,"BHI" => -1159,"BHM" => 466,"BIH" => -919,"BKK" => -1720,"BKO" => 864,"OHH" => -1139,"OHM" => -181,"OIH" => 153,"UHI" => -1146}
+    @BQ3 = {"BHH" => -792,"BHI" => 2664,"BII" => -299,"BKI" => 419,"BMH" => 937,"BMM" => 8335,"BNN" => 998,"BOH" => 775,"OHH" => 2174,"OHM" => 439,"OII" => 280,"OKH" => 1798,"OKI" => -793,"OKO" => -2242,"OMH" => -2402,"OOO" => 11699}
+    @BQ4 = {"BHH" => -3895,"BIH" => 3761,"BII" => -4654,"BIK" => 1348,"BKK" => -1806,"BMI" => -3385,"BOO" => -12396,"OAH" => 926,"OHH" => 266,"OHK" => -2036,"ONN" => -973}
+    @BW1 = {",と" => 660,",同" => 727,"B1あ" => 1404,"B1同" => 542,"、と" => 660,"、同" => 727,"」と" => 1682,"あっ" => 1505,"いう" => 1743,"いっ" => -2055,"いる" => 672,"うし" => -4817,"うん" => 665,"から" => 3472,"がら" => 600,"こう" => -790,"こと" => 2083,"こん" => -1262,"さら" => -4143,"さん" => 4573,"した" => 2641,"して" => 1104,"すで" => -3399,"そこ" => 1977,"それ" => -871,"たち" => 1122,"ため" => 601,"った" => 3463,"つい" => -802,"てい" => 805,"てき" => 1249,"でき" => 1127,"です" => 3445,"では" => 844,"とい" => -4915,"とみ" => 1922,"どこ" => 3887,"ない" => 5713,"なっ" => 3015,"など" => 7379,"なん" => -1113,"にし" => 2468,"には" => 1498,"にも" => 1671,"に対" => -912,"の一" => -501,"の中" => 741,"ませ" => 2448,"まで" => 1711,"まま" => 2600,"まる" => -2155,"やむ" => -1947,"よっ" => -2565,"れた" => 2369,"れで" => -913,"をし" => 1860,"を見" => 731,"亡く" => -1886,"京都" => 2558,"取り" => -2784,"大き" => -2604,"大阪" => 1497,"平方" => -2314,"引き" => -1336,"日本" => -195,"本当" => -2423,"毎日" => -2113,"目指" => -724,"Ｂ１あ" => 1404,"Ｂ１同" => 542,"｣と" => 1682}
+    @BW2 = {".." => -11822,"11" => -669,"――" => -5730,"−−" => -13175,"いう" => -1609,"うか" => 2490,"かし" => -1350,"かも" => -602,"から" => -7194,"かれ" => 4612,"がい" => 853,"がら" => -3198,"きた" => 1941,"くな" => -1597,"こと" => -8392,"この" => -4193,"させ" => 4533,"され" => 13168,"さん" => -3977,"しい" => -1819,"しか" => -545,"した" => 5078,"して" => 972,"しな" => 939,"その" => -3744,"たい" => -1253,"たた" => -662,"ただ" => -3857,"たち" => -786,"たと" => 1224,"たは" => -939,"った" => 4589,"って" => 1647,"っと" => -2094,"てい" => 6144,"てき" => 3640,"てく" => 2551,"ては" => -3110,"ても" => -3065,"でい" => 2666,"でき" => -1528,"でし" => -3828,"です" => -4761,"でも" => -4203,"とい" => 1890,"とこ" => -1746,"とと" => -2279,"との" => 720,"とみ" => 5168,"とも" => -3941,"ない" => -2488,"なが" => -1313,"など" => -6509,"なの" => 2614,"なん" => 3099,"にお" => -1615,"にし" => 2748,"にな" => 2454,"によ" => -7236,"に対" => -14943,"に従" => -4688,"に関" => -11388,"のか" => 2093,"ので" => -7059,"のに" => -6041,"のの" => -6125,"はい" => 1073,"はが" => -1033,"はず" => -2532,"ばれ" => 1813,"まし" => -1316,"まで" => -6621,"まれ" => 5409,"めて" => -3153,"もい" => 2230,"もの" => -10713,"らか" => -944,"らし" => -1611,"らに" => -1897,"りし" => 651,"りま" => 1620,"れた" => 4270,"れて" => 849,"れば" => 4114,"ろう" => 6067,"われ" => 7901,"を通" => -11877,"んだ" => 728,"んな" => -4115,"一人" => 602,"一方" => -1375,"一日" => 970,"一部" => -1051,"上が" => -4479,"会社" => -1116,"出て" => 2163,"分の" => -7758,"同党" => 970,"同日" => -913,"大阪" => -2471,"委員" => -1250,"少な" => -1050,"年度" => -8669,"年間" => -1626,"府県" => -2363,"手権" => -1982,"新聞" => -4066,"日新" => -722,"日本" => -7068,"日米" => 3372,"曜日" => -601,"朝鮮" => -2355,"本人" => -2697,"東京" => -1543,"然と" => -1384,"社会" => -1276,"立て" => -990,"第に" => -1612,"米国" => -4268,"１１" => -669}
+    @BW3 = {"あた" => -2194,"あり" => 719,"ある" => 3846,"い." => -1185,"い。" => -1185,"いい" => 5308,"いえ" => 2079,"いく" => 3029,"いた" => 2056,"いっ" => 1883,"いる" => 5600,"いわ" => 1527,"うち" => 1117,"うと" => 4798,"えと" => 1454,"か." => 2857,"か。" => 2857,"かけ" => -743,"かっ" => -4098,"かに" => -669,"から" => 6520,"かり" => -2670,"が," => 1816,"が、" => 1816,"がき" => -4855,"がけ" => -1127,"がっ" => -913,"がら" => -4977,"がり" => -2064,"きた" => 1645,"けど" => 1374,"こと" => 7397,"この" => 1542,"ころ" => -2757,"さい" => -714,"さを" => 976,"し," => 1557,"し、" => 1557,"しい" => -3714,"した" => 3562,"して" => 1449,"しな" => 2608,"しま" => 1200,"す." => -1310,"す。" => -1310,"する" => 6521,"ず," => 3426,"ず、" => 3426,"ずに" => 841,"そう" => 428,"た." => 8875,"た。" => 8875,"たい" => -594,"たの" => 812,"たり" => -1183,"たる" => -853,"だ." => 4098,"だ。" => 4098,"だっ" => 1004,"った" => -4748,"って" => 300,"てい" => 6240,"てお" => 855,"ても" => 302,"です" => 1437,"でに" => -1482,"では" => 2295,"とう" => -1387,"とし" => 2266,"との" => 541,"とも" => -3543,"どう" => 4664,"ない" => 1796,"なく" => -903,"など" => 2135,"に," => -1021,"に、" => -1021,"にし" => 1771,"にな" => 1906,"には" => 2644,"の," => -724,"の、" => -724,"の子" => -1000,"は," => 1337,"は、" => 1337,"べき" => 2181,"まし" => 1113,"ます" => 6943,"まっ" => -1549,"まで" => 6154,"まれ" => -793,"らし" => 1479,"られ" => 6820,"るる" => 3818,"れ," => 854,"れ、" => 854,"れた" => 1850,"れて" => 1375,"れば" => -3246,"れる" => 1091,"われ" => -605,"んだ" => 606,"んで" => 798,"カ月" => 990,"会議" => 860,"入り" => 1232,"大会" => 2217,"始め" => 1681,"市" => 965,"新聞" => -5055,"日," => 974,"日、" => 974,"社会" => 2024,"ｶ月" => 990}
+    @TC1 = {"AAA" => 1093,"HHH" => 1029,"HHM" => 580,"HII" => 998,"HOH" => -390,"HOM" => -331,"IHI" => 1169,"IOH" => -142,"IOI" => -1015,"IOM" => 467,"MMH" => 187,"OOI" => -1832}
+    @TC2 = {"HHO" => 2088,"HII" => -1023,"HMM" => -1154,"IHI" => -1965,"KKH" => 703,"OII" => -2649}
+    @TC3 = {"AAA" => -294,"HHH" => 346,"HHI" => -341,"HII" => -1088,"HIK" => 731,"HOH" => -1486,"IHH" => 128,"IHI" => -3041,"IHO" => -1935,"IIH" => -825,"IIM" => -1035,"IOI" => -542,"KHH" => -1216,"KKA" => 491,"KKH" => -1217,"KOK" => -1009,"MHH" => -2694,"MHM" => -457,"MHO" => 123,"MMH" => -471,"NNH" => -1689,"NNO" => 662,"OHO" => -3393}
+    @TC4 = {"HHH" => -203,"HHI" => 1344,"HHK" => 365,"HHM" => -122,"HHN" => 182,"HHO" => 669,"HIH" => 804,"HII" => 679,"HOH" => 446,"IHH" => 695,"IHO" => -2324,"IIH" => 321,"III" => 1497,"IIO" => 656,"IOO" => 54,"KAK" => 4845,"KKA" => 3386,"KKK" => 3065,"MHH" => -405,"MHI" => 201,"MMH" => -241,"MMM" => 661,"MOM" => 841}
+    @TQ1 = {"BHHH" => -227,"BHHI" => 316,"BHIH" => -132,"BIHH" => 60,"BIII" => 1595,"BNHH" => -744,"BOHH" => 225,"BOOO" => -908,"OAKK" => 482,"OHHH" => 281,"OHIH" => 249,"OIHI" => 200,"OIIH" => -68}
+    @TQ2 = {"BIHH" => -1401,"BIII" => -1033,"BKAK" => -543,"BOOO" => -5591}
+    @TQ3 = {"BHHH" => 478,"BHHM" => -1073,"BHIH" => 222,"BHII" => -504,"BIIH" => -116,"BIII" => -105,"BMHI" => -863,"BMHM" => -464,"BOMH" => 620,"OHHH" => 346,"OHHI" => 1729,"OHII" => 997,"OHMH" => 481,"OIHH" => 623,"OIIH" => 1344,"OKAK" => 2792,"OKHH" => 587,"OKKA" => 679,"OOHH" => 110,"OOII" => -685}
+    @TQ4 = {"BHHH" => -721,"BHHM" => -3604,"BHII" => -966,"BIIH" => -607,"BIII" => -2181,"OAAA" => -2763,"OAKK" => 180,"OHHH" => -294,"OHHI" => 2446,"OHHO" => 480,"OHIH" => -1573,"OIHH" => 1935,"OIHI" => -493,"OIIH" => 626,"OIII" => -4007,"OKAK" => -8156}
+    @TW1 = {"につい" => -4681,"東京都" => 2026}
+    @TW2 = {"ある程" => -2049,"いった" => -1256,"ころが" => -2434,"しょう" => 3873,"その後" => -4430,"だって" => -1049,"ていた" => 1833,"として" => -4657,"ともに" => -4517,"もので" => 1882,"一気に" => -792,"初めて" => -1512,"同時に" => -8097,"大きな" => -1255,"対して" => -2721,"社会党" => -3216}
+    @TW3 = {"いただ" => -1734,"してい" => 1314,"として" => -4314,"につい" => -5483,"にとっ" => -5989,"に当た" => -6247,"ので," => -727,"ので、" => -727,"のもの" => -600,"れから" => -3752,"十二月" => -2287}
+    @TW4 = {"いう." => 8576,"いう。" => 8576,"からな" => -2348,"してい" => 2958,"たが," => 1516,"たが、" => 1516,"ている" => 1538,"という" => 1349,"ました" => 5543,"ません" => 1097,"ようと" => -4258,"よると" => 5865}
+    @UC1 = {"A" => 484,"K" => 93,"M" => 645,"O" => -505}
+    @UC2 = {"A" => 819,"H" => 1059,"I" => 409,"M" => 3987,"N" => 5775,"O" => 646}
+    @UC3 = {"A" => -1370,"I" => 2311}
+    @UC4 = {"A" => -2643,"H" => 1809,"I" => -1032,"K" => -3450,"M" => 3565,"N" => 3876,"O" => 6646}
+    @UC5 = {"H" => 313,"I" => -1238,"K" => -799,"M" => 539,"O" => -831}
+    @UC6 = {"H" => -506,"I" => -253,"K" => 87,"M" => 247,"O" => -387}
+    @UP1 = {"O" => -214}
+    @UP2 = {"B" => 69,"O" => 935}
+    @UP3 = {"B" => 189}
+    @UQ1 = {"BH" => 21,"BI" => -12,"BK" => -99,"BN" => 142,"BO" => -56,"OH" => -95,"OI" => 477,"OK" => 410,"OO" => -2422}
+    @UQ2 = {"BH" => 216,"BI" => 113,"OK" => 1759}
+    @UQ3 = {"BA" => -479,"BH" => 42,"BI" => 1913,"BK" => -7198,"BM" => 3160,"BN" => 6427,"BO" => 14761,"OI" => -827,"ON" => -3212}
+    @UW1 = {"," => 156,"、" => 156,"「" => -463,"あ" => -941,"う" => -127,"が" => -553,"き" => 121,"こ" => 505,"で" => -201,"と" => -547,"ど" => -123,"に" => -789,"の" => -185,"は" => -847,"も" => -466,"や" => -470,"よ" => 182,"ら" => -292,"り" => 208,"れ" => 169,"を" => -446,"ん" => -137,"・" => -135,"主" => -402,"京" => -268,"区" => -912,"午" => 871,"国" => -460,"大" => 561,"委" => 729,"市" => -411,"日" => -141,"理" => 361,"生" => -408,"県" => -386,"都" => -718,"｢" => -463,"･" => -135}
+    @UW2 = {"," => -829,"、" => -829,"〇" => 892,"「" => -645,"」" => 3145,"あ" => -538,"い" => 505,"う" => 134,"お" => -502,"か" => 1454,"が" => -856,"く" => -412,"こ" => 1141,"さ" => 878,"ざ" => 540,"し" => 1529,"す" => -675,"せ" => 300,"そ" => -1011,"た" => 188,"だ" => 1837,"つ" => -949,"て" => -291,"で" => -268,"と" => -981,"ど" => 1273,"な" => 1063,"に" => -1764,"の" => 130,"は" => -409,"ひ" => -1273,"べ" => 1261,"ま" => 600,"も" => -1263,"や" => -402,"よ" => 1639,"り" => -579,"る" => -694,"れ" => 571,"を" => -2516,"ん" => 2095,"ア" => -587,"カ" => 306,"キ" => 568,"ッ" => 831,"三" => -758,"不" => -2150,"世" => -302,"中" => -968,"主" => -861,"事" => 492,"人" => -123,"会" => 978,"保" => 362,"入" => 548,"初" => -3025,"副" => -1566,"北" => -3414,"区" => -422,"大" => -1769,"天" => -865,"太" => -483,"子" => -1519,"学" => 760,"実" => 1023,"小" => -2009,"市" => -813,"年" => -1060,"強" => 1067,"手" => -1519,"揺" => -1033,"政" => 1522,"文" => -1355,"新" => -1682,"日" => -1815,"明" => -1462,"最" => -630,"朝" => -1843,"本" => -1650,"東" => -931,"果" => -665,"次" => -2378,"民" => -180,"気" => -1740,"理" => 752,"発" => 529,"目" => -1584,"相" => -242,"県" => -1165,"立" => -763,"第" => 810,"米" => 509,"自" => -1353,"行" => 838,"西" => -744,"見" => -3874,"調" => 1010,"議" => 1198,"込" => 3041,"開" => 1758,"間" => -1257,"｢" => -645,"｣" => 3145,"ｯ" => 831,"ｱ" => -587,"ｶ" => 306,"ｷ" => 568}
+    @UW3 = {"," => 4889,"1" => -800,"−" => -1723,"、" => 4889,"々" => -2311,"〇" => 5827,"」" => 2670,"〓" => -3573,"あ" => -2696,"い" => 1006,"う" => 2342,"え" => 1983,"お" => -4864,"か" => -1163,"が" => 3271,"く" => 1004,"け" => 388,"げ" => 401,"こ" => -3552,"ご" => -3116,"さ" => -1058,"し" => -395,"す" => 584,"せ" => 3685,"そ" => -5228,"た" => 842,"ち" => -521,"っ" => -1444,"つ" => -1081,"て" => 6167,"で" => 2318,"と" => 1691,"ど" => -899,"な" => -2788,"に" => 2745,"の" => 4056,"は" => 4555,"ひ" => -2171,"ふ" => -1798,"へ" => 1199,"ほ" => -5516,"ま" => -4384,"み" => -120,"め" => 1205,"も" => 2323,"や" => -788,"よ" => -202,"ら" => 727,"り" => 649,"る" => 5905,"れ" => 2773,"わ" => -1207,"を" => 6620,"ん" => -518,"ア" => 551,"グ" => 1319,"ス" => 874,"ッ" => -1350,"ト" => 521,"ム" => 1109,"ル" => 1591,"ロ" => 2201,"ン" => 278,"・" => -3794,"一" => -1619,"下" => -1759,"世" => -2087,"両" => 3815,"中" => 653,"主" => -758,"予" => -1193,"二" => 974,"人" => 2742,"今" => 792,"他" => 1889,"以" => -1368,"低" => 811,"何" => 4265,"作" => -361,"保" => -2439,"元" => 4858,"党" => 3593,"全" => 1574,"公" => -3030,"六" => 755,"共" => -1880,"円" => 5807,"再" => 3095,"分" => 457,"初" => 2475,"別" => 1129,"前" => 2286,"副" => 4437,"力" => 365,"動" => -949,"務" => -1872,"化" => 1327,"北" => -1038,"区" => 4646,"千" => -2309,"午" => -783,"協" => -1006,"口" => 483,"右" => 1233,"各" => 3588,"合" => -241,"同" => 3906,"和" => -837,"員" => 4513,"国" => 642,"型" => 1389,"場" => 1219,"外" => -241,"妻" => 2016,"学" => -1356,"安" => -423,"実" => -1008,"家" => 1078,"小" => -513,"少" => -3102,"州" => 1155,"市" => 3197,"平" => -1804,"年" => 2416,"広" => -1030,"府" => 1605,"度" => 1452,"建" => -2352,"当" => -3885,"得" => 1905,"思" => -1291,"性" => 1822,"戸" => -488,"指" => -3973,"政" => -2013,"教" => -1479,"数" => 3222,"文" => -1489,"新" => 1764,"日" => 2099,"旧" => 5792,"昨" => -661,"時" => -1248,"曜" => -951,"最" => -937,"月" => 4125,"期" => 360,"李" => 3094,"村" => 364,"東" => -805,"核" => 5156,"森" => 2438,"業" => 484,"氏" => 2613,"民" => -1694,"決" => -1073,"法" => 1868,"海" => -495,"無" => 979,"物" => 461,"特" => -3850,"生" => -273,"用" => 914,"町" => 1215,"的" => 7313,"直" => -1835,"省" => 792,"県" => 6293,"知" => -1528,"私" => 4231,"税" => 401,"立" => -960,"第" => 1201,"米" => 7767,"系" => 3066,"約" => 3663,"級" => 1384,"統" => -4229,"総" => 1163,"線" => 1255,"者" => 6457,"能" => 725,"自" => -2869,"英" => 785,"見" => 1044,"調" => -562,"財" => -733,"費" => 1777,"車" => 1835,"軍" => 1375,"込" => -1504,"通" => -1136,"選" => -681,"郎" => 1026,"郡" => 4404,"部" => 1200,"金" => 2163,"長" => 421,"開" => -1432,"間" => 1302,"関" => -1282,"雨" => 2009,"電" => -1045,"非" => 2066,"駅" => 1620,"１" => -800,"｣" => 2670,"･" => -3794,"ｯ" => -1350,"ｱ" => 551,"ｸﾞ" => 1319,"ｽ" => 874,"ﾄ" => 521,"ﾑ" => 1109,"ﾙ" => 1591,"ﾛ" => 2201,"ﾝ" => 278}
+    @UW4 = {"," => 3930,"." => 3508,"―" => -4841,"、" => 3930,"。" => 3508,"〇" => 4999,"「" => 1895,"」" => 3798,"〓" => -5156,"あ" => 4752,"い" => -3435,"う" => -640,"え" => -2514,"お" => 2405,"か" => 530,"が" => 6006,"き" => -4482,"ぎ" => -3821,"く" => -3788,"け" => -4376,"げ" => -4734,"こ" => 2255,"ご" => 1979,"さ" => 2864,"し" => -843,"じ" => -2506,"す" => -731,"ず" => 1251,"せ" => 181,"そ" => 4091,"た" => 5034,"だ" => 5408,"ち" => -3654,"っ" => -5882,"つ" => -1659,"て" => 3994,"で" => 7410,"と" => 4547,"な" => 5433,"に" => 6499,"ぬ" => 1853,"ね" => 1413,"の" => 7396,"は" => 8578,"ば" => 1940,"ひ" => 4249,"び" => -4134,"ふ" => 1345,"へ" => 6665,"べ" => -744,"ほ" => 1464,"ま" => 1051,"み" => -2082,"む" => -882,"め" => -5046,"も" => 4169,"ゃ" => -2666,"や" => 2795,"ょ" => -1544,"よ" => 3351,"ら" => -2922,"り" => -9726,"る" => -14896,"れ" => -2613,"ろ" => -4570,"わ" => -1783,"を" => 13150,"ん" => -2352,"カ" => 2145,"コ" => 1789,"セ" => 1287,"ッ" => -724,"ト" => -403,"メ" => -1635,"ラ" => -881,"リ" => -541,"ル" => -856,"ン" => -3637,"・" => -4371,"ー" => -11870,"一" => -2069,"中" => 2210,"予" => 782,"事" => -190,"井" => -1768,"人" => 1036,"以" => 544,"会" => 950,"体" => -1286,"作" => 530,"側" => 4292,"先" => 601,"党" => -2006,"共" => -1212,"内" => 584,"円" => 788,"初" => 1347,"前" => 1623,"副" => 3879,"力" => -302,"動" => -740,"務" => -2715,"化" => 776,"区" => 4517,"協" => 1013,"参" => 1555,"合" => -1834,"和" => -681,"員" => -910,"器" => -851,"回" => 1500,"国" => -619,"園" => -1200,"地" => 866,"場" => -1410,"塁" => -2094,"士" => -1413,"多" => 1067,"大" => 571,"子" => -4802,"学" => -1397,"定" => -1057,"寺" => -809,"小" => 1910,"屋" => -1328,"山" => -1500,"島" => -2056,"川" => -2667,"市" => 2771,"年" => 374,"庁" => -4556,"後" => 456,"性" => 553,"感" => 916,"所" => -1566,"支" => 856,"改" => 787,"政" => 2182,"教" => 704,"文" => 522,"方" => -856,"日" => 1798,"時" => 1829,"最" => 845,"月" => -9066,"木" => -485,"来" => -442,"校" => -360,"業" => -1043,"氏" => 5388,"民" => -2716,"気" => -910,"沢" => -939,"済" => -543,"物" => -735,"率" => 672,"球" => -1267,"生" => -1286,"産" => -1101,"田" => -2900,"町" => 1826,"的" => 2586,"目" => 922,"省" => -3485,"県" => 2997,"空" => -867,"立" => -2112,"第" => 788,"米" => 2937,"系" => 786,"約" => 2171,"経" => 1146,"統" => -1169,"総" => 940,"線" => -994,"署" => 749,"者" => 2145,"能" => -730,"般" => -852,"行" => -792,"規" => 792,"警" => -1184,"議" => -244,"谷" => -1000,"賞" => 730,"車" => -1481,"軍" => 1158,"輪" => -1433,"込" => -3370,"近" => 929,"道" => -1291,"選" => 2596,"郎" => -4866,"都" => 1192,"野" => -1100,"銀" => -2213,"長" => 357,"間" => -2344,"院" => -2297,"際" => -2604,"電" => -878,"領" => -1659,"題" => -792,"館" => -1984,"首" => 1749,"高" => 2120,"｢" => 1895,"｣" => 3798,"･" => -4371,"ｯ" => -724,"ｰ" => -11870,"ｶ" => 2145,"ｺ" => 1789,"ｾ" => 1287,"ﾄ" => -403,"ﾒ" => -1635,"ﾗ" => -881,"ﾘ" => -541,"ﾙ" => -856,"ﾝ" => -3637}
+    @UW5 = {"," => 465,"." => -299,"1" => -514,"E2" => -32768,"]" => -2762,"、" => 465,"。" => -299,"「" => 363,"あ" => 1655,"い" => 331,"う" => -503,"え" => 1199,"お" => 527,"か" => 647,"が" => -421,"き" => 1624,"ぎ" => 1971,"く" => 312,"げ" => -983,"さ" => -1537,"し" => -1371,"す" => -852,"だ" => -1186,"ち" => 1093,"っ" => 52,"つ" => 921,"て" => -18,"で" => -850,"と" => -127,"ど" => 1682,"な" => -787,"に" => -1224,"の" => -635,"は" => -578,"べ" => 1001,"み" => 502,"め" => 865,"ゃ" => 3350,"ょ" => 854,"り" => -208,"る" => 429,"れ" => 504,"わ" => 419,"を" => -1264,"ん" => 327,"イ" => 241,"ル" => 451,"ン" => -343,"中" => -871,"京" => 722,"会" => -1153,"党" => -654,"務" => 3519,"区" => -901,"告" => 848,"員" => 2104,"大" => -1296,"学" => -548,"定" => 1785,"嵐" => -1304,"市" => -2991,"席" => 921,"年" => 1763,"思" => 872,"所" => -814,"挙" => 1618,"新" => -1682,"日" => 218,"月" => -4353,"査" => 932,"格" => 1356,"機" => -1508,"氏" => -1347,"田" => 240,"町" => -3912,"的" => -3149,"相" => 1319,"省" => -1052,"県" => -4003,"研" => -997,"社" => -278,"空" => -813,"統" => 1955,"者" => -2233,"表" => 663,"語" => -1073,"議" => 1219,"選" => -1018,"郎" => -368,"長" => 786,"間" => 1191,"題" => 2368,"館" => -689,"１" => -514,"Ｅ２" => -32768,"｢" => 363,"ｲ" => 241,"ﾙ" => 451,"ﾝ" => -343}
+    @UW6 = {"," => 227,"." => 808,"1" => -270,"E1" => 306,"、" => 227,"。" => 808,"あ" => -307,"う" => 189,"か" => 241,"が" => -73,"く" => -121,"こ" => -200,"じ" => 1782,"す" => 383,"た" => -428,"っ" => 573,"て" => -1014,"で" => 101,"と" => -105,"な" => -253,"に" => -149,"の" => -417,"は" => -236,"も" => -206,"り" => 187,"る" => -135,"を" => 195,"ル" => -673,"ン" => -496,"一" => -277,"中" => 201,"件" => -800,"会" => 624,"前" => 302,"区" => 1792,"員" => -1212,"委" => 798,"学" => -960,"市" => 887,"広" => -695,"後" => 535,"業" => -697,"相" => 753,"社" => -507,"福" => 974,"空" => -822,"者" => 1811,"連" => 463,"郎" => 1082,"１" => -270,"Ｅ１" => 306,"ﾙ" => -673,"ﾝ" => -496}
+  end
+end

metadata

@@ -0,0 +1,46 @@
+--- !ruby/object:Gem::Specification
+name: tiny_segmenter
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Peter Graham
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-08-19 00:00:00.000000000 Z
+dependencies: []
+description: Ruby port of TinySegmenter.js for tokenizing Japanese text.
+email:
+- pete@gigadrill.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/tiny_segmenter.rb
+homepage: http://github.com/6/tiny_segmenter
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.21
+signing_key: 
+specification_version: 3
+summary: Ruby port of TinySegmenter.js for tokenizing Japanese text.
+test_files: []