jtcg_locale_detector 1.0.1

data/python/src/locale_detector_c.c

@@ -0,0 +1,403 @@
+/*
+ * Pure C implementation of locale detector
+ * High-performance character set analysis with bitmap lookup
+ */
+
+#include <Python.h>
+#include <string.h>
+#include <stdint.h>
+#include <time.h>
+
+/* Result constants */
+typedef enum {
+    LOCALE_UNKNOWN = 0,
+    LOCALE_TRADITIONAL = 1,
+    LOCALE_SIMPLIFIED = 2,
+    LOCALE_BOTH = 3,
+    LOCALE_MIXED = 4
+} locale_type_t;
+
+/* Character count structure */
+typedef struct {
+    long traditional;
+    long simplified;
+    long shared;
+    long total;
+} hanzi_counts_t;
+
+/* Static character set definitions as const arrays */
+static const char* TRAD_CHARS = "制咖片型超聲盤鑒定仔點他命書歌粉巾字帳恤手指記憶棒形轉彎溝光○〇㐄㐅㐆㐌㐖毒㐜㐡㐤㐰㐺㑇㑳㒳㒸㔾㗂㗎㝵㞎㞙㞞㠯㢲㢴㤅㥁㥯㨗㫺㬎㮎㮚㮸㲋㲱㲾㳮㵎㵪㶸㷖㷭㹢㹴犬㺢狓㺵㼝㽮㿝䍃䔢䖟䖸䗈䗥䗪䝓䠶䥯䦉䯝䰾魚䲔䳗䳘䵹鼄䶑一對應映射丁不識下兒子做二休世丘之貉並中台原則串為甚謂乾淨了百事無成八變五十些人得道雞升天代如併來去個國政策勁幽靈在歐洲遊蕩接樣蘿蔔坑側化傳價元論醇共再准刀兩斷切分耕耘收穫錢貨物向看舊就緒險刻千金動勞永逸匙零夜半卡通回復返影蹤反常態口咬氣句話同吐快吹周味呼諾嗚品紅鍋哄而散起唱和問三知生熟團漆黑火糟堆場空塊麵塌糊塗塵染壁廂夔已足多情露水大早到晚夫妻當關萬莫開失古恨套所料既往孔見提師要家主審寸陰難買鬥牛小撮部陣局展身層巴掌帆風順席地帶過年計於春頭載四季期被蛇怕井繩度願式份彈頃深前律徑心意念差愁孤行俱全房廳交遮打技長把抓死拿眼淚鼻涕鑰鎖折段抿拍即合掃排掬揮撥擁上入擊洞擲攬改故轍敗文值名斑方面旁族日秋餐隔雅里終父旦時晌會霎間晃暴寒曝更月望垠際朝夕本正經利杯羹東西板枝獨秀根筋桿進條龍服務概模次函數又性程總付步腳印趨登毛拔呵氧氮碳決雌雄波未平派謊言流清楚白準溜煙潭有獲聞是處降琴鶴甲病發可拾沙目然瞭直以相眨穿睹瞥瞬矢的解石鳥神教秉虔誠秘種窩蜂窮竅笑置筆苟勾銷抹殺煞等獎箍節吃箭仇雙鵰詩籌籮筐系列紙級士官統絲毫掛維網盡線微吭響股腦胎脈承腔臂力致效資源址器舉功投般說講規貿易葉障著慎滿皆輸號木電池衣傾鐘高低視仁覺醒覽遺角銀幣觸潰九鼎蔽抄出駟馬追重語破貧洗貫走路安蹴至幾蹶振躍役膽汗較輩輪辭贊退六連遍遞邊針血錘音錯門思閃真倒項栽霧類保護川先驚乍體鬨鱗爪鳴滴泡鄰域黨專鼓作齊炒丑烯亥克內酯冬加奴卯肝炎基尺梁街褲鎬客寵庭巳汝昌烷玲磊糖肇酉醛啷青縣韙良香骨鯛";
+
+static const char* SIMP_CHARS = "制咖片型超声盘鉴定仔点他命书歌粉巾字帐恤手指记忆棒形转弯沟光○〇㐄㐅㐆㐌㐖毒㐜㐡㐤㐰㐺㑇㑳㒳㒸㔾㗂㗎㝵㞎㞙㞞以㢲㢴㤅㥁㥯㨗㫺㬎㮎㮚㮸㲋㲱㲾㳮涧㵪㶸㷖㷭㹢㹴犬㺢狓㺵碗㽮㿝䍃䔢䖟䖸䗈䗥䗪䝓射䥯䦉䯝鲃鱼䲔䳗鹅䵹鼄䶑一对应映射丁不识下儿子做二休世丘之貉并中台原则串为甚谓干净了百事无成八变五十些人得道鸡升天代如并来去个国政策劲幽灵在欧洲游荡接样萝卜坑侧化传价元论醇共再准刀两断切分耕耘收获钱货物向看旧就绪险刻千金动劳永逸匙零夜半卡通回复返影踪反常态口咬气句话同吐快吹周味呼诺呜品红锅哄而散起唱和问三知生熟团漆黑火糟堆场空块面塌糊涂尘染壁厢夔已足多情露水大早到晚夫妻当关万莫开失古恨套所料既往孔见提师要家主审寸阴难买斗牛小撮部阵局展身层巴掌帆风顺席地带过年计于春头载四季期被蛇怕井绳度愿式份弹顷深前律径心意念差愁孤行俱全房厅交遮打技长把抓死拿眼泪鼻涕钥锁折段抿拍即合扫排掬挥拨拥上入击洞掷揽改故辙败文值名斑方面旁族日秋餐隔雅里终父旦时晌会霎间晃暴寒曝更月望垠际朝夕本正经利杯羹东西板枝独秀根筋杆进条龙服务概模次函数又性程总付步脚印趋登毛拔呵氧氮碳决雌雄波未平派谎言流清楚白准溜烟潭有获闻是处降琴鹤甲病发可拾沙目然了直以相眨穿睹瞥瞬矢的解石鸟神教秉虔诚秘种窝蜂穷窍笑置笔苟勾销抹杀煞等奖箍节吃箭仇双雕诗筹箩筐系列纸级士官统丝毫挂维网尽线微吭响股脑胎脉承腔臂力致效资源址器举功投般说讲规贸易叶障着慎满皆输号木电池衣倾钟高低视仁觉醒览遗角银币触溃九鼎蔽抄出驷马追重语破贫洗贯走路安蹴至几蹶振跃役胆汗较辈轮辞赞退六连遍递边针血锤音错门思闪真倒项栽雾类保护川先惊乍体哄鳞爪鸣滴泡邻域党专鼓作齐炒丑烯亥克内酯冬加奴卯肝炎基尺梁街裤镐客宠庭巳汝昌烷玲磊糖肇酉醛啷青县韪良香骨鲷";
+
+/* Global bitmap arrays for fast lookup - static allocation for BMP (65536 codepoints) */
+static unsigned char trad_bitmap[65536] = {0};
+static unsigned char simp_bitmap[65536] = {0};
+static unsigned char shared_bitmap[65536] = {0};
+static int bitmap_initialized = 0;
+
+/* Unicode utilities */
+static uint32_t utf8_to_codepoint(const char* str, int* bytes_consumed) {
+    const unsigned char* s = (const unsigned char*)str;
+    uint32_t cp = 0;
+    
+    if (s[0] < 0x80) {
+        /* ASCII: 0xxxxxxx */
+        cp = s[0];
+        *bytes_consumed = 1;
+    } else if ((s[0] & 0xE0) == 0xC0) {
+        /* 2-byte: 110xxxxx 10xxxxxx */
+        cp = ((s[0] & 0x1F) << 6) | (s[1] & 0x3F);
+        *bytes_consumed = 2;
+    } else if ((s[0] & 0xF0) == 0xE0) {
+        /* 3-byte: 1110xxxx 10xxxxxx 10xxxxxx */
+        cp = ((s[0] & 0x0F) << 12) | ((s[1] & 0x3F) << 6) | (s[2] & 0x3F);
+        *bytes_consumed = 3;
+    } else if ((s[0] & 0xF8) == 0xF0) {
+        /* 4-byte: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */
+        cp = ((s[0] & 0x07) << 18) | ((s[1] & 0x3F) << 12) | ((s[2] & 0x3F) << 6) | (s[3] & 0x3F);
+        *bytes_consumed = 4;
+    } else {
+        /* Invalid UTF-8 */
+        cp = 0xFFFD; /* Replacement character */
+        *bytes_consumed = 1;
+    }
+    
+    return cp;
+}
+
+/* Initialize bitmap arrays from character strings */
+static void init_bitmap() {
+    if (bitmap_initialized) return;
+    
+    const char* p;
+    int bytes;
+    uint32_t cp;
+    
+    /* Initialize traditional characters bitmap */
+    p = TRAD_CHARS;
+    while (*p) {
+        cp = utf8_to_codepoint(p, &bytes);
+        if (cp < 65536) {
+            trad_bitmap[cp] = 1;
+        }
+        p += bytes;
+    }
+    
+    /* Initialize simplified characters bitmap */
+    p = SIMP_CHARS;
+    while (*p) {
+        cp = utf8_to_codepoint(p, &bytes);
+        if (cp < 65536) {
+            simp_bitmap[cp] = 1;
+        }
+        p += bytes;
+    }
+    
+    /* Initialize shared characters bitmap */
+    for (int i = 0; i < 65536; i++) {
+        if (trad_bitmap[i] && simp_bitmap[i]) {
+            shared_bitmap[i] = 1;
+        }
+    }
+    
+    bitmap_initialized = 1;
+}
+
+/* Fast character classification functions */
+static inline int is_traditional_char(uint32_t cp) {
+    return (cp < 65536) ? trad_bitmap[cp] : 0;
+}
+
+static inline int is_simplified_char(uint32_t cp) {
+    return (cp < 65536) ? simp_bitmap[cp] : 0;
+}
+
+static inline int is_shared_char(uint32_t cp) {
+    return (cp < 65536) ? shared_bitmap[cp] : 0;
+}
+
+static inline int is_hanzi_char(uint32_t cp) {
+    return (cp < 65536) ? (trad_bitmap[cp] || simp_bitmap[cp]) : 0;
+}
+
+/* Core analysis functions */
+static hanzi_counts_t count_hanzi_chars(const char* text) {
+    if (!bitmap_initialized) init_bitmap();
+    
+    hanzi_counts_t counts = {0, 0, 0, 0};
+    
+    const char* p = text;
+    int bytes;
+    uint32_t cp;
+    
+    while (*p) {
+        cp = utf8_to_codepoint(p, &bytes);
+        
+        if (is_hanzi_char(cp)) {
+            counts.total++;
+            
+            if (is_shared_char(cp)) {
+                counts.shared++;
+            } else if (is_traditional_char(cp)) {
+                counts.traditional++;
+            } else if (is_simplified_char(cp)) {
+                counts.simplified++;
+            }
+        }
+        
+        p += bytes;
+    }
+    
+    return counts;
+}
+
+static locale_type_t identify_locale_type(const char* text) {
+    hanzi_counts_t counts = count_hanzi_chars(text);
+    
+    if (counts.total == 0) {
+        return LOCALE_UNKNOWN;
+    }
+    
+    /* Pure traditional (only traditional chars, no simplified) */
+    if (counts.traditional > 0 && counts.simplified == 0 && counts.shared == 0) {
+        return LOCALE_TRADITIONAL;
+    }
+    
+    /* Pure simplified (only simplified chars, no traditional) */
+    if (counts.simplified > 0 && counts.traditional == 0 && counts.shared == 0) {
+        return LOCALE_SIMPLIFIED;
+    }
+    
+    /* Only shared characters */
+    if (counts.shared > 0 && counts.traditional == 0 && counts.simplified == 0) {
+        return LOCALE_BOTH;
+    }
+    
+    /* Mixed content */
+    return LOCALE_MIXED;
+}
+
+static const char* identify_locale_string(const char* text) {
+    locale_type_t type = identify_locale_type(text);
+    
+    switch (type) {
+        case LOCALE_TRADITIONAL: return "TRADITIONAL";
+        case LOCALE_SIMPLIFIED: return "SIMPLIFIED";
+        case LOCALE_BOTH: return "BOTH";
+        case LOCALE_MIXED: return "MIXED";
+        default: return "UNKNOWN";
+    }
+}
+
+static int is_traditional_text(const char* text) {
+    hanzi_counts_t counts = count_hanzi_chars(text);
+    
+    if (counts.total == 0) return 0;
+    
+    /* Traditional if: no simplified chars present */
+    return (counts.simplified == 0);
+}
+
+static int is_simplified_text(const char* text) {
+    hanzi_counts_t counts = count_hanzi_chars(text);
+    
+    if (counts.total == 0) return 0;
+    
+    /* Simplified if: no traditional chars present */
+    return (counts.traditional == 0);
+}
+
+static const char* detect_chinese_locale(const char* text) {
+    hanzi_counts_t counts = count_hanzi_chars(text);
+    
+    if (counts.total == 0) {
+        return "unknown";
+    }
+    
+    /* Calculate ratios */
+    double trad_ratio = (double)counts.traditional / counts.total;
+    double simp_ratio = (double)counts.simplified / counts.total;
+    
+    /* Decision logic based on character analysis */
+    if (counts.simplified == 0 && (counts.traditional > 0 || counts.shared > 0)) {
+        return "zh-TW";  /* No simplified chars present */
+    }
+    
+    if (counts.traditional == 0 && (counts.simplified > 0 || counts.shared > 0)) {
+        return "zh-CN";  /* No traditional chars present */
+    }
+    
+    /* For mixed content, use ratio analysis */
+    if (trad_ratio > 0.6 && trad_ratio > simp_ratio) {
+        return "zh-TW";
+    }
+    
+    if (simp_ratio > 0.6 && simp_ratio > trad_ratio) {
+        return "zh-CN";
+    }
+    
+    /* Default to traditional if ratios are close */
+    return trad_ratio >= simp_ratio ? "zh-TW" : "zh-CN";
+}
+
+/* Fast extraction of hanzi characters */
+static int extract_hanzi_chars(const char* text, char* result, int max_result_len) {
+    if (!bitmap_initialized) init_bitmap();
+    
+    const char* p = text;
+    char* out = result;
+    int result_len = 0;
+    int bytes;
+    uint32_t cp;
+    
+    while (*p && result_len < max_result_len - 4) {  /* Reserve space for UTF-8 char + null */
+        cp = utf8_to_codepoint(p, &bytes);
+        
+        if (is_hanzi_char(cp)) {
+            /* Copy the UTF-8 bytes */
+            for (int i = 0; i < bytes; i++) {
+                *out++ = p[i];
+                result_len++;
+            }
+        }
+        
+        p += bytes;
+    }
+    
+    *out = '\0';
+    return result_len;
+}
+
+/* Python C API wrapper functions */
+static PyObject* py_count_hanzi_chars(PyObject *self, PyObject *args) {
+    const char* text;
+    if (!PyArg_ParseTuple(args, "s", &text)) {
+        return NULL;
+    }
+    
+    hanzi_counts_t counts = count_hanzi_chars(text);
+    
+    return Py_BuildValue("{s:l,s:l,s:l,s:l}",
+                         "traditional", counts.traditional,
+                         "simplified", counts.simplified,
+                         "shared", counts.shared,
+                         "total", counts.total);
+}
+
+static PyObject* py_identify_locale(PyObject *self, PyObject *args) {
+    const char* text;
+    if (!PyArg_ParseTuple(args, "s", &text)) {
+        return NULL;
+    }
+    
+    const char* result = identify_locale_string(text);
+    return PyUnicode_FromString(result);
+}
+
+static PyObject* py_detect_chinese_locale(PyObject *self, PyObject *args) {
+    const char* text;
+    if (!PyArg_ParseTuple(args, "s", &text)) {
+        return NULL;
+    }
+    
+    const char* result = detect_chinese_locale(text);
+    return PyUnicode_FromString(result);
+}
+
+static PyObject* py_is_traditional(PyObject *self, PyObject *args) {
+    const char* text;
+    if (!PyArg_ParseTuple(args, "s", &text)) {
+        return NULL;
+    }
+    
+    int result = is_traditional_text(text);
+    return PyBool_FromLong(result);
+}
+
+static PyObject* py_is_simplified(PyObject *self, PyObject *args) {
+    const char* text;
+    if (!PyArg_ParseTuple(args, "s", &text)) {
+        return NULL;
+    }
+    
+    int result = is_simplified_text(text);
+    return PyBool_FromLong(result);
+}
+
+static PyObject* py_extract_hanzi(PyObject *self, PyObject *args) {
+    const char* text;
+    if (!PyArg_ParseTuple(args, "s", &text)) {
+        return NULL;
+    }
+    
+    /* Allocate buffer for result (conservatively estimate UTF-8 size) */
+    int text_len = strlen(text);
+    char* result = malloc(text_len + 1);
+    if (!result) {
+        return PyErr_NoMemory();
+    }
+    
+    extract_hanzi_chars(text, result, text_len + 1);
+    
+    PyObject* py_result = PyUnicode_FromString(result);
+    free(result);
+    
+    return py_result;
+}
+
+static PyObject* py_benchmark(PyObject *self, PyObject *args) {
+    const char* text;
+    int iterations = 10000;
+    
+    if (!PyArg_ParseTuple(args, "s|i", &text, &iterations)) {
+        return NULL;
+    }
+    
+    clock_t start = clock();
+    
+    for (int i = 0; i < iterations; i++) {
+        const char* result = detect_chinese_locale(text);
+        (void)result;  /* Suppress unused variable warning */
+    }
+    
+    clock_t end = clock();
+    double time_taken = ((double)(end - start)) / CLOCKS_PER_SEC;
+    double us_per_iteration = (time_taken * 1000000) / iterations;
+    
+    return Py_BuildValue("{s:i,s:d,s:d}",
+                         "iterations", iterations,
+                         "total_time", time_taken,
+                         "microseconds_per_iteration", us_per_iteration);
+}
+
+static int is_in_charset(Py_UCS4 ch, const char *charset) {
+    // 線性查找，極短文本下最快
+    for (const char *p = charset; *p; ++p) {
+        if ((unsigned char)*p == ch) return 1;
+    }
+    return 0;
+}
+
+static PyMethodDef LocaleDetectorMethods[] = {
+    {"count_hanzi_chars", py_count_hanzi_chars, METH_VARARGS, "統計中文字符類型數量"},
+    {"identify_locale", py_identify_locale, METH_VARARGS, "識別文本中文類型"},
+    {"detect_chinese_locale", py_detect_chinese_locale, METH_VARARGS, "檢測中文地區設定"},
+    {"is_traditional", py_is_traditional, METH_VARARGS, "判斷是否為繁體文本"},
+    {"is_simplified", py_is_simplified, METH_VARARGS, "判斷是否為簡體文本"},
+    {"extract_hanzi", py_extract_hanzi, METH_VARARGS, "提取中文字符"},
+    {"benchmark", py_benchmark, METH_VARARGS, "效能測試"},
+    {NULL, NULL, 0, NULL}
+};
+
+static struct PyModuleDef locale_detectormodule = {
+    PyModuleDef_HEAD_INIT,
+    "locale_detector_c",   // module name
+    NULL,                  // module doc
+    -1,                    // size of per-interpreter state of the module
+    LocaleDetectorMethods
+};
+
+PyMODINIT_FUNC PyInit_locale_detector_c(void) {
+    return PyModule_Create(&locale_detectormodule);
+}

data/python/src/locale_detector_c.h

@@ -0,0 +1,37 @@
+/*
+ * Pure C locale detector header file
+ */
+
+#ifndef LOCALE_DETECTOR_C_H
+#define LOCALE_DETECTOR_C_H
+
+#include <stdint.h>
+
+/* Result constants */
+typedef enum {
+    LOCALE_UNKNOWN = 0,
+    LOCALE_TRADITIONAL = 1,
+    LOCALE_SIMPLIFIED = 2,
+    LOCALE_BOTH = 3,
+    LOCALE_MIXED = 4
+} locale_type_t;
+
+/* Character count structure */
+typedef struct {
+    long traditional;
+    long simplified;
+    long shared;
+    long total;
+} hanzi_counts_t;
+
+/* Core C functions - can be used directly without Python */
+hanzi_counts_t count_hanzi_chars(const char* text);
+locale_type_t identify_locale_type(const char* text);
+const char* identify_locale_string(const char* text);
+int is_traditional_text(const char* text);
+int is_simplified_text(const char* text);
+const char* detect_chinese_locale(const char* text);
+int extract_hanzi_chars(const char* text, char* result, int max_result_len);
+void benchmark_locale_detection(const char* text, int iterations);
+
+#endif /* LOCALE_DETECTOR_C_H */