garu-ko 0.3.1 → 0.3.2

package/dist/index.d.ts

@@ -11,6 +11,8 @@ export interface AnalyzeResult {
     score: number;
     elapsed: number;
 }
+export { normalizeText, splitSentences } from './normalize.js';
+export type { NormalizeOptions, Segment } from './normalize.js';
 export interface AnalyzeOptions {
     topN?: number;
 }

package/dist/index.js

@@ -1,3 +1,4 @@
+export { normalizeText, splitSentences } from './normalize.js';
 const isNode = typeof process !== 'undefined' &&
     process.versions != null &&
     process.versions.node != null;
@@ -132,7 +133,7 @@ export class Garu {
         return {
             version: this._wasm.constructor.version(),
             size: this._modelSize,
-            accuracy: 0.953,
+            accuracy: 0.908,
         };
     }
     /**

package/dist/normalize.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Text normalization utilities for Garu Korean morphological analyzer.
+ * Pre-processes informal/non-standard Korean text before analysis.
+ */
+export interface NormalizeOptions {
+    /** Collapse repeated jamo (ㅋㅋㅋ→ㅋㅋ). Default: true */
+    jamo?: boolean;
+    /** Expand 2-char jamo abbreviations (ㄱㅅ→감사). Default: false */
+    jamoAbbrev?: boolean;
+    /** Normalize common dialect stems (워디→어디). Default: false */
+    dialect?: boolean;
+}
+export declare function normalizeText(text: string, opts?: NormalizeOptions): string;
+export interface Segment {
+    text: string;
+    /** Character offset of this segment in the original text */
+    offset: number;
+}
+export declare function splitSentences(text: string): Segment[];

package/dist/normalize.js

@@ -0,0 +1,71 @@
+/**
+ * Text normalization utilities for Garu Korean morphological analyzer.
+ * Pre-processes informal/non-standard Korean text before analysis.
+ */
+const COMPAT_JAMO_RE = /([\u3131-\u3163])\1{2,}/g;
+const JAMO_ABBREV_MAP = {
+    'ㄱㅅ': '감사',
+    'ㅊㅋ': '축하',
+    'ㄴㄴ': '노노',
+    'ㅂㅂ': '바이바이',
+    'ㅈㅅ': '죄송',
+    'ㄱㅊ': '괜찮',
+    'ㅇㅈ': '인정',
+    'ㅎㅇ': '하이',
+    'ㄹㅇ': '리얼',
+    'ㅇㅋ': '오케이',
+    'ㅁㄹ': '모름',
+    'ㅅㄱ': '수고',
+    'ㅊㅊ': '추천',
+    'ㅍㅌ': '파이팅',
+    'ㄷㄷ': '덜덜',
+};
+const DIALECT_MAP = {
+    '워디': '어디',
+    '와케': '왜이렇게',
+    '우예': '어떻게',
+    '긍게': '그러니까',
+    '워메': '어머',
+    '가유': '가요',
+    '허유': '해요',
+};
+export function normalizeText(text, opts = {}) {
+    const { jamo = true, jamoAbbrev = false, dialect = false } = opts;
+    let result = text;
+    if (jamo) {
+        result = result.replace(COMPAT_JAMO_RE, '$1$1');
+    }
+    if (jamoAbbrev) {
+        for (const [abbrev, expansion] of Object.entries(JAMO_ABBREV_MAP)) {
+            const re = new RegExp(`(?<![가-힣ㄱ-ㅣ])${abbrev}(?![가-힣ㄱ-ㅣ])`, 'g');
+            result = result.replace(re, expansion);
+        }
+    }
+    if (dialect) {
+        for (const [dialectForm, standard] of Object.entries(DIALECT_MAP)) {
+            result = result.replace(new RegExp(dialectForm, 'g'), standard);
+        }
+    }
+    return result;
+}
+export function splitSentences(text) {
+    const segments = [];
+    const re = /([.!?…]+)\s+(?=[가-힣A-Z"'([{])/g;
+    let lastCharOffset = 0;
+    let lastByteIdx = 0;
+    let match;
+    while ((match = re.exec(text)) !== null) {
+        const segText = text.slice(lastByteIdx, match.index + match[1].length).trim();
+        if (segText.length > 0) {
+            segments.push({ text: segText, offset: lastCharOffset });
+        }
+        const newStart = match.index + match[0].length;
+        lastCharOffset += [...text.slice(lastByteIdx, newStart)].length;
+        lastByteIdx = newStart;
+    }
+    const last = text.slice(lastByteIdx).trim();
+    if (last.length > 0) {
+        segments.push({ text: last, offset: lastCharOffset });
+    }
+    return segments.length > 0 ? segments : [{ text: text.trim(), offset: 0 }];
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "garu-ko",
-  "version": "0.3.1",
+  "version": "0.3.2",
   "description": "Ultra-lightweight Korean morphological analyzer for the web (1.7MB model, WASM 93KB, F1 95.3%)",
   "type": "module",
   "main": "dist/index.js",

package/pkg/garu_wasm.d.ts

@@ -1,52 +0,0 @@
-/* tslint:disable */
-/* eslint-disable */
-
-export class GaruWasm {
-    free(): void;
-    [Symbol.dispose](): void;
-    analyze(text: string): any;
-    analyze_topn(text: string, n: number): any;
-    constructor(model_data: Uint8Array);
-    tokenize(text: string): any;
-    static version(): string;
-}
-
-export type InitInput = RequestInfo | URL | Response | BufferSource | WebAssembly.Module;
-
-export interface InitOutput {
-    readonly memory: WebAssembly.Memory;
-    readonly __wbg_garuwasm_free: (a: number, b: number) => void;
-    readonly garuwasm_analyze: (a: number, b: number, c: number) => [number, number, number];
-    readonly garuwasm_analyze_topn: (a: number, b: number, c: number, d: number) => [number, number, number];
-    readonly garuwasm_new: (a: number, b: number) => [number, number, number];
-    readonly garuwasm_tokenize: (a: number, b: number, c: number) => [number, number, number];
-    readonly garuwasm_version: () => [number, number];
-    readonly __wbindgen_malloc: (a: number, b: number) => number;
-    readonly __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;
-    readonly __wbindgen_externrefs: WebAssembly.Table;
-    readonly __externref_table_dealloc: (a: number) => void;
-    readonly __wbindgen_free: (a: number, b: number, c: number) => void;
-    readonly __wbindgen_start: () => void;
-}
-
-export type SyncInitInput = BufferSource | WebAssembly.Module;
-
-/**
- * Instantiates the given `module`, which can either be bytes or
- * a precompiled `WebAssembly.Module`.
- *
- * @param {{ module: SyncInitInput }} module - Passing `SyncInitInput` directly is deprecated.
- *
- * @returns {InitOutput}
- */
-export function initSync(module: { module: SyncInitInput } | SyncInitInput): InitOutput;
-
-/**
- * If `module_or_path` is {RequestInfo} or {URL}, makes a request and
- * for everything else, calls `WebAssembly.instantiate` directly.
- *
- * @param {{ module_or_path: InitInput | Promise<InitInput> }} module_or_path - Passing `InitInput` directly is deprecated.
- *
- * @returns {Promise<InitOutput>}
- */
-export default function __wbg_init (module_or_path?: { module_or_path: InitInput | Promise<InitInput> } | InitInput | Promise<InitInput>): Promise<InitOutput>;

package/pkg/garu_wasm.js

@@ -1,345 +0,0 @@
-/* @ts-self-types="./garu_wasm.d.ts" */
-
-export class GaruWasm {
-    __destroy_into_raw() {
-        const ptr = this.__wbg_ptr;
-        this.__wbg_ptr = 0;
-        GaruWasmFinalization.unregister(this);
-        return ptr;
-    }
-    free() {
-        const ptr = this.__destroy_into_raw();
-        wasm.__wbg_garuwasm_free(ptr, 0);
-    }
-    /**
-     * @param {string} text
-     * @returns {any}
-     */
-    analyze(text) {
-        const ptr0 = passStringToWasm0(text, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
-        const len0 = WASM_VECTOR_LEN;
-        const ret = wasm.garuwasm_analyze(this.__wbg_ptr, ptr0, len0);
-        if (ret[2]) {
-            throw takeFromExternrefTable0(ret[1]);
-        }
-        return takeFromExternrefTable0(ret[0]);
-    }
-    /**
-     * @param {string} text
-     * @param {number} n
-     * @returns {any}
-     */
-    analyze_topn(text, n) {
-        const ptr0 = passStringToWasm0(text, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
-        const len0 = WASM_VECTOR_LEN;
-        const ret = wasm.garuwasm_analyze_topn(this.__wbg_ptr, ptr0, len0, n);
-        if (ret[2]) {
-            throw takeFromExternrefTable0(ret[1]);
-        }
-        return takeFromExternrefTable0(ret[0]);
-    }
-    /**
-     * @param {Uint8Array} model_data
-     */
-    constructor(model_data) {
-        const ptr0 = passArray8ToWasm0(model_data, wasm.__wbindgen_malloc);
-        const len0 = WASM_VECTOR_LEN;
-        const ret = wasm.garuwasm_new(ptr0, len0);
-        if (ret[2]) {
-            throw takeFromExternrefTable0(ret[1]);
-        }
-        this.__wbg_ptr = ret[0] >>> 0;
-        GaruWasmFinalization.register(this, this.__wbg_ptr, this);
-        return this;
-    }
-    /**
-     * @param {string} text
-     * @returns {any}
-     */
-    tokenize(text) {
-        const ptr0 = passStringToWasm0(text, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
-        const len0 = WASM_VECTOR_LEN;
-        const ret = wasm.garuwasm_tokenize(this.__wbg_ptr, ptr0, len0);
-        if (ret[2]) {
-            throw takeFromExternrefTable0(ret[1]);
-        }
-        return takeFromExternrefTable0(ret[0]);
-    }
-    /**
-     * @returns {string}
-     */
-    static version() {
-        let deferred1_0;
-        let deferred1_1;
-        try {
-            const ret = wasm.garuwasm_version();
-            deferred1_0 = ret[0];
-            deferred1_1 = ret[1];
-            return getStringFromWasm0(ret[0], ret[1]);
-        } finally {
-            wasm.__wbindgen_free(deferred1_0, deferred1_1, 1);
-        }
-    }
-}
-if (Symbol.dispose) GaruWasm.prototype[Symbol.dispose] = GaruWasm.prototype.free;
-import * as import1 from "./snippets/garu-core-d0bd202007f7afa3/inline0.js"
-
-function __wbg_get_imports() {
-    const import0 = {
-        __proto__: null,
-        __wbg_Error_83742b46f01ce22d: function(arg0, arg1) {
-            const ret = Error(getStringFromWasm0(arg0, arg1));
-            return ret;
-        },
-        __wbg_String_8564e559799eccda: function(arg0, arg1) {
-            const ret = String(arg1);
-            const ptr1 = passStringToWasm0(ret, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
-            const len1 = WASM_VECTOR_LEN;
-            getDataViewMemory0().setInt32(arg0 + 4 * 1, len1, true);
-            getDataViewMemory0().setInt32(arg0 + 4 * 0, ptr1, true);
-        },
-        __wbg___wbindgen_throw_6ddd609b62940d55: function(arg0, arg1) {
-            throw new Error(getStringFromWasm0(arg0, arg1));
-        },
-        __wbg_new_a70fbab9066b301f: function() {
-            const ret = new Array();
-            return ret;
-        },
-        __wbg_new_ab79df5bd7c26067: function() {
-            const ret = new Object();
-            return ret;
-        },
-        __wbg_set_282384002438957f: function(arg0, arg1, arg2) {
-            arg0[arg1 >>> 0] = arg2;
-        },
-        __wbg_set_6be42768c690e380: function(arg0, arg1, arg2) {
-            arg0[arg1] = arg2;
-        },
-        __wbindgen_cast_0000000000000001: function(arg0) {
-            // Cast intrinsic for `F64 -> Externref`.
-            const ret = arg0;
-            return ret;
-        },
-        __wbindgen_cast_0000000000000002: function(arg0, arg1) {
-            // Cast intrinsic for `Ref(String) -> Externref`.
-            const ret = getStringFromWasm0(arg0, arg1);
-            return ret;
-        },
-        __wbindgen_cast_0000000000000003: function(arg0) {
-            // Cast intrinsic for `U64 -> Externref`.
-            const ret = BigInt.asUintN(64, arg0);
-            return ret;
-        },
-        __wbindgen_init_externref_table: function() {
-            const table = wasm.__wbindgen_externrefs;
-            const offset = table.grow(4);
-            table.set(0, undefined);
-            table.set(offset + 0, undefined);
-            table.set(offset + 1, null);
-            table.set(offset + 2, true);
-            table.set(offset + 3, false);
-        },
-    };
-    return {
-        __proto__: null,
-        "./garu_wasm_bg.js": import0,
-        "./snippets/garu-core-d0bd202007f7afa3/inline0.js": import1,
-    };
-}
-
-const GaruWasmFinalization = (typeof FinalizationRegistry === 'undefined')
-    ? { register: () => {}, unregister: () => {} }
-    : new FinalizationRegistry(ptr => wasm.__wbg_garuwasm_free(ptr >>> 0, 1));
-
-let cachedDataViewMemory0 = null;
-function getDataViewMemory0() {
-    if (cachedDataViewMemory0 === null || cachedDataViewMemory0.buffer.detached === true || (cachedDataViewMemory0.buffer.detached === undefined && cachedDataViewMemory0.buffer !== wasm.memory.buffer)) {
-        cachedDataViewMemory0 = new DataView(wasm.memory.buffer);
-    }
-    return cachedDataViewMemory0;
-}
-
-function getStringFromWasm0(ptr, len) {
-    ptr = ptr >>> 0;
-    return decodeText(ptr, len);
-}
-
-let cachedUint8ArrayMemory0 = null;
-function getUint8ArrayMemory0() {
-    if (cachedUint8ArrayMemory0 === null || cachedUint8ArrayMemory0.byteLength === 0) {
-        cachedUint8ArrayMemory0 = new Uint8Array(wasm.memory.buffer);
-    }
-    return cachedUint8ArrayMemory0;
-}
-
-function passArray8ToWasm0(arg, malloc) {
-    const ptr = malloc(arg.length * 1, 1) >>> 0;
-    getUint8ArrayMemory0().set(arg, ptr / 1);
-    WASM_VECTOR_LEN = arg.length;
-    return ptr;
-}
-
-function passStringToWasm0(arg, malloc, realloc) {
-    if (realloc === undefined) {
-        const buf = cachedTextEncoder.encode(arg);
-        const ptr = malloc(buf.length, 1) >>> 0;
-        getUint8ArrayMemory0().subarray(ptr, ptr + buf.length).set(buf);
-        WASM_VECTOR_LEN = buf.length;
-        return ptr;
-    }
-
-    let len = arg.length;
-    let ptr = malloc(len, 1) >>> 0;
-
-    const mem = getUint8ArrayMemory0();
-
-    let offset = 0;
-
-    for (; offset < len; offset++) {
-        const code = arg.charCodeAt(offset);
-        if (code > 0x7F) break;
-        mem[ptr + offset] = code;
-    }
-    if (offset !== len) {
-        if (offset !== 0) {
-            arg = arg.slice(offset);
-        }
-        ptr = realloc(ptr, len, len = offset + arg.length * 3, 1) >>> 0;
-        const view = getUint8ArrayMemory0().subarray(ptr + offset, ptr + len);
-        const ret = cachedTextEncoder.encodeInto(arg, view);
-
-        offset += ret.written;
-        ptr = realloc(ptr, len, offset, 1) >>> 0;
-    }
-
-    WASM_VECTOR_LEN = offset;
-    return ptr;
-}
-
-function takeFromExternrefTable0(idx) {
-    const value = wasm.__wbindgen_externrefs.get(idx);
-    wasm.__externref_table_dealloc(idx);
-    return value;
-}
-
-let cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true });
-cachedTextDecoder.decode();
-const MAX_SAFARI_DECODE_BYTES = 2146435072;
-let numBytesDecoded = 0;
-function decodeText(ptr, len) {
-    numBytesDecoded += len;
-    if (numBytesDecoded >= MAX_SAFARI_DECODE_BYTES) {
-        cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true });
-        cachedTextDecoder.decode();
-        numBytesDecoded = len;
-    }
-    return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len));
-}
-
-const cachedTextEncoder = new TextEncoder();
-
-if (!('encodeInto' in cachedTextEncoder)) {
-    cachedTextEncoder.encodeInto = function (arg, view) {
-        const buf = cachedTextEncoder.encode(arg);
-        view.set(buf);
-        return {
-            read: arg.length,
-            written: buf.length
-        };
-    };
-}
-
-let WASM_VECTOR_LEN = 0;
-
-let wasmModule, wasm;
-function __wbg_finalize_init(instance, module) {
-    wasm = instance.exports;
-    wasmModule = module;
-    cachedDataViewMemory0 = null;
-    cachedUint8ArrayMemory0 = null;
-    wasm.__wbindgen_start();
-    return wasm;
-}
-
-async function __wbg_load(module, imports) {
-    if (typeof Response === 'function' && module instanceof Response) {
-        if (typeof WebAssembly.instantiateStreaming === 'function') {
-            try {
-                return await WebAssembly.instantiateStreaming(module, imports);
-            } catch (e) {
-                const validResponse = module.ok && expectedResponseType(module.type);
-
-                if (validResponse && module.headers.get('Content-Type') !== 'application/wasm') {
-                    console.warn("`WebAssembly.instantiateStreaming` failed because your server does not serve Wasm with `application/wasm` MIME type. Falling back to `WebAssembly.instantiate` which is slower. Original error:\n", e);
-
-                } else { throw e; }
-            }
-        }
-
-        const bytes = await module.arrayBuffer();
-        return await WebAssembly.instantiate(bytes, imports);
-    } else {
-        const instance = await WebAssembly.instantiate(module, imports);
-
-        if (instance instanceof WebAssembly.Instance) {
-            return { instance, module };
-        } else {
-            return instance;
-        }
-    }
-
-    function expectedResponseType(type) {
-        switch (type) {
-            case 'basic': case 'cors': case 'default': return true;
-        }
-        return false;
-    }
-}
-
-function initSync(module) {
-    if (wasm !== undefined) return wasm;
-
-
-    if (module !== undefined) {
-        if (Object.getPrototypeOf(module) === Object.prototype) {
-            ({module} = module)
-        } else {
-            console.warn('using deprecated parameters for `initSync()`; pass a single object instead')
-        }
-    }
-
-    const imports = __wbg_get_imports();
-    if (!(module instanceof WebAssembly.Module)) {
-        module = new WebAssembly.Module(module);
-    }
-    const instance = new WebAssembly.Instance(module, imports);
-    return __wbg_finalize_init(instance, module);
-}
-
-async function __wbg_init(module_or_path) {
-    if (wasm !== undefined) return wasm;
-
-
-    if (module_or_path !== undefined) {
-        if (Object.getPrototypeOf(module_or_path) === Object.prototype) {
-            ({module_or_path} = module_or_path)
-        } else {
-            console.warn('using deprecated parameters for the initialization function; pass a single object instead')
-        }
-    }
-
-    if (module_or_path === undefined) {
-        module_or_path = new URL('garu_wasm_bg.wasm', import.meta.url);
-    }
-    const imports = __wbg_get_imports();
-
-    if (typeof module_or_path === 'string' || (typeof Request === 'function' && module_or_path instanceof Request) || (typeof URL === 'function' && module_or_path instanceof URL)) {
-        module_or_path = fetch(module_or_path);
-    }
-
-    const { instance, module } = await __wbg_load(await module_or_path, imports);
-
-    return __wbg_finalize_init(instance, module);
-}
-
-export { initSync, __wbg_init as default };

package/pkg/garu_wasm_bg.wasm.d.ts

@@ -1,15 +0,0 @@
-/* tslint:disable */
-/* eslint-disable */
-export const memory: WebAssembly.Memory;
-export const __wbg_garuwasm_free: (a: number, b: number) => void;
-export const garuwasm_analyze: (a: number, b: number, c: number) => [number, number, number];
-export const garuwasm_analyze_topn: (a: number, b: number, c: number, d: number) => [number, number, number];
-export const garuwasm_new: (a: number, b: number) => [number, number, number];
-export const garuwasm_tokenize: (a: number, b: number, c: number) => [number, number, number];
-export const garuwasm_version: () => [number, number];
-export const __wbindgen_malloc: (a: number, b: number) => number;
-export const __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;
-export const __wbindgen_externrefs: WebAssembly.Table;
-export const __externref_table_dealloc: (a: number) => void;
-export const __wbindgen_free: (a: number, b: number, c: number) => void;
-export const __wbindgen_start: () => void;

package/pkg/package.json

@@ -1,15 +0,0 @@
-{
-  "name": "garu-wasm",
-  "type": "module",
-  "version": "0.2.10",
-  "files": [
-    "garu_wasm_bg.wasm",
-    "garu_wasm.js",
-    "garu_wasm.d.ts"
-  ],
-  "main": "garu_wasm.js",
-  "types": "garu_wasm.d.ts",
-  "sideEffects": [
-    "./snippets/*"
-  ]
-}

package/pkg/snippets/garu-core-82ebac66ba12d856/inline0.js

@@ -1 +0,0 @@
-export function performance_now() { return performance.now(); }

package/pkg/snippets/garu-core-8bb32ce2ac7b2d7b/inline0.js

@@ -1 +0,0 @@
-export function performance_now() { return performance.now(); }

package/pkg/snippets/garu-core-c91d368e7251f313/inline0.js

@@ -1 +0,0 @@
-export function performance_now() { return performance.now(); }

package/pkg/snippets/garu-core-d0bd202007f7afa3/inline0.js

@@ -1 +0,0 @@
-export function performance_now() { return performance.now(); }

package/pkg/snippets/garu-core-ea50edba4dc5fdae/inline0.js

@@ -1 +0,0 @@
-export function performance_now() { return performance.now(); }