@oomfware/lang-detect 0.1.0

package/dist/nn/forward.js

@@ -0,0 +1,154 @@
+// #region types
+// #endregion
+// #region dequantization
+/**
+ * dequantizes an int8 array back to float32 using its absmax scale.
+ *
+ * @param data quantized int8 values
+ * @param scale the scale factor used during quantization (scaleMax / absmax)
+ * @returns dequantized float32 array
+ */
+const dequantize = (data, scale) => {
+    const result = new Float32Array(data.length);
+    for (let i = 0; i < data.length; i++) {
+        result[i] = data[i] / scale;
+    }
+    return result;
+};
+/**
+ * unpacks 6-bit packed bytes into signed int8 values.
+ *
+ * packing scheme: 4 values (6 bits each, unsigned offset by +31) → 3 bytes.
+ * byte0 = (u0 << 2) | (u1 >> 4)
+ * byte1 = ((u1 & 0x0F) << 4) | (u2 >> 2)
+ * byte2 = ((u2 & 0x03) << 6) | u3
+ *
+ * @param packed packed 6-bit data
+ * @param count number of original values
+ * @returns signed int8 values in [-31, 31]
+ */
+const unpack6 = (packed, count) => {
+    const result = new Int8Array(count);
+    let ri = 0;
+    let pi = 0;
+    // process full groups of 4
+    const fullGroups = (count >> 2) << 2;
+    while (ri < fullGroups) {
+        const b0 = packed[pi];
+        const b1 = packed[pi + 1];
+        const b2 = packed[pi + 2];
+        result[ri] = (b0 >> 2) - 31;
+        result[ri + 1] = (((b0 & 0x03) << 4) | (b1 >> 4)) - 31;
+        result[ri + 2] = (((b1 & 0x0f) << 2) | (b2 >> 6)) - 31;
+        result[ri + 3] = (b2 & 0x3f) - 31;
+        ri += 4;
+        pi += 3;
+    }
+    // remainder (1-3 values)
+    const rem = count - fullGroups;
+    if (rem >= 1) {
+        result[ri] = (packed[pi] >> 2) - 31;
+    }
+    if (rem >= 2) {
+        result[ri + 1] = (((packed[pi] & 0x03) << 4) | (packed[pi + 1] >> 4)) - 31;
+    }
+    if (rem >= 3) {
+        result[ri + 2] = (((packed[pi + 1] & 0x0f) << 2) | (packed[pi + 2] >> 6)) - 31;
+    }
+    return result;
+};
+/**
+ * loads int8 quantized weights from a binary buffer and dequantizes to float32.
+ *
+ * binary format: 2 × f32 scales (wScale, bScale), then weight bytes, then bias bytes.
+ *
+ * @param bin raw binary weight data
+ * @param inputSize number of input features
+ * @param outputSize number of output classes
+ * @returns dequantized model weights
+ */
+export const loadWeights = (bin, inputSize, outputSize) => {
+    const view = new DataView(bin);
+    const wScale = view.getFloat32(0, true);
+    const bScale = view.getFloat32(4, true);
+    const wSize = outputSize * inputSize;
+    const w = new Int8Array(bin, 8, wSize);
+    const b = new Int8Array(bin, 8 + wSize, outputSize);
+    return {
+        w: dequantize(w, wScale),
+        b: dequantize(b, bScale),
+        inputSize,
+        outputSize,
+    };
+};
+/**
+ * loads int6 packed quantized weights from a binary buffer and dequantizes to float32.
+ *
+ * same header as int8 (2 × f32 scales), but payload is 6-bit packed.
+ *
+ * @param bin raw binary weight data
+ * @param inputSize number of input features
+ * @param outputSize number of output classes
+ * @returns dequantized model weights
+ */
+export const loadWeights6 = (bin, inputSize, outputSize) => {
+    const view = new DataView(bin);
+    const wScale = view.getFloat32(0, true);
+    const bScale = view.getFloat32(4, true);
+    const wCount = outputSize * inputSize;
+    const wPackedSize = Math.ceil((wCount * 3) / 4);
+    const bPackedSize = Math.ceil((outputSize * 3) / 4);
+    const wPacked = new Uint8Array(bin, 8, wPackedSize);
+    const bPacked = new Uint8Array(bin, 8 + wPackedSize, bPackedSize);
+    return {
+        w: dequantize(unpack6(wPacked, wCount), wScale),
+        b: dequantize(unpack6(bPacked, outputSize), bScale),
+        inputSize,
+        outputSize,
+    };
+};
+// #endregion
+// #region forward pass
+/**
+ * applies softmax in-place to an output array.
+ *
+ * @param output logit array to convert to probabilities
+ */
+const softmax = (output) => {
+    let max = -Infinity;
+    for (let i = 0; i < output.length; i++) {
+        if (output[i] > max) {
+            max = output[i];
+        }
+    }
+    let expSum = 0;
+    for (let i = 0; i < output.length; i++) {
+        output[i] = Math.exp(output[i] - max);
+        expSum += output[i];
+    }
+    for (let i = 0; i < output.length; i++) {
+        output[i] /= expSum;
+    }
+};
+/**
+ * forward pass for a linear model: dense → softmax.
+ *
+ * @param input input feature vector (ngram frequencies)
+ * @param m model weights
+ * @returns output probabilities (one per language in the group)
+ */
+export const forward = (input, m) => {
+    const output = new Float32Array(m.outputSize);
+    for (let i = 0; i < m.outputSize; i++) {
+        let sum = m.b[i];
+        const off = i * m.inputSize;
+        for (let j = 0; j < m.inputSize; j++) {
+            sum += input[j] * m.w[off + j];
+        }
+        output[i] = sum;
+    }
+    softmax(output);
+    return output;
+};
+// #endregion
+//# sourceMappingURL=forward.js.map

package/dist/nn/forward.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"forward.js","sourceRoot":"","sources":["../../src/nn/forward.ts"],"names":[],"mappings":"AAAA,gBAAgB;AAUhB,aAAa;AAEb,yBAAyB;AAEzB;;;;;;GAMG;AACH,MAAM,UAAU,GAAG,CAAC,IAAe,EAAE,KAAa,EAAgB,EAAE;IACnE,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC7C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC;IAC7B,CAAC;IACD,OAAO,MAAM,CAAC;AACf,CAAC,CAAC;AAEF;;;;;;;;;;;GAWG;AACH,MAAM,OAAO,GAAG,CAAC,MAAkB,EAAE,KAAa,EAAa,EAAE;IAChE,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC;IACpC,IAAI,EAAE,GAAG,CAAC,CAAC;IACX,IAAI,EAAE,GAAG,CAAC,CAAC;IAEX,2BAA2B;IAC3B,MAAM,UAAU,GAAG,CAAC,KAAK,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;IACrC,OAAO,EAAE,GAAG,UAAU,EAAE,CAAC;QACxB,MAAM,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC;QACtB,MAAM,EAAE,GAAG,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;QAC1B,MAAM,EAAE,GAAG,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;QAC1B,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC;QAC5B,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;QACvD,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;QACvD,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAClC,EAAE,IAAI,CAAC,CAAC;QACR,EAAE,IAAI,CAAC,CAAC;IACT,CAAC;IAED,yBAAyB;IACzB,MAAM,GAAG,GAAG,KAAK,GAAG,UAAU,CAAC;IAC/B,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC;QACd,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC;IACrC,CAAC;IACD,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC;QACd,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;IAC5E,CAAC;IACD,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC;QACd,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;IAChF,CAAC;IAED,OAAO,MAAM,CAAC;AACf,CAAC,CAAC;AAEF;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,GAAgB,EAAE,SAAiB,EAAE,UAAkB,EAAgB,EAAE;IACpG,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,GAAG,CAAC,CAAC;IAC/B,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IAExC,MAAM,KAAK,GAAG,UAAU,GAAG,SAAS,CAAC;IACrC,MAAM,CAAC,GAAG,IAAI,SAAS,CAAC,GAAG,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC;IACvC,MAAM,CAAC,GAAG,IAAI,SAAS,CAAC,GAAG,EAAE,CAAC,GAAG,KAAK,EAAE,UAAU,CAAC,CAAC;IAEpD,OAAO;QACN,CAAC,EAAE,UAAU,CAAC,CAAC,EAAE,MAAM,CAAC;QACxB,CAAC,EAAE,UAAU,CAAC,CAAC,EAAE,MAAM,CAAC;QACxB,SAAS;QACT,UAAU;KACV,CAAC;AACH,CAAC,CAAC;AAEF;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,GAAgB,EAAE,SAAiB,EAAE,UAAkB,EAAgB,EAAE;IACrG,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,GAAG,CAAC,CAAC;IAC/B,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IAExC,MAAM,MAAM,GAAG,UAAU,GAAG,SAAS,CAAC;IACtC,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAChD,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAEpD,MAAM,OAAO,GAAG,IAAI,UAAU,CAAC,GAAG,EAAE,CAAC,EAAE,WAAW,CAAC,CAAC;IACpD,MAAM,OAAO,GAAG,IAAI,UAAU,CAAC,GAAG,EAAE,CAAC,GAAG,WAAW,EAAE,WAAW,CAAC,CAAC;IAElE,OAAO;QACN,CAAC,EAAE,UAAU,CAAC,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC,EAAE,MAAM,CAAC;QAC/C,CAAC,EAAE,UAAU,CAAC,OAAO,CAAC,OAAO,EAAE,UAAU,CAAC,EAAE,MAAM,CAAC;QACnD,SAAS;QACT,UAAU;KACV,CAAC;AACH,CAAC,CAAC;AAEF,aAAa;AAEb,uBAAuB;AAEvB;;;;GAIG;AACH,MAAM,OAAO,GAAG,CAAC,MAAoB,EAAQ,EAAE;IAC9C,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC;IACpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,IAAI,MAAM,CAAC,CAAC,CAAC,GAAG,GAAG,EAAE,CAAC;YACrB,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACjB,CAAC;IACF,CAAC;IACD,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC;QACtC,MAAM,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;IACrB,CAAC;IACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC;IACrB,CAAC;AACF,CAAC,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,KAAmB,EAAE,CAAe,EAAgB,EAAE;IAC7E,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;IAC9C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACjB,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC;QAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,GAAG,IAAI,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;QAChC,CAAC;QACD,MAAM,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC;IACjB,CAAC;IAED,OAAO,CAAC,MAAM,CAAC,CAAC;IAChB,OAAO,MAAM,CAAC;AACf,CAAC,CAAC;AAEF,aAAa"}

package/dist/nn/groups.d.ts

@@ -0,0 +1,23 @@
+/** languages identifiable by unique Unicode script ranges alone (no NN needed). */
+declare const UNIQUE_SCRIPT_LANGS: Record<string, (text: string) => boolean>;
+/**
+ * detects CJK languages by script. Japanese is identified by kana; if only
+ * Han characters are present, defaults to Mandarin Chinese.
+ *
+ * @param text input text
+ * @returns `'jpn'`, `'cmn'`, or `null` if no CJK script detected
+ */
+declare const detectCJK: (text: string) => string | null;
+/** a script group with its languages and detection test. */
+type Group = {
+    langs: string[];
+    test: (text: string) => boolean;
+};
+/**
+ * script-family groups for NN-based detection.
+ * architecture sizes are defined in the weight metadata (.json files).
+ */
+declare const GROUPS: Record<string, Group>;
+export type { Group };
+export { UNIQUE_SCRIPT_LANGS, detectCJK, GROUPS };
+//# sourceMappingURL=groups.d.ts.map

package/dist/nn/groups.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"groups.d.ts","sourceRoot":"","sources":["../../src/nn/groups.ts"],"names":[],"mappings":"AAEA,mFAAmF;AACnF,QAAA,MAAM,mBAAmB,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAOlE,CAAC;AAEF;;;;;;GAMG;AACH,QAAA,MAAM,SAAS,GAAI,MAAM,MAAM,KAAG,MAAM,GAAG,IAQ1C,CAAC;AAMF,4DAA4D;AAC5D,KAAK,KAAK,GAAG;IACZ,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,IAAI,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC;CAChC,CAAC;AAEF;;;GAGG;AACH,QAAA,MAAM,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,CAgDjC,CAAC;AAIF,YAAY,EAAE,KAAK,EAAE,CAAC;AACtB,OAAO,EAAE,mBAAmB,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC"}

package/dist/nn/groups.js

@@ -0,0 +1,81 @@
+// #region script detection
+/** languages identifiable by unique Unicode script ranges alone (no NN needed). */
+const UNIQUE_SCRIPT_LANGS = {
+    kor: (t) => /[\uAC00-\uD7AF\u1100-\u11FF]/.test(t),
+    kat: (t) => /[\u10A0-\u10FF\u2D00-\u2D2F]/.test(t),
+    hye: (t) => /[\u0530-\u058F]/.test(t),
+    ben: (t) => /[\u0980-\u09FF]/.test(t),
+    ell: (t) => /[\u0370-\u03FF\u1F00-\u1FFF]/.test(t),
+    heb: (t) => /[\u0590-\u05FF]/.test(t),
+};
+/**
+ * detects CJK languages by script. Japanese is identified by kana; if only
+ * Han characters are present, defaults to Mandarin Chinese.
+ *
+ * @param text input text
+ * @returns `'jpn'`, `'cmn'`, or `null` if no CJK script detected
+ */
+const detectCJK = (text) => {
+    if (/[\u3040-\u309F\u30A0-\u30FF]/.test(text)) {
+        return 'jpn';
+    }
+    if (/[\u4E00-\u9FFF\u3400-\u4DBF]/.test(text)) {
+        return 'cmn';
+    }
+    return null;
+};
+/**
+ * script-family groups for NN-based detection.
+ * architecture sizes are defined in the weight metadata (.json files).
+ */
+const GROUPS = {
+    cyrillic: {
+        langs: ['bel', 'bul', 'kaz', 'mkd', 'rus', 'srp', 'ukr'],
+        test: (t) => /[\u0400-\u04FF]/.test(t),
+    },
+    arabic: {
+        langs: ['ara', 'ckb', 'pes'],
+        test: (t) => /[\u0600-\u06FF\u0750-\u077F]/.test(t),
+    },
+    devanagari: {
+        langs: ['hin', 'mar'],
+        test: (t) => /[\u0900-\u097F]/.test(t),
+    },
+    latin: {
+        langs: [
+            'afr',
+            'aze',
+            'cat',
+            'ces',
+            'dan',
+            'deu',
+            'eng',
+            'est',
+            'eus',
+            'fin',
+            'fra',
+            'hau',
+            'hrv',
+            'hun',
+            'ind',
+            'isl',
+            'ita',
+            'lit',
+            'nld',
+            'nob',
+            'pol',
+            'por',
+            'ron',
+            'run',
+            'slk',
+            'spa',
+            'swe',
+            'tgl',
+            'tur',
+            'vie',
+        ],
+        test: (t) => /[a-zA-Z\u00C0-\u024F]/.test(t),
+    },
+};
+export { UNIQUE_SCRIPT_LANGS, detectCJK, GROUPS };
+//# sourceMappingURL=groups.js.map

package/dist/nn/groups.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"groups.js","sourceRoot":"","sources":["../../src/nn/groups.ts"],"names":[],"mappings":"AAAA,2BAA2B;AAE3B,mFAAmF;AACnF,MAAM,mBAAmB,GAA8C;IACtE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,8BAA8B,CAAC,IAAI,CAAC,CAAC,CAAC;IAClD,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,8BAA8B,CAAC,IAAI,CAAC,CAAC,CAAC;IAClD,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC;IACrC,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC;IACrC,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,8BAA8B,CAAC,IAAI,CAAC,CAAC,CAAC;IAClD,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC;CACrC,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,SAAS,GAAG,CAAC,IAAY,EAAiB,EAAE;IACjD,IAAI,8BAA8B,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC/C,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IAAI,8BAA8B,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC/C,OAAO,KAAK,CAAC;IACd,CAAC;IACD,OAAO,IAAI,CAAC;AACb,CAAC,CAAC;AAYF;;;GAGG;AACH,MAAM,MAAM,GAA0B;IACrC,QAAQ,EAAE;QACT,KAAK,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC;QACxD,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC;KACtC;IACD,MAAM,EAAE;QACP,KAAK,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC;QAC5B,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,8BAA8B,CAAC,IAAI,CAAC,CAAC,CAAC;KACnD;IACD,UAAU,EAAE;QACX,KAAK,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC;QACrB,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC;KACtC;IACD,KAAK,EAAE;QACN,KAAK,EAAE;YACN,KAAK;YACL,KAAK;YACL,KAAK;YACL,KAAK;YACL,KAAK;YACL,KAAK;YACL,KAAK;YACL,KAAK;YACL,KAAK;YACL,KAAK;YACL,KAAK;YACL,KAAK;YACL,KAAK;YACL,KAAK;YACL,KAAK;YACL,KAAK;YACL,KAAK;YACL,KAAK;YACL,KAAK;YACL,KAAK;YACL,KAAK;YACL,KAAK;YACL,KAAK;YACL,KAAK;YACL,KAAK;YACL,KAAK;YACL,KAAK;YACL,KAAK;YACL,KAAK;YACL,KAAK;SACL;QACD,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,uBAAuB,CAAC,IAAI,CAAC,CAAC,CAAC;KAC5C;CACD,CAAC;AAKF,OAAO,EAAE,mBAAmB,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC"}

package/dist/nn/load.d.ts

@@ -0,0 +1,15 @@
+/**
+ * loads binary data from a URL via fetch.
+ *
+ * @param url URL to fetch
+ * @returns the response body as an ArrayBuffer
+ */
+export declare const loadBinary: (url: URL) => Promise<ArrayBuffer>;
+/**
+ * loads and parses JSON from a URL via fetch.
+ *
+ * @param url URL to fetch
+ * @returns the parsed JSON value
+ */
+export declare const loadJson: (url: URL) => Promise<unknown>;
+//# sourceMappingURL=load.d.ts.map

package/dist/nn/load.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"load.d.ts","sourceRoot":"","sources":["../../src/nn/load.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,eAAO,MAAM,UAAU,GAAU,KAAK,GAAG,KAAG,OAAO,CAAC,WAAW,CAG9D,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,QAAQ,GAAU,KAAK,GAAG,KAAG,OAAO,CAAC,OAAO,CAGxD,CAAC"}

package/dist/nn/load.js

@@ -0,0 +1,21 @@
+/**
+ * loads binary data from a URL via fetch.
+ *
+ * @param url URL to fetch
+ * @returns the response body as an ArrayBuffer
+ */
+export const loadBinary = async (url) => {
+    const response = await fetch(url);
+    return response.arrayBuffer();
+};
+/**
+ * loads and parses JSON from a URL via fetch.
+ *
+ * @param url URL to fetch
+ * @returns the parsed JSON value
+ */
+export const loadJson = async (url) => {
+    const response = await fetch(url);
+    return response.json();
+};
+//# sourceMappingURL=load.js.map

package/dist/nn/load.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"load.js","sourceRoot":"","sources":["../../src/nn/load.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,KAAK,EAAE,GAAQ,EAAwB,EAAE;IAClE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;IAClC,OAAO,QAAQ,CAAC,WAAW,EAAE,CAAC;AAC/B,CAAC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,QAAQ,GAAG,KAAK,EAAE,GAAQ,EAAoB,EAAE;IAC5D,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;IAClC,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;AACxB,CAAC,CAAC"}

package/dist/nn/load.node.d.ts

@@ -0,0 +1,15 @@
+/**
+ * loads binary data from a file URL using node:fs.
+ *
+ * @param url file URL to load
+ * @returns the file contents as an ArrayBuffer
+ */
+export declare const loadBinary: (url: URL) => Promise<ArrayBuffer>;
+/**
+ * loads and parses JSON from a file URL using node:fs.
+ *
+ * @param url file URL to load
+ * @returns the parsed JSON value
+ */
+export declare const loadJson: (url: URL) => Promise<unknown>;
+//# sourceMappingURL=load.node.d.ts.map

package/dist/nn/load.node.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"load.node.d.ts","sourceRoot":"","sources":["../../src/nn/load.node.ts"],"names":[],"mappings":"AAGA;;;;;GAKG;AACH,eAAO,MAAM,UAAU,GAAU,KAAK,GAAG,KAAG,OAAO,CAAC,WAAW,CAG9D,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,QAAQ,GAAU,KAAK,GAAG,KAAG,OAAO,CAAC,OAAO,CAGxD,CAAC"}

package/dist/nn/load.node.js

@@ -0,0 +1,23 @@
+import { readFileSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+/**
+ * loads binary data from a file URL using node:fs.
+ *
+ * @param url file URL to load
+ * @returns the file contents as an ArrayBuffer
+ */
+export const loadBinary = async (url) => {
+    const buffer = readFileSync(fileURLToPath(url));
+    return buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);
+};
+/**
+ * loads and parses JSON from a file URL using node:fs.
+ *
+ * @param url file URL to load
+ * @returns the parsed JSON value
+ */
+export const loadJson = async (url) => {
+    const content = readFileSync(fileURLToPath(url), 'utf-8');
+    return JSON.parse(content);
+};
+//# sourceMappingURL=load.node.js.map

package/dist/nn/load.node.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"load.node.js","sourceRoot":"","sources":["../../src/nn/load.node.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC;;;;;GAKG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,KAAK,EAAE,GAAQ,EAAwB,EAAE;IAClE,MAAM,MAAM,GAAG,YAAY,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC;IAChD,OAAO,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;AACtF,CAAC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,QAAQ,GAAG,KAAK,EAAE,GAAQ,EAAoB,EAAE;IAC5D,MAAM,OAAO,GAAG,YAAY,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC,CAAC;IAC1D,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;AAC5B,CAAC,CAAC"}

package/dist/nn/normalize.d.ts

@@ -0,0 +1,17 @@
+/**
+ * normalizes text for ngram extraction: lowercases, strips non-letter/non-mark
+ * characters, collapses whitespace, and pads with spaces.
+ *
+ * @param text raw input text
+ * @returns normalized text padded with leading/trailing spaces
+ */
+export declare const normalize: (text: string) => string;
+/**
+ * extracts ngram frequencies from a string.
+ *
+ * @param text normalized text (from {@link normalize})
+ * @param length ngram length (1 for unigrams, 2 for bigrams, etc.)
+ * @returns map of ngram string to its relative frequency (count / total)
+ */
+export declare const extractNgrams: (text: string, length: number) => Record<string, number>;
+//# sourceMappingURL=normalize.d.ts.map

package/dist/nn/normalize.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"normalize.d.ts","sourceRoot":"","sources":["../../src/nn/normalize.ts"],"names":[],"mappings":"AAIA;;;;;;GAMG;AACH,eAAO,MAAM,SAAS,GAAI,MAAM,MAAM,KAAG,MAExC,CAAC;AAEF;;;;;;GAMG;AACH,eAAO,MAAM,aAAa,GAAI,MAAM,MAAM,EAAE,QAAQ,MAAM,KAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAejF,CAAC"}

package/dist/nn/normalize.js

@@ -0,0 +1,34 @@
+const HYPHEN_RE = /-+/g;
+const NON_LETTER_RE = /[^\p{L}\p{M}\s]/gu;
+const MULTI_SPACE_RE = /\s{2,}/g;
+/**
+ * normalizes text for ngram extraction: lowercases, strips non-letter/non-mark
+ * characters, collapses whitespace, and pads with spaces.
+ *
+ * @param text raw input text
+ * @returns normalized text padded with leading/trailing spaces
+ */
+export const normalize = (text) => {
+    return ` ${text.replace(HYPHEN_RE, ' ').replace(NON_LETTER_RE, '').replace(MULTI_SPACE_RE, ' ').toLowerCase().trim()} `;
+};
+/**
+ * extracts ngram frequencies from a string.
+ *
+ * @param text normalized text (from {@link normalize})
+ * @param length ngram length (1 for unigrams, 2 for bigrams, etc.)
+ * @returns map of ngram string to its relative frequency (count / total)
+ */
+export const extractNgrams = (text, length) => {
+    const ngrams = {};
+    let total = 0;
+    for (let i = 0, l = text.length - length; i <= l; i++) {
+        const value = text.slice(i, i + length);
+        ngrams[value] = (ngrams[value] || 0) + 1;
+        total++;
+    }
+    for (const value in ngrams) {
+        ngrams[value] /= total;
+    }
+    return ngrams;
+};
+//# sourceMappingURL=normalize.js.map

package/dist/nn/normalize.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"normalize.js","sourceRoot":"","sources":["../../src/nn/normalize.ts"],"names":[],"mappings":"AAAA,MAAM,SAAS,GAAG,KAAK,CAAC;AACxB,MAAM,aAAa,GAAG,mBAAmB,CAAC;AAC1C,MAAM,cAAc,GAAG,SAAS,CAAC;AAEjC;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,IAAY,EAAU,EAAE;IACjD,OAAO,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,GAAG,CAAC;AACzH,CAAC,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,IAAY,EAAE,MAAc,EAA0B,EAAE;IACrF,MAAM,MAAM,GAA2B,EAAE,CAAC;IAC1C,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACvD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,CAAC;QACxC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QACzC,KAAK,EAAE,CAAC;IACT,CAAC;IAED,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,MAAM,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC;IACxB,CAAC;IAED,OAAO,MAAM,CAAC;AACf,CAAC,CAAC"}

package/package.json

@@ -0,0 +1,61 @@
+{
+  "name": "@oomfware/lang-detect",
+  "version": "0.1.0",
+  "description": "natural language detection library",
+  "license": "0BSD",
+  "repository": {
+    "type": "git",
+    "url": "https://codeberg.org/oomfware/lang-detect"
+  },
+  "files": [
+    "dist/",
+    "src/",
+    "weights/",
+    "!src/**/*.bench.ts",
+    "!src/**/*.test.ts"
+  ],
+  "type": "module",
+  "types": "./dist/index.d.ts",
+  "imports": {
+    "#load": {
+      "source": "./src/nn/load.node.ts",
+      "node": "./dist/nn/load.node.js",
+      "default": "./dist/nn/load.js"
+    }
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    },
+    "./lite": {
+      "types": "./dist/lite.d.ts",
+      "default": "./dist/lite.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "devDependencies": {
+    "@types/node": "^25.3.3",
+    "bumpp": "^10.4.1",
+    "lande": "^1.0.10",
+    "oxfmt": "^0.36.0",
+    "oxlint": "^1.51.0",
+    "typescript": "^5.9.3"
+  },
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsc --watch",
+    "typecheck": "tsc --noEmit",
+    "fmt": "oxfmt",
+    "lint": "oxlint",
+    "eval": "node --conditions source src/eval.ts",
+    "test": "node --test --conditions source src/**/*.test.ts",
+    "download-dataset": "cd train && bash download_dataset.sh",
+    "export": "pnpm run export:standard && pnpm run export:lite",
+    "export:standard": "cd train && uv run export.py -e pruned_mega_aug50 -o ../weights/standard",
+    "export:lite": "cd train && uv run export.py -e pruned_mega_aug50 -o ../weights/lite --quant-bits 6"
+  }
+}

package/src/eval.ts

@@ -0,0 +1,173 @@
+/**
+ * evaluate detection accuracy against the UDHR dataset.
+ *
+ * usage:
+ *   node --conditions source src/eval.ts [--lite] [--lande]
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { parseArgs } from 'node:util';
+
+import { create } from './nn/detect.ts';
+
+const { values: args } = parseArgs({
+	options: {
+		lite: { type: 'boolean', default: false },
+		lande: { type: 'boolean', default: false },
+	},
+});
+
+const variant = args.lite ? 'lite' : 'standard';
+const quantBits = args.lite ? 6 : 8;
+
+const weightsDir = path.resolve(import.meta.dirname!, '..', 'weights', variant);
+const { initialize, detect } = create(
+	{
+		cyrillic: {
+			weights: new URL(`file://${path.join(weightsDir, 'cyrillic.bin')}`),
+			meta: new URL(`file://${path.join(weightsDir, 'cyrillic.json')}`),
+		},
+		arabic: {
+			weights: new URL(`file://${path.join(weightsDir, 'arabic.bin')}`),
+			meta: new URL(`file://${path.join(weightsDir, 'arabic.json')}`),
+		},
+		devanagari: {
+			weights: new URL(`file://${path.join(weightsDir, 'devanagari.bin')}`),
+			meta: new URL(`file://${path.join(weightsDir, 'devanagari.json')}`),
+		},
+		latin: {
+			weights: new URL(`file://${path.join(weightsDir, 'latin.bin')}`),
+			meta: new URL(`file://${path.join(weightsDir, 'latin.json')}`),
+		},
+	},
+	quantBits,
+);
+
+// ── UDHR code → ISO 639-3 mapping ──
+
+const UDHR_CODE_TO_LANG: Record<string, string> = {
+	afr: 'afr',
+	bel: 'bel',
+	ben: 'ben',
+	bul: 'bul',
+	cat: 'cat',
+	ces: 'ces',
+	ckb: 'ckb',
+	cmn_hans: 'cmn',
+	dan: 'dan',
+	deu_1996: 'deu',
+	ell_monotonic: 'ell',
+	eng: 'eng',
+	eus: 'eus',
+	fin: 'fin',
+	fra: 'fra',
+	hau_NG: 'hau',
+	heb: 'heb',
+	hin: 'hin',
+	hrv: 'hrv',
+	hun: 'hun',
+	hye: 'hye',
+	ind: 'ind',
+	isl: 'isl',
+	ita: 'ita',
+	jpn: 'jpn',
+	kat: 'kat',
+	kaz: 'kaz',
+	kor: 'kor',
+	lit: 'lit',
+	mar: 'mar',
+	mkd: 'mkd',
+	nld: 'nld',
+	nob: 'nob',
+	pes_1: 'pes',
+	pol: 'pol',
+	por_BR: 'por',
+	por_PT: 'por',
+	ron_2006: 'ron',
+	run: 'run',
+	rus: 'rus',
+	slk: 'slk',
+	spa: 'spa',
+	srp_cyrl: 'srp',
+	srp_latn: 'srp',
+	swe: 'swe',
+	tgl: 'tgl',
+	tur: 'tur',
+	ukr: 'ukr',
+	vie: 'vie',
+};
+
+const TAG_RE = /<[^>]+>/g;
+
+// ── load UDHR sentences ──
+
+const declDir = path.resolve(import.meta.dirname!, '..', 'train', 'resources', 'udhr', 'declaration');
+const sentences: { lang: string; text: string }[] = [];
+
+for (const [code, lang] of Object.entries(UDHR_CODE_TO_LANG)) {
+	const htmlFile = path.join(declDir, `${code}.html`);
+	if (!fs.existsSync(htmlFile)) {
+		continue;
+	}
+
+	const content = fs.readFileSync(htmlFile, 'utf-8');
+	for (const match of content.matchAll(/<p>(.*?)<\/p>/gs)) {
+		const text = match[1].replace(TAG_RE, '').trim();
+		if (text.length < 10) {
+			continue;
+		}
+		sentences.push({ lang, text });
+	}
+}
+
+// ── helpers ──
+
+type Stats = { pass: number; total: number };
+
+const evaluate = (name: string, detectFn: (text: string) => string | undefined) => {
+	const perLang: Record<string, Stats> = {};
+	let totalPass = 0;
+
+	for (const { lang, text } of sentences) {
+		perLang[lang] ??= { pass: 0, total: 0 };
+		perLang[lang].total++;
+
+		if (detectFn(text) === lang) {
+			perLang[lang].pass++;
+			totalPass++;
+		}
+	}
+
+	const overallAcc = (totalPass / sentences.length) * 100;
+
+	console.log(`\n=== ${name} ===`);
+	console.log(`${sentences.length} sentences, ${Object.keys(perLang).length} languages`);
+	console.log(`overall accuracy: ${overallAcc.toFixed(2)}%`);
+
+	const sorted = Object.entries(perLang).sort((a, b) => a[1].pass / a[1].total - b[1].pass / b[1].total);
+	for (const [lang, stats] of sorted) {
+		const acc = (stats.pass / stats.total) * 100;
+		if (acc < 100) {
+			console.log(`  ${lang}: ${acc.toFixed(1)}% (${stats.total})`);
+		}
+	}
+};
+
+// ── evaluate ──
+
+await initialize();
+
+evaluate(`UDHR: ${variant} (${quantBits}-bit)`, (text) => {
+	const result = detect(text);
+	return result[0]?.[0];
+});
+
+if (args.lande) {
+	// eslint-disable-next-line @typescript-eslint/no-require-imports
+	const { default: lande } = await import('lande');
+	evaluate('UDHR: lande', (text) => {
+		const result = lande(text);
+		return result?.[0]?.[0];
+	});
+}