@try-works/tool-mode-runtime-litert 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/lib/ear-frontend.d.ts +113 -0
  2. package/lib/ear-frontend.d.ts.map +1 -0
  3. package/lib/ear-frontend.js +258 -0
  4. package/lib/ear.d.ts +51 -0
  5. package/lib/ear.d.ts.map +1 -0
  6. package/lib/ear.js +296 -0
  7. package/lib/emo-tokenizer.d.ts +87 -0
  8. package/lib/emo-tokenizer.d.ts.map +1 -0
  9. package/lib/emo-tokenizer.js +365 -0
  10. package/lib/emo.d.ts +53 -0
  11. package/lib/emo.d.ts.map +1 -0
  12. package/lib/emo.js +218 -0
  13. package/lib/errors.d.ts +14 -0
  14. package/lib/errors.d.ts.map +1 -0
  15. package/lib/errors.js +29 -0
  16. package/lib/index.d.ts +28 -0
  17. package/lib/index.d.ts.map +1 -0
  18. package/lib/index.js +34 -0
  19. package/lib/model.d.ts +103 -0
  20. package/lib/model.d.ts.map +1 -0
  21. package/lib/model.js +112 -0
  22. package/lib/moderator.d.ts +59 -0
  23. package/lib/moderator.d.ts.map +1 -0
  24. package/lib/moderator.js +189 -0
  25. package/lib/pillow.d.ts +75 -0
  26. package/lib/pillow.d.ts.map +1 -0
  27. package/lib/pillow.js +187 -0
  28. package/lib/png.d.ts +27 -0
  29. package/lib/png.d.ts.map +1 -0
  30. package/lib/png.js +176 -0
  31. package/lib/processor.d.ts +43 -0
  32. package/lib/processor.d.ts.map +1 -0
  33. package/lib/processor.js +48 -0
  34. package/lib/provider-factory.d.ts +44 -0
  35. package/lib/provider-factory.d.ts.map +1 -0
  36. package/lib/provider-factory.js +219 -0
  37. package/lib/runtime.d.ts +41 -0
  38. package/lib/runtime.d.ts.map +1 -0
  39. package/lib/runtime.js +66 -0
  40. package/lib/shapes.d.ts +28 -0
  41. package/lib/shapes.d.ts.map +1 -0
  42. package/lib/shapes.js +187 -0
  43. package/lib/shim.d.ts +45 -0
  44. package/lib/shim.d.ts.map +1 -0
  45. package/lib/shim.js +215 -0
  46. package/lib/wav.d.ts +39 -0
  47. package/lib/wav.d.ts.map +1 -0
  48. package/lib/wav.js +134 -0
  49. package/package.json +47 -4
  50. package/README.md +0 -3
package/lib/ear.js ADDED
@@ -0,0 +1,296 @@
1
+ /**
2
+ * Processor ear/mel-v1: spoken-language identification over the ear LiteRT artifact.
3
+ *
4
+ * PRIMARY SOURCE (Desert-Ant-Labs/desert-ant-core @ baf4e6e3b4655c8e54aa8d5a25215b5f35f899a7):
5
+ * - Sources/Ear/Model.swift:19 languages from languages.json; :72-102 identify (linear resample,
6
+ * window offsets, per-window softmax and probability averaging, alias folding, top-5);
7
+ * :104-114 tensor name/shape [1, 80, 3000]; :119-123 linear resample.
8
+ * - Sources/Ear/Ear.swift:29-34 isReliable; :54 reliableMargin = 0.25; :113 defaultWindows = 3;
9
+ * :156-165 non-empty audio and windows > 0.
10
+ * - Sources/Ear/Languages.swift:4-8 aliases (tl->fil, nb->no, yue->zh); :20 confusables
11
+ * (no, sv, da); :22-24 canonicalLanguage.
12
+ * - Sources/Ear/Frontend.swift + AudioDSP: the log-mel front end and window selection, ported
13
+ * in ear-frontend.ts.
14
+ * - .assets/ear/ear_meta.json: 16 kHz, 30 s windows, recommended 3, n_fft 400, hop 160,
15
+ * mels 80, frames 3000, clamp 1e-10, floor 8 decades, affine (x + 4) / 4, input 'mel',
16
+ * output 'output_0', 103 logits in languages.json order.
17
+ *
18
+ * INPUT closed: { audio_path: string 1..4096, windows?: integer 1..16 }.
19
+ * WAV only (RIFF/WAVE), PCM 16-bit or IEEE float32, mono or stereo, any declared rate (linear
20
+ * resample to 16 kHz). Short audio is zero-padded to one window; long audio is searched for the
21
+ * most speech-like windows (window/3 candidate stride) and only the selected ones are scored.
22
+ *
23
+ * OUTPUT exactly: { language, confidence, isReliable,
24
+ * candidates: [{language, probability}] (top 5, folded, descending),
25
+ * windows } (integer count actually listened to).
26
+ *
27
+ * HONEST LIMITS: 103 model labels = 102 canonical languages after yue -> zh folding (this label
28
+ * set has no tl or nb, so only yue -> zh fires; the full alias map is kept for future label
29
+ * sets). Scores are decision probabilities averaged over a few 30 s windows, not calibrated
30
+ * confidence, and this is not a speech-recognition or nationality claim.
31
+ */
32
+ import { readFile } from 'node:fs/promises';
33
+ import path from 'node:path';
34
+ import { Effect, Schema } from 'effect';
35
+ import { InferenceFailed, InvalidInput, RuntimeUnavailable } from '@try-works/tool-mode-runtime-core';
36
+ import { EAR_BINS, EAR_DEFAULT_WINDOWS, EAR_FRAMES, EAR_HOP, EAR_MELS, EAR_N_FFT, EAR_SAMPLE_RATE, EAR_WINDOW_SAMPLES, logMelFeatures, parseMelFilters, windowOffsets, } from "./ear-frontend.js";
37
+ import { describeCause } from "./errors.js";
38
+ import { closedKeyFailure, inputCeilingFailure } from "./processor.js";
39
+ import { LitertSession } from "./runtime.js";
40
+ import { decodeWav, resampleLinear } from "./wav.js";
41
+ /** The processor id this module owns. */
42
+ export const EAR_PROCESSOR = 'ear/mel-v1';
43
+ /** Codes the detector emits that differ from the codes callers expect (Languages.swift:4-8). */
44
+ export const EAR_LANGUAGE_ALIASES = { tl: 'fil', nb: 'no', yue: 'zh' };
45
+ /** Languages the detector confuses confidently rather than uncertainly (Languages.swift:20). */
46
+ export const EAR_CONFUSABLE_LANGUAGES = ['no', 'sv', 'da'];
47
+ /** How far ahead the top candidate must be before routing on it (Ear.swift:54). */
48
+ export const EAR_RELIABLE_MARGIN = 0.25;
49
+ /** The label count at the pinned revision (103 logits, languages.json order). */
50
+ export const EAR_LANGUAGE_COUNT = 103;
51
+ /** The largest windows value the tool accepts. */
52
+ export const EAR_MAX_WINDOWS = 16;
53
+ const MAX_PATH_CHARACTERS = 4096;
54
+ const INPUT_NAME = 'mel';
55
+ const OUTPUT_NAME = 'output_0';
56
+ const INPUT_KEYS = new Set(['audio_path', 'windows']);
57
+ const EarMetaSchema = Schema.Struct({
58
+ audio: Schema.Struct({ sample_rate: Schema.Finite, recommended_windows: Schema.Finite }),
59
+ frontend: Schema.Struct({
60
+ n_fft: Schema.Finite,
61
+ hop_length: Schema.Finite,
62
+ n_mels: Schema.Finite,
63
+ frames: Schema.Finite,
64
+ clamp_min: Schema.Finite,
65
+ floor_below_peak_db: Schema.Finite,
66
+ affine: Schema.Struct({ add: Schema.Finite, divide: Schema.Finite }),
67
+ }),
68
+ artifacts: Schema.Struct({ litert: Schema.Struct({ input: Schema.String, output: Schema.String }) }),
69
+ output: Schema.Struct({ logits: Schema.Finite }),
70
+ });
71
+ const LanguagesSchema = Schema.Array(Schema.NonEmptyString);
72
+ /** The canonical code a raw label reports as (Languages.swift:22-24). */
73
+ const canonicalLanguage = (code) => {
74
+ if (code === 'tl')
75
+ return 'fil';
76
+ if (code === 'nb')
77
+ return 'no';
78
+ if (code === 'yue')
79
+ return 'zh';
80
+ return code;
81
+ };
82
+ function parseInput(input, maxInputBytes) {
83
+ const ceiling = inputCeilingFailure(input, { maxInputBytes, timeoutMs: 0 });
84
+ if (ceiling !== undefined)
85
+ return Effect.fail(ceiling);
86
+ const keys = closedKeyFailure(input, INPUT_KEYS, 'audio_path and windows');
87
+ if (keys !== undefined)
88
+ return Effect.fail(keys);
89
+ const audioPath = input['audio_path'];
90
+ if (audioPath === undefined)
91
+ return Effect.fail(new InvalidInput({ field: 'audio_path', message: 'audio_path is required' }));
92
+ if (!Schema.is(Schema.String)(audioPath) || audioPath.length === 0) {
93
+ return Effect.fail(new InvalidInput({ field: 'audio_path', message: 'audio_path must be a non-empty string' }));
94
+ }
95
+ if (audioPath.length > MAX_PATH_CHARACTERS) {
96
+ return Effect.fail(new InvalidInput({ field: 'audio_path', message: 'audio_path must be at most ' + String(MAX_PATH_CHARACTERS) + ' characters' }));
97
+ }
98
+ const windowsValue = input['windows'];
99
+ if (windowsValue !== undefined) {
100
+ if (!Schema.is(Schema.Int)(windowsValue) || windowsValue < 1 || windowsValue > EAR_MAX_WINDOWS) {
101
+ return Effect.fail(new InvalidInput({
102
+ field: 'windows',
103
+ message: 'windows must be an integer between 1 and ' + String(EAR_MAX_WINDOWS) + ', or omitted for the default of ' + String(EAR_DEFAULT_WINDOWS),
104
+ }));
105
+ }
106
+ }
107
+ return Effect.succeed({ audioPath, windows: windowsValue ?? EAR_DEFAULT_WINDOWS });
108
+ }
109
+ /** Read one JSON asset beside the model, as a lossless JSON value for schema decoding. */
110
+ function readAssetJson(assetDirectory, name) {
111
+ return Effect.tryPromise({
112
+ try: async () => JSON.parse(await readFile(path.join(assetDirectory, name), 'utf8')),
113
+ catch: (cause) => new RuntimeUnavailable({ provider: 'litert', reason: 'the ear asset "' + name + '" cannot be read as JSON: ' + describeCause(cause) }),
114
+ });
115
+ }
116
+ /** Read the pinned meta and refuse a document whose geometry or IO disagrees with the source. */
117
+ function loadMeta(assetDirectory) {
118
+ return Effect.flatMap(readAssetJson(assetDirectory, 'ear_meta.json'), (raw) => {
119
+ const decoded = Schema.decodeUnknownOption(EarMetaSchema)(raw);
120
+ if (decoded._tag === 'None') {
121
+ return Effect.fail(new RuntimeUnavailable({ provider: 'litert', reason: 'the ear ear_meta.json is not the pinned document' }));
122
+ }
123
+ const meta = decoded.value;
124
+ const problems = [];
125
+ if (meta.audio.sample_rate !== EAR_SAMPLE_RATE)
126
+ problems.push('audio.sample_rate must be ' + String(EAR_SAMPLE_RATE));
127
+ if (meta.audio.recommended_windows !== EAR_DEFAULT_WINDOWS)
128
+ problems.push('audio.recommended_windows must be ' + String(EAR_DEFAULT_WINDOWS));
129
+ if (meta.frontend.n_fft !== EAR_N_FFT)
130
+ problems.push('n_fft must be ' + String(EAR_N_FFT));
131
+ if (meta.frontend.hop_length !== EAR_HOP)
132
+ problems.push('hop_length must be ' + String(EAR_HOP));
133
+ if (meta.frontend.n_mels !== EAR_MELS)
134
+ problems.push('n_mels must be ' + String(EAR_MELS));
135
+ if (meta.frontend.frames !== EAR_FRAMES)
136
+ problems.push('frames must be ' + String(EAR_FRAMES));
137
+ if (meta.frontend.clamp_min !== 1e-10)
138
+ problems.push('clamp_min must be 1e-10');
139
+ if (meta.frontend.floor_below_peak_db !== 8)
140
+ problems.push('floor_below_peak_db must be 8');
141
+ if (meta.frontend.affine.add !== 4)
142
+ problems.push('affine.add must be 4');
143
+ if (meta.frontend.affine.divide !== 4)
144
+ problems.push('affine.divide must be 4');
145
+ if (meta.artifacts.litert.input !== INPUT_NAME)
146
+ problems.push('artifacts.litert.input must be "' + INPUT_NAME + '"');
147
+ if (meta.artifacts.litert.output !== OUTPUT_NAME)
148
+ problems.push('artifacts.litert.output must be "' + OUTPUT_NAME + '"');
149
+ if (meta.output.logits !== EAR_LANGUAGE_COUNT)
150
+ problems.push('output.logits must be ' + String(EAR_LANGUAGE_COUNT));
151
+ return problems.length > 0
152
+ ? Effect.fail(new RuntimeUnavailable({ provider: 'litert', reason: 'ear_meta.json does not match the pinned front end: ' + problems.join('; ') }))
153
+ : Effect.succeed(meta);
154
+ });
155
+ }
156
+ /** Read the model's label order (languages.json). */
157
+ function loadLanguages(assetDirectory) {
158
+ return Effect.flatMap(readAssetJson(assetDirectory, 'languages.json'), (raw) => {
159
+ const decoded = Schema.decodeUnknownOption(LanguagesSchema)(raw);
160
+ if (decoded._tag === 'None') {
161
+ return Effect.fail(new RuntimeUnavailable({ provider: 'litert', reason: 'the ear languages.json is not an array of language codes' }));
162
+ }
163
+ if (decoded.value.length !== EAR_LANGUAGE_COUNT) {
164
+ return Effect.fail(new RuntimeUnavailable({
165
+ provider: 'litert',
166
+ reason: 'languages.json declares ' + String(decoded.value.length) + ' languages; the pinned model emits ' + String(EAR_LANGUAGE_COUNT),
167
+ }));
168
+ }
169
+ return Effect.succeed(decoded.value);
170
+ });
171
+ }
172
+ /** Read and validate the mel filterbank beside the model. */
173
+ function loadFilters(assetDirectory) {
174
+ return Effect.tryPromise({
175
+ try: async () => {
176
+ const bytes = new Uint8Array(await readFile(path.join(assetDirectory, 'mel_filters.f32')));
177
+ const filters = parseMelFilters(bytes);
178
+ if (filters.mels !== EAR_MELS || filters.bins !== EAR_BINS) {
179
+ throw new Error(String(filters.mels) + 'x' + String(filters.bins) + ' is not the pinned ' + String(EAR_MELS) + 'x' + String(EAR_BINS) + ' filterbank');
180
+ }
181
+ return filters;
182
+ },
183
+ catch: (cause) => new RuntimeUnavailable({ provider: 'litert', reason: 'the ear mel filterbank cannot be used: ' + describeCause(cause) }),
184
+ });
185
+ }
186
+ /** Resolve the caller's path through the host authority, decode the WAV and resample to 16 kHz. */
187
+ function loadAudio(audioPath, context) {
188
+ return Effect.tryPromise({
189
+ try: async () => {
190
+ const resolved = context.resolvePath(audioPath);
191
+ const decoded = decodeWav(new Uint8Array(await readFile(resolved)));
192
+ if (decoded.samples.length === 0)
193
+ throw new Error('the recording contains no audio samples');
194
+ return resampleLinear(decoded.samples, decoded.sampleRate, EAR_SAMPLE_RATE);
195
+ },
196
+ catch: (cause) => new InvalidInput({ field: 'audio_path', message: 'audio_path cannot be used: ' + describeCause(cause) }),
197
+ });
198
+ }
199
+ /** Validate one logits vector (Model.swift:104-114). */
200
+ function logitsOf(raw, expected) {
201
+ return Effect.try({
202
+ try: () => {
203
+ const logits = raw[OUTPUT_NAME];
204
+ if (!(logits instanceof Float32Array) || logits.length !== expected) {
205
+ throw new Error('the model returned ' + String(logits?.length ?? 'no') + ' logits; expected ' + String(expected));
206
+ }
207
+ for (const value of logits) {
208
+ if (!Number.isFinite(value))
209
+ throw new Error('the model returned a non-finite logit');
210
+ }
211
+ return logits;
212
+ },
213
+ catch: (cause) => new InferenceFailed({ provider: 'litert', phase: 'post', message: describeCause(cause) }),
214
+ });
215
+ }
216
+ /** Softmax in double precision (Model.swift:125-131). */
217
+ function softmax(logits) {
218
+ let peak = -Number.MAX_VALUE;
219
+ for (const value of logits)
220
+ peak = Math.max(peak, value);
221
+ const exps = new Float64Array(logits.length);
222
+ let total = 0;
223
+ for (let index = 0; index < logits.length; index += 1) {
224
+ const value = Math.exp((logits[index] ?? 0) - peak);
225
+ exps[index] = value;
226
+ total += value;
227
+ }
228
+ if (!(total > 0))
229
+ return exps;
230
+ for (let index = 0; index < exps.length; index += 1)
231
+ exps[index] = (exps[index] ?? 0) / total;
232
+ return exps;
233
+ }
234
+ /** Fold aliases, rank, and take the top five (Model.swift:90-101). */
235
+ function candidatesOf(totals, windows, languages) {
236
+ const scale = 1 / Math.max(windows, 1);
237
+ const mass = new Map();
238
+ for (let index = 0; index < languages.length; index += 1) {
239
+ const code = canonicalLanguage(languages[index] ?? '');
240
+ mass.set(code, (mass.get(code) ?? 0) + (totals[index] ?? 0) * scale);
241
+ }
242
+ return [...mass.entries()]
243
+ .map(([language, probability]) => ({ language, probability }))
244
+ .sort((a, b) => b.probability - a.probability || (a.language < b.language ? -1 : a.language > b.language ? 1 : 0))
245
+ .slice(0, 5);
246
+ }
247
+ /** The reliability rule (Ear.swift:29-34). */
248
+ function reliable(candidates) {
249
+ const top = candidates[0];
250
+ if (top === undefined)
251
+ return false;
252
+ if (EAR_CONFUSABLE_LANGUAGES.some((code) => code === top.language))
253
+ return false;
254
+ const runnerUp = candidates[1]?.probability ?? 0;
255
+ return top.probability - runnerUp >= EAR_RELIABLE_MARGIN;
256
+ }
257
+ /** The ear processor. */
258
+ export const earMelProcessor = {
259
+ id: EAR_PROCESSOR,
260
+ features: [EAR_PROCESSOR],
261
+ invoke: ({ input, context, limits, assetDirectory }) => Effect.gen(function* () {
262
+ const options = yield* parseInput(input, limits.maxInputBytes);
263
+ yield* loadMeta(assetDirectory);
264
+ const languages = yield* loadLanguages(assetDirectory);
265
+ const filters = yield* loadFilters(assetDirectory);
266
+ const samples = yield* loadAudio(options.audioPath, context);
267
+ const offsets = windowOffsets(samples, options.windows);
268
+ const session = yield* LitertSession;
269
+ const totals = new Float64Array(languages.length);
270
+ for (const offset of offsets) {
271
+ const window = samples.subarray(offset, Math.min(offset + EAR_WINDOW_SAMPLES, samples.length));
272
+ const features = logMelFeatures(window, filters);
273
+ const raw = yield* session.run({
274
+ [INPUT_NAME]: { data: features, dims: [1, EAR_MELS, EAR_FRAMES] },
275
+ });
276
+ const logits = yield* logitsOf(raw, languages.length);
277
+ const probabilities = softmax(logits);
278
+ for (let index = 0; index < totals.length; index += 1) {
279
+ totals[index] = (totals[index] ?? 0) + (probabilities[index] ?? 0);
280
+ }
281
+ }
282
+ const candidates = candidatesOf(totals, offsets.length, languages);
283
+ const top = candidates[0];
284
+ if (top === undefined) {
285
+ return yield* Effect.fail(new InferenceFailed({ provider: 'litert', phase: 'post', message: 'the model produced no language candidates' }));
286
+ }
287
+ return {
288
+ language: top.language,
289
+ confidence: top.probability,
290
+ isReliable: reliable(candidates),
291
+ candidates: candidates.map((candidate) => ({ language: candidate.language, probability: candidate.probability })),
292
+ windows: offsets.length,
293
+ };
294
+ }),
295
+ };
296
+ //# sourceMappingURL=ear.js.map
@@ -0,0 +1,87 @@
1
+ /**
2
+ * The emo front end: the EMTK semantic tokenizer and the hashed n-gram featurizer.
3
+ *
4
+ * PRIMARY SOURCE (Desert-Ant-Labs/desert-ant-core @ baf4e6e3b4655c8e54aa8d5a25215b5f35f899a7):
5
+ * - Sources/Emo/Tokenizer.swift:5-13 FNV-1a (offset 0xCBF29CE484222325, prime 0x100000001B3)
6
+ * - Sources/Emo/Tokenizer.swift:15-43 NGram.encode: five bucket seeds, impSeed
7
+ * 0xFF51AFD7ED558CCD, bucket = h % nBuckets,
8
+ * sign from bit 63, importance = h(impSeed) % nImportance
9
+ * - Sources/Emo/Tokenizer.swift:45-96 per-script features: Latin w:/g:, Indic a:/k:,
10
+ * CJK c:/j:, SEA s:
11
+ * - Sources/Emo/Tokenizer.swift:98-110 NFKC + lowercase normalization and word runs
12
+ * - Sources/Emo/Tokenizer.swift:112-163 clusters, Jamo, category predicates
13
+ * - Sources/Emo/Tokenizer.swift:166-203 EMTK container: magic EMTK, offset 6, u32 unkID,
14
+ * u32 k, k float32 scores, k uint16 byte lengths, UTF-8
15
+ * pieces back to back, maxLen = min(maxScalars, 24)
16
+ * - Sources/Emo/Tokenizer.swift:205-242 Viterbi: window maxLen, strict-greater tie-break,
17
+ * unk fallback with the unk score
18
+ * - Sources/Emo/Tokenizer.swift:276-366 byte-exact vocabulary keys; duplicate byte pieces
19
+ * make the container malformed
20
+ *
21
+ * EQUIVALENCE RISK (recorded verbatim in the lock qualification): the Swift front end uses
22
+ * TextNormalization NFKC plus Swift Unicode category predicates; this port uses the JavaScript
23
+ * engine's NFKC/Unicode data. Arithmetic is pinned by independent Python FNV goldens and by our
24
+ * own tokenizer vectors, but Swift <-> JS NFKC equality and Viterbi tie-order equivalence are
25
+ * NOT independently proven (no ground-truth tokenizer vectors were available).
26
+ */
27
+ /** FNV-1a 64-bit offset basis. */
28
+ export declare const EMO_OFFSET_BASIS = 14695981039346656037n;
29
+ /** FNV-1a 64-bit prime. */
30
+ export declare const EMO_PRIME = 1099511628211n;
31
+ /** The five bucket seeds (Tokenizer.swift:38-41). */
32
+ export declare const EMO_BUCKET_SEEDS: readonly bigint[];
33
+ /** The importance seed (Tokenizer.swift:42). */
34
+ export declare const EMO_IMPORTANCE_SEED = 18397679294719823053n;
35
+ /** The Viterbi window cap (Tokenizer.swift:201). */
36
+ export declare const EMO_MAX_TOKEN_SCALARS = 24;
37
+ /** FNV-1a over the UTF-8 bytes of a string (Tokenizer.swift:5-13). */
38
+ export declare function fnv64(text: string, seed: bigint): bigint;
39
+ /** A parsed EMTK tokenizer. */
40
+ export interface EmtTokenizer {
41
+ readonly vocabSize: number;
42
+ readonly unkID: number;
43
+ /** The float32 piece scores, in id order. */
44
+ readonly scores: readonly number[];
45
+ /** The longest piece in scalars, capped at 24. */
46
+ readonly maxLen: number;
47
+ /** Tokenize one phrase to ids (Tokenizer.swift:205-242). */
48
+ readonly encode: (text: string) => readonly number[];
49
+ }
50
+ /**
51
+ * Parse the EMTK container (Tokenizer.swift:166-203).
52
+ *
53
+ * @param bytes - the whole emo_tokenizer.bin
54
+ * @returns the tokenizer
55
+ * @throws Error naming the malformed part
56
+ */
57
+ export declare const parseEmtTokenizer: (bytes: Uint8Array) => EmtTokenizer;
58
+ /**
59
+ * The ordered feature strings for one phrase (Tokenizer.swift:45-96).
60
+ *
61
+ * Latin runs emit w: + the word, then g: n-grams over <word> sizes 3..5; Indic runs emit a:,
62
+ * k: cluster unigrams and k: cluster bigrams/trigrams over <clusters>; CJK runs emit j: Jamo
63
+ * n-grams for Hangul plus c: scalar n-grams sizes 1..2; SEA runs emit s: scalar n-grams 2..4.
64
+ * Empty text produces the w:\u0000 sentinel.
65
+ */
66
+ export declare const emoFeatureList: (text: string) => readonly string[];
67
+ /** The geometry the meta pins for the n-gram tensors. */
68
+ export interface EmoNgramGeometry {
69
+ readonly nBuckets: number;
70
+ readonly nHashes: number;
71
+ readonly nImportance: number;
72
+ readonly maxFeatures: number;
73
+ }
74
+ /** The flattened n-gram tensors for one phrase (before fixed-window padding). */
75
+ export interface EmoNgrams {
76
+ /** featureCount * nHashes, feature-major. */
77
+ readonly buckets: Int32Array;
78
+ /** featureCount * nHashes, feature-major. */
79
+ readonly signs: Float32Array;
80
+ /** featureCount. */
81
+ readonly importance: Int32Array;
82
+ /** The tensor value max(featureCount, 1) (Model.swift:41, :71). */
83
+ readonly count: number;
84
+ }
85
+ /** Hash the feature list into the model's bucket/sign/importance arrays (Tokenizer.swift:15-36). */
86
+ export declare const encodeNgrams: (text: string, geometry: EmoNgramGeometry) => EmoNgrams;
87
+ //# sourceMappingURL=emo-tokenizer.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"emo-tokenizer.d.ts","sourceRoot":"","sources":["../src/emo-tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,kCAAkC;AAClC,eAAO,MAAM,gBAAgB,wBAAsB,CAAA;AAEnD,2BAA2B;AAC3B,eAAO,MAAM,SAAS,iBAAiB,CAAA;AAIvC,qDAAqD;AACrD,eAAO,MAAM,gBAAgB,EAAE,SAAS,MAAM,EAM7C,CAAA;AAED,gDAAgD;AAChD,eAAO,MAAM,mBAAmB,wBAAsB,CAAA;AAEtD,oDAAoD;AACpD,eAAO,MAAM,qBAAqB,KAAK,CAAA;AAIvC,sEAAsE;AACtE,wBAAgB,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAOxD;AAcD,+BAA+B;AAC/B,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,6CAA6C;IAC7C,QAAQ,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAA;IAClC,kDAAkD;IAClD,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,4DAA4D;IAC5D,QAAQ,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,SAAS,MAAM,EAAE,CAAA;CACrD;AAED;;;;;;GAMG;AACH,eAAO,MAAM,iBAAiB,GAAI,OAAO,UAAU,KAAG,YAwGrD,CAAA;AAoID;;;;;;;GAOG;AACH,eAAO,MAAM,cAAc,GAAI,MAAM,MAAM,KAAG,SAAS,MAAM,EAyB5D,CAAA;AAED,yDAAyD;AACzD,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAC5B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;CAC7B;AAED,iFAAiF;AACjF,MAAM,WAAW,SAAS;IACxB,6CAA6C;IAC7C,QAAQ,CAAC,OAAO,EAAE,UAAU,CAAA;IAC5B,6CAA6C;IAC7C,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAA;IAC5B,oBAAoB;IACpB,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAA;IAC/B,mEAAmE;IACnE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CACvB;AAED,oGAAoG;AACpG,eAAO,MAAM,YAAY,GAAI,MAAM,MAAM,EAAE,UAAU,gBAAgB,KAAG,SAmBvE,CAAA"}