@keymanapp/kmc-model 17.0.85-alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.nyc_output/coverage-10524-1681239236645-0.json +1 -0
- package/Makefile +38 -0
- package/build/cjs-src/lexical-model-compiler.cjs +152688 -0
- package/build/src/build-trie.d.ts +40 -0
- package/build/src/build-trie.d.ts.map +1 -0
- package/build/src/build-trie.js +362 -0
- package/build/src/build-trie.js.map +1 -0
- package/build/src/join-word-breaker-decorator.d.ts +10 -0
- package/build/src/join-word-breaker-decorator.d.ts.map +1 -0
- package/build/src/join-word-breaker-decorator.js +121 -0
- package/build/src/join-word-breaker-decorator.js.map +1 -0
- package/build/src/lexical-model-compiler.d.ts +19 -0
- package/build/src/lexical-model-compiler.d.ts.map +1 -0
- package/build/src/lexical-model-compiler.js +155 -0
- package/build/src/lexical-model-compiler.js.map +1 -0
- package/build/src/lexical-model.d.ts +135 -0
- package/build/src/lexical-model.d.ts.map +1 -0
- package/build/src/lexical-model.js +6 -0
- package/build/src/lexical-model.js.map +1 -0
- package/build/src/main.d.ts +15 -0
- package/build/src/main.d.ts.map +1 -0
- package/build/src/main.js +46 -0
- package/build/src/main.js.map +1 -0
- package/build/src/model-compiler-errors.d.ts +77 -0
- package/build/src/model-compiler-errors.d.ts.map +1 -0
- package/build/src/model-compiler-errors.js +156 -0
- package/build/src/model-compiler-errors.js.map +1 -0
- package/build/src/model-defaults.d.ts +56 -0
- package/build/src/model-defaults.d.ts.map +1 -0
- package/build/src/model-defaults.js +106 -0
- package/build/src/model-defaults.js.map +1 -0
- package/build/src/model-definitions.d.ts +71 -0
- package/build/src/model-definitions.d.ts.map +1 -0
- package/build/src/model-definitions.js +189 -0
- package/build/src/model-definitions.js.map +1 -0
- package/build/src/script-overrides-decorator.d.ts +4 -0
- package/build/src/script-overrides-decorator.d.ts.map +1 -0
- package/build/src/script-overrides-decorator.js +63 -0
- package/build/src/script-overrides-decorator.js.map +1 -0
- package/build/test/helpers/index.d.ts +69 -0
- package/build/test/helpers/index.d.ts.map +1 -0
- package/build/test/helpers/index.js +160 -0
- package/build/test/helpers/index.js.map +1 -0
- package/build/test/test-compile-model-with-pseudoclosure.d.ts +2 -0
- package/build/test/test-compile-model-with-pseudoclosure.d.ts.map +1 -0
- package/build/test/test-compile-model-with-pseudoclosure.js +200 -0
- package/build/test/test-compile-model-with-pseudoclosure.js.map +1 -0
- package/build/test/test-compile-model.d.ts +2 -0
- package/build/test/test-compile-model.d.ts.map +1 -0
- package/build/test/test-compile-model.js +30 -0
- package/build/test/test-compile-model.js.map +1 -0
- package/build/test/test-compile-trie.d.ts +2 -0
- package/build/test/test-compile-trie.d.ts.map +1 -0
- package/build/test/test-compile-trie.js +125 -0
- package/build/test/test-compile-trie.js.map +1 -0
- package/build/test/test-default-apply-case.d.ts +2 -0
- package/build/test/test-default-apply-case.d.ts.map +1 -0
- package/build/test/test-default-apply-case.js +105 -0
- package/build/test/test-default-apply-case.js.map +1 -0
- package/build/test/test-default-search-term-to-key.d.ts +2 -0
- package/build/test/test-default-search-term-to-key.d.ts.map +1 -0
- package/build/test/test-default-search-term-to-key.js +148 -0
- package/build/test/test-default-search-term-to-key.js.map +1 -0
- package/build/test/test-error-logger.d.ts +2 -0
- package/build/test/test-error-logger.d.ts.map +1 -0
- package/build/test/test-error-logger.js +26 -0
- package/build/test/test-error-logger.js.map +1 -0
- package/build/test/test-join-word-breaker.d.ts +2 -0
- package/build/test/test-join-word-breaker.d.ts.map +1 -0
- package/build/test/test-join-word-breaker.js +84 -0
- package/build/test/test-join-word-breaker.js.map +1 -0
- package/build/test/test-model-definitions.d.ts +2 -0
- package/build/test/test-model-definitions.d.ts.map +1 -0
- package/build/test/test-model-definitions.js +165 -0
- package/build/test/test-model-definitions.js.map +1 -0
- package/build/test/test-override-script-defaults.d.ts +2 -0
- package/build/test/test-override-script-defaults.d.ts.map +1 -0
- package/build/test/test-override-script-defaults.js +28 -0
- package/build/test/test-override-script-defaults.js.map +1 -0
- package/build/test/test-parse-wordlist.d.ts +2 -0
- package/build/test/test-parse-wordlist.d.ts.map +1 -0
- package/build/test/test-parse-wordlist.js +110 -0
- package/build/test/test-parse-wordlist.js.map +1 -0
- package/build/test/test-punctuation.d.ts +2 -0
- package/build/test/test-punctuation.d.ts.map +1 -0
- package/build/test/test-punctuation.js +31 -0
- package/build/test/test-punctuation.js.map +1 -0
- package/build/test/tsconfig.tsbuildinfo +1 -0
- package/build/test/wordbreakers/data.d.ts +35 -0
- package/build/test/wordbreakers/data.d.ts.map +1 -0
- package/build/test/wordbreakers/data.js +1778 -0
- package/build/test/wordbreakers/data.js.map +1 -0
- package/build/test/wordbreakers/default-wordbreaker-esm.d.ts +10 -0
- package/build/test/wordbreakers/default-wordbreaker-esm.d.ts.map +1 -0
- package/build/test/wordbreakers/default-wordbreaker-esm.js +354 -0
- package/build/test/wordbreakers/default-wordbreaker-esm.js.map +1 -0
- package/build/tsconfig.tsbuildinfo +1 -0
- package/build.sh +73 -0
- package/coverage/lcov-report/base.css +224 -0
- package/coverage/lcov-report/block-navigation.js +87 -0
- package/coverage/lcov-report/favicon.png +0 -0
- package/coverage/lcov-report/index.html +161 -0
- package/coverage/lcov-report/prettify.css +1 -0
- package/coverage/lcov-report/prettify.js +2 -0
- package/coverage/lcov-report/sort-arrow-sprite.png +0 -0
- package/coverage/lcov-report/sorter.js +196 -0
- package/coverage/lcov-report/src/build-trie.ts.html +1618 -0
- package/coverage/lcov-report/src/index.html +221 -0
- package/coverage/lcov-report/src/join-word-breaker-decorator.ts.html +487 -0
- package/coverage/lcov-report/src/lexical-model-compiler.ts.html +622 -0
- package/coverage/lcov-report/src/main.ts.html +271 -0
- package/coverage/lcov-report/src/model-compiler-errors.ts.html +691 -0
- package/coverage/lcov-report/src/model-defaults.ts.html +415 -0
- package/coverage/lcov-report/src/model-definitions.ts.html +748 -0
- package/coverage/lcov-report/src/script-overrides-decorator.ts.html +310 -0
- package/coverage/lcov-report/test/helpers/index.html +116 -0
- package/coverage/lcov-report/test/helpers/index.ts.html +646 -0
- package/coverage/lcov-report/test/index.html +266 -0
- package/coverage/lcov-report/test/test-compile-model-with-pseudoclosure.ts.html +802 -0
- package/coverage/lcov-report/test/test-compile-model.ts.html +187 -0
- package/coverage/lcov-report/test/test-compile-trie.ts.html +541 -0
- package/coverage/lcov-report/test/test-default-apply-case.ts.html +466 -0
- package/coverage/lcov-report/test/test-default-search-term-to-key.ts.html +628 -0
- package/coverage/lcov-report/test/test-error-logger.ts.html +196 -0
- package/coverage/lcov-report/test/test-join-word-breaker.ts.html +376 -0
- package/coverage/lcov-report/test/test-model-definitions.ts.html +676 -0
- package/coverage/lcov-report/test/test-override-script-defaults.ts.html +184 -0
- package/coverage/lcov-report/test/test-parse-wordlist.ts.html +466 -0
- package/coverage/lcov-report/test/test-punctuation.ts.html +190 -0
- package/coverage/lcov-report/test/wordbreakers/data.ts.html +5413 -0
- package/coverage/lcov-report/test/wordbreakers/default-wordbreaker-esm.ts.html +1234 -0
- package/coverage/lcov-report/test/wordbreakers/index.html +131 -0
- package/coverage/lcov.info +5969 -0
- package/package.json +61 -0
- package/src/build-trie.ts +511 -0
- package/src/join-word-breaker-decorator.ts +134 -0
- package/src/lexical-model-compiler.ts +179 -0
- package/src/lexical-model.ts +150 -0
- package/src/main.ts +62 -0
- package/src/model-compiler-errors.ts +203 -0
- package/src/model-defaults.ts +111 -0
- package/src/model-definitions.ts +222 -0
- package/src/script-overrides-decorator.ts +75 -0
- package/test/README.md +15 -0
- package/test/fixtures/example.qaa.joinwordbreaker/example.qaa.joinwordbreaker.model.ts +10 -0
- package/test/fixtures/example.qaa.joinwordbreaker/wordlist.tsv +3 -0
- package/test/fixtures/example.qaa.scriptusesspaces/example.qaa.scriptusesspaces.model.ts +10 -0
- package/test/fixtures/example.qaa.scriptusesspaces/wordlist.tsv +8 -0
- package/test/fixtures/example.qaa.sencoten/example.qaa.sencoten.model.kmp.json +45 -0
- package/test/fixtures/example.qaa.sencoten/example.qaa.sencoten.model.kps +35 -0
- package/test/fixtures/example.qaa.sencoten/example.qaa.sencoten.model.ts +6 -0
- package/test/fixtures/example.qaa.sencoten/wordlist.tsv +10 -0
- package/test/fixtures/example.qaa.smp/example.qaa.smp.model.ts +6 -0
- package/test/fixtures/example.qaa.smp/wordlist.tsv +5 -0
- package/test/fixtures/example.qaa.trivial/example.qaa.trivial.model.ts +5 -0
- package/test/fixtures/example.qaa.trivial/wordlist.tsv +3 -0
- package/test/fixtures/example.qaa.utf16be/example.qaa.utf16be.model.ts +5 -0
- package/test/fixtures/example.qaa.utf16be/wordlist.txt +0 -0
- package/test/fixtures/example.qaa.utf16le/example.qaa.utf16le.model.ts +5 -0
- package/test/fixtures/example.qaa.utf16le/wordlist.txt +0 -0
- package/test/fixtures/example.qaa.wordbreaker/example.qaa.wordbreaker.model.ts +9 -0
- package/test/fixtures/example.qaa.wordbreaker/wordlist.tsv +3 -0
- package/test/helpers/index.ts +187 -0
- package/test/test-compile-model-with-pseudoclosure.ts +239 -0
- package/test/test-compile-model.ts +34 -0
- package/test/test-compile-trie.ts +152 -0
- package/test/test-default-apply-case.ts +128 -0
- package/test/test-default-search-term-to-key.ts +181 -0
- package/test/test-error-logger.ts +38 -0
- package/test/test-join-word-breaker.ts +97 -0
- package/test/test-model-definitions.ts +198 -0
- package/test/test-override-script-defaults.ts +33 -0
- package/test/test-parse-wordlist.ts +127 -0
- package/test/test-punctuation.ts +35 -0
- package/test/tsconfig.json +22 -0
- package/test/wordbreakers/README.md +3 -0
- package/test/wordbreakers/data.ts +1776 -0
- package/test/wordbreakers/default-wordbreaker-esm.ts +383 -0
- package/tools/create-override-script-regexp.ts +145 -0
- package/tsconfig.json +17 -0
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
/*
|
|
2
|
+
lexical-model-compiler.ts: base file for lexical model compiler.
|
|
3
|
+
*/
|
|
4
|
+
/// <reference path="./lexical-model.ts" />
|
|
5
|
+
import * as ts from "typescript";
|
|
6
|
+
import * as fs from "fs";
|
|
7
|
+
import * as path from "path";
|
|
8
|
+
import { createTrieDataStructure } from "./build-trie.js";
|
|
9
|
+
import { ModelDefinitions } from "./model-definitions.js";
|
|
10
|
+
import { decorateWithJoin } from "./join-word-breaker-decorator.js";
|
|
11
|
+
import { decorateWithScriptOverrides } from "./script-overrides-decorator.js";
|
|
12
|
+
export default class LexicalModelCompiler {
|
|
13
|
+
/**
|
|
14
|
+
* Returns the generated code for the model that will ultimately be loaded by
|
|
15
|
+
* the LMLayer worker. This code contains all model parameters, and specifies
|
|
16
|
+
* word breakers and auxilary functions that may be required.
|
|
17
|
+
*
|
|
18
|
+
* @param model_id The model ID. TODO: not sure if this is actually required!
|
|
19
|
+
* @param modelSource A specification of the model to compile
|
|
20
|
+
* @param sourcePath Where to find auxilary sources files
|
|
21
|
+
*/
|
|
22
|
+
generateLexicalModelCode(model_id, modelSource, sourcePath) {
|
|
23
|
+
// TODO: add metadata in comment
|
|
24
|
+
const filePrefix = `(function() {\n'use strict';\n`;
|
|
25
|
+
const fileSuffix = `})();`;
|
|
26
|
+
let func = filePrefix;
|
|
27
|
+
//
|
|
28
|
+
// Emit the model as code and data
|
|
29
|
+
//
|
|
30
|
+
switch (modelSource.format) {
|
|
31
|
+
case "custom-1.0":
|
|
32
|
+
let sources = modelSource.sources.map(function (source) {
|
|
33
|
+
return fs.readFileSync(path.join(sourcePath, source), 'utf8');
|
|
34
|
+
});
|
|
35
|
+
func += this.transpileSources(sources).join('\n');
|
|
36
|
+
func += `LMLayerWorker.loadModel(new ${modelSource.rootClass}());\n`;
|
|
37
|
+
break;
|
|
38
|
+
case "fst-foma-1.0":
|
|
39
|
+
throw new ModelSourceError(`Unimplemented model format: ${modelSource.format}`);
|
|
40
|
+
case "trie-1.0":
|
|
41
|
+
// Convert all relative path names to paths relative to the enclosing
|
|
42
|
+
// directory. This way, we'll read the files relative to the model.ts
|
|
43
|
+
// file, rather than the current working directory.
|
|
44
|
+
let filenames = modelSource.sources.map(filename => path.join(sourcePath, filename));
|
|
45
|
+
let definitions = new ModelDefinitions(modelSource);
|
|
46
|
+
func += definitions.compileDefinitions();
|
|
47
|
+
// Needs the actual searchTermToKey closure...
|
|
48
|
+
// Which needs the actual applyCasing closure as well.
|
|
49
|
+
func += `LMLayerWorker.loadModel(new models.TrieModel(${createTrieDataStructure(filenames, definitions.searchTermToKey)}, {\n`;
|
|
50
|
+
let wordBreakerSourceCode = compileWordBreaker(normalizeWordBreakerSpec(modelSource.wordBreaker));
|
|
51
|
+
func += ` wordBreaker: ${wordBreakerSourceCode},\n`;
|
|
52
|
+
// START - the lexical mapping option block
|
|
53
|
+
func += ` searchTermToKey: ${definitions.compileSearchTermToKey()},\n`;
|
|
54
|
+
if (modelSource.languageUsesCasing != null) {
|
|
55
|
+
func += ` languageUsesCasing: ${modelSource.languageUsesCasing},\n`;
|
|
56
|
+
} // else leave undefined.
|
|
57
|
+
if (modelSource.languageUsesCasing) {
|
|
58
|
+
func += ` applyCasing: ${definitions.compileApplyCasing()},\n`;
|
|
59
|
+
}
|
|
60
|
+
// END - the lexical mapping option block.
|
|
61
|
+
if (modelSource.punctuation) {
|
|
62
|
+
func += ` punctuation: ${JSON.stringify(modelSource.punctuation)},\n`;
|
|
63
|
+
}
|
|
64
|
+
func += `}));\n`;
|
|
65
|
+
break;
|
|
66
|
+
default:
|
|
67
|
+
throw new ModelSourceError(`Unknown model format: ${modelSource.format}`);
|
|
68
|
+
}
|
|
69
|
+
func += fileSuffix;
|
|
70
|
+
return func;
|
|
71
|
+
}
|
|
72
|
+
transpileSources(sources) {
|
|
73
|
+
return sources.map((source) => ts.transpileModule(source, {
|
|
74
|
+
compilerOptions: {
|
|
75
|
+
target: ts.ScriptTarget.ES3,
|
|
76
|
+
module: ts.ModuleKind.None,
|
|
77
|
+
}
|
|
78
|
+
}).outputText);
|
|
79
|
+
}
|
|
80
|
+
;
|
|
81
|
+
logError(s) {
|
|
82
|
+
console.error(require('chalk').red(s));
|
|
83
|
+
}
|
|
84
|
+
;
|
|
85
|
+
}
|
|
86
|
+
;
|
|
87
|
+
export class ModelSourceError extends Error {
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Returns a JavaScript expression (as a string) that can serve as a word
|
|
91
|
+
* breaking function.
|
|
92
|
+
*/
|
|
93
|
+
function compileWordBreaker(spec) {
|
|
94
|
+
let wordBreakerCode = compileInnerWordBreaker(spec.use);
|
|
95
|
+
if (spec.joinWordsAt) {
|
|
96
|
+
wordBreakerCode = compileJoinDecorator(spec, wordBreakerCode);
|
|
97
|
+
}
|
|
98
|
+
if (spec.overrideScriptDefaults) {
|
|
99
|
+
wordBreakerCode = compileScriptOverrides(spec, wordBreakerCode);
|
|
100
|
+
}
|
|
101
|
+
return wordBreakerCode;
|
|
102
|
+
}
|
|
103
|
+
function compileJoinDecorator(spec, existingWordBreakerCode) {
|
|
104
|
+
// Bundle the source of the join decorator, as an IIFE,
|
|
105
|
+
// like this: (function join(breaker, joiners) {/*...*/}(breaker, joiners))
|
|
106
|
+
// The decorator will run IMMEDIATELY when the model is loaded,
|
|
107
|
+
// by the LMLayer returning the decorated word breaker to the
|
|
108
|
+
// LMLayer model.
|
|
109
|
+
let joinerExpr = JSON.stringify(spec.joinWordsAt);
|
|
110
|
+
return `(${decorateWithJoin.toString()}(${existingWordBreakerCode}, ${joinerExpr}))`;
|
|
111
|
+
}
|
|
112
|
+
function compileScriptOverrides(spec, existingWordBreakerCode) {
|
|
113
|
+
return `(${decorateWithScriptOverrides.toString()}(${existingWordBreakerCode}, '${spec.overrideScriptDefaults}'))`;
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Compiles the base word breaker, that may be decorated later.
|
|
117
|
+
* Returns the source code of a JavaScript expression.
|
|
118
|
+
*/
|
|
119
|
+
function compileInnerWordBreaker(spec) {
|
|
120
|
+
if (typeof spec === "string") {
|
|
121
|
+
// It must be a builtin word breaker, so just instantiate it.
|
|
122
|
+
return `wordBreakers['${spec}']`;
|
|
123
|
+
}
|
|
124
|
+
else {
|
|
125
|
+
// It must be a function:
|
|
126
|
+
return spec.toString()
|
|
127
|
+
// Note: the .toString() might just be the property name, but we want a
|
|
128
|
+
// plain function:
|
|
129
|
+
.replace(/^wordBreak(ing|er)\b/, 'function');
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* Given a word breaker specification in any of the messy ways,
|
|
134
|
+
* normalizes it to a common form that the compiler can deal with.
|
|
135
|
+
*/
|
|
136
|
+
function normalizeWordBreakerSpec(wordBreakerSpec) {
|
|
137
|
+
if (wordBreakerSpec == undefined) {
|
|
138
|
+
// Use the default word breaker when it's unspecified
|
|
139
|
+
return { use: 'default' };
|
|
140
|
+
}
|
|
141
|
+
else if (isSimpleWordBreaker(wordBreakerSpec)) {
|
|
142
|
+
// The word breaker was passed as a literal function; use its source code.
|
|
143
|
+
return { use: wordBreakerSpec };
|
|
144
|
+
}
|
|
145
|
+
else if (wordBreakerSpec.use) {
|
|
146
|
+
return wordBreakerSpec;
|
|
147
|
+
}
|
|
148
|
+
else {
|
|
149
|
+
throw new Error(`Unknown word breaker: ${wordBreakerSpec}`);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
function isSimpleWordBreaker(spec) {
|
|
153
|
+
return typeof spec === "function" || spec === "default" || spec === "ascii";
|
|
154
|
+
}
|
|
155
|
+
//# sourceMappingURL=lexical-model-compiler.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"lexical-model-compiler.js","sourceRoot":"","sources":["../../src/lexical-model-compiler.ts"],"names":[],"mappings":"AAAA;;EAEE;AAEF,2CAA2C;AAE3C,OAAO,KAAK,EAAE,MAAM,YAAY,CAAC;AACjC,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,uBAAuB,EAAE,MAAM,iBAAiB,CAAC;AAC1D,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAC,gBAAgB,EAAC,MAAM,kCAAkC,CAAC;AAClE,OAAO,EAAC,2BAA2B,EAAC,MAAM,iCAAiC,CAAC;AAG5E,MAAM,CAAC,OAAO,OAAO,oBAAoB;IAEvC;;;;;;;;OAQG;IACH,wBAAwB,CAAC,QAAgB,EAAE,WAA+B,EAAE,UAAkB;QAC5F,gCAAgC;QAChC,MAAM,UAAU,GAAW,gCAAgC,CAAC;QAC5D,MAAM,UAAU,GAAW,OAAO,CAAC;QACnC,IAAI,IAAI,GAAG,UAAU,CAAC;QAEtB,EAAE;QACF,kCAAkC;QAClC,EAAE;QAEF,QAAO,WAAW,CAAC,MAAM,EAAE;YACzB,KAAK,YAAY;gBACf,IAAI,OAAO,GAAa,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,UAAS,MAAM;oBAC7D,OAAO,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC;gBAChE,CAAC,CAAC,CAAC;gBACH,IAAI,IAAI,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAClD,IAAI,IAAI,+BAA+B,WAAW,CAAC,SAAS,QAAQ,CAAC;gBACrE,MAAM;YACR,KAAK,cAAc;gBACjB,MAAM,IAAI,gBAAgB,CAAC,+BAA+B,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC;YAClF,KAAK,UAAU;gBACb,qEAAqE;gBACrE,qEAAqE;gBACrE,mDAAmD;gBACnD,IAAI,SAAS,GAAG,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC,CAAC;gBAErF,IAAI,WAAW,GAAG,IAAI,gBAAgB,CAAC,WAAW,CAAC,CAAC;gBAEpD,IAAI,IAAI,WAAW,CAAC,kBAAkB,EAAE,CAAC;gBAEzC,8CAA8C;gBAC9C,sDAAsD;gBACtD,IAAI,IAAI,gDACN,uBAAuB,CAAC,SAAS,EAAE,WAAW,CAAC,eAAe,CAChE,OAAO,CAAC;gBAER,IAAI,qBAAqB,GAAG,kBAAkB,CAAC,wBAAwB,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,CAAC;gBAClG,IAAI,IAAI,kBAAkB,qBAAqB,KAAK,CAAC;gBAErD,2CAA2C;gBAC3C,IAAI,IAAI,sBAAsB,WAAW,CAAC,sBAAsB,EAAE,KAAK,CAAC;gBAExE,IAAG,WAAW,CAAC,kBAAkB,IAAI,IAAI,EAAE;oBACzC,IAAI,IAAI,yBAAyB,WAAW,CAAC,kBAAkB,KAAK,CAAC;iBACtE,CAAC,wBAAwB;gBAE1B,IAAG,WAAW,CAAC,kBAAkB,EAAE;oBACjC,IAAI,IAAI,kBAAkB,WAAW,CAAC,kBAAkB,EAAE,KAAK,CAAC;iBACjE;gBACD,0CAA0C;gBAE1C,IAAI,WAAW,CAAC,WAAW,EAAE;oBAC3B,IAAI,IAAI,kBAAkB,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,WAAW,CAAC,KAAK,CAAC;iBACxE;gBACD,IAAI,IAAI,QAAQ,CAAC;gBACjB,MAAM;YACR;gBACE,MAAM,IAAI,gBAAgB,CAAC,yBAAyB,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC;SAC7E;QAED,IAAI,IAAI,UAAU,CAAC;QAEnB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,gBAAgB,CAAC,OAAsB;QACrC,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,eAAe,CAAC,MAAM,EAAE;YACtD,eAAe,EAAE;gBACf,MAAM,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG;gBAC3B,MAAM,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI;aAC3B;SACF,CAAC,CAAC,UAAU,CACd,CAAC;IACJ,CAAC;IAAA,CAAC;IAEF,QAAQ,CAAC,CAAS;QAChB,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACzC,CAAC;IAAA,CAAC;CACH;AAAA,CAAC;AAEF,MAAM,OAAO,gBAAiB,SAAQ,KAAK;CAC1C;AAED;;;GAGG;AACH,SAAS,kBAAkB,CAAC,IAAqB;IAC/C,IAAI,eAAe,GAAG,uBAAuB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAExD,IAAI,IAAI,CAAC,WAAW,EAAE;QACpB,eAAe,GAAG,oBAAoB,CAAC,IAAI,EAAE,eAAe,CAAC,CAAC;KAC/D;IAED,IAAI,IAAI,CAAC,sBAAsB,EAAE;QAC/B,eAAe,GAAG,sBAAsB,CAAC,IAAI,EAAE,eAAe,CAAC,CAAC;KACjE;IAED,OAAO,eAAe,CAAC;AACzB,CAAC;AAED,SAAS,oBAAoB,CAAC,IAAqB,EAAE,uBAA+B;IAClF,uDAAuD;IACvD,2EAA2E;IAC3E,+DAA+D;IAC/D,6DAA6D;IAC7D,iBAAiB;IACjB,IAAI,UAAU,GAAW,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA;IACzD,OAAO,IAAI,gBAAgB,CAAC,QAAQ,EAAE,IAAI,uBAAuB,KAAK,UAAU,IAAI,CAAC;AACvF,CAAC;AAED,SAAS,sBAAsB,CAAC,IAAqB,EAAE,uBAA+B;IACpF,OAAO,IAAI,2BAA2B,CAAC,QAAQ,EAAE,IAAI,uBAAuB,MAAM,IAAI,CAAC,sBAAsB,KAAK,CAAC;AACrH,CAAC;AAED;;;GAGG;AACH,SAAS,uBAAuB,CAAC,IAA2B;IAC1D,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE;QAC5B,6DAA6D;QAC7D,OAAO,iBAAiB,IAAI,IAAI,CAAC;KAClC;SAAM;QACL,yBAAyB;QACzB,OAAO,IAAI,CAAC,QAAQ,EAAE;YACpB,uEAAuE;YACvE,kBAAkB;aACjB,OAAO,CAAC,sBAAsB,EAAE,UAAU,CAAC,CAAC;KAChD;AACH,CAAC;AAED;;;GAGG;AACH,SAAS,wBAAwB,CAAC,eAAkD;IAClF,IAAI,eAAe,IAAI,SAAS,EAAE;QAChC,qDAAqD;QACrD,OAAO,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC;KAC3B;SAAM,IAAI,mBAAmB,CAAC,eAAe,CAAC,EAAE;QAC/C,0EAA0E;QAC1E,OAAO,EAAE,GAAG,EAAE,eAAe,EAAE,CAAC;KACjC;SAAM,IAAI,eAAe,CAAC,GAAG,EAAE;QAC9B,OAAO,eAAe,CAAC;KACxB;SAAM;QACL,MAAM,IAAI,KAAK,CAAC,yBAAyB,eAAe,EAAE,CAAC,CAAC;KAC7D;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,IAA6C;IACxE,OAAO,OAAO,IAAI,KAAK,UAAU,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,OAAO,CAAC;AAC9E,CAAC"}
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Interfaces and constants used by the lexical model compiler. These target
|
|
3
|
+
* the LMLayer's internal worker code, so we provide those definitions too.
|
|
4
|
+
*/
|
|
5
|
+
/// <reference types="@keymanapp/models-types" />
|
|
6
|
+
export interface LexicalModelDeclaration {
|
|
7
|
+
readonly format: 'trie-1.0' | 'fst-foma-1.0' | 'custom-1.0';
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* Keyman 14.0+ word breaker specification:
|
|
11
|
+
*
|
|
12
|
+
* Can support all old word breaking specification,
|
|
13
|
+
* but can also be extended with options.
|
|
14
|
+
*
|
|
15
|
+
* @since 14.0
|
|
16
|
+
*/
|
|
17
|
+
export interface WordBreakerSpec {
|
|
18
|
+
readonly use: SimpleWordBreakerSpec;
|
|
19
|
+
/**
|
|
20
|
+
* If present, joins words that were split by the word breaker
|
|
21
|
+
* together at the given strings. e.g.,
|
|
22
|
+
*
|
|
23
|
+
* joinWordsAt: ['-'] // to keep hyphenated items together
|
|
24
|
+
*
|
|
25
|
+
* @since 14.0
|
|
26
|
+
*/
|
|
27
|
+
readonly joinWordsAt?: string[];
|
|
28
|
+
/**
|
|
29
|
+
* Overrides word splitting behaviour for certain scripts.
|
|
30
|
+
* For example, specifing that spaces break words in certain South-East
|
|
31
|
+
* Asian scripts that otherwise do not use spaces.
|
|
32
|
+
*
|
|
33
|
+
* @since 14.0
|
|
34
|
+
*/
|
|
35
|
+
readonly overrideScriptDefaults?: OverrideScriptDefaults;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Simplified word breaker specification.
|
|
39
|
+
*
|
|
40
|
+
* @since 11.0
|
|
41
|
+
*/
|
|
42
|
+
export type SimpleWordBreakerSpec = 'default' | 'ascii' | WordBreakingFunction;
|
|
43
|
+
/**
|
|
44
|
+
* Simplifies input text to facilitate finding entries within a lexical model's
|
|
45
|
+
* lexicon.
|
|
46
|
+
* @since 11.0
|
|
47
|
+
*/
|
|
48
|
+
export type SimpleWordformToKeySpec = (term: string) => string;
|
|
49
|
+
/**
|
|
50
|
+
* Simplifies input text to facilitate finding entries within a lexical model's
|
|
51
|
+
* lexicon, using the model's `applyCasing` function to assist in the keying process.
|
|
52
|
+
* @since 14.0
|
|
53
|
+
*/
|
|
54
|
+
export type CasedWordformToKeySpec = (term: string, applyCasing?: CasingFunction) => string;
|
|
55
|
+
/**
|
|
56
|
+
* Simplifies input text to facilitate finding entries within a lexical model's
|
|
57
|
+
* lexicon.
|
|
58
|
+
*/
|
|
59
|
+
export type WordformToKeySpec = SimpleWordformToKeySpec | CasedWordformToKeySpec;
|
|
60
|
+
/**
|
|
61
|
+
* Override the default word breaking behaviour for some scripts.
|
|
62
|
+
*
|
|
63
|
+
* There is currently only one option:
|
|
64
|
+
*
|
|
65
|
+
* 'break-words-at-spaces'
|
|
66
|
+
* : some South-East Asian scripts conventionally do not use space or any
|
|
67
|
+
* explicit word boundary character to write word breaks. These scripts are:
|
|
68
|
+
*
|
|
69
|
+
* * Burmese
|
|
70
|
+
* * Khmer
|
|
71
|
+
* * Thai
|
|
72
|
+
* * Laos
|
|
73
|
+
*
|
|
74
|
+
* (this list may be incomplete and extended in the future)
|
|
75
|
+
*
|
|
76
|
+
* For these scripts, the default word breaker breaks at **every**
|
|
77
|
+
* letter/syllable/ideograph. However, in languages that use these scripts BUT
|
|
78
|
+
* use spaces (or some other delimier) as word breaks, enable
|
|
79
|
+
* 'break-words-at-spaces'; enabling 'break-words-at-spaces' prevents the word
|
|
80
|
+
* breaker from making too many breaks in these scripts.
|
|
81
|
+
*
|
|
82
|
+
* @since 14.0
|
|
83
|
+
*/
|
|
84
|
+
export type OverrideScriptDefaults = 'break-words-at-spaces';
|
|
85
|
+
export interface LexicalModelSource extends LexicalModelDeclaration {
|
|
86
|
+
readonly sources: Array<string>;
|
|
87
|
+
/**
|
|
88
|
+
* The name of the type to instantiate (without parameters) as the base object for a custom predictive model.
|
|
89
|
+
*/
|
|
90
|
+
readonly rootClass?: string;
|
|
91
|
+
/**
|
|
92
|
+
* When set to `true`, suggestions will attempt to match the case of the input text even if
|
|
93
|
+
* the lexicon entries use a different casing scheme due to search term keying effects.
|
|
94
|
+
* @since 14.0
|
|
95
|
+
*/
|
|
96
|
+
readonly languageUsesCasing?: boolean;
|
|
97
|
+
/**
|
|
98
|
+
* Specifies the casing rules for a language. Should implement three casing forms:
|
|
99
|
+
* - 'lower' -- a fully-lowercased version of the text appropriate for the language's
|
|
100
|
+
* use of the writing system.
|
|
101
|
+
* - 'upper' -- a fully-uppercased version of the text
|
|
102
|
+
* - 'initial' -- a version preserving the input casing aside from the initial character,
|
|
103
|
+
* which is uppercased (like with proper nouns and sentence-initial words in English
|
|
104
|
+
* sentences.)
|
|
105
|
+
*
|
|
106
|
+
* This is only utilized if `languageUsesCasing` is defined and set to `true`.
|
|
107
|
+
* @since 14.0
|
|
108
|
+
*/
|
|
109
|
+
readonly applyCasing?: CasingFunction;
|
|
110
|
+
/**
|
|
111
|
+
* Which word breaker to use. Choose from:
|
|
112
|
+
*
|
|
113
|
+
* - 'default' -- breaks according to Unicode UAX #29 §4.1 Default Word
|
|
114
|
+
* Boundary Specification, which works well for *most* languages.
|
|
115
|
+
* - 'ascii' -- a very simple word breaker, for demonstration purposes only.
|
|
116
|
+
* - word breaking function -- provide your own function that breaks words.
|
|
117
|
+
* - class-based word-breaker - may be supported in the future.
|
|
118
|
+
*/
|
|
119
|
+
readonly wordBreaker?: WordBreakerSpec | SimpleWordBreakerSpec;
|
|
120
|
+
/**
|
|
121
|
+
* How to simplify words, to convert them into simplified search keys
|
|
122
|
+
* This often involves removing accents, lowercasing, etc.
|
|
123
|
+
*/
|
|
124
|
+
readonly searchTermToKey?: WordformToKeySpec;
|
|
125
|
+
/**
|
|
126
|
+
* Punctuation and spacing suggested by the model.
|
|
127
|
+
*
|
|
128
|
+
* @see LexicalModelPunctuation
|
|
129
|
+
*/
|
|
130
|
+
readonly punctuation?: LexicalModelPunctuation;
|
|
131
|
+
}
|
|
132
|
+
export interface LexicalModelCompiled extends LexicalModelDeclaration {
|
|
133
|
+
readonly id: string;
|
|
134
|
+
}
|
|
135
|
+
//# sourceMappingURL=lexical-model.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"lexical-model.d.ts","sourceRoot":"","sources":["../../src/lexical-model.ts"],"names":[],"mappings":"AAAA;;;GAGG;;AAEH,MAAM,WAAW,uBAAuB;IACtC,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAC,cAAc,GAAC,YAAY,CAAC;CAEzD;AAED;;;;;;;GAOG;AACF,MAAM,WAAW,eAAe;IAC/B,QAAQ,CAAC,GAAG,EAAE,qBAAqB,CAAC;IACpC;;;;;;;OAOG;IACH,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IAEhC;;;;;;OAMG;IACH,QAAQ,CAAC,sBAAsB,CAAC,EAAE,sBAAsB,CAAC;CAC1D;AAED;;;;GAIG;AACF,MAAM,MAAM,qBAAqB,GAAG,SAAS,GAAG,OAAO,GAAG,oBAAoB,CAAC;AAEhF;;;;GAIG;AACF,MAAM,MAAM,uBAAuB,GAAG,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC;AAEhE;;;;GAIG;AACH,MAAM,MAAM,sBAAsB,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,WAAW,CAAC,EAAE,cAAc,KAAK,MAAM,CAAC;AAE5F;;;GAGG;AACF,MAAM,MAAM,iBAAiB,GAAG,uBAAuB,GAAG,sBAAsB,CAAC;AAElF;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,MAAM,sBAAsB,GAAG,uBAAuB,CAAC;AAG7D,MAAM,WAAW,kBAAmB,SAAQ,uBAAuB;IACjE,QAAQ,CAAC,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAChC;;OAEG;IACH,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAA;IAE3B;;;;OAIG;IACH,QAAQ,CAAC,kBAAkB,CAAC,EAAE,OAAO,CAAA;IAErC;;;;;;;;;;;OAWG;IACH,QAAQ,CAAC,WAAW,CAAC,EAAE,cAAc,CAAA;IAErC;;;;;;;;OAQG;IACH,QAAQ,CAAC,WAAW,CAAC,EAAE,eAAe,GAAG,qBAAqB,CAAC;IAE/D;;;OAGG;IACH,QAAQ,CAAC,eAAe,CAAC,EAAE,iBAAiB,CAAC;IAE7C;;;;OAIG;IACH,QAAQ,CAAC,WAAW,CAAC,EAAE,uBAAuB,CAAC;CAChD;AAED,MAAM,WAAW,oBAAqB,SAAQ,uBAAuB;IACnE,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;CACrB"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"lexical-model.js","sourceRoot":"","sources":["../../src/lexical-model.ts"],"names":[],"mappings":"AAAA;;;GAGG"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { LexicalModelSource } from './lexical-model.js';
|
|
2
|
+
/**
|
|
3
|
+
* Compiles a model.ts file, using paths relative to its location.
|
|
4
|
+
*
|
|
5
|
+
* @param filename path to model.ts source.
|
|
6
|
+
* @return model source code
|
|
7
|
+
*/
|
|
8
|
+
export declare function compileModel(filename: string): string;
|
|
9
|
+
/**
|
|
10
|
+
* Loads a lexical model's source module from the given filename.
|
|
11
|
+
*
|
|
12
|
+
* @param filename path to the model source file.
|
|
13
|
+
*/
|
|
14
|
+
export declare function loadFromFilename(filename: string): LexicalModelSource;
|
|
15
|
+
//# sourceMappingURL=main.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../../src/main.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAExD;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAMrD;AAWD;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,kBAAkB,CA0BrE"}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import * as fs from 'fs';
|
|
2
|
+
import * as path from 'path';
|
|
3
|
+
import ts from 'typescript';
|
|
4
|
+
import LexicalModelCompiler from './lexical-model-compiler.js';
|
|
5
|
+
/**
|
|
6
|
+
* Compiles a model.ts file, using paths relative to its location.
|
|
7
|
+
*
|
|
8
|
+
* @param filename path to model.ts source.
|
|
9
|
+
* @return model source code
|
|
10
|
+
*/
|
|
11
|
+
export function compileModel(filename) {
|
|
12
|
+
let modelSource = loadFromFilename(filename);
|
|
13
|
+
let containingDirectory = path.dirname(filename);
|
|
14
|
+
return (new LexicalModelCompiler)
|
|
15
|
+
.generateLexicalModelCode('<unknown>', modelSource, containingDirectory);
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Loads a lexical model's source module from the given filename.
|
|
19
|
+
*
|
|
20
|
+
* @param filename path to the model source file.
|
|
21
|
+
*/
|
|
22
|
+
export function loadFromFilename(filename) {
|
|
23
|
+
let sourceCode = fs.readFileSync(filename, 'utf8');
|
|
24
|
+
// Compile the module to JavaScript code.
|
|
25
|
+
// NOTE: transpile module does a very simple TS to JS compilation.
|
|
26
|
+
// It DOES NOT check for types!
|
|
27
|
+
let compilationOutput = ts.transpile(sourceCode, {
|
|
28
|
+
// Our runtime only supports ES3 with Node/CommonJS modules on Android 5.0.
|
|
29
|
+
// When we drop Android 5.0 support, we can update this to a `ScriptTarget`
|
|
30
|
+
// matrix against target version of Keyman, here and in
|
|
31
|
+
// lexical-model-compiler.ts.
|
|
32
|
+
target: ts.ScriptTarget.ES3,
|
|
33
|
+
module: ts.ModuleKind.CommonJS,
|
|
34
|
+
});
|
|
35
|
+
// Turn the module into a function in which we can inject a global.
|
|
36
|
+
let moduleCode = '(function(exports){' + compilationOutput + '})';
|
|
37
|
+
// Run the module; its exports will be assigned to `moduleExports`.
|
|
38
|
+
let moduleExports = {};
|
|
39
|
+
let module = eval(moduleCode);
|
|
40
|
+
module(moduleExports);
|
|
41
|
+
if (!moduleExports['__esModule'] || !moduleExports['default']) {
|
|
42
|
+
throw new Error(`Model source '${filename}' does have a default export. Did you remember to write \`export default source;\`?`);
|
|
43
|
+
}
|
|
44
|
+
return moduleExports['default'];
|
|
45
|
+
}
|
|
46
|
+
//# sourceMappingURL=main.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"main.js","sourceRoot":"","sources":["../../src/main.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,MAAM,YAAY,CAAC;AAE5B,OAAO,oBAAoB,MAAM,6BAA6B,CAAC;AAG/D;;;;;GAKG;AACH,MAAM,UAAU,YAAY,CAAC,QAAgB;IAC3C,IAAI,WAAW,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IAC7C,IAAI,mBAAmB,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAEjD,OAAO,CAAC,IAAI,oBAAoB,CAAC;SAC9B,wBAAwB,CAAC,WAAW,EAAE,WAAW,EAAE,mBAAmB,CAAC,CAAC;AAC7E,CAAC;AAWD;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAAC,QAAgB;IAC/C,IAAI,UAAU,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACnD,yCAAyC;IACzC,kEAAkE;IAClE,+BAA+B;IAC/B,IAAI,iBAAiB,GAAG,EAAE,CAAC,SAAS,CAAC,UAAU,EAAE;QAC/C,2EAA2E;QAC3E,4EAA4E;QAC5E,wDAAwD;QACxD,6BAA6B;QAC7B,MAAM,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG;QAC3B,MAAM,EAAE,EAAE,CAAC,UAAU,CAAC,QAAQ;KAC/B,CAAC,CAAC;IACH,mEAAmE;IACnE,IAAI,UAAU,GAAG,qBAAqB,GAAG,iBAAiB,GAAG,IAAI,CAAC;IAElE,mEAAmE;IACnE,IAAI,aAAa,GAA0B,EAAE,CAAC;IAC9C,IAAI,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC;IAC9B,MAAM,CAAC,aAAa,CAAC,CAAC;IAEtB,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,EAAE;QAC7D,MAAM,IAAI,KAAK,CAAC,iBAAiB,QAAQ,qFAAqF,CAAC,CAAC;KACjI;IAED,OAAO,aAAa,CAAC,SAAS,CAAuB,CAAC;AACxD,CAAC"}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Log levels.
|
|
3
|
+
*
|
|
4
|
+
* Note: Currently, this acts like a bit set, where the upper 4 bits of an
|
|
5
|
+
* unsigned 16 bit value are the log level flags.
|
|
6
|
+
*
|
|
7
|
+
* Warning: even though these look like bitmasks, these flags may not be
|
|
8
|
+
* combined.
|
|
9
|
+
*/
|
|
10
|
+
export declare enum LogLevel {
|
|
11
|
+
CERR_FATAL = 32768,
|
|
12
|
+
CERR_ERROR = 16384,
|
|
13
|
+
CERR_WARNING = 8192,
|
|
14
|
+
CERR_HINT = 4096
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Error codes. Use these when logging messages.
|
|
18
|
+
*
|
|
19
|
+
* Extends https://github.com/keymanapp/keyman/blob/99db3c0d2448f448242e6397f9d72e9a7ccee4b9/windows/src/global/inc/Comperr.h
|
|
20
|
+
*/
|
|
21
|
+
export declare enum KeymanCompilerError {
|
|
22
|
+
CERR_LEXICAL_MODEL_MIN = 2048,
|
|
23
|
+
CERR_LEXICAL_MODEL_MAX = 2303,
|
|
24
|
+
CERR_FATAL_LM = 34816,
|
|
25
|
+
CERR_ERROR_LM = 18432,
|
|
26
|
+
CERR_WARN_LM = 10240,
|
|
27
|
+
CWARN_MixedNormalizationForms = 10241,
|
|
28
|
+
CWARN_DuplicateWordInSameFile = 10242,
|
|
29
|
+
CERR_HINT_LM = 6144,
|
|
30
|
+
CWARN_TooManyErrorsOrWarnings = 8359
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* How many errors or warnings are too many!
|
|
34
|
+
*/
|
|
35
|
+
export declare const MAX_MESSAGES = 100;
|
|
36
|
+
/**
|
|
37
|
+
* Logs compiler messages (warnings, errors, logs).
|
|
38
|
+
*
|
|
39
|
+
* @param code Error code
|
|
40
|
+
* @param message A helpful message!
|
|
41
|
+
* @param source [optional] the filename/line number in the source that induced this error
|
|
42
|
+
*
|
|
43
|
+
* @see https://github.com/keymanapp/keyman/blob/99db3c0d2448f448242e6397f9d72e9a7ccee4b9/windows/src/developer/TIKE/project/Keyman.Developer.System.Project.ProjectLog.pas#L60-L77
|
|
44
|
+
*/
|
|
45
|
+
export declare function log(code: KeymanCompilerError, message: string, source?: FilenameAndLineNo): void;
|
|
46
|
+
/**
|
|
47
|
+
* Override where log messages go.
|
|
48
|
+
*
|
|
49
|
+
* @param fn The desired log message handler.
|
|
50
|
+
*/
|
|
51
|
+
export declare function redirectLogMessagesTo(fn: (log: LogMessage) => void): void;
|
|
52
|
+
/**
|
|
53
|
+
* Reset the log message handler to the default.
|
|
54
|
+
*/
|
|
55
|
+
export declare function resetLogMessageHandler(): void;
|
|
56
|
+
/**
|
|
57
|
+
* Prints log messages to stdout. The default log action.
|
|
58
|
+
*/
|
|
59
|
+
export declare function printLogs(log: LogMessage): void;
|
|
60
|
+
/**
|
|
61
|
+
* Duct tapes together a filename and a line number of a log.
|
|
62
|
+
*/
|
|
63
|
+
interface FilenameAndLineNo {
|
|
64
|
+
readonly filename: string;
|
|
65
|
+
readonly lineno: number;
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* A log message that knows how to format itself.
|
|
69
|
+
*/
|
|
70
|
+
export interface LogMessage {
|
|
71
|
+
readonly code: KeymanCompilerError;
|
|
72
|
+
readonly level: LogLevel;
|
|
73
|
+
readonly message: string;
|
|
74
|
+
format(): string;
|
|
75
|
+
}
|
|
76
|
+
export {};
|
|
77
|
+
//# sourceMappingURL=model-compiler-errors.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"model-compiler-errors.d.ts","sourceRoot":"","sources":["../../src/model-compiler-errors.ts"],"names":[],"mappings":"AACA;;;;;;;;GAQG;AACH,oBAAY,QAAQ;IAClB,UAAU,QAAS;IACnB,UAAU,QAAS;IACnB,YAAY,OAAS;IACrB,SAAS,OAAS;CACnB;AAED;;;;GAIG;AACH,oBAAY,mBAAmB;IAC7B,sBAAsB,OAAS;IAC/B,sBAAsB,OAAS;IAG/B,aAAa,QAA+C;IAG5D,aAAa,QAA+C;IAG5D,YAAY,QAAiD;IAE7D,6BAA6B,QAAS;IACtC,6BAA6B,QAAS;IAEtC,YAAY,OAA8C;IAI1D,6BAA6B,OAAS;CACvC;AAcD;;GAEG;AACH,eAAO,MAAM,YAAY,MAAM,CAAC;AAYhC;;;;;;;;GAQG;AACH,wBAAgB,GAAG,CAAC,IAAI,EAAE,mBAAmB,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,iBAAiB,QAkBzF;AAED;;;;GAIG;AACH,wBAAgB,qBAAqB,CAAC,EAAE,EAAE,CAAC,GAAG,EAAE,UAAU,KAAK,IAAI,QAElE;AAED;;GAEG;AACH,wBAAgB,sBAAsB,SAGrC;AAED;;GAEG;AACH,wBAAgB,SAAS,CAAC,GAAG,EAAE,UAAU,GAAG,IAAI,CAE/C;AAED;;GAEG;AACH,UAAU,iBAAiB;IACzB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,IAAI,EAAE,mBAAmB,CAAC;IACnC,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IAEzB,MAAM,IAAI,MAAM,CAAC;CAClB"}
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
// TODO: merge with kmc keyboard-compiler errors
|
|
2
|
+
/**
|
|
3
|
+
* Log levels.
|
|
4
|
+
*
|
|
5
|
+
* Note: Currently, this acts like a bit set, where the upper 4 bits of an
|
|
6
|
+
* unsigned 16 bit value are the log level flags.
|
|
7
|
+
*
|
|
8
|
+
* Warning: even though these look like bitmasks, these flags may not be
|
|
9
|
+
* combined.
|
|
10
|
+
*/
|
|
11
|
+
export var LogLevel;
|
|
12
|
+
(function (LogLevel) {
|
|
13
|
+
LogLevel[LogLevel["CERR_FATAL"] = 32768] = "CERR_FATAL";
|
|
14
|
+
LogLevel[LogLevel["CERR_ERROR"] = 16384] = "CERR_ERROR";
|
|
15
|
+
LogLevel[LogLevel["CERR_WARNING"] = 8192] = "CERR_WARNING";
|
|
16
|
+
LogLevel[LogLevel["CERR_HINT"] = 4096] = "CERR_HINT";
|
|
17
|
+
})(LogLevel || (LogLevel = {}));
|
|
18
|
+
;
|
|
19
|
+
/**
|
|
20
|
+
* Error codes. Use these when logging messages.
|
|
21
|
+
*
|
|
22
|
+
* Extends https://github.com/keymanapp/keyman/blob/99db3c0d2448f448242e6397f9d72e9a7ccee4b9/windows/src/global/inc/Comperr.h
|
|
23
|
+
*/
|
|
24
|
+
export var KeymanCompilerError;
|
|
25
|
+
(function (KeymanCompilerError) {
|
|
26
|
+
KeymanCompilerError[KeymanCompilerError["CERR_LEXICAL_MODEL_MIN"] = 2048] = "CERR_LEXICAL_MODEL_MIN";
|
|
27
|
+
KeymanCompilerError[KeymanCompilerError["CERR_LEXICAL_MODEL_MAX"] = 2303] = "CERR_LEXICAL_MODEL_MAX";
|
|
28
|
+
KeymanCompilerError[KeymanCompilerError["CERR_FATAL_LM"] = 34816] = "CERR_FATAL_LM";
|
|
29
|
+
/* Place all fatal LM compiler errors here! */
|
|
30
|
+
KeymanCompilerError[KeymanCompilerError["CERR_ERROR_LM"] = 18432] = "CERR_ERROR_LM";
|
|
31
|
+
/* Place all recoverable LM compiler errors here! */
|
|
32
|
+
KeymanCompilerError[KeymanCompilerError["CERR_WARN_LM"] = 10240] = "CERR_WARN_LM";
|
|
33
|
+
/* Place all LM compiler warnings here! */
|
|
34
|
+
KeymanCompilerError[KeymanCompilerError["CWARN_MixedNormalizationForms"] = 10241] = "CWARN_MixedNormalizationForms";
|
|
35
|
+
KeymanCompilerError[KeymanCompilerError["CWARN_DuplicateWordInSameFile"] = 10242] = "CWARN_DuplicateWordInSameFile";
|
|
36
|
+
KeymanCompilerError[KeymanCompilerError["CERR_HINT_LM"] = 6144] = "CERR_HINT_LM";
|
|
37
|
+
/* Place all LM compiler hints here! */
|
|
38
|
+
/* Errors that are not specific to the lexical model compiler, from comperr.h: */
|
|
39
|
+
KeymanCompilerError[KeymanCompilerError["CWARN_TooManyErrorsOrWarnings"] = 8359] = "CWARN_TooManyErrorsOrWarnings";
|
|
40
|
+
})(KeymanCompilerError || (KeymanCompilerError = {}));
|
|
41
|
+
/**
|
|
42
|
+
* Human-readable titles for the various log levels.
|
|
43
|
+
*
|
|
44
|
+
* Taken from https://github.com/keymanapp/keyman/blob/d83cfffe511ce65b781f919e89e3693146844849/windows/src/developer/TIKE/project/Keyman.Developer.System.Project.ProjectLog.pas#L39-L46
|
|
45
|
+
*/
|
|
46
|
+
const LOG_LEVEL_TITLE = {
|
|
47
|
+
[LogLevel.CERR_HINT]: 'Hint',
|
|
48
|
+
[LogLevel.CERR_WARNING]: 'Warning',
|
|
49
|
+
[LogLevel.CERR_ERROR]: 'Error',
|
|
50
|
+
[LogLevel.CERR_FATAL]: 'Fatal Error',
|
|
51
|
+
};
|
|
52
|
+
/**
|
|
53
|
+
* How many errors or warnings are too many!
|
|
54
|
+
*/
|
|
55
|
+
export const MAX_MESSAGES = 100;
|
|
56
|
+
/**
|
|
57
|
+
* Direct where log messages go.
|
|
58
|
+
*/
|
|
59
|
+
let _logHandler = printLogs;
|
|
60
|
+
/**
|
|
61
|
+
* How many logs or warnings have been witnessed so far.
|
|
62
|
+
*/
|
|
63
|
+
let _messagesSeen = 0;
|
|
64
|
+
/**
|
|
65
|
+
* Logs compiler messages (warnings, errors, logs).
|
|
66
|
+
*
|
|
67
|
+
* @param code Error code
|
|
68
|
+
* @param message A helpful message!
|
|
69
|
+
* @param source [optional] the filename/line number in the source that induced this error
|
|
70
|
+
*
|
|
71
|
+
* @see https://github.com/keymanapp/keyman/blob/99db3c0d2448f448242e6397f9d72e9a7ccee4b9/windows/src/developer/TIKE/project/Keyman.Developer.System.Project.ProjectLog.pas#L60-L77
|
|
72
|
+
*/
|
|
73
|
+
export function log(code, message, source) {
|
|
74
|
+
// Ignore the request if there are too many messages
|
|
75
|
+
if (_messagesSeen > MAX_MESSAGES)
|
|
76
|
+
return;
|
|
77
|
+
let logMessage = source
|
|
78
|
+
? new LogMessageFromSource(code, message, source)
|
|
79
|
+
: new OrdinaryLogMessage(code, message);
|
|
80
|
+
_logHandler(logMessage);
|
|
81
|
+
_messagesSeen++;
|
|
82
|
+
if (_messagesSeen > MAX_MESSAGES) {
|
|
83
|
+
_logHandler(new OrdinaryLogMessage(KeymanCompilerError.CWARN_TooManyErrorsOrWarnings, `More than ${MAX_MESSAGES} warnings or errors received; suppressing further messages`));
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Override where log messages go.
|
|
88
|
+
*
|
|
89
|
+
* @param fn The desired log message handler.
|
|
90
|
+
*/
|
|
91
|
+
export function redirectLogMessagesTo(fn) {
|
|
92
|
+
_logHandler = fn;
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Reset the log message handler to the default.
|
|
96
|
+
*/
|
|
97
|
+
export function resetLogMessageHandler() {
|
|
98
|
+
_logHandler = printLogs;
|
|
99
|
+
_messagesSeen = 0;
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* Prints log messages to stdout. The default log action.
|
|
103
|
+
*/
|
|
104
|
+
export function printLogs(log) {
|
|
105
|
+
console.log(log.format());
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Concrete implementation of the log message.
|
|
109
|
+
*/
|
|
110
|
+
class OrdinaryLogMessage {
|
|
111
|
+
code;
|
|
112
|
+
message;
|
|
113
|
+
constructor(code, message) {
|
|
114
|
+
this.code = code;
|
|
115
|
+
this.message = message;
|
|
116
|
+
}
|
|
117
|
+
get level() {
|
|
118
|
+
return this.code & 0xF000;
|
|
119
|
+
}
|
|
120
|
+
determineLogLevelTitle() {
|
|
121
|
+
return LOG_LEVEL_TITLE[this.level] || '';
|
|
122
|
+
}
|
|
123
|
+
format() {
|
|
124
|
+
let prefix = this.determineLogLevelTitle();
|
|
125
|
+
if (prefix)
|
|
126
|
+
prefix = `${prefix}: `;
|
|
127
|
+
return `${prefix}${h(this.code)} ${this.message}`;
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* A log message with a filename and line number.
|
|
132
|
+
*/
|
|
133
|
+
class LogMessageFromSource extends OrdinaryLogMessage {
|
|
134
|
+
filename;
|
|
135
|
+
lineno;
|
|
136
|
+
constructor(code, message, source) {
|
|
137
|
+
super(code, message);
|
|
138
|
+
this.filename = source.filename;
|
|
139
|
+
this.lineno = source.lineno;
|
|
140
|
+
}
|
|
141
|
+
format() {
|
|
142
|
+
let originalMessage = super.format();
|
|
143
|
+
return `${this.filename} (${this.lineno}): ${originalMessage}`;
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
/**
|
|
147
|
+
* Format a number as a zero-padded 4 digit hexadecimal.
|
|
148
|
+
*/
|
|
149
|
+
function h(n) {
|
|
150
|
+
let formatted = n.toString(16).toUpperCase();
|
|
151
|
+
if (formatted.length < 4) {
|
|
152
|
+
formatted = '0'.repeat(4 - formatted.length);
|
|
153
|
+
}
|
|
154
|
+
return formatted;
|
|
155
|
+
}
|
|
156
|
+
//# sourceMappingURL=model-compiler-errors.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"model-compiler-errors.js","sourceRoot":"","sources":["../../src/model-compiler-errors.ts"],"names":[],"mappings":"AAAA,gDAAgD;AAChD;;;;;;;;GAQG;AACH,MAAM,CAAN,IAAY,QAKX;AALD,WAAY,QAAQ;IAClB,uDAAmB,CAAA;IACnB,uDAAmB,CAAA;IACnB,0DAAqB,CAAA;IACrB,oDAAkB,CAAA;AACpB,CAAC,EALW,QAAQ,KAAR,QAAQ,QAKnB;AAAA,CAAC;AAEF;;;;GAIG;AACH,MAAM,CAAN,IAAY,mBAqBX;AArBD,WAAY,mBAAmB;IAC7B,oGAA+B,CAAA;IAC/B,oGAA+B,CAAA;IAG/B,mFAA4D,CAAA;IAC1D,8CAA8C;IAEhD,mFAA4D,CAAA;IAC1D,oDAAoD;IAEtD,iFAA6D,CAAA;IAC3D,0CAA0C;IAC5C,mHAAsC,CAAA;IACtC,mHAAsC,CAAA;IAEtC,gFAA0D,CAAA;IACxD,uCAAuC;IAEzC,iFAAiF;IACjF,kHAAsC,CAAA;AACxC,CAAC,EArBW,mBAAmB,KAAnB,mBAAmB,QAqB9B;AAED;;;;GAIG;AACH,MAAM,eAAe,GAAkC;IACrD,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM;IAC5B,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,SAAS;IAClC,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,OAAO;IAC9B,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,aAAa;CACrC,CAAC;AAEF;;GAEG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,GAAG,CAAC;AAEhC;;GAEG;AACH,IAAI,WAAW,GAA8B,SAAS,CAAC;AAEvD;;GAEG;AACH,IAAI,aAAa,GAAG,CAAC,CAAC;AAEtB;;;;;;;;GAQG;AACH,MAAM,UAAU,GAAG,CAAC,IAAyB,EAAE,OAAe,EAAE,MAA0B;IACxF,oDAAoD;IACpD,IAAI,aAAa,GAAG,YAAY;QAC9B,OAAO;IAET,IAAI,UAAU,GAAG,MAAM;QACrB,CAAC,CAAC,IAAI,oBAAoB,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC;QACjD,CAAC,CAAC,IAAI,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAE1C,WAAW,CAAC,UAAU,CAAC,CAAA;IACvB,aAAa,EAAE,CAAC;IAEhB,IAAI,aAAa,GAAG,YAAY,EAAE;QAC9B,WAAW,CAAC,IAAI,kBAAkB,CAChC,mBAAmB,CAAC,6BAA6B,EACjD,aAAa,YAAY,4DAA4D,CACtF,CAAC,CAAC;KACN;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,qBAAqB,CAAC,EAA6B;IACjE,WAAW,GAAG,EAAE,CAAC;AACnB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,sBAAsB;IACpC,WAAW,GAAG,SAAS,CAAC;IACxB,aAAa,GAAG,CAAC,CAAC;AACpB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,SAAS,CAAC,GAAe;IACvC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;AAC5B,CAAC;AAqBD;;GAEG;AACH,MAAM,kBAAkB;IACb,IAAI,CAAsB;IAC1B,OAAO,CAAS;IAEzB,YAAY,IAAyB,EAAE,OAAe;QACpD,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;IAED,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC;IAC5B,CAAC;IAED,sBAAsB;QACpB,OAAO,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;IAC3C,CAAC;IAED,MAAM;QACJ,IAAI,MAAM,GAAG,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC3C,IAAI,MAAM;YACR,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC;QAEzB,OAAO,GAAG,MAAM,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAA;IACnD,CAAC;CACF;AAED;;GAEG;AACH,MAAM,oBAAqB,SAAQ,kBAAkB;IAC1C,QAAQ,CAAS;IACjB,MAAM,CAAS;IAExB,YAAY,IAAyB,EAAE,OAAe,EAAE,MAAyB;QAC/E,KAAK,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACrB,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;QAChC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;IAC9B,CAAC;IAED,MAAM;QACJ,IAAI,eAAe,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;QACrC,OAAO,GAAG,IAAI,CAAC,QAAQ,KAAK,IAAI,CAAC,MAAM,MAAM,eAAe,EAAE,CAAC;IACjE,CAAC;CACF;AAED;;GAEG;AACH,SAAS,CAAC,CAAC,CAAS;IAClB,IAAI,SAAS,GAAG,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IAC7C,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE;QACxB,SAAS,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;KAC9C;IAED,OAAO,SAAS,CAAC;AACnB,CAAC"}
|