@keymanapp/kmc-model 18.0.53-alpha → 18.0.55-alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/src/build-trie.d.ts +39 -39
- package/build/src/build-trie.js +374 -374
- package/build/src/build-trie.js.map +1 -1
- package/build/src/compiler-callbacks.d.ts +3 -3
- package/build/src/compiler-callbacks.js +4 -4
- package/build/src/join-word-breaker-decorator.d.ts +9 -9
- package/build/src/join-word-breaker-decorator.js +120 -120
- package/build/src/join-word-breaker-decorator.js.map +1 -1
- package/build/src/lexical-model-compiler.d.ts +87 -87
- package/build/src/lexical-model-compiler.js +260 -260
- package/build/src/lexical-model-compiler.js.map +1 -1
- package/build/src/lexical-model.d.ts +143 -143
- package/build/src/lexical-model.js +5 -5
- package/build/src/main.d.ts +3 -3
- package/build/src/main.js +2 -2
- package/build/src/model-compiler-messages.d.ts +56 -56
- package/build/src/model-compiler-messages.js +63 -63
- package/build/src/model-defaults.d.ts +55 -55
- package/build/src/model-defaults.js +101 -101
- package/build/src/model-defaults.js.map +1 -1
- package/build/src/model-definitions.d.ts +70 -70
- package/build/src/model-definitions.js +182 -182
- package/build/src/model-definitions.js.map +1 -1
- package/build/src/script-overrides-decorator.d.ts +3 -3
- package/build/src/script-overrides-decorator.js +63 -63
- package/build/src/script-overrides-decorator.js.map +1 -1
- package/package.json +8 -9
|
@@ -1,261 +1,261 @@
|
|
|
1
|
-
/*
|
|
2
|
-
lexical-model-compiler.ts: base file for lexical model compiler.
|
|
3
|
-
*/
|
|
4
|
-
import ts from "typescript";
|
|
5
|
-
import { createTrieDataStructure } from "./build-trie.js";
|
|
6
|
-
import { ModelDefinitions } from "./model-definitions.js";
|
|
7
|
-
import { decorateWithJoin } from "./join-word-breaker-decorator.js";
|
|
8
|
-
import { decorateWithScriptOverrides } from "./script-overrides-decorator.js";
|
|
9
|
-
import { ModelCompilerError, ModelCompilerMessageContext, ModelCompilerMessages } from "./model-compiler-messages.js";
|
|
10
|
-
import { callbacks, setCompilerCallbacks } from "./compiler-callbacks.js";
|
|
11
|
-
;
|
|
12
|
-
;
|
|
13
|
-
;
|
|
14
|
-
/**
|
|
15
|
-
* @public
|
|
16
|
-
* Compiles a .model.ts file to a .model.js. The compiler does not read or write
|
|
17
|
-
* from filesystem or network directly, but relies on callbacks for all external
|
|
18
|
-
* IO.
|
|
19
|
-
*/
|
|
20
|
-
export class LexicalModelCompiler {
|
|
21
|
-
/**
|
|
22
|
-
* Initialize the compiler. There are currently no options
|
|
23
|
-
* specific to the lexical model compiler
|
|
24
|
-
* @param callbacks - Callbacks for external interfaces, including message
|
|
25
|
-
* reporting and file io
|
|
26
|
-
* @param options - Compiler options
|
|
27
|
-
* @returns always succeeds and returns true
|
|
28
|
-
*/
|
|
29
|
-
async init(callbacks, _options) {
|
|
30
|
-
setCompilerCallbacks(callbacks);
|
|
31
|
-
return true;
|
|
32
|
-
}
|
|
33
|
-
/**
|
|
34
|
-
* Compiles a .model.ts file to .model.js. Returns an object containing binary
|
|
35
|
-
* artifacts on success. The files are passed in by name, and the compiler
|
|
36
|
-
* will use callbacks as passed to the {@link LexicalModelCompiler.init}
|
|
37
|
-
* function to read any input files by disk.
|
|
38
|
-
* @param infile - Path to source file. Path will be parsed to find relative
|
|
39
|
-
* references in the .kmn file, such as icon or On Screen
|
|
40
|
-
* Keyboard file
|
|
41
|
-
* @param outfile - Path to output file. The file will not be written to, but
|
|
42
|
-
* will be included in the result for use by
|
|
43
|
-
* {@link LexicalModelCompiler.write}.
|
|
44
|
-
* @returns Binary artifacts on success, null on failure.
|
|
45
|
-
*/
|
|
46
|
-
async run(inputFilename, outputFilename) {
|
|
47
|
-
try {
|
|
48
|
-
let modelSource = this.loadFromFilename(inputFilename);
|
|
49
|
-
let containingDirectory = callbacks.path.dirname(inputFilename);
|
|
50
|
-
let code = this.generateLexicalModelCode('<unknown>', modelSource, containingDirectory);
|
|
51
|
-
const result = {
|
|
52
|
-
artifacts: {
|
|
53
|
-
js: {
|
|
54
|
-
data: new TextEncoder().encode(code),
|
|
55
|
-
filename: outputFilename ?? inputFilename.replace(/\.model\.ts$/, '.model.js')
|
|
56
|
-
}
|
|
57
|
-
}
|
|
58
|
-
};
|
|
59
|
-
return result;
|
|
60
|
-
}
|
|
61
|
-
catch (e) {
|
|
62
|
-
callbacks.reportMessage(e instanceof ModelCompilerError
|
|
63
|
-
? e.event
|
|
64
|
-
: ModelCompilerMessages.Fatal_UnexpectedException({ e: e }));
|
|
65
|
-
return null;
|
|
66
|
-
}
|
|
67
|
-
}
|
|
68
|
-
/**
|
|
69
|
-
* Write artifacts from a successful compile to disk, via callbacks methods.
|
|
70
|
-
* The artifacts written may include:
|
|
71
|
-
*
|
|
72
|
-
* - .model.js file - Javascript lexical model for web and touch platforms
|
|
73
|
-
*
|
|
74
|
-
* @param artifacts - object containing artifact binary data to write out
|
|
75
|
-
* @returns always returns true
|
|
76
|
-
*/
|
|
77
|
-
async write(artifacts) {
|
|
78
|
-
callbacks.fs.writeFileSync(artifacts.js.filename, artifacts.js.data);
|
|
79
|
-
return true;
|
|
80
|
-
}
|
|
81
|
-
/**
|
|
82
|
-
* @internal
|
|
83
|
-
* Loads a lexical model's source module from the given filename.
|
|
84
|
-
*
|
|
85
|
-
* @param filename - path to the model source file.
|
|
86
|
-
*/
|
|
87
|
-
loadFromFilename(filename) {
|
|
88
|
-
let sourceCode = new TextDecoder().decode(callbacks.loadFile(filename));
|
|
89
|
-
// Compile the module to JavaScript code.
|
|
90
|
-
// NOTE: transpile module does a very simple TS to JS compilation.
|
|
91
|
-
// It DOES NOT check for types!
|
|
92
|
-
let compilationOutput = ts.transpile(sourceCode, {
|
|
93
|
-
// Our runtime only supports ES3 with Node/CommonJS modules on Android 5.0.
|
|
94
|
-
// When we drop Android 5.0 support, we can update this to a `ScriptTarget`
|
|
95
|
-
// matrix against target version of Keyman, here and in
|
|
96
|
-
// lexical-model-compiler.ts.
|
|
97
|
-
target: ts.ScriptTarget.ES3,
|
|
98
|
-
module: ts.ModuleKind.CommonJS,
|
|
99
|
-
});
|
|
100
|
-
// Turn the module into a function in which we can inject a global.
|
|
101
|
-
let moduleCode = '(function(exports){' + compilationOutput + '})';
|
|
102
|
-
// Run the module; its exports will be assigned to `moduleExports`.
|
|
103
|
-
let moduleExports = {};
|
|
104
|
-
let module = eval(moduleCode);
|
|
105
|
-
module(moduleExports);
|
|
106
|
-
if (!moduleExports['__esModule'] || !moduleExports['default']) {
|
|
107
|
-
ModelCompilerMessageContext.filename = filename;
|
|
108
|
-
throw new ModelCompilerError(ModelCompilerMessages.Error_NoDefaultExport());
|
|
109
|
-
}
|
|
110
|
-
return moduleExports['default'];
|
|
111
|
-
}
|
|
112
|
-
/**
|
|
113
|
-
* @internal
|
|
114
|
-
* Returns the generated code for the model that will ultimately be loaded by
|
|
115
|
-
* the LMLayer worker. This code contains all model parameters, and specifies
|
|
116
|
-
* word breakers and auxilary functions that may be required.
|
|
117
|
-
*
|
|
118
|
-
* @param model_id - The model ID. TODO: not sure if this is actually required!
|
|
119
|
-
* @param modelSource - A specification of the model to compile
|
|
120
|
-
* @param sourcePath - Where to find auxilary sources files
|
|
121
|
-
*/
|
|
122
|
-
generateLexicalModelCode(model_id, modelSource, sourcePath) {
|
|
123
|
-
// TODO: add metadata in comment
|
|
124
|
-
const filePrefix = `(function() {\n'use strict';\n`;
|
|
125
|
-
const fileSuffix = `})();`;
|
|
126
|
-
let func = filePrefix;
|
|
127
|
-
//
|
|
128
|
-
// Emit the model as code and data
|
|
129
|
-
//
|
|
130
|
-
switch (modelSource.format) {
|
|
131
|
-
case "custom-1.0":
|
|
132
|
-
let sources = modelSource.sources.map(function (source) {
|
|
133
|
-
return new TextDecoder().decode(callbacks.loadFile(callbacks.path.join(sourcePath, source)));
|
|
134
|
-
});
|
|
135
|
-
func += this.transpileSources(sources).join('\n');
|
|
136
|
-
func += `LMLayerWorker.loadModel(new ${modelSource.rootClass}());\n`;
|
|
137
|
-
break;
|
|
138
|
-
case "fst-foma-1.0":
|
|
139
|
-
throw new ModelCompilerError(ModelCompilerMessages.Error_UnimplementedModelFormat({ format: modelSource.format }));
|
|
140
|
-
case "trie-1.0":
|
|
141
|
-
// Convert all relative path names to paths relative to the enclosing
|
|
142
|
-
// directory. This way, we'll read the files relative to the model.ts
|
|
143
|
-
// file, rather than the current working directory.
|
|
144
|
-
let filenames = modelSource.sources.map(filename => callbacks.path.join(sourcePath, filename));
|
|
145
|
-
let definitions = new ModelDefinitions(modelSource);
|
|
146
|
-
func += definitions.compileDefinitions();
|
|
147
|
-
// Needs the actual searchTermToKey closure...
|
|
148
|
-
// Which needs the actual applyCasing closure as well.
|
|
149
|
-
func += `LMLayerWorker.loadModel(new models.TrieModel(${createTrieDataStructure(filenames, definitions.searchTermToKey)}, {\n`;
|
|
150
|
-
let wordBreakerSourceCode = compileWordBreaker(normalizeWordBreakerSpec(modelSource.wordBreaker));
|
|
151
|
-
func += ` wordBreaker: ${wordBreakerSourceCode},\n`;
|
|
152
|
-
// START - the lexical mapping option block
|
|
153
|
-
func += ` searchTermToKey: ${definitions.compileSearchTermToKey()},\n`;
|
|
154
|
-
if (modelSource.languageUsesCasing != null) {
|
|
155
|
-
func += ` languageUsesCasing: ${modelSource.languageUsesCasing},\n`;
|
|
156
|
-
} // else leave undefined.
|
|
157
|
-
if (modelSource.languageUsesCasing) {
|
|
158
|
-
func += ` applyCasing: ${definitions.compileApplyCasing()},\n`;
|
|
159
|
-
}
|
|
160
|
-
// END - the lexical mapping option block.
|
|
161
|
-
if (modelSource.punctuation) {
|
|
162
|
-
func += ` punctuation: ${JSON.stringify(modelSource.punctuation)},\n`;
|
|
163
|
-
}
|
|
164
|
-
func += `}));\n`;
|
|
165
|
-
break;
|
|
166
|
-
default:
|
|
167
|
-
throw new ModelCompilerError(ModelCompilerMessages.Error_UnknownModelFormat({ format: modelSource.format }));
|
|
168
|
-
}
|
|
169
|
-
func += fileSuffix;
|
|
170
|
-
return func;
|
|
171
|
-
}
|
|
172
|
-
/**
|
|
173
|
-
* @internal
|
|
174
|
-
*/
|
|
175
|
-
transpileSources(sources) {
|
|
176
|
-
return sources.map((source) => ts.transpileModule(source, {
|
|
177
|
-
compilerOptions: {
|
|
178
|
-
target: ts.ScriptTarget.ES3,
|
|
179
|
-
module: ts.ModuleKind.None,
|
|
180
|
-
}
|
|
181
|
-
}).outputText);
|
|
182
|
-
}
|
|
183
|
-
;
|
|
184
|
-
}
|
|
185
|
-
;
|
|
186
|
-
/**
|
|
187
|
-
* @internal
|
|
188
|
-
* Returns a JavaScript expression (as a string) that can serve as a word
|
|
189
|
-
* breaking function.
|
|
190
|
-
*/
|
|
191
|
-
function compileWordBreaker(spec) {
|
|
192
|
-
let wordBreakerCode = compileInnerWordBreaker(spec.use);
|
|
193
|
-
if (spec.joinWordsAt) {
|
|
194
|
-
wordBreakerCode = compileJoinDecorator(spec, wordBreakerCode);
|
|
195
|
-
}
|
|
196
|
-
if (spec.overrideScriptDefaults) {
|
|
197
|
-
wordBreakerCode = compileScriptOverrides(spec, wordBreakerCode);
|
|
198
|
-
}
|
|
199
|
-
return wordBreakerCode;
|
|
200
|
-
}
|
|
201
|
-
/**
|
|
202
|
-
* @internal
|
|
203
|
-
*/
|
|
204
|
-
function compileJoinDecorator(spec, existingWordBreakerCode) {
|
|
205
|
-
// Bundle the source of the join decorator, as an IIFE,
|
|
206
|
-
// like this: (function join(breaker, joiners) {/*...*/}(breaker, joiners))
|
|
207
|
-
// The decorator will run IMMEDIATELY when the model is loaded,
|
|
208
|
-
// by the LMLayer returning the decorated word breaker to the
|
|
209
|
-
// LMLayer model.
|
|
210
|
-
let joinerExpr = JSON.stringify(spec.joinWordsAt);
|
|
211
|
-
return `(${decorateWithJoin.toString()}(${existingWordBreakerCode}, ${joinerExpr}))`;
|
|
212
|
-
}
|
|
213
|
-
function compileScriptOverrides(spec, existingWordBreakerCode) {
|
|
214
|
-
return `(${decorateWithScriptOverrides.toString()}(${existingWordBreakerCode}, '${spec.overrideScriptDefaults}'))`;
|
|
215
|
-
}
|
|
216
|
-
/**
|
|
217
|
-
* @internal
|
|
218
|
-
* Compiles the base word breaker, that may be decorated later.
|
|
219
|
-
* Returns the source code of a JavaScript expression.
|
|
220
|
-
*/
|
|
221
|
-
function compileInnerWordBreaker(spec) {
|
|
222
|
-
if (typeof spec === "string") {
|
|
223
|
-
// It must be a builtin word breaker, so just instantiate it.
|
|
224
|
-
return `wordBreakers['${spec}']`;
|
|
225
|
-
}
|
|
226
|
-
else {
|
|
227
|
-
// It must be a function:
|
|
228
|
-
return spec.toString()
|
|
229
|
-
// Note: the .toString() might just be the property name, but we want a
|
|
230
|
-
// plain function:
|
|
231
|
-
.replace(/^wordBreak(ing|er)\b/, 'function');
|
|
232
|
-
}
|
|
233
|
-
}
|
|
234
|
-
/**
|
|
235
|
-
* @internal
|
|
236
|
-
* Given a word breaker specification in any of the messy ways,
|
|
237
|
-
* normalizes it to a common form that the compiler can deal with.
|
|
238
|
-
*/
|
|
239
|
-
function normalizeWordBreakerSpec(wordBreakerSpec) {
|
|
240
|
-
if (wordBreakerSpec == undefined) {
|
|
241
|
-
// Use the default word breaker when it's unspecified
|
|
242
|
-
return { use: 'default' };
|
|
243
|
-
}
|
|
244
|
-
else if (isSimpleWordBreaker(wordBreakerSpec)) {
|
|
245
|
-
// The word breaker was passed as a literal function; use its source code.
|
|
246
|
-
return { use: wordBreakerSpec };
|
|
247
|
-
}
|
|
248
|
-
else if (wordBreakerSpec.use) {
|
|
249
|
-
return wordBreakerSpec;
|
|
250
|
-
}
|
|
251
|
-
else {
|
|
252
|
-
throw new ModelCompilerError(ModelCompilerMessages.Error_UnknownWordBreaker({ spec: wordBreakerSpec.toString() }));
|
|
253
|
-
}
|
|
254
|
-
}
|
|
255
|
-
/**
|
|
256
|
-
* @internal
|
|
257
|
-
*/
|
|
258
|
-
function isSimpleWordBreaker(spec) {
|
|
259
|
-
return typeof spec === "function" || spec === "default" || spec === "ascii";
|
|
260
|
-
}
|
|
1
|
+
/*
|
|
2
|
+
lexical-model-compiler.ts: base file for lexical model compiler.
|
|
3
|
+
*/
|
|
4
|
+
import ts from "typescript";
|
|
5
|
+
import { createTrieDataStructure } from "./build-trie.js";
|
|
6
|
+
import { ModelDefinitions } from "./model-definitions.js";
|
|
7
|
+
import { decorateWithJoin } from "./join-word-breaker-decorator.js";
|
|
8
|
+
import { decorateWithScriptOverrides } from "./script-overrides-decorator.js";
|
|
9
|
+
import { ModelCompilerError, ModelCompilerMessageContext, ModelCompilerMessages } from "./model-compiler-messages.js";
|
|
10
|
+
import { callbacks, setCompilerCallbacks } from "./compiler-callbacks.js";
|
|
11
|
+
;
|
|
12
|
+
;
|
|
13
|
+
;
|
|
14
|
+
/**
|
|
15
|
+
* @public
|
|
16
|
+
* Compiles a .model.ts file to a .model.js. The compiler does not read or write
|
|
17
|
+
* from filesystem or network directly, but relies on callbacks for all external
|
|
18
|
+
* IO.
|
|
19
|
+
*/
|
|
20
|
+
export class LexicalModelCompiler {
|
|
21
|
+
/**
|
|
22
|
+
* Initialize the compiler. There are currently no options
|
|
23
|
+
* specific to the lexical model compiler
|
|
24
|
+
* @param callbacks - Callbacks for external interfaces, including message
|
|
25
|
+
* reporting and file io
|
|
26
|
+
* @param options - Compiler options
|
|
27
|
+
* @returns always succeeds and returns true
|
|
28
|
+
*/
|
|
29
|
+
async init(callbacks, _options) {
|
|
30
|
+
setCompilerCallbacks(callbacks);
|
|
31
|
+
return true;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Compiles a .model.ts file to .model.js. Returns an object containing binary
|
|
35
|
+
* artifacts on success. The files are passed in by name, and the compiler
|
|
36
|
+
* will use callbacks as passed to the {@link LexicalModelCompiler.init}
|
|
37
|
+
* function to read any input files by disk.
|
|
38
|
+
* @param infile - Path to source file. Path will be parsed to find relative
|
|
39
|
+
* references in the .kmn file, such as icon or On Screen
|
|
40
|
+
* Keyboard file
|
|
41
|
+
* @param outfile - Path to output file. The file will not be written to, but
|
|
42
|
+
* will be included in the result for use by
|
|
43
|
+
* {@link LexicalModelCompiler.write}.
|
|
44
|
+
* @returns Binary artifacts on success, null on failure.
|
|
45
|
+
*/
|
|
46
|
+
async run(inputFilename, outputFilename) {
|
|
47
|
+
try {
|
|
48
|
+
let modelSource = this.loadFromFilename(inputFilename);
|
|
49
|
+
let containingDirectory = callbacks.path.dirname(inputFilename);
|
|
50
|
+
let code = this.generateLexicalModelCode('<unknown>', modelSource, containingDirectory);
|
|
51
|
+
const result = {
|
|
52
|
+
artifacts: {
|
|
53
|
+
js: {
|
|
54
|
+
data: new TextEncoder().encode(code),
|
|
55
|
+
filename: outputFilename ?? inputFilename.replace(/\.model\.ts$/, '.model.js')
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
};
|
|
59
|
+
return result;
|
|
60
|
+
}
|
|
61
|
+
catch (e) {
|
|
62
|
+
callbacks.reportMessage(e instanceof ModelCompilerError
|
|
63
|
+
? e.event
|
|
64
|
+
: ModelCompilerMessages.Fatal_UnexpectedException({ e: e }));
|
|
65
|
+
return null;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Write artifacts from a successful compile to disk, via callbacks methods.
|
|
70
|
+
* The artifacts written may include:
|
|
71
|
+
*
|
|
72
|
+
* - .model.js file - Javascript lexical model for web and touch platforms
|
|
73
|
+
*
|
|
74
|
+
* @param artifacts - object containing artifact binary data to write out
|
|
75
|
+
* @returns always returns true
|
|
76
|
+
*/
|
|
77
|
+
async write(artifacts) {
|
|
78
|
+
callbacks.fs.writeFileSync(artifacts.js.filename, artifacts.js.data);
|
|
79
|
+
return true;
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* @internal
|
|
83
|
+
* Loads a lexical model's source module from the given filename.
|
|
84
|
+
*
|
|
85
|
+
* @param filename - path to the model source file.
|
|
86
|
+
*/
|
|
87
|
+
loadFromFilename(filename) {
|
|
88
|
+
let sourceCode = new TextDecoder().decode(callbacks.loadFile(filename));
|
|
89
|
+
// Compile the module to JavaScript code.
|
|
90
|
+
// NOTE: transpile module does a very simple TS to JS compilation.
|
|
91
|
+
// It DOES NOT check for types!
|
|
92
|
+
let compilationOutput = ts.transpile(sourceCode, {
|
|
93
|
+
// Our runtime only supports ES3 with Node/CommonJS modules on Android 5.0.
|
|
94
|
+
// When we drop Android 5.0 support, we can update this to a `ScriptTarget`
|
|
95
|
+
// matrix against target version of Keyman, here and in
|
|
96
|
+
// lexical-model-compiler.ts.
|
|
97
|
+
target: ts.ScriptTarget.ES3,
|
|
98
|
+
module: ts.ModuleKind.CommonJS,
|
|
99
|
+
});
|
|
100
|
+
// Turn the module into a function in which we can inject a global.
|
|
101
|
+
let moduleCode = '(function(exports){' + compilationOutput + '})';
|
|
102
|
+
// Run the module; its exports will be assigned to `moduleExports`.
|
|
103
|
+
let moduleExports = {};
|
|
104
|
+
let module = eval(moduleCode);
|
|
105
|
+
module(moduleExports);
|
|
106
|
+
if (!moduleExports['__esModule'] || !moduleExports['default']) {
|
|
107
|
+
ModelCompilerMessageContext.filename = filename;
|
|
108
|
+
throw new ModelCompilerError(ModelCompilerMessages.Error_NoDefaultExport());
|
|
109
|
+
}
|
|
110
|
+
return moduleExports['default'];
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* @internal
|
|
114
|
+
* Returns the generated code for the model that will ultimately be loaded by
|
|
115
|
+
* the LMLayer worker. This code contains all model parameters, and specifies
|
|
116
|
+
* word breakers and auxilary functions that may be required.
|
|
117
|
+
*
|
|
118
|
+
* @param model_id - The model ID. TODO: not sure if this is actually required!
|
|
119
|
+
* @param modelSource - A specification of the model to compile
|
|
120
|
+
* @param sourcePath - Where to find auxilary sources files
|
|
121
|
+
*/
|
|
122
|
+
generateLexicalModelCode(model_id, modelSource, sourcePath) {
|
|
123
|
+
// TODO: add metadata in comment
|
|
124
|
+
const filePrefix = `(function() {\n'use strict';\n`;
|
|
125
|
+
const fileSuffix = `})();`;
|
|
126
|
+
let func = filePrefix;
|
|
127
|
+
//
|
|
128
|
+
// Emit the model as code and data
|
|
129
|
+
//
|
|
130
|
+
switch (modelSource.format) {
|
|
131
|
+
case "custom-1.0":
|
|
132
|
+
let sources = modelSource.sources.map(function (source) {
|
|
133
|
+
return new TextDecoder().decode(callbacks.loadFile(callbacks.path.join(sourcePath, source)));
|
|
134
|
+
});
|
|
135
|
+
func += this.transpileSources(sources).join('\n');
|
|
136
|
+
func += `LMLayerWorker.loadModel(new ${modelSource.rootClass}());\n`;
|
|
137
|
+
break;
|
|
138
|
+
case "fst-foma-1.0":
|
|
139
|
+
throw new ModelCompilerError(ModelCompilerMessages.Error_UnimplementedModelFormat({ format: modelSource.format }));
|
|
140
|
+
case "trie-1.0":
|
|
141
|
+
// Convert all relative path names to paths relative to the enclosing
|
|
142
|
+
// directory. This way, we'll read the files relative to the model.ts
|
|
143
|
+
// file, rather than the current working directory.
|
|
144
|
+
let filenames = modelSource.sources.map(filename => callbacks.path.join(sourcePath, filename));
|
|
145
|
+
let definitions = new ModelDefinitions(modelSource);
|
|
146
|
+
func += definitions.compileDefinitions();
|
|
147
|
+
// Needs the actual searchTermToKey closure...
|
|
148
|
+
// Which needs the actual applyCasing closure as well.
|
|
149
|
+
func += `LMLayerWorker.loadModel(new models.TrieModel(${createTrieDataStructure(filenames, definitions.searchTermToKey)}, {\n`;
|
|
150
|
+
let wordBreakerSourceCode = compileWordBreaker(normalizeWordBreakerSpec(modelSource.wordBreaker));
|
|
151
|
+
func += ` wordBreaker: ${wordBreakerSourceCode},\n`;
|
|
152
|
+
// START - the lexical mapping option block
|
|
153
|
+
func += ` searchTermToKey: ${definitions.compileSearchTermToKey()},\n`;
|
|
154
|
+
if (modelSource.languageUsesCasing != null) {
|
|
155
|
+
func += ` languageUsesCasing: ${modelSource.languageUsesCasing},\n`;
|
|
156
|
+
} // else leave undefined.
|
|
157
|
+
if (modelSource.languageUsesCasing) {
|
|
158
|
+
func += ` applyCasing: ${definitions.compileApplyCasing()},\n`;
|
|
159
|
+
}
|
|
160
|
+
// END - the lexical mapping option block.
|
|
161
|
+
if (modelSource.punctuation) {
|
|
162
|
+
func += ` punctuation: ${JSON.stringify(modelSource.punctuation)},\n`;
|
|
163
|
+
}
|
|
164
|
+
func += `}));\n`;
|
|
165
|
+
break;
|
|
166
|
+
default:
|
|
167
|
+
throw new ModelCompilerError(ModelCompilerMessages.Error_UnknownModelFormat({ format: modelSource.format }));
|
|
168
|
+
}
|
|
169
|
+
func += fileSuffix;
|
|
170
|
+
return func;
|
|
171
|
+
}
|
|
172
|
+
/**
|
|
173
|
+
* @internal
|
|
174
|
+
*/
|
|
175
|
+
transpileSources(sources) {
|
|
176
|
+
return sources.map((source) => ts.transpileModule(source, {
|
|
177
|
+
compilerOptions: {
|
|
178
|
+
target: ts.ScriptTarget.ES3,
|
|
179
|
+
module: ts.ModuleKind.None,
|
|
180
|
+
}
|
|
181
|
+
}).outputText);
|
|
182
|
+
}
|
|
183
|
+
;
|
|
184
|
+
}
|
|
185
|
+
;
|
|
186
|
+
/**
|
|
187
|
+
* @internal
|
|
188
|
+
* Returns a JavaScript expression (as a string) that can serve as a word
|
|
189
|
+
* breaking function.
|
|
190
|
+
*/
|
|
191
|
+
function compileWordBreaker(spec) {
|
|
192
|
+
let wordBreakerCode = compileInnerWordBreaker(spec.use);
|
|
193
|
+
if (spec.joinWordsAt) {
|
|
194
|
+
wordBreakerCode = compileJoinDecorator(spec, wordBreakerCode);
|
|
195
|
+
}
|
|
196
|
+
if (spec.overrideScriptDefaults) {
|
|
197
|
+
wordBreakerCode = compileScriptOverrides(spec, wordBreakerCode);
|
|
198
|
+
}
|
|
199
|
+
return wordBreakerCode;
|
|
200
|
+
}
|
|
201
|
+
/**
|
|
202
|
+
* @internal
|
|
203
|
+
*/
|
|
204
|
+
function compileJoinDecorator(spec, existingWordBreakerCode) {
|
|
205
|
+
// Bundle the source of the join decorator, as an IIFE,
|
|
206
|
+
// like this: (function join(breaker, joiners) {/*...*/}(breaker, joiners))
|
|
207
|
+
// The decorator will run IMMEDIATELY when the model is loaded,
|
|
208
|
+
// by the LMLayer returning the decorated word breaker to the
|
|
209
|
+
// LMLayer model.
|
|
210
|
+
let joinerExpr = JSON.stringify(spec.joinWordsAt);
|
|
211
|
+
return `(${decorateWithJoin.toString()}(${existingWordBreakerCode}, ${joinerExpr}))`;
|
|
212
|
+
}
|
|
213
|
+
function compileScriptOverrides(spec, existingWordBreakerCode) {
|
|
214
|
+
return `(${decorateWithScriptOverrides.toString()}(${existingWordBreakerCode}, '${spec.overrideScriptDefaults}'))`;
|
|
215
|
+
}
|
|
216
|
+
/**
|
|
217
|
+
* @internal
|
|
218
|
+
* Compiles the base word breaker, that may be decorated later.
|
|
219
|
+
* Returns the source code of a JavaScript expression.
|
|
220
|
+
*/
|
|
221
|
+
function compileInnerWordBreaker(spec) {
|
|
222
|
+
if (typeof spec === "string") {
|
|
223
|
+
// It must be a builtin word breaker, so just instantiate it.
|
|
224
|
+
return `wordBreakers['${spec}']`;
|
|
225
|
+
}
|
|
226
|
+
else {
|
|
227
|
+
// It must be a function:
|
|
228
|
+
return spec.toString()
|
|
229
|
+
// Note: the .toString() might just be the property name, but we want a
|
|
230
|
+
// plain function:
|
|
231
|
+
.replace(/^wordBreak(ing|er)\b/, 'function');
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
/**
|
|
235
|
+
* @internal
|
|
236
|
+
* Given a word breaker specification in any of the messy ways,
|
|
237
|
+
* normalizes it to a common form that the compiler can deal with.
|
|
238
|
+
*/
|
|
239
|
+
function normalizeWordBreakerSpec(wordBreakerSpec) {
|
|
240
|
+
if (wordBreakerSpec == undefined) {
|
|
241
|
+
// Use the default word breaker when it's unspecified
|
|
242
|
+
return { use: 'default' };
|
|
243
|
+
}
|
|
244
|
+
else if (isSimpleWordBreaker(wordBreakerSpec)) {
|
|
245
|
+
// The word breaker was passed as a literal function; use its source code.
|
|
246
|
+
return { use: wordBreakerSpec };
|
|
247
|
+
}
|
|
248
|
+
else if (wordBreakerSpec.use) {
|
|
249
|
+
return wordBreakerSpec;
|
|
250
|
+
}
|
|
251
|
+
else {
|
|
252
|
+
throw new ModelCompilerError(ModelCompilerMessages.Error_UnknownWordBreaker({ spec: wordBreakerSpec.toString() }));
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
/**
|
|
256
|
+
* @internal
|
|
257
|
+
*/
|
|
258
|
+
function isSimpleWordBreaker(spec) {
|
|
259
|
+
return typeof spec === "function" || spec === "default" || spec === "ascii";
|
|
260
|
+
}
|
|
261
261
|
//# sourceMappingURL=lexical-model-compiler.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"lexical-model-compiler.js","sourceRoot":"","sources":["../../src/lexical-model-compiler.ts"],"names":[],"mappings":"AAAA;;EAEE;AAEF,OAAO,EAAE,MAAM,YAAY,CAAC;AAC5B,OAAO,EAAE,uBAAuB,EAAE,MAAM,iBAAiB,CAAC;AAC1D,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAC,gBAAgB,EAAC,MAAM,kCAAkC,CAAC;AAClE,OAAO,EAAC,2BAA2B,EAAC,MAAM,iCAAiC,CAAC;AAE5E,OAAO,EAAE,kBAAkB,EAAE,2BAA2B,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AACtH,OAAO,EAAE,SAAS,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAUzE,CAAC;AAYD,CAAC;AAYD,CAAC;AAEF;;;;;GAKG;AACH,MAAM,OAAO,oBAAoB;IAE/B;;;;;;;OAOG;IACH,KAAK,CAAC,IAAI,CAAC,SAA4B,EAAE,QAAyB;QAChE,oBAAoB,CAAC,SAAS,CAAC,CAAC;QAChC,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,GAAG,CAAC,aAAqB,EAAE,cAAuB;QACtD,IAAI;
|
|
1
|
+
{"version":3,"file":"lexical-model-compiler.js","sourceRoot":"","sources":["../../src/lexical-model-compiler.ts"],"names":[],"mappings":"AAAA;;EAEE;AAEF,OAAO,EAAE,MAAM,YAAY,CAAC;AAC5B,OAAO,EAAE,uBAAuB,EAAE,MAAM,iBAAiB,CAAC;AAC1D,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAC,gBAAgB,EAAC,MAAM,kCAAkC,CAAC;AAClE,OAAO,EAAC,2BAA2B,EAAC,MAAM,iCAAiC,CAAC;AAE5E,OAAO,EAAE,kBAAkB,EAAE,2BAA2B,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AACtH,OAAO,EAAE,SAAS,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAUzE,CAAC;AAYD,CAAC;AAYD,CAAC;AAEF;;;;;GAKG;AACH,MAAM,OAAO,oBAAoB;IAE/B;;;;;;;OAOG;IACH,KAAK,CAAC,IAAI,CAAC,SAA4B,EAAE,QAAyB;QAChE,oBAAoB,CAAC,SAAS,CAAC,CAAC;QAChC,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,GAAG,CAAC,aAAqB,EAAE,cAAuB;QACtD,IAAI,CAAC;YACH,IAAI,WAAW,GAAG,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC,CAAC;YACvD,IAAI,mBAAmB,GAAG,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;YAChE,IAAI,IAAI,GAAG,IAAI,CAAC,wBAAwB,CAAC,WAAW,EAAE,WAAW,EAAE,mBAAmB,CAAC,CAAC;YACxF,MAAM,MAAM,GAA+B;gBACzC,SAAS,EAAE;oBACT,EAAE,EAAE;wBACF,IAAI,EAAE,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC;wBACpC,QAAQ,EAAE,cAAc,IAAI,aAAa,CAAC,OAAO,CAAC,cAAc,EAAE,WAAW,CAAC;qBAC/E;iBACF;aACF,CAAA;YACD,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAM,CAAC,EAAE,CAAC;YACV,SAAS,CAAC,aAAa,CACrB,CAAC,YAAY,kBAAkB;gBAC/B,CAAC,CAAC,CAAC,CAAC,KAAK;gBACT,CAAC,CAAC,qBAAqB,CAAC,yBAAyB,CAAC,EAAC,CAAC,EAAC,CAAC,EAAC,CAAC,CACzD,CAAC;YACF,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,KAAK,CAAC,SAAwC;QAClD,SAAS,CAAC,EAAE,CAAC,aAAa,CAAC,SAAS,CAAC,EAAE,CAAC,QAAQ,EAAE,SAAS,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;QACrE,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;;;OAKG;IACI,gBAAgB,CAAC,QAAgB;QAEtC,IAAI,UAAU,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;QACxE,yCAAyC;QACzC,kEAAkE;QAClE,+BAA+B;QAC/B,IAAI,iBAAiB,GAAG,EAAE,CAAC,SAAS,CAAC,UAAU,EAAE;YAC/C,2EAA2E;YAC3E,2EAA2E;YAC3E,uDAAuD;YACvD,6BAA6B;YAC7B,MAAM,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG;YAC3B,MAAM,EAAE,EAAE,CAAC,UAAU,CAAC,QAAQ;SAC/B,CAAC,CAAC;QACH,mEAAmE;QACnE,IAAI,UAAU,GAAG,qBAAqB,GAAG,iBAAiB,GAAG,IAAI,CAAC;QAElE,mEAAmE;QACnE,IAAI,aAAa,GAA0B,EAAE,CAAC;QAC9C,IAAI,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC;QAC9B,MAAM,CAAC,aAAa,CAAC,CAAC;QAEtB,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,EAAE,CAAC;YAC9D,2BAA2B,CAAC,QAAQ,GAAG,QAAQ,CAAC;YAChD,MAAM,IAAI,kBAAkB,CAAC,qBAAqB,CAAC,qBAAqB,EAAE,CAAC,CAAC;QAC9E,CAAC;QAED,OAAO,aAAa,CAAC,SAAS,CAAuB,CAAC;IACxD,CAAC;IAED;;;;;;;;;OASG;IACH,wBAAwB,CAAC,QAAgB,EAAE,WAA+B,EAAE,UAAkB;QAC5F,gCAAgC;QAChC,MAAM,UAAU,GAAW,gCAAgC,CAAC;QAC5D,MAAM,UAAU,GAAW,OAAO,CAAC;QACnC,IAAI,IAAI,GAAG,UAAU,CAAC;QAEtB,EAAE;QACF,kCAAkC;QAClC,EAAE;QAEF,QAAO,WAAW,CAAC,MAAM,EAAE,CAAC;YAC1B,KAAK,YAAY;gBACf,IAAI,OAAO,GAAa,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,UAAS,MAAM;oBAC7D,OAAO,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;gBAC/F,CAAC,CAAC,CAAC;gBACH,IAAI,IAAI,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAClD,IAAI,IAAI,+BAA+B,WAAW,CAAC,SAAS,QAAQ,CAAC;gBACrE,MAAM;YACR,KAAK,cAAc;gBACjB,MAAM,IAAI,kBAAkB,CAAC,qBAAqB,CAAC,8BAA8B,CAAC,EAAC,MAAM,EAAC,WAAW,CAAC,MAAM,EAAC,CAAC,CAAC,CAAC;YAClH,KAAK,UAAU;gBACb,qEAAqE;gBACrE,qEAAqE;gBACrE,mDAAmD;gBACnD,IAAI,SAAS,GAAG,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC,CAAC;gBAE/F,IAAI,WAAW,GAAG,IAAI,gBAAgB,CAAC,WAAW,CAAC,CAAC;gBAEpD,IAAI,IAAI,WAAW,CAAC,kBAAkB,EAAE,CAAC;gBAEzC,8CAA8C;gBAC9C,sDAAsD;gBACtD,IAAI,IAAI,gDACN,uBAAuB,CAAC,SAAS,EAAE,WAAW,CAAC,eAAe,CAChE,OAAO,CAAC;gBAER,IAAI,qBAAqB,GAAG,kBAAkB,CAAC,wBAAwB,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,CAAC;gBAClG,IAAI,IAAI,kBAAkB,qBAAqB,KAAK,CAAC;gBAErD,2CAA2C;gBAC3C,IAAI,IAAI,sBAAsB,WAAW,CAAC,sBAAsB,EAAE,KAAK,CAAC;gBAExE,IAAG,WAAW,CAAC,kBAAkB,IAAI,IAAI,EAAE,CAAC;oBAC1C,IAAI,IAAI,yBAAyB,WAAW,CAAC,kBAAkB,KAAK,CAAC;gBACvE,CAAC,CAAC,wBAAwB;gBAE1B,IAAG,WAAW,CAAC,kBAAkB,EAAE,CAAC;oBAClC,IAAI,IAAI,kBAAkB,WAAW,CAAC,kBAAkB,EAAE,KAAK,CAAC;gBAClE,CAAC;gBACD,0CAA0C;gBAE1C,IAAI,WAAW,CAAC,WAAW,EAAE,CAAC;oBAC5B,IAAI,IAAI,kBAAkB,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,WAAW,CAAC,KAAK,CAAC;gBACzE,CAAC;gBACD,IAAI,IAAI,QAAQ,CAAC;gBACjB,MAAM;YACR;gBACE,MAAM,IAAI,kBAAkB,CAAC,qBAAqB,CAAC,wBAAwB,CAAC,EAAC,MAAM,EAAE,WAAW,CAAC,MAAM,EAAC,CAAC,CAAC,CAAC;QAC/G,CAAC;QAED,IAAI,IAAI,UAAU,CAAC;QAEnB,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACH,gBAAgB,CAAC,OAAsB;QACrC,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,eAAe,CAAC,MAAM,EAAE;YACtD,eAAe,EAAE;gBACf,MAAM,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG;gBAC3B,MAAM,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI;aAC3B;SACF,CAAC,CAAC,UAAU,CACd,CAAC;IACJ,CAAC;IAAA,CAAC;CAEH;AAAA,CAAC;AAEF;;;;GAIG;AACH,SAAS,kBAAkB,CAAC,IAAqB;IAC/C,IAAI,eAAe,GAAG,uBAAuB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAExD,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;QACrB,eAAe,GAAG,oBAAoB,CAAC,IAAI,EAAE,eAAe,CAAC,CAAC;IAChE,CAAC;IAED,IAAI,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAChC,eAAe,GAAG,sBAAsB,CAAC,IAAI,EAAE,eAAe,CAAC,CAAC;IAClE,CAAC;IAED,OAAO,eAAe,CAAC;AACzB,CAAC;AAED;;GAEG;AACH,SAAS,oBAAoB,CAAC,IAAqB,EAAE,uBAA+B;IAClF,uDAAuD;IACvD,2EAA2E;IAC3E,+DAA+D;IAC/D,6DAA6D;IAC7D,iBAAiB;IACjB,IAAI,UAAU,GAAW,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA;IACzD,OAAO,IAAI,gBAAgB,CAAC,QAAQ,EAAE,IAAI,uBAAuB,KAAK,UAAU,IAAI,CAAC;AACvF,CAAC;AAED,SAAS,sBAAsB,CAAC,IAAqB,EAAE,uBAA+B;IACpF,OAAO,IAAI,2BAA2B,CAAC,QAAQ,EAAE,IAAI,uBAAuB,MAAM,IAAI,CAAC,sBAAsB,KAAK,CAAC;AACrH,CAAC;AAED;;;;GAIG;AACH,SAAS,uBAAuB,CAAC,IAA2B;IAC1D,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC7B,6DAA6D;QAC7D,OAAO,iBAAiB,IAAI,IAAI,CAAC;IACnC,CAAC;SAAM,CAAC;QACN,yBAAyB;QACzB,OAAO,IAAI,CAAC,QAAQ,EAAE;YACpB,uEAAuE;YACvE,kBAAkB;aACjB,OAAO,CAAC,sBAAsB,EAAE,UAAU,CAAC,CAAC;IACjD,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,SAAS,wBAAwB,CAAC,eAAkD;IAClF,IAAI,eAAe,IAAI,SAAS,EAAE,CAAC;QACjC,qDAAqD;QACrD,OAAO,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC;IAC5B,CAAC;SAAM,IAAI,mBAAmB,CAAC,eAAe,CAAC,EAAE,CAAC;QAChD,0EAA0E;QAC1E,OAAO,EAAE,GAAG,EAAE,eAAe,EAAE,CAAC;IAClC,CAAC;SAAM,IAAI,eAAe,CAAC,GAAG,EAAE,CAAC;QAC/B,OAAO,eAAe,CAAC;IACzB,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,kBAAkB,CAAC,qBAAqB,CAAC,wBAAwB,CAAC,EAAC,IAAI,EAAE,eAAe,CAAC,QAAQ,EAAE,EAAC,CAAC,CAAC,CAAC;IACnH,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,mBAAmB,CAAC,IAA6C;IACxE,OAAO,OAAO,IAAI,KAAK,UAAU,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,OAAO,CAAC;AAC9E,CAAC"}
|