@keymanapp/kmc-model 17.0.85-alpha

package/src/lexical-model-compiler.ts

@@ -0,0 +1,179 @@
+/*
+  lexical-model-compiler.ts: base file for lexical model compiler.
+*/
+
+/// <reference path="./lexical-model.ts" />
+
+import * as ts from "typescript";
+import * as fs from "fs";
+import * as path from "path";
+import { createTrieDataStructure } from "./build-trie.js";
+import { ModelDefinitions } from "./model-definitions.js";
+import {decorateWithJoin} from "./join-word-breaker-decorator.js";
+import {decorateWithScriptOverrides} from "./script-overrides-decorator.js";
+import { LexicalModelSource, WordBreakerSpec, SimpleWordBreakerSpec } from "./lexical-model.js";
+
+export default class LexicalModelCompiler {
+
+  /**
+   * Returns the generated code for the model that will ultimately be loaded by
+   * the LMLayer worker. This code contains all model parameters, and specifies
+   * word breakers and auxilary functions that may be required.
+   *
+   * @param model_id      The model ID. TODO: not sure if this is actually required!
+   * @param modelSource   A specification of the model to compile
+   * @param sourcePath    Where to find auxilary sources files
+   */
+  generateLexicalModelCode(model_id: string, modelSource: LexicalModelSource, sourcePath: string) {
+    // TODO: add metadata in comment
+    const filePrefix: string = `(function() {\n'use strict';\n`;
+    const fileSuffix: string = `})();`;
+    let func = filePrefix;
+
+    //
+    // Emit the model as code and data
+    //
+
+    switch(modelSource.format) {
+      case "custom-1.0":
+        let sources: string[] = modelSource.sources.map(function(source) {
+          return fs.readFileSync(path.join(sourcePath, source), 'utf8');
+        });
+        func += this.transpileSources(sources).join('\n');
+        func += `LMLayerWorker.loadModel(new ${modelSource.rootClass}());\n`;
+        break;
+      case "fst-foma-1.0":
+        throw new ModelSourceError(`Unimplemented model format: ${modelSource.format}`);
+      case "trie-1.0":
+        // Convert all relative path names to paths relative to the enclosing
+        // directory. This way, we'll read the files relative to the model.ts
+        // file, rather than the current working directory.
+        let filenames = modelSource.sources.map(filename => path.join(sourcePath, filename));
+
+        let definitions = new ModelDefinitions(modelSource);
+
+        func += definitions.compileDefinitions();
+
+        // Needs the actual searchTermToKey closure...
+        // Which needs the actual applyCasing closure as well.
+        func += `LMLayerWorker.loadModel(new models.TrieModel(${
+          createTrieDataStructure(filenames, definitions.searchTermToKey)
+        }, {\n`;
+
+        let wordBreakerSourceCode = compileWordBreaker(normalizeWordBreakerSpec(modelSource.wordBreaker));
+        func += `  wordBreaker: ${wordBreakerSourceCode},\n`;
+
+        // START - the lexical mapping option block
+        func += `  searchTermToKey: ${definitions.compileSearchTermToKey()},\n`;
+
+        if(modelSource.languageUsesCasing != null) {
+          func += `  languageUsesCasing: ${modelSource.languageUsesCasing},\n`;
+        } // else leave undefined.
+
+        if(modelSource.languageUsesCasing) {
+          func += `  applyCasing: ${definitions.compileApplyCasing()},\n`;
+        }
+        // END - the lexical mapping option block.
+
+        if (modelSource.punctuation) {
+          func += `  punctuation: ${JSON.stringify(modelSource.punctuation)},\n`;
+        }
+        func += `}));\n`;
+        break;
+      default:
+        throw new ModelSourceError(`Unknown model format: ${modelSource.format}`);
+    }
+
+    func += fileSuffix;
+
+    return func;
+  }
+
+  transpileSources(sources: Array<string>): Array<string> {
+    return sources.map((source) => ts.transpileModule(source, {
+        compilerOptions: {
+          target: ts.ScriptTarget.ES3,
+          module: ts.ModuleKind.None,
+        }
+      }).outputText
+    );
+  };
+
+  logError(s: string) {
+    console.error(require('chalk').red(s));
+  };
+};
+
+export class ModelSourceError extends Error {
+}
+
+/**
+ * Returns a JavaScript expression (as a string) that can serve as a word
+ * breaking function.
+ */
+function compileWordBreaker(spec: WordBreakerSpec): string {
+  let wordBreakerCode = compileInnerWordBreaker(spec.use);
+
+  if (spec.joinWordsAt) {
+    wordBreakerCode = compileJoinDecorator(spec, wordBreakerCode);
+  }
+
+  if (spec.overrideScriptDefaults) {
+    wordBreakerCode = compileScriptOverrides(spec, wordBreakerCode);
+  }
+
+  return wordBreakerCode;
+}
+
+function compileJoinDecorator(spec: WordBreakerSpec, existingWordBreakerCode: string) {
+  // Bundle the source of the join decorator, as an IIFE,
+  // like this: (function join(breaker, joiners) {/*...*/}(breaker, joiners))
+  // The decorator will run IMMEDIATELY when the model is loaded,
+  // by the LMLayer returning the decorated word breaker to the
+  // LMLayer model.
+  let joinerExpr: string = JSON.stringify(spec.joinWordsAt)
+  return `(${decorateWithJoin.toString()}(${existingWordBreakerCode}, ${joinerExpr}))`;
+}
+
+function compileScriptOverrides(spec: WordBreakerSpec, existingWordBreakerCode: string) {
+  return `(${decorateWithScriptOverrides.toString()}(${existingWordBreakerCode}, '${spec.overrideScriptDefaults}'))`;
+}
+
+/**
+ * Compiles the base word breaker, that may be decorated later.
+ * Returns the source code of a JavaScript expression.
+ */
+function compileInnerWordBreaker(spec: SimpleWordBreakerSpec): string {
+  if (typeof spec === "string") {
+    // It must be a builtin word breaker, so just instantiate it.
+    return `wordBreakers['${spec}']`;
+  } else {
+    // It must be a function:
+    return spec.toString()
+      // Note: the .toString() might just be the property name, but we want a
+      // plain function:
+      .replace(/^wordBreak(ing|er)\b/, 'function');
+  }
+}
+
+/**
+ * Given a word breaker specification in any of the messy ways,
+ * normalizes it to a common form that the compiler can deal with.
+ */
+function normalizeWordBreakerSpec(wordBreakerSpec: LexicalModelSource["wordBreaker"]): WordBreakerSpec {
+  if (wordBreakerSpec == undefined) {
+    // Use the default word breaker when it's unspecified
+    return { use: 'default' };
+  } else if (isSimpleWordBreaker(wordBreakerSpec)) {
+    // The word breaker was passed as a literal function; use its source code.
+    return { use: wordBreakerSpec };
+  } else if (wordBreakerSpec.use) {
+    return wordBreakerSpec;
+  } else {
+    throw new Error(`Unknown word breaker: ${wordBreakerSpec}`);
+  }
+}
+
+function isSimpleWordBreaker(spec: WordBreakerSpec | SimpleWordBreakerSpec): spec is SimpleWordBreakerSpec  {
+  return typeof spec === "function" || spec === "default" || spec === "ascii";
+}

package/src/lexical-model.ts

@@ -0,0 +1,150 @@
+/**
+ * Interfaces and constants used by the lexical model compiler. These target
+ * the LMLayer's internal worker code, so we provide those definitions too.
+ */
+
+export interface LexicalModelDeclaration {
+  readonly format: 'trie-1.0'|'fst-foma-1.0'|'custom-1.0',
+  //... metadata ...
+}
+
+/**
+ * Keyman 14.0+ word breaker specification:
+ *
+ * Can support all old word breaking specification,
+ * but can also be extended with options.
+ *
+ * @since 14.0
+ */
+ export interface WordBreakerSpec {
+  readonly use: SimpleWordBreakerSpec;
+  /**
+   * If present, joins words that were split by the word breaker
+   * together at the given strings. e.g.,
+   *
+   *    joinWordsAt: ['-'] // to keep hyphenated items together
+   *
+   * @since 14.0
+   */
+  readonly joinWordsAt?: string[];
+
+  /**
+   * Overrides word splitting behaviour for certain scripts.
+   * For example, specifing that spaces break words in certain South-East
+   * Asian scripts that otherwise do not use spaces.
+   *
+   * @since 14.0
+   */
+  readonly overrideScriptDefaults?: OverrideScriptDefaults;
+}
+
+/**
+ * Simplified word breaker specification.
+ *
+ * @since 11.0
+ */
+ export type SimpleWordBreakerSpec = 'default' | 'ascii' | WordBreakingFunction;
+
+/**
+ * Simplifies input text to facilitate finding entries within a lexical model's
+ * lexicon.
+ * @since 11.0
+ */
+ export type SimpleWordformToKeySpec = (term: string) => string;
+
+/**
+ * Simplifies input text to facilitate finding entries within a lexical model's
+ * lexicon, using the model's `applyCasing` function to assist in the keying process.
+ * @since 14.0
+ */
+export type CasedWordformToKeySpec = (term: string, applyCasing?: CasingFunction) => string;
+
+/**
+ * Simplifies input text to facilitate finding entries within a lexical model's
+ * lexicon.
+ */
+ export type WordformToKeySpec = SimpleWordformToKeySpec | CasedWordformToKeySpec;
+
+/**
+ * Override the default word breaking behaviour for some scripts.
+ *
+ * There is currently only one option:
+ *
+ * 'break-words-at-spaces'
+ * : some South-East Asian scripts conventionally do not use space or any
+ * explicit word boundary character to write word breaks. These scripts are:
+ *
+ *   * Burmese
+ *   * Khmer
+ *   * Thai
+ *   * Laos
+ *
+ * (this list may be incomplete and extended in the future)
+ *
+ * For these scripts, the default word breaker breaks at **every**
+ * letter/syllable/ideograph. However, in languages that use these scripts BUT
+ * use spaces (or some other delimier) as word breaks, enable
+ * 'break-words-at-spaces'; enabling 'break-words-at-spaces' prevents the word
+ * breaker from making too many breaks in these scripts.
+ *
+ * @since 14.0
+ */
+export type OverrideScriptDefaults = 'break-words-at-spaces';
+
+
+export interface LexicalModelSource extends LexicalModelDeclaration {
+  readonly sources: Array<string>;
+  /**
+   * The name of the type to instantiate (without parameters) as the base object for a custom predictive model.
+   */
+  readonly rootClass?: string
+
+  /**
+   * When set to `true`, suggestions will attempt to match the case of the input text even if
+   * the lexicon entries use a different casing scheme due to search term keying effects.
+   * @since 14.0
+   */
+  readonly languageUsesCasing?: boolean
+
+  /**
+   * Specifies the casing rules for a language.  Should implement three casing forms:
+   * - 'lower' -- a fully-lowercased version of the text appropriate for the language's
+   *   use of the writing system.
+   * - 'upper' -- a fully-uppercased version of the text
+   * - 'initial' -- a version preserving the input casing aside from the initial character,
+   *   which is uppercased (like with proper nouns and sentence-initial words in English
+   *   sentences.)
+   *
+   * This is only utilized if `languageUsesCasing` is defined and set to `true`.
+   * @since 14.0
+   */
+  readonly applyCasing?: CasingFunction
+
+  /**
+   * Which word breaker to use. Choose from:
+   *
+   *  - 'default' -- breaks according to Unicode UAX #29 §4.1 Default Word
+   *    Boundary Specification, which works well for *most* languages.
+   *  - 'ascii' -- a very simple word breaker, for demonstration purposes only.
+   *  - word breaking function -- provide your own function that breaks words.
+   *  - class-based word-breaker - may be supported in the future.
+   */
+  readonly wordBreaker?: WordBreakerSpec | SimpleWordBreakerSpec;
+
+  /**
+   * How to simplify words, to convert them into simplified search keys
+   * This often involves removing accents, lowercasing, etc.
+   */
+  readonly searchTermToKey?: WordformToKeySpec;
+
+  /**
+   * Punctuation and spacing suggested by the model.
+   *
+   * @see LexicalModelPunctuation
+   */
+  readonly punctuation?: LexicalModelPunctuation;
+}
+
+export interface LexicalModelCompiled extends LexicalModelDeclaration {
+  readonly id: string;
+}

package/src/main.ts

@@ -0,0 +1,62 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import ts from 'typescript';
+
+import LexicalModelCompiler from './lexical-model-compiler.js';
+import { LexicalModelSource } from './lexical-model.js';
+
+/**
+ * Compiles a model.ts file, using paths relative to its location.
+ *
+ * @param filename path to model.ts source.
+ * @return model source code
+ */
+export function compileModel(filename: string): string {
+  let modelSource = loadFromFilename(filename);
+  let containingDirectory = path.dirname(filename);
+
+  return (new LexicalModelCompiler)
+    .generateLexicalModelCode('<unknown>', modelSource, containingDirectory);
+}
+
+/**
+ * An ECMAScript module as emitted by the TypeScript compiler.
+ */
+interface ES2015Module {
+  /** This is always true. */
+  __esModule: boolean;
+  'default'?: unknown;
+}
+
+/**
+ * Loads a lexical model's source module from the given filename.
+ *
+ * @param filename path to the model source file.
+ */
+export function loadFromFilename(filename: string): LexicalModelSource {
+  let sourceCode = fs.readFileSync(filename, 'utf8');
+  // Compile the module to JavaScript code.
+  // NOTE: transpile module does a very simple TS to JS compilation.
+  // It DOES NOT check for types!
+  let compilationOutput = ts.transpile(sourceCode, {
+    // Our runtime only supports ES3 with Node/CommonJS modules on Android 5.0.
+    // When we drop Android 5.0 support, we can update this to a `ScriptTarget` 
+    // matrix against target version of Keyman, here and in 
+    // lexical-model-compiler.ts.
+    target: ts.ScriptTarget.ES3,
+    module: ts.ModuleKind.CommonJS,
+  });
+  // Turn the module into a function in which we can inject a global.
+  let moduleCode = '(function(exports){' + compilationOutput + '})';
+
+  // Run the module; its exports will be assigned to `moduleExports`.
+  let moduleExports: Partial<ES2015Module> = {};
+  let module = eval(moduleCode);
+  module(moduleExports);
+
+  if (!moduleExports['__esModule'] || !moduleExports['default']) {
+    throw new Error(`Model source '${filename}' does have a default export. Did you remember to write \`export default source;\`?`);
+  }
+
+  return moduleExports['default'] as LexicalModelSource;
+}

package/src/model-compiler-errors.ts

@@ -0,0 +1,203 @@
+// TODO: merge with kmc keyboard-compiler errors
+/**
+ * Log levels.
+ *
+ * Note: Currently, this acts like a bit set, where the upper 4 bits of an
+ * unsigned 16 bit value are the log level flags.
+ *
+ * Warning: even though these look like bitmasks, these flags may not be
+ * combined.
+ */
+export enum LogLevel {
+  CERR_FATAL = 0x8000,
+  CERR_ERROR = 0x4000,
+  CERR_WARNING = 0x2000,
+  CERR_HINT = 0x1000
+};
+
+/**
+ * Error codes. Use these when logging messages.
+ *
+ * Extends https://github.com/keymanapp/keyman/blob/99db3c0d2448f448242e6397f9d72e9a7ccee4b9/windows/src/global/inc/Comperr.h
+ */
+export enum KeymanCompilerError {
+  CERR_LEXICAL_MODEL_MIN = 0x0800,
+  CERR_LEXICAL_MODEL_MAX = 0x08FF,
+
+
+  CERR_FATAL_LM = LogLevel.CERR_FATAL | CERR_LEXICAL_MODEL_MIN,
+    /* Place all fatal LM compiler errors here! */
+
+  CERR_ERROR_LM = LogLevel.CERR_ERROR | CERR_LEXICAL_MODEL_MIN,
+    /* Place all recoverable LM compiler errors here! */
+
+  CERR_WARN_LM = LogLevel.CERR_WARNING | CERR_LEXICAL_MODEL_MIN,
+    /* Place all LM compiler warnings here! */
+  CWARN_MixedNormalizationForms = 0x2801, /* CERR_WARN_LM + 1 */
+  CWARN_DuplicateWordInSameFile = 0x2802, /* CERR_WARN_LM + 2 */
+
+  CERR_HINT_LM = LogLevel.CERR_HINT | CERR_LEXICAL_MODEL_MIN,
+    /* Place all LM compiler hints here! */
+
+  /* Errors that are not specific to the lexical model compiler, from comperr.h: */
+  CWARN_TooManyErrorsOrWarnings = 0x20A7,
+}
+
+/**
+ * Human-readable titles for the various log levels.
+ *
+ * Taken from https://github.com/keymanapp/keyman/blob/d83cfffe511ce65b781f919e89e3693146844849/windows/src/developer/TIKE/project/Keyman.Developer.System.Project.ProjectLog.pas#L39-L46
+ */
+const LOG_LEVEL_TITLE: {[level in LogLevel]: string} = {
+  [LogLevel.CERR_HINT]: 'Hint',
+  [LogLevel.CERR_WARNING]: 'Warning',
+  [LogLevel.CERR_ERROR]: 'Error',
+  [LogLevel.CERR_FATAL]: 'Fatal Error',
+};
+
+/**
+ * How many errors or warnings are too many!
+ */
+export const MAX_MESSAGES = 100;
+
+/**
+ * Direct where log messages go.
+ */
+let _logHandler: (log: LogMessage) => void = printLogs;
+
+/**
+ * How many logs or warnings have been witnessed so far.
+ */
+let _messagesSeen = 0;
+
+/**
+ * Logs compiler messages (warnings, errors, logs).
+ *
+ * @param code Error code
+ * @param message A helpful message!
+ * @param source [optional] the filename/line number in the source that induced this error
+ *
+ * @see https://github.com/keymanapp/keyman/blob/99db3c0d2448f448242e6397f9d72e9a7ccee4b9/windows/src/developer/TIKE/project/Keyman.Developer.System.Project.ProjectLog.pas#L60-L77
+ */
+export function log(code: KeymanCompilerError, message: string, source?: FilenameAndLineNo) {
+  // Ignore the request if there are too many messages
+  if (_messagesSeen > MAX_MESSAGES)
+    return;
+
+  let logMessage = source
+    ? new LogMessageFromSource(code, message, source)
+    : new OrdinaryLogMessage(code, message);
+
+  _logHandler(logMessage)
+  _messagesSeen++;
+
+  if (_messagesSeen > MAX_MESSAGES) {
+      _logHandler(new OrdinaryLogMessage(
+        KeymanCompilerError.CWARN_TooManyErrorsOrWarnings,
+        `More than ${MAX_MESSAGES} warnings or errors received; suppressing further messages`
+      ));
+  }
+}
+
+/**
+ * Override where log messages go.
+ *
+ * @param fn The desired log message handler.
+ */
+export function redirectLogMessagesTo(fn: (log: LogMessage) => void) {
+  _logHandler = fn;
+}
+
+/**
+ * Reset the log message handler to the default.
+ */
+export function resetLogMessageHandler() {
+  _logHandler = printLogs;
+  _messagesSeen = 0;
+}
+
+/**
+ * Prints log messages to stdout. The default log action.
+ */
+export function printLogs(log: LogMessage): void {
+  console.log(log.format());
+}
+
+/**
+ * Duct tapes together a filename and a line number of a log.
+ */
+interface FilenameAndLineNo {
+  readonly filename: string;
+  readonly lineno: number;
+}
+
+/**
+ * A log message that knows how to format itself.
+ */
+export interface LogMessage {
+  readonly code: KeymanCompilerError;
+  readonly level: LogLevel;
+  readonly message: string;
+
+  format(): string;
+}
+
+/**
+ * Concrete implementation of the log message.
+ */
+class OrdinaryLogMessage implements LogMessage {
+  readonly code: KeymanCompilerError;
+  readonly message: string;
+
+  constructor(code: KeymanCompilerError, message: string) {
+    this.code = code;
+    this.message = message;
+  }
+
+  get level(): LogLevel {
+    return this.code & 0xF000;
+  }
+
+  determineLogLevelTitle(): string {
+    return LOG_LEVEL_TITLE[this.level] || '';
+  }
+
+  format(): string {
+    let prefix = this.determineLogLevelTitle();
+    if (prefix)
+      prefix = `${prefix}: `;
+
+    return `${prefix}${h(this.code)} ${this.message}`
+  }
+}
+
+/**
+ * A log message with a filename and line number.
+ */
+class LogMessageFromSource extends OrdinaryLogMessage {
+  readonly filename: string;
+  readonly lineno: number;
+
+  constructor(code: KeymanCompilerError, message: string, source: FilenameAndLineNo) {
+    super(code, message);
+    this.filename = source.filename;
+    this.lineno = source.lineno;
+  }
+
+  format(): string {
+    let originalMessage = super.format();
+    return `${this.filename} (${this.lineno}): ${originalMessage}`;
+  }
+}
+
+/**
+ * Format a number as a zero-padded 4 digit hexadecimal.
+ */
+function h(n: number) {
+  let formatted = n.toString(16).toUpperCase();
+  if (formatted.length < 4) {
+    formatted = '0'.repeat(4 - formatted.length);
+  }
+
+  return formatted;
+}