@keymanapp/kmc-model 17.0.85-alpha

package/src/model-defaults.ts

@@ -0,0 +1,111 @@
+/**
+ * Converts wordforms into an indexable form. It does this by
+ * normalizing the letter case of characters INDIVIDUALLY (to disregard
+ * context-sensitive case transformations), normalizing to NFKD form,
+ * and removing common diacritical marks.
+ *
+ * This is a very speculative implementation, that might work with
+ * your language. We don't guarantee that this will be perfect for your
+ * language, but it's a start.
+ *
+ * This uses String.prototype.normalize() to convert normalize into NFKD.
+ * NFKD neutralizes some funky distinctions, e.g., ꬲ, ｅ, e should all be the
+ * same character; plus, it's an easy way to separate a Latin character from
+ * its diacritics; Even then, orthographies regularly use code points
+ * that, under NFKD normalization, do NOT decompose appropriately for your
+ * language (e.g., SENĆOŦEN, Plains Cree in syllabics).
+ *
+ * Use this in early iterations of the model. For a production lexical model,
+ * you will probably write/generate your own key function, tailored to your
+ * language. There is a chance the default will work properly out of the box.
+ */
+export function defaultSearchTermToKey(wordform: string): string {
+  return wordform
+      .normalize('NFKD')
+      // Remove any combining diacritics (if input is in NFKD)
+      .replace(/[\u0300-\u036F]/g, '')
+      // Replace directional quotation marks with plain apostrophes
+      .replace(/‘/, "'")
+      .replace(/’/, "'")
+      // Also double-quote marks.
+      .replace(/“/, '"')
+      .replace(/”/, '"');
+}
+
+/**
+ * Converts wordforms into an indexable form. It does this by
+ * normalizing the letter case of characters INDIVIDUALLY (to disregard
+ * context-sensitive case transformations), normalizing to NFKD form,
+ * and removing common diacritical marks.
+ *
+ * This is a very speculative implementation, that might work with
+ * your language. We don't guarantee that this will be perfect for your
+ * language, but it's a start.
+ *
+ * This uses String.prototype.normalize() to convert normalize into NFKD.
+ * NFKD neutralizes some funky distinctions, e.g., ꬲ, ｅ, e should all be the
+ * same character; plus, it's an easy way to separate a Latin character from
+ * its diacritics; Even then, orthographies regularly use code points
+ * that, under NFKD normalization, do NOT decompose appropriately for your
+ * language (e.g., SENĆOŦEN, Plains Cree in syllabics).
+ *
+ * Use this in early iterations of the model. For a production lexical model,
+ * you will probably write/generate your own key function, tailored to your
+ * language. There is a chance the default will work properly out of the box.
+ */
+export function defaultCasedSearchTermToKey(wordform: string, applyCasing: CasingFunction): string {
+  // While this is a bit WET, as the basic `defaultSearchTermToKey` exists and performs some of
+  // the same functions, repetition is the easiest way to allow the function to be safely compiled
+  // with ease by use of `.toString()`.
+  return Array.from(wordform
+        .normalize('NFKD')
+        // Remove any combining diacritics (if input is in NFKD)
+        .replace(/[\u0300-\u036F]/g, '')
+      ) // end of `Array.from`
+      .map(function(c) { return applyCasing('lower', c)})
+      .join('')
+      // Replace directional quotation marks with plain apostrophes
+      .replace(/‘/, "'")
+      .replace(/’/, "'")
+      // Also double-quote marks.
+      .replace(/“/, '"')
+      .replace(/”/, '"');
+}
+
+/**
+ * Specifies default casing behavior for lexical models when `languageUsesCasing` is
+ * set to true.
+ * @param casing One of 'lower' (lowercased), 'upper' (uppercased), or 'initial'.
+ *
+ * 'initial' is designed to cover cases like sentence-initial & proper noun capitalization in English.
+ * This may be overwritten as appropriate in model-specific implementations.
+ * @param text The text to be modified.
+ */
+export function defaultApplyCasing(casing: CasingForm, text: string): string {
+  switch(casing) {
+    case 'lower':
+      return text.toLowerCase();
+    case 'upper':
+      return text.toUpperCase();
+    case 'initial':
+      var headCode = text.charCodeAt(0);
+      // The length of the first code unit, as measured in code points.
+      var headUnitLength = 1;
+
+      // Is the first character a high surrogate, indicating possible use of UTF-16
+      // surrogate pairs?  Also, is the string long enough for there to BE a pair?
+      if(text.length > 1 && headCode >= 0xD800 && headCode <= 0xDBFF) {
+        // It's possible, so now we check for low surrogates.
+        var lowSurrogateCode = text.charCodeAt(1);
+
+        if(lowSurrogateCode >= 0xDC00 && lowSurrogateCode <= 0xDFFF) {
+          // We have a surrogate pair; this pair is the 'first' character.
+          headUnitLength++;
+        }
+      }
+
+      // Capitalizes the first code unit of the string, leaving the rest intact.
+      return text.substring(0, headUnitLength).toUpperCase() // head - uppercased
+             .concat(text.substring(headUnitLength));        // tail - lowercased
+  }
+}

package/src/model-definitions.ts

@@ -0,0 +1,222 @@
+import { defaultApplyCasing,
+         defaultCasedSearchTermToKey,
+         defaultSearchTermToKey
+       } from "./model-defaults.js";
+
+import KEYMAN_VERSION from "@keymanapp/keyman-version";
+import { LexicalModelSource, WordformToKeySpec } from "./lexical-model.js";
+
+/**
+ * Processes certain defined model behaviors in such a way that the needed closures
+ * may be safely compiled to a JS file and loaded within the LMLayer.
+ *
+ * This is accomplished by writing out a 'pseudoclosure' within the model's IIFE,
+ * then used to build _actual_ closures at LMLayer load time.  This 'pseudoclosure'
+ * will very closely match the organizational patterns of this class in order to
+ * facilitate the maintenance of this approach.
+ */
+export class ModelDefinitions {
+  static readonly COMPILED_NAME = 'definitions';
+  /**
+   * A closure fully implementing the model's defined `applyCasing` behavior with
+   * the function parameter preset to the version-appropriate default.
+   * `defaults.applyCasing` is captured as part of the closure.
+   *
+   * During compilation of some models (such as Trie-based wordlist templated models),
+   * this closure will be directly used as part of searchTermToKey.
+   *
+   * In compiled code, this will instead be defined in-line as an autogenerated closure
+   * using the other properties of the pseudoclosure.
+   */
+  applyCasing?: CasingFunction;
+
+  /**
+   * A closure fully implementing the model's defined `searchTermToKey` behavior
+   * based upon the model's specified casing rules.  The `applyCasing` closure is
+   * itself captured within this closure.
+   *
+   * During compilation of some models (such as Trie-based wordlist templated models),
+   * this closure will be directly utilized when compiling the lexicon.
+   *
+   * In compiled code, this will instead be defined in-line as an autogenerated closure
+   * using the other properties of the pseudoclosure.
+   */
+  searchTermToKey?: WordformToKeySpec;
+
+  /**
+   * Contains embedded 'default' implementations that may be needed for
+   * closures in the compiled version, annotated with the current version
+   * of Developer.
+   */
+  private defaults: {
+    version: string;
+    applyCasing?: CasingFunction;
+  } = {
+    version: KEYMAN_VERSION.VERSION_WITH_TAG
+  };
+
+  /**
+   * Contains the model-specific definitions specified in the model's source.
+   *
+   * These definitions may expect `defaults.applyCasing` as a parameter in
+   * their final closures.
+   */
+  private model: {
+    applyCasing?: CasingFunction;
+    searchTermToKey?: WordformToKeySpec;
+  } = {};
+
+  constructor(modelSource: LexicalModelSource) {
+    // Determine the model's `applyCasing` function / implementation.
+    if(modelSource.languageUsesCasing) {
+      this.defaults.applyCasing = defaultApplyCasing;
+
+      if(modelSource.applyCasing) {
+        this.model.applyCasing = modelSource.applyCasing;
+        let _this = this;
+
+        // Since the defined casing function may expect to take our default implementation
+        // as a parameter, we can define the full implementation via closure capture.
+        this.applyCasing = function(casing: CasingForm, text: string) {
+          return _this.model.applyCasing(casing, text, _this.defaults.applyCasing);
+        };
+      } else {
+        this.applyCasing = this.defaults.applyCasing;
+      }
+    }
+
+    // START: if(model type uses keying)...
+
+    // Use the default search term to key function, if left unspecified.
+    if(modelSource.searchTermToKey) {
+      this.model.searchTermToKey = modelSource.searchTermToKey;
+    } else if(modelSource.languageUsesCasing) {
+      // applyCasing is defined here.
+      // Unfortunately, this only works conceptually.  .toString on a closure
+      // does not result in proper compilation.
+      this.model.searchTermToKey = defaultCasedSearchTermToKey;
+    } else if(modelSource.languageUsesCasing == false) {
+      this.model.searchTermToKey = defaultSearchTermToKey;
+    } else {
+      // If languageUsesCasing is not defined, then we use pre-14.0 behavior,
+      // which expects a lowercased default.
+      this.model.searchTermToKey = defaultCasedSearchTermToKey;
+      // Needed to provide pre-14.0 default lowercasing as part of the
+      // search-term keying operation.
+      this.defaults.applyCasing = defaultApplyCasing;
+      // For compile-time use.
+      this.applyCasing = this.defaults.applyCasing;
+    }
+
+    let _this = this;
+    this.searchTermToKey = function(text: string) {
+      return _this.model.searchTermToKey(text, _this.applyCasing);
+    }
+
+    // END: if(model type uses keying)...
+  }
+
+  // ------------ end:  common compile-time / run-time code ---------------
+
+  // START:  handwritten compilation code (to accomplish the 'common' pattern defined above)
+
+  /**
+   * Writes out a compiled JS version of the pseudoclosure, preserving all function
+   * implementations.
+   *
+   * This should be written to the file within the same IIFE as the model but BEFORE
+   * the model itself, as the model will need to refer to the definitions herein.
+   */
+  compileDefinitions(): string {
+    let defn: string = '';
+    defn += `var ${PSEUDOCLOSURE} = {\n`
+
+    // ----------------------
+    // START - the 'defaults', which are common within the same Developer version.
+    defn += `  defaults: {\n    version: "${this.defaults.version}"`;
+
+    // Only write out `applyCasing` if and when it is needed.
+    if(this.defaults.applyCasing) {
+      defn += `,\n    applyCasing: ${this.defaults.applyCasing.toString()}`;
+    }
+
+    // Finalizes `defaults`
+    defn += `\n  },`;
+    // END - the 'defaults'
+
+    // ----------------------
+    // START - model-specific definitions (when defined)
+    defn += `  model: {\n`;
+    defn += `    searchTermToKey: ${this.model.searchTermToKey.toString()}`;
+
+    if(this.model.applyCasing) {
+      defn += `,\n    applyCasing: ${this.model.applyCasing.toString()}`;
+    }
+    defn += `\n  }`
+    // END - model-specific definitions
+
+    // ----------------------
+    // START - compiled closures.  Given those definitions, write out the
+    // pseudoclosure-referencing closures for the needed methods.
+
+    // We should be able to define these closures in-line with the object's
+    // initialization.  Worst-case, we simply move the definitions outside
+    // of the pseudoclosure's init and THEN define/assign these closures to
+    // the object, as references will be available then for sure.
+    if(this.model.applyCasing) {
+      // A major potential issue:  if the user wants to call extra custom functions that they've written.
+      //
+      // `applyCasing` recursion SHOULD be fine if they write `this.applyCasing() and forward all arguments
+      // appropriately, as it will be known as `applyCasing` on the runtime `this` (`model`) object.
+      //
+      // Similarly, as long as any helper functions are similarly compiled and stored as part of `model`,
+      // they should be accessible too.  The issue would be to actually allow use of extra custom funcs
+      // and include them as part of this object as part of compilation.
+      defn += `,\n  applyCasing: function(caseToApply, text) {
+        return ${PSEUDOCLOSURE}.model.applyCasing(caseToApply, text, ${PSEUDOCLOSURE}.defaults.applyCasing);
+      }`;
+    } else if(this.defaults.applyCasing) {
+      // We can't directly assign from `.defaults`, as initialization-time field reads
+      // are not permitted within JS.  Function references, however, are valid.
+      defn += `,\n  applyCasing: function(caseToApply, text) {
+        return ${PSEUDOCLOSURE}.defaults.applyCasing(caseToApply, text);
+      }`;
+    }
+
+    // if(this.searchTermToKey) {
+    defn += `,\n  searchTermToKey: function(text) {
+      return ${PSEUDOCLOSURE}.model.searchTermToKey(text, ${PSEUDOCLOSURE}.applyCasing);
+    }`;
+    // }
+
+    // END - compiled closures.
+
+    // ----------------------
+    // Finalize the definition of... `definitions`.
+    defn += `\n};\n`;
+
+    return defn;
+  }
+
+  /**
+   * Compiles the model-options entry for `searchTermToKey` in reference to the
+   * compiled pseudoclosure.
+   */
+  compileSearchTermToKey(): string {
+    // Simply point the model to the constructed closure defined by `compilePseudoclosure`.
+    // See "START - compiled closures" section.
+    return `${PSEUDOCLOSURE}.searchTermToKey`;
+  }
+
+  /**
+   * Compiles the model-options entry for `applyCasing` in reference to the
+   * compiled pseudoclosure.
+   */
+  compileApplyCasing(): string {// Simply point the model to the constructed closure defined by `compilePseudoclosure`.
+    // See "START - compiled closures" section.
+    return `${PSEUDOCLOSURE}.applyCasing`;
+  }
+}
+
+// Because it references the class field, this line must come afterward.
+const PSEUDOCLOSURE = ModelDefinitions.COMPILED_NAME;

package/src/script-overrides-decorator.ts

@@ -0,0 +1,75 @@
+import { OverrideScriptDefaults } from "./lexical-model.js";
+
+export function decorateWithScriptOverrides(breaker: WordBreakingFunction, option: OverrideScriptDefaults) {
+  if (option !== 'break-words-at-spaces') {
+    throw new Error(`Unsupported script override: ${option}`)
+  }
+
+  /**
+   * Matches if when a span contains a Southeast-Asian letter or mark anywhere.
+   * This makes it a candidate for joining.
+   *
+   * See: tools/create-override-script-regexp.ts for how this RegExp was
+   * generated.
+   *
+   * Last updated for Unicode 13.0.0.
+   */
+  const HAS_SOUTHEAST_ASIAN_LETTER = /[\u0E01-\u0E3A\u0E40-\u0E4E\u0E81\u0E82\u0E84\u0E86-\u0E8A\u0E8C-\u0EA3\u0EA5\u0EA7-\u0EBD\u0EC0-\u0EC4\u0EC6\u0EC8-\u0ECD\u0EDC-\u0EDF\u1000-\u103F\u1050-\u108F\u109A-\u109D\u1780-\u17D3\u17D7\u17DC\u17DD\u30A1-\u30FA\u30FC-\u30FF]/;
+
+  return function enhancedBreaker(phrase: string): Span[] {
+    let originalSpans = breaker(phrase);
+
+    if (originalSpans.length === 0) {
+      return [];
+    }
+
+    let outputSpans = [originalSpans.shift()];
+    for (let currentSpan of originalSpans) {
+      let previousSpan = lastFrom(outputSpans);
+
+      if (spansAreBackToBack(previousSpan, currentSpan) &&
+          hasSouthEastAsianLetter(previousSpan) &&
+          hasSouthEastAsianLetter(currentSpan)
+      ) {
+        // previous span SHOULD be joined with current!
+        outputSpans[outputSpans.length - 1] = concatenateSpans(previousSpan, currentSpan);
+      } else {
+        outputSpans.push(currentSpan);
+      }
+    }
+
+    return outputSpans;
+  }
+
+  function hasSouthEastAsianLetter(span: Span) {
+    return HAS_SOUTHEAST_ASIAN_LETTER.test(span.text);
+  }
+
+  /**
+   * Returns true when the spans are contiguous.
+   * Order matters when calling this function!
+   */
+  function spansAreBackToBack(former: Span, latter: Span): boolean {
+    return former.end === latter.start;
+  }
+
+  function concatenateSpans(former: Span, latter: Span) {
+    if (latter.start !== former.end) {
+      throw new Error(`Cannot concatenate non-contiguous spans: ${JSON.stringify(former)}/${JSON.stringify(latter)}`);
+    }
+
+    return {
+      start: former.start,
+      end: latter.end,
+      length: former.length + latter.length,
+      text: former.text + latter.text
+    };
+  }
+
+  /**
+   * Get the last element from the array.
+   */
+  function lastFrom<T>(array: T[]): T | undefined {
+    return array[array.length - 1];
+  }
+}

package/test/README.md

@@ -0,0 +1,15 @@
+Lexical Model and Package Compiler
+==================================
+
+Build
+-----
+
+    npm run build
+
+
+Test
+----
+
+    npm test
+
+**NOTE**: ensure you run `npm run build` before running the tests!

package/test/fixtures/example.qaa.joinwordbreaker/example.qaa.joinwordbreaker.model.ts

@@ -0,0 +1,10 @@
+const source: LexicalModelSource = {
+  format: 'trie-1.0',
+  sources: ['wordlist.tsv'],
+  /* Keyman 14.0+ word breaker specification: */
+  wordBreaker: {
+    use: 'default',
+    joinWordsAt: [':', '-', '·']
+  }
+};
+export default source;

package/test/fixtures/example.qaa.joinwordbreaker/wordlist.tsv

@@ -0,0 +1,3 @@
+Kanien'kehá:ka	1000
+amiskwaciy-wâskahikan	100
+cel·la	10

package/test/fixtures/example.qaa.scriptusesspaces/example.qaa.scriptusesspaces.model.ts

@@ -0,0 +1,10 @@
+const source: LexicalModelSource = {
+  format: 'trie-1.0',
+  sources: ['wordlist.tsv'],
+  /* Keyman 14.0+ word breaker specification: */
+  wordBreaker: {
+    use: 'default',
+    overrideScriptDefaults: 'break-words-at-spaces',
+  }
+};
+export default source;

package/test/fixtures/example.qaa.scriptusesspaces/wordlist.tsv

@@ -0,0 +1,8 @@
+ເກ໌າະ	675
+ເຢາະ	285
+ຢັອຫ	246
+ສນາ	99
+ບຣອມ	69
+ລະ	40
+ຈາກ	26
+ກ໌ນິ	26

package/test/fixtures/example.qaa.sencoten/example.qaa.sencoten.model.kmp.json

@@ -0,0 +1,45 @@
+{
+  "system": {
+    "keymanDeveloperVersion": "12.0.1500.0",
+    "fileVersion": "12.0"
+  },
+  "options": {},
+  "info": {
+    "author": {
+      "description": "Eddie Antonio Santos",
+      "url": "mailto:Eddie.Santos@nrc-cnrc.gc.ca"
+    },
+    "copyright": {
+      "description": "© 2019 National Research Council Canada"
+    },
+    "name": {
+      "description": "SENĆOŦEN (Saanich Dialect) Lexical Model"
+    },
+    "version": {
+      "description": "1.0.3"
+    }
+  },
+  "files": [
+    {
+      "name": "example.qaa.sencoten.model.js",
+      "description": "Lexical model example.qaa.sencoten.model.js",
+      "copyLocation": 0
+    }
+  ],
+  "lexicalModels": [
+    {
+      "name": "SENĆOŦEN dictionary",
+      "id": "example.qaa.sencoten",
+      "languages": [
+        {
+          "name": "North Straits Salish",
+          "id": "qaa"
+        },
+        {
+          "name": "SENĆOŦEN",
+          "id": "qaa-Latn"
+        }
+      ]
+    }
+  ]
+}

package/test/fixtures/example.qaa.sencoten/example.qaa.sencoten.model.kps

@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Package>
+  <System>
+    <KeymanDeveloperVersion>12.0.1500.0</KeymanDeveloperVersion>
+    <FileVersion>12.0</FileVersion>
+  </System>
+  <Options>
+    <FollowKeyboardVersion/>
+  </Options>
+  <Info>
+    <Name URL="">SENĆOŦEN (Saanich Dialect) Lexical Model</Name>
+    <Copyright URL="">© 2019 National Research Council Canada</Copyright>
+    <Author URL="mailto:Eddie.Santos@nrc-cnrc.gc.ca">Eddie Antonio Santos</Author>
+    <Version>1.0.3</Version>
+  </Info>
+  <Files>
+    <File>
+      <Name>..\build\example.qaa.sencoten.model.js</Name>
+      <Description>Lexical model example.qaa.sencoten.model.js</Description>
+      <CopyLocation>0</CopyLocation>
+      <FileType>.model.js</FileType>
+    </File>
+  </Files>
+  <LexicalModels>
+    <LexicalModel>
+      <Name>SENĆOŦEN dictionary</Name>
+      <ID>example.qaa.sencoten</ID>
+      <Version>1.0.3</Version>
+      <Languages>
+        <Language ID="qaa">North Straits Salish</Language>
+        <Language ID="qaa-Latn">SENĆOŦEN</Language>
+      </Languages>
+    </LexicalModel>
+  </LexicalModels>
+</Package>

package/test/fixtures/example.qaa.sencoten/example.qaa.sencoten.model.ts

@@ -0,0 +1,6 @@
+const source: LexicalModelSource = {
+  format: 'trie-1.0',
+  sources: ['wordlist.tsv'],
+  wordBreaker: 'default',
+};
+export default source;

package/test/fixtures/example.qaa.sencoten/wordlist.tsv

@@ -0,0 +1,10 @@
+TŦE	13644
+E	9134
+SEN	4816
+Ȼ	3479
+SW̱	2621
+NIȽ	2314
+U¸	2298
+I¸	1988
+ȻSE	1925
+I	1884

package/test/fixtures/example.qaa.smp/example.qaa.smp.model.ts

@@ -0,0 +1,6 @@
+const source: LexicalModelSource = {
+  format: 'trie-1.0',
+  sources: ['wordlist.tsv'],
+  wordBreaker: 'default',
+};
+export default source;

package/test/fixtures/example.qaa.smp/wordlist.tsv

@@ -0,0 +1,5 @@
+CRAZ🤪	13644
+🙄	9134
+😇	4816
+🇸
+🇺

package/test/fixtures/example.qaa.trivial/example.qaa.trivial.model.ts

@@ -0,0 +1,5 @@
+const source: LexicalModelSource = {
+  format: 'trie-1.0',
+  sources: ['wordlist.tsv'],
+};
+export default source;

package/test/fixtures/example.qaa.trivial/wordlist.tsv

@@ -0,0 +1,3 @@
+I
+like
+turtles

package/test/fixtures/example.qaa.utf16be/example.qaa.utf16be.model.ts

@@ -0,0 +1,5 @@
+const source: LexicalModelSource = {
+  format: 'trie-1.0',
+  sources: ['wordlist.txt'],
+};
+export default source;

package/test/fixtures/example.qaa.utf16le/example.qaa.utf16le.model.ts

@@ -0,0 +1,5 @@
+const source: LexicalModelSource = {
+  format: 'trie-1.0',
+  sources: ['wordlist.txt'],
+};
+export default source;

package/test/fixtures/example.qaa.wordbreaker/example.qaa.wordbreaker.model.ts

@@ -0,0 +1,9 @@
+const source: LexicalModelSource = {
+  format: 'trie-1.0',
+  sources: ['wordlist.tsv'],
+  /* Keyman 14.0+ word breaker specification: */
+  wordBreaker: {
+    use: 'default'
+  }
+};
+export default source;

package/test/fixtures/example.qaa.wordbreaker/wordlist.tsv

@@ -0,0 +1,3 @@
+I
+like
+turtles