@keymanapp/kmc-package 17.0.85-alpha

package/src/kmp-compiler.ts

@@ -0,0 +1,273 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import * as xml2js from 'xml2js';
+import JSZip from 'jszip';
+import KEYMAN_VERSION from "@keymanapp/keyman-version";
+
+import type { KpsFile, KpsFileContentFile, KpsFileInfo, KpsFileKeyboard, KpsFileLanguage, KpsFileLexicalModel, KpsFileOptions, KpsPackage } from './kps-file.js';
+import type { KmpJsonFile, KmpJsonFileInfo, KmpJsonFileLanguage, KmpJsonFileOptions } from './kmp-json-file.js';
+
+export { type KmpJsonFile } from './kmp-json-file.js';
+
+const FILEVERSION_KMP_JSON = '12.0';
+
+export default class KmpCompiler {
+
+  public transformKpsToKmpObject(kpsString: string, kpsPath: string): KmpJsonFile {
+
+    // Load the KPS data from XML as JS structured data.
+
+    let kpsPackage = (() => {
+        let a: KpsPackage;
+        let parser = new xml2js.Parser({
+          tagNameProcessors: [xml2js.processors.firstCharLowerCase],
+          explicitArray: false
+        });
+        parser.parseString(kpsString, (e: unknown, r: unknown) => { a = r as KpsPackage });
+        return a;
+    })();
+
+    let kps: KpsFile = kpsPackage.package;
+
+    //
+    // To convert to kmp.json, we need to:
+    //
+    //   1. Unwrap arrays (and convert to array where single object)
+    //   2. Fix casing on `iD`
+    //   3. Rewrap info, keyboard.languages, lexicalModel.languages, startMenu.items elements
+    //   4. Remove options.followKeyboardVersion, file.fileType
+    //   5. Convert file.copyLocation to a Number
+    //   6. Filenames need to be basenames (but this comes after processing)
+    //
+
+    // Start to construct the kmp.json file from the .kps file
+
+    let kmp: KmpJsonFile = {
+      system: {
+        fileVersion: FILEVERSION_KMP_JSON,
+        keymanDeveloperVersion: KEYMAN_VERSION.VERSION
+      },
+      options: {}
+    };
+
+    //
+    // Fill in additional fields
+    //
+
+    let keys: [keyof KpsFileOptions, keyof KmpJsonFileOptions][] = [
+      ['executeProgram','executeProgram'],
+      ['graphicFile', 'graphicFile'],
+      ['msiFileName','msiFilename'],
+      ['msiOptions', 'msiOptions'],
+      ['readMeFile', 'readmeFile']
+    ];
+    for (let [src,dst] of keys) {
+      if (kps.options[src]) {
+        kmp.options[dst] = kps.options[src];
+      }
+    }
+
+    //
+    // Add basic metadata
+    //
+
+    if(kps.info) {
+      kmp.info = this.kpsInfoToKmpInfo(kps.info);
+    }
+
+    // FollowKeyboardVersion support
+
+    if(kps.options?.followKeyboardVersion !== undefined) {
+      kmp.info.version = {
+        description: this.extractKeyboardVersionFromKmx(kpsPath, kps)
+      };
+    }
+
+    //
+    // Add file metadata
+    //
+
+    if(kps.files && kps.files.file) {
+      kmp.files = this.arrayWrap(kps.files.file).map((file: KpsFileContentFile) => {
+        return {
+          name: file.name,
+          description: file.description,
+          copyLocation: parseInt(file.copyLocation, 10) || undefined
+          // note: we don't emit fileType as that is not permitted in kmp.json
+        };
+      });
+    }
+    kmp.files = kmp.files ?? [];
+
+    // Add the standard kmp.json self-referential to match existing implementations
+    kmp.files.push({
+      name: "kmp.json",
+      description: "Package information (JSON)"
+    });
+
+    //
+    // Add keyboard metadata
+    //
+
+    if(kps.keyboards && kps.keyboards.keyboard) {
+      kmp.keyboards = this.arrayWrap(kps.keyboards.keyboard).map((keyboard: KpsFileKeyboard) => {
+        return {
+          displayFont: keyboard.displayFont ? path.basename(keyboard.displayFont.replaceAll('\\', '/')) : undefined,
+          oskFont: keyboard.oSKFont ? path.basename(keyboard.oSKFont.replaceAll('\\', '/')) : undefined,
+          name:keyboard.name,
+          id:keyboard.iD,
+          version:keyboard.version,
+          languages: this.kpsLanguagesToKmpLanguages(this.arrayWrap(keyboard.languages.language) as KpsFileLanguage[])
+        };
+      });
+    }
+
+    //
+    // Add lexical-model metadata
+    //
+
+    if(kps.lexicalModels && kps.lexicalModels.lexicalModel) {
+      kmp.lexicalModels = this.arrayWrap(kps.lexicalModels.lexicalModel).map((model: KpsFileLexicalModel) => {
+        return { name:model.name, id:model.iD, languages: this.kpsLanguagesToKmpLanguages(this.arrayWrap(model.languages.language) as KpsFileLanguage[]) }
+      });
+    }
+
+    //
+    // Add Windows Start Menu metadata
+    //
+
+    if(kps.startMenu && kps.startMenu.items) {
+      kmp.startMenu = {};
+      if(kps.startMenu.addUninstallEntry) kmp.startMenu.addUninstallEntry = kps.startMenu.addUninstallEntry === '';
+      if(kps.startMenu.folder) kmp.startMenu.folder = kps.startMenu.folder;
+      if(kps.startMenu.items && kps.startMenu.items.item) kmp.startMenu.items = this.arrayWrap(kps.startMenu.items.item);
+    }
+
+    //
+    // Add translation strings
+    //
+
+    if(kps.strings && kps.strings.string) {
+      kmp.strings = this.arrayWrap(kps.strings.string);
+    }
+
+    kmp = this.stripUndefined(kmp) as KmpJsonFile;
+
+    return kmp;
+  }
+
+    // Helper functions
+
+  private kpsInfoToKmpInfo(info: KpsFileInfo): KmpJsonFileInfo {
+    let ni: KmpJsonFileInfo = {};
+
+    const keys: [(keyof KpsFileInfo), (keyof KmpJsonFileInfo)][] = [
+      ['author','author'],
+      ['copyright','copyright'],
+      ['name','name'],
+      ['version','version'],
+      ['webSite','website']
+    ];
+
+    for (let [src,dst] of keys) {
+      if (info[src]) {
+        ni[dst] = {description: info[src]._ ?? (typeof info[src] == 'string' ? info[src].toString() : '')};
+        if(info[src].$ && info[src].$.URL) ni[dst].url = info[src].$.URL;
+      }
+    }
+
+    return ni;
+  };
+
+  private arrayWrap(a: unknown) {
+    if (Array.isArray(a)) {
+      return a;
+    }
+    return [a];
+  };
+
+  private kpsLanguagesToKmpLanguages(language: KpsFileLanguage[]): KmpJsonFileLanguage[] {
+    return language.map((element) => { return { name: element._, id: element.$.ID } });
+  };
+
+  private extractKeyboardVersionFromKmx(kpsPath: string, kps: KpsFile) {
+    // TODO-LDML: #7340 this is incomplete; we need to read from first .kmx
+    // When we do this, we'll need to update fixtures to have the correct
+    // version string also
+    if(!kps.keyboards || !kps.keyboards.keyboard) {
+      // We don't read from the kps metadata because we want the keyboard
+      // version data here;
+      // TODO: currently model files don't support follow-version?
+      return '0.0.0';
+    }
+    let k: KpsFileKeyboard[] = this.arrayWrap(kps.keyboards.keyboard);
+    return k?.[0]?.version ?? '0.0.0';
+  }
+
+  private stripUndefined(o: any) {
+    for(const key in o) {
+      if(o[key] === undefined) {
+        delete o[key];
+      } else if(typeof o[key] == 'object') {
+        o[key] = this.stripUndefined(o[key]);
+      }
+    }
+    return o;
+  }
+
+  /**
+   * Returns a Promise to the serialized data which can then be written to a .kmp file.
+   *
+   * @param kpsFilename - Filename of the kps, not read, used only for calculating relative paths
+   * @param kmpJsonData - The kmp.json Object
+   */
+  public buildKmpFile(kpsFilename: string, kmpJsonData: KmpJsonFile): Promise<string> {
+    const zip = JSZip();
+
+    const kmpJsonFileName = 'kmp.json';
+
+    const basePath = path.dirname(kpsFilename);
+
+    // Make a copy of kmpJsonData, as we mutate paths for writing
+    const data: KmpJsonFile = JSON.parse(JSON.stringify(kmpJsonData));
+    if(!data.files) {
+      data.files = [];
+    }
+
+    data.files.forEach(function(value) {
+      // Get the path of the file
+      let filename = value.name;
+
+      // We add this separately after zipping all other files
+      if(filename == 'kmp.json') {
+        return;
+      }
+
+      if(path.isAbsolute(value.name)) {
+        // absolute paths are not very cross-platform compatible -- we are going to have trouble
+        // with path separators and roots
+        // TODO: emit a warning
+      } else {
+        // Transform separators to platform separators -- kps files may use
+        // either / or \, although older kps files were always \.
+        if(path.sep == '/') {
+          filename = filename.replace(/\\/g, '/');
+        } else {
+          filename = filename.replace(/\//g, '\\');
+        }
+        filename = path.resolve(basePath, filename);
+      }
+      const basename = path.basename(filename);
+      let data = fs.readFileSync(filename);
+      zip.file(basename, data);
+
+      // Remove path data from files before JSON save
+      value.name = basename;
+    });
+
+    zip.file(kmpJsonFileName, JSON.stringify(data, null, 2));
+
+    // Generate kmp file
+    return zip.generateAsync({type: 'binarystring', compression:'DEFLATE'});
+  }
+}

package/src/kmp-json-file.ts

@@ -0,0 +1,77 @@
+export interface KmpJsonFile {
+  system: KmpJsonFileSystem;
+  options: KmpJsonFileOptions;
+  info?: KmpJsonFileInfo;
+  files?: KmpJsonFileContentFile[];
+  lexicalModels?: KmpJsonFileLexicalModel[];
+  startMenu?: KmpJsonFileStartMenu;
+  keyboards?: KmpJsonFileKeyboard[];
+  strings?: string[];
+}
+
+export interface KmpJsonFileSystem {
+  keymanDeveloperVersion: string;
+  fileVersion: string;
+}
+
+export interface KmpJsonFileOptions {
+  readmeFile?: string;
+  graphicFile?: string;
+  executeProgram?: string;
+  msiFilename?: string;
+  msiOptions?: string;
+}
+
+export interface KmpJsonFileInfo {
+  website?: KmpJsonFileInfoItem;
+  version?: KmpJsonFileInfoItem;
+  name?: KmpJsonFileInfoItem;
+  copyright?: KmpJsonFileInfoItem;
+  author?: KmpJsonFileInfoItem;
+}
+
+export interface KmpJsonFileInfoItem {
+  description: string;
+  url?: string;
+}
+
+export interface KmpJsonFileContentFile {
+  name: string;
+  description: string;
+  copyLocation?: number;
+}
+
+export interface KmpJsonFileLexicalModel {
+  name: string;
+  id: string;
+  languages: KmpJsonFileLanguage[];
+}
+
+export interface KmpJsonFileLanguage {
+  name: string;
+  id: string;
+}
+
+export interface KmpJsonFileKeyboard {
+  name: string;
+  id: string;
+  version: string;
+  oskFont?: string;
+  displayFont?: string;
+  rtl?: boolean;
+  languages?: KmpJsonFileLanguage[];
+}
+
+export interface KmpJsonFileStartMenu {
+  folder?: string;
+  addUninstallEntry?: boolean;
+  items?: KmpJsonFileStartMenuItem[];
+}
+
+export interface KmpJsonFileStartMenuItem {
+  name: string;
+  filename: string;
+  arguments?: string;
+  icon?: string;
+  location?: string;
+}

package/src/kps-file.ts

@@ -0,0 +1,126 @@
+//
+// The interfaces in this file are designed with reference to the
+// mapped structures produced by xml2js when passed a .kps file.
+//
+// A few notes:
+//
+// * Casing is updated to camelCase during load (leaving `iD` as a
+//   mixed up beastie).
+// * Arrays are buried a layer too deep (e.g. <Files><File/><File/></Files>
+//   leads to KpsFiles.KpsFile[]
+// * Properties such as used in Info Items use `_` and `$` and must be
+//   extracted.
+// * Strings element is not yet checked to be correct
+//
+
+export interface KpsPackage {
+  /**
+   * <Package> -- the root element.
+   */
+  package: KpsFile;
+}
+
+export interface KpsFile {
+  system: KpsFileSystem;
+  options: KpsFileOptions;
+  info?: KpsFileInfo;
+  files?: KpsFileContentFiles;
+  keyboards?: KpsFileKeyboards;
+  lexicalModels?: KpsFileLexicalModels;
+  startMenu?: KpsFileStartMenu;
+  strings?: KpsFileStrings;
+}
+
+export interface KpsFileSystem {
+  keymanDeveloperVersion: string;
+  fileVersion: string;
+}
+
+export interface KpsFileOptions {
+  followKeyboardVersion?: string;
+  readMeFile?: string;
+  graphicFile?: string;
+  executeProgram?: string;
+  msiFileName?: string;
+  msiOptions?: string;
+}
+
+export interface KpsFileInfo {
+  name?: KpsFileInfoItem;
+  copyright?: KpsFileInfoItem;
+  author?: KpsFileInfoItem;
+  webSite?: KpsFileInfoItem;
+  version?: KpsFileInfoItem;
+}
+
+export interface KpsFileInfoItem {
+  _: string;
+  $: { URL: string };
+}
+
+export interface KpsFileContentFiles {
+  file: KpsFileContentFile[] | KpsFileContentFile;
+}
+
+export interface KpsFileContentFile {
+  name: string;
+  description: string;
+  copyLocation: string;
+  fileType: string;
+}
+
+export interface KpsFileLexicalModel {
+  name: string;
+  iD: string;
+  languages: KpsFileLanguages;
+}
+
+export interface KpsFileLexicalModels {
+  lexicalModel: KpsFileLexicalModel[] | KpsFileLexicalModel;
+}
+
+export interface KpsFileLanguages {
+  language: KpsFileLanguage[] | KpsFileLanguage;
+}
+
+export interface KpsFileLanguage {
+  _: string;
+  $: { ID: string }
+}
+
+export interface KpsFileKeyboard {
+  name: string;
+  iD: string;
+  version: string;
+  oSKFont?: string;
+  displayFont?: string;
+  rtl?: boolean;
+  languages?: KpsFileLanguages;
+}
+
+export interface KpsFileKeyboards {
+  keyboard: KpsFileKeyboard[] | KpsFileKeyboard;
+}
+
+export interface KpsFileStartMenu {
+  folder?: string;
+  addUninstallEntry?: string;
+  items?: KpsFileStartMenuItems;
+}
+
+export interface KpsFileStartMenuItem {
+  name: string;
+  filename: string;
+  arguments?: string;
+  icon?: string;
+  location?: string;
+}
+
+export interface KpsFileStartMenuItems {
+  item: KpsFileStartMenuItem[] | KpsFileStartMenuItem;
+}
+
+export interface KpsFileStrings {
+  //TODO: validate this structure
+  string: string[] | string;
+}

package/test/fixtures/example.qaa.sencoten/example.qaa.sencoten.model.js

@@ -0,0 +1,55 @@
+(function() {
+'use strict';
+var definitions = {
+  defaults: {
+    version: "16.0.60-alpha",
+    applyCasing: function defaultApplyCasing(casing, text) {
+    switch (casing) {
+        case 'lower':
+            return text.toLowerCase();
+        case 'upper':
+            return text.toUpperCase();
+        case 'initial':
+            var headCode = text.charCodeAt(0);
+            // The length of the first code unit, as measured in code points.
+            var headUnitLength = 1;
+            // Is the first character a high surrogate, indicating possible use of UTF-16
+            // surrogate pairs?  Also, is the string long enough for there to BE a pair?
+            if (text.length > 1 && headCode >= 0xD800 && headCode <= 0xDBFF) {
+                // It's possible, so now we check for low surrogates.
+                var lowSurrogateCode = text.charCodeAt(1);
+                if (lowSurrogateCode >= 0xDC00 && lowSurrogateCode <= 0xDFFF) {
+                    // We have a surrogate pair; this pair is the 'first' character.
+                    headUnitLength++;
+                }
+            }
+            // Capitalizes the first code unit of the string, leaving the rest intact.
+            return text.substring(0, headUnitLength).toUpperCase() // head - uppercased
+                .concat(text.substring(headUnitLength)); // tail - lowercased
+    }
+}
+  },  model: {
+    searchTermToKey: function defaultCasedSearchTermToKey(wordform, applyCasing) {
+    // While this is a bit WET, as the basic `defaultSearchTermToKey` exists and performs some of
+    // the same functions, repetition is the easiest way to allow the function to be safely compiled
+    // with ease by use of `.toString()`.
+    return Array.from(wordform
+        .normalize('NFKD')
+        // Remove any combining diacritics (if input is in NFKD)
+        .replace(/[\u0300-\u036F]/g, '')) // end of `Array.from`
+        .map(function (c) { return applyCasing('lower', c); })
+        .join('');
+}
+  },
+  applyCasing: function(caseToApply, text) {
+        return definitions.defaults.applyCasing(caseToApply, text);
+      },
+  searchTermToKey: function(text) {
+      return definitions.model.searchTermToKey(text, definitions.applyCasing);
+    }
+};
+LMLayerWorker.loadModel(new models.TrieModel({"totalWeight":44103,"root":{"type":"internal","weight":13644,"values":["t","e","s","ȼ","n","u","i"],"children":{"t":{"type":"leaf","weight":13644,"entries":[{"key":"tŧe","weight":13644,"content":"TŦE"}]},"e":{"type":"leaf","weight":9134,"entries":[{"key":"e","weight":9134,"content":"E"}]},"s":{"type":"internal","weight":4816,"values":["e","w"],"children":{"e":{"type":"leaf","weight":4816,"entries":[{"key":"sen","weight":4816,"content":"SEN"}]},"w":{"type":"leaf","weight":2621,"entries":[{"key":"sw","weight":2621,"content":"SW̱"}]}}},"ȼ":{"type":"internal","weight":3479,"values":["﷐","s"],"children":{"﷐":{"type":"leaf","weight":3479,"entries":[{"key":"ȼ","weight":3479,"content":"Ȼ"}]},"s":{"type":"leaf","weight":1925,"entries":[{"key":"ȼse","weight":1925,"content":"ȻSE"}]}}},"n":{"type":"leaf","weight":2314,"entries":[{"key":"niƚ","weight":2314,"content":"NIȽ"}]},"u":{"type":"leaf","weight":2298,"entries":[{"key":"u ","weight":2298,"content":"U¸"}]},"i":{"type":"leaf","weight":1988,"entries":[{"key":"i ","weight":1988,"content":"I¸"},{"key":"i","weight":1884,"content":"I"}]}}}}, {
+  wordBreaker: wordBreakers['default'],
+  searchTermToKey: definitions.searchTermToKey,
+}));
+})();

package/test/fixtures/example.qaa.sencoten/example.qaa.sencoten.model.kmp.intermediate.json

@@ -0,0 +1,48 @@
+{
+  "system": {
+    "keymanDeveloperVersion": "12.0.1500.0",
+    "fileVersion": "12.0"
+  },
+  "options": {},
+  "info": {
+    "author": {
+      "description": "Eddie Antonio Santos",
+      "url": "mailto:Eddie.Santos@nrc-cnrc.gc.ca"
+    },
+    "copyright": {
+      "description": "© 2019 National Research Council Canada"
+    },
+    "name": {
+      "description": "SENĆOŦEN (Saanich Dialect) Lexical Model"
+    },
+    "version": {
+      "description": "0.0.0"
+    }
+  },
+  "files": [
+    {
+      "name": "example.qaa.sencoten.model.js",
+      "description": "Lexical model example.qaa.sencoten.model.js"
+    },
+    {
+      "description": "Package information (JSON)",
+      "name": "kmp.json"
+    }
+  ],
+  "lexicalModels": [
+    {
+      "name": "SENĆOŦEN dictionary",
+      "id": "example.qaa.sencoten",
+      "languages": [
+        {
+          "name": "North Straits Salish",
+          "id": "qaa"
+        },
+        {
+          "name": "SENĆOŦEN",
+          "id": "qaa-Latn"
+        }
+      ]
+    }
+  ]
+}

package/test/fixtures/example.qaa.sencoten/example.qaa.sencoten.model.kmp.json

@@ -0,0 +1,48 @@
+{
+  "system": {
+    "keymanDeveloperVersion": "12.0.1500.0",
+    "fileVersion": "12.0"
+  },
+  "options": {},
+  "info": {
+    "author": {
+      "description": "Eddie Antonio Santos",
+      "url": "mailto:Eddie.Santos@nrc-cnrc.gc.ca"
+    },
+    "copyright": {
+      "description": "© 2019 National Research Council Canada"
+    },
+    "name": {
+      "description": "SENĆOŦEN (Saanich Dialect) Lexical Model"
+    },
+    "version": {
+      "description": "0.0.0"
+    }
+  },
+  "files": [
+    {
+      "name": "..\\build\\example.qaa.sencoten.model.js",
+      "description": "Lexical model example.qaa.sencoten.model.js"
+    },
+    {
+      "name": "kmp.json",
+      "description": "Package information (JSON)"
+    }
+  ],
+  "lexicalModels": [
+    {
+      "name": "SENĆOŦEN dictionary",
+      "id": "example.qaa.sencoten",
+      "languages": [
+        {
+          "name": "North Straits Salish",
+          "id": "qaa"
+        },
+        {
+          "name": "SENĆOŦEN",
+          "id": "qaa-Latn"
+        }
+      ]
+    }
+  ]
+}

package/test/fixtures/example.qaa.sencoten/example.qaa.sencoten.model.kmp.zipped.json

@@ -0,0 +1,45 @@
+{
+  "system": {
+    "keymanDeveloperVersion": "12.0.1500.0",
+    "fileVersion": "12.0"
+  },
+  "options": {},
+  "info": {
+    "author": {
+      "description": "Eddie Antonio Santos",
+      "url": "mailto:Eddie.Santos@nrc-cnrc.gc.ca"
+    },
+    "copyright": {
+      "description": "© 2019 National Research Council Canada"
+    },
+    "name": {
+      "description": "SENĆOŦEN (Saanich Dialect) Lexical Model"
+    },
+    "version": {
+      "description": "1.0.3"
+    }
+  },
+  "files": [
+    {
+      "name": "example.qaa.sencoten.model.js",
+      "description": "Lexical model example.qaa.sencoten.model.js",
+      "copyLocation": 0
+    }
+  ],
+  "lexicalModels": [
+    {
+      "name": "SENĆOŦEN dictionary",
+      "id": "example.qaa.sencoten",
+      "languages": [
+        {
+          "name": "North Straits Salish",
+          "id": "qaa"
+        },
+        {
+          "name": "SENĆOŦEN",
+          "id": "qaa-Latn"
+        }
+      ]
+    }
+  ]
+}