@willwade/aac-processors 0.2.16 → 0.2.18

package/dist/browser/utilities/analytics/morphology/tdsnapLexiconParser.js

@@ -0,0 +1,182 @@
+export class TDSnapLexiconParser {
+    parseDb(dbPath, locale) {
+        const detectedLocale = locale || this.inferLocale(dbPath);
+        // eslint-disable-next-line @typescript-eslint/no-var-requires
+        const Database = require('better-sqlite3');
+        // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+        const db = new Database(dbPath, { readonly: true });
+        try {
+            return this.extractAll(db, detectedLocale);
+        }
+        finally {
+            db.close();
+        }
+    }
+    inferLocale(dbPath) {
+        const match = dbPath.match(/lang_([a-z]{2}_[A-Z]{2})/i);
+        return match ? match[1] : 'unknown';
+    }
+    extractAll(db, locale) {
+        const words = new Map();
+        const subclassCache = new Map();
+        const getSubclass = (id) => {
+            let name = subclassCache.get(id);
+            if (name !== undefined)
+                return name;
+            const row = db.prepare('SELECT Name FROM PosSubclass WHERE Id = ?').get(id);
+            name = row?.Name;
+            if (name) {
+                subclassCache.set(id, name);
+                return name;
+            }
+            return undefined;
+        };
+        const allWords = db
+            .prepare(`SELECT w.Id as wordId, w.Text as text,
+                i.Id as inflectionId, i.LexemeId as lexemeId, i.PosSubclassId as posSubclassId
+         FROM Word w
+         JOIN Spelling s ON s.WordId = w.Id
+         JOIN Inflection i ON i.Id = s.InflectionId
+         WHERE i.PosSubclassId != 0
+         ORDER BY w.Text`)
+            .all();
+        const lexemeForms = new Map();
+        for (const row of allWords) {
+            const tag = getSubclass(row.posSubclassId);
+            if (!tag)
+                continue;
+            let formsByTag = lexemeForms.get(row.lexemeId);
+            if (!formsByTag) {
+                formsByTag = new Map();
+                lexemeForms.set(row.lexemeId, formsByTag);
+            }
+            const existing = formsByTag.get(tag);
+            if (existing) {
+                if (!existing.includes(row.text))
+                    existing.push(row.text);
+            }
+            else {
+                formsByTag.set(tag, [row.text]);
+            }
+        }
+        const wordToLexeme = new Map();
+        for (const row of allWords) {
+            if (!wordToLexeme.has(row.text.toLowerCase())) {
+                wordToLexeme.set(row.text.toLowerCase(), row.lexemeId);
+            }
+        }
+        for (const [text, lexemeId] of wordToLexeme) {
+            const formsByTag = lexemeForms.get(lexemeId);
+            if (!formsByTag || formsByTag.size === 0)
+                continue;
+            const forms = [];
+            for (const [tag, formTexts] of formsByTag) {
+                for (const formText of formTexts) {
+                    if (formText.toLowerCase() !== text) {
+                        forms.push({ tag, form: formText });
+                    }
+                }
+            }
+            if (forms.length > 0) {
+                words.set(text, { lexemeId, forms });
+            }
+        }
+        return { locale, words };
+    }
+    lookupWord(data, word) {
+        const entry = data.words.get(word.toLowerCase());
+        if (!entry)
+            return [];
+        return entry.forms.map((f) => f.form);
+    }
+    lookupWordByTag(data, word, tag) {
+        const entry = data.words.get(word.toLowerCase());
+        if (!entry)
+            return [];
+        return entry.forms.filter((f) => f.tag === tag).map((f) => f.form);
+    }
+    static parseContentTypeHandler(handler) {
+        if (!handler)
+            return null;
+        const colonIdx = handler.indexOf(':');
+        if (colonIdx === -1) {
+            const parts = handler.split(',');
+            return { category: parts[0], subtype: '', params: parts.slice(1) };
+        }
+        const category = handler.substring(0, colonIdx);
+        const rest = handler.substring(colonIdx + 1);
+        const commaIdx = rest.indexOf(',');
+        if (commaIdx === -1) {
+            return { category, subtype: rest, params: [] };
+        }
+        const subtype = rest.substring(0, commaIdx);
+        const paramsStr = rest.substring(commaIdx + 1);
+        const params = paramsStr.split(',').map((p) => p.trim());
+        return { category, subtype, params };
+    }
+    static tagToPos(tag) {
+        return TDSnapLexiconParser.TAG_TO_POS[tag] || 'Unknown';
+    }
+    static handlerToPos(handler) {
+        const parsed = TDSnapLexiconParser.parseContentTypeHandler(handler);
+        if (!parsed)
+            return 'Unknown';
+        if (parsed.category === 'RESET' || parsed.category === 'SPECIAL')
+            return 'Ignore';
+        const key = `${parsed.category}:${parsed.subtype}`;
+        const tag = TDSnapLexiconParser.HANDLER_TAG_MAP[key];
+        if (tag)
+            return TDSnapLexiconParser.TAG_TO_POS[tag] || 'Unknown';
+        return TDSnapLexiconParser.TAG_TO_POS[parsed.subtype] || 'Unknown';
+    }
+}
+TDSnapLexiconParser.TAG_TO_POS = {
+    V0: 'Verb',
+    VZ: 'Verb',
+    VG: 'Verb',
+    VD: 'Verb',
+    VN: 'Verb',
+    SNG: 'Noun',
+    PLU: 'Noun',
+    ADJ: 'Adjective',
+    ADJR: 'Adjective',
+    ADJT: 'Adjective',
+    ADV: 'Adjective',
+    SUB: 'Pronoun',
+    OBJ: 'Pronoun',
+    POS: 'Pronoun',
+    NPOS: 'Pronoun',
+    REF: 'Pronoun',
+    B0: 'Verb',
+    BZ: 'Verb',
+    BM: 'Verb',
+    BR: 'Verb',
+    BDZ: 'Verb',
+    BDR: 'Verb',
+    BG: 'Verb',
+    BN: 'Verb',
+};
+TDSnapLexiconParser.HANDLER_TAG_MAP = {
+    'NOUN:PLU': 'PLU',
+    'DESCRIBE:ADJR': 'ADJR',
+    'DESCRIBE:ADJT': 'ADJT',
+    'DESCRIBE:ADV': 'ADV',
+    'VERB:V0': 'V0',
+    'VERB:VZ': 'VZ',
+    'VERB:VG': 'VG',
+    'VERB:VD': 'VD',
+    'VERB:VN': 'VN',
+    'PRONOUN:SUB': 'SUB',
+    'PRONOUN:OBJ': 'OBJ',
+    'PRONOUN:POS': 'POS',
+    'PRONOUN:NPOS': 'NPOS',
+    'PRONOUN:REF': 'REF',
+    'BE:B0': 'B0',
+    'BE:BZ': 'BZ',
+    'BE:BM': 'BM',
+    'BE:BR': 'BR',
+    'BE:BDZ': 'BDZ',
+    'BE:BDR': 'BDR',
+    'BE:BG': 'BG',
+    'BE:BN': 'BN',
+};

package/dist/core/treeStructure.d.ts

@@ -121,7 +121,7 @@ export declare class AACButton {
         identifier?: string;
         metadata?: string;
     };
-    contentType?: 'Normal' | 'AutoContent' | 'Workspace' | 'LiveCell';
+    contentType?: 'Normal' | 'AutoContent' | 'Workspace' | 'LiveCell' | 'Inflector' | 'Prediction';
     contentSubType?: string;
     image?: string;
     resolvedImageEntry?: string;
@@ -169,7 +169,7 @@ export declare class AACButton {
             metadata?: string;
         };
         style?: AACStyle;
-        contentType?: 'Normal' | 'AutoContent' | 'Workspace' | 'LiveCell';
+        contentType?: 'Normal' | 'AutoContent' | 'Workspace' | 'LiveCell' | 'Inflector' | 'Prediction';
         contentSubType?: string;
         image?: string;
         resolvedImageEntry?: string;
@@ -222,6 +222,7 @@ export declare class AACPage {
     descriptionHtml?: string;
     images?: any[];
     sounds?: any[];
+    wordListItems?: import('../types/aac').AACWordListItem[];
     semantic_ids?: string[];
     clone_ids?: string[];
     scanningConfig?: import('../types/aac').ScanningConfig;

package/dist/index.node.d.ts

@@ -13,6 +13,7 @@ export * as Analytics from './analytics';
 export * as Validation from './validation';
 export * as Metrics from './metrics';
 export { Grid3VerbsParser } from './utilities/analytics/morphology/grid3VerbsParser';
+export { TDSnapLexiconParser } from './utilities/analytics/morphology/tdsnapLexiconParser';
 export { WordFormGenerator } from './utilities/analytics/morphology/wordFormGenerator';
 export * as Gridset from './gridset';
 export * as Snap from './snap';

package/dist/index.node.js

@@ -33,7 +33,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.Translation = exports.AstericsGrid = exports.ApplePanels = exports.Opml = exports.Excel = exports.Dot = exports.TouchChat = exports.Obfset = exports.OBF = exports.Snap = exports.Gridset = exports.WordFormGenerator = exports.Grid3VerbsParser = exports.Metrics = exports.Validation = exports.Analytics = void 0;
+exports.Translation = exports.AstericsGrid = exports.ApplePanels = exports.Opml = exports.Excel = exports.Dot = exports.TouchChat = exports.Obfset = exports.OBF = exports.Snap = exports.Gridset = exports.WordFormGenerator = exports.TDSnapLexiconParser = exports.Grid3VerbsParser = exports.Metrics = exports.Validation = exports.Analytics = void 0;
 exports.getProcessor = getProcessor;
 exports.getSupportedExtensions = getSupportedExtensions;
 exports.isExtensionSupported = isExtensionSupported;
@@ -56,9 +56,11 @@ exports.Analytics = __importStar(require("./analytics"));
 exports.Validation = __importStar(require("./validation"));
 // Metrics namespace (pageset analytics)
 exports.Metrics = __importStar(require("./metrics"));
-// Node-only morphology utilities (Grid 3 verbs parser)
+// Node-only morphology utilities (Grid 3 verbs parser, TDSnap lexicon parser)
 var grid3VerbsParser_1 = require("./utilities/analytics/morphology/grid3VerbsParser");
 Object.defineProperty(exports, "Grid3VerbsParser", { enumerable: true, get: function () { return grid3VerbsParser_1.Grid3VerbsParser; } });
+var tdsnapLexiconParser_1 = require("./utilities/analytics/morphology/tdsnapLexiconParser");
+Object.defineProperty(exports, "TDSnapLexiconParser", { enumerable: true, get: function () { return tdsnapLexiconParser_1.TDSnapLexiconParser; } });
 var wordFormGenerator_1 = require("./utilities/analytics/morphology/wordFormGenerator");
 Object.defineProperty(exports, "WordFormGenerator", { enumerable: true, get: function () { return wordFormGenerator_1.WordFormGenerator; } });
 // Processor namespaces (platform-specific utilities)

package/dist/processors/gridset/commands.js

@@ -920,6 +920,61 @@ function getAllPluginIds() {
     const plugins = new Set(Object.values(exports.GRID3_COMMANDS).map((cmd) => cmd.pluginId));
     return Array.from(plugins).sort();
 }
+function textOfStructured(val) {
+    if (!val || typeof val !== 'object')
+        return undefined;
+    const parts = [];
+    const processS = (s) => {
+        if (!s)
+            return;
+        if (s.r !== undefined) {
+            const rElements = Array.isArray(s.r) ? s.r : [s.r];
+            for (const r of rElements) {
+                if (typeof r === 'number') {
+                    if (r !== 0)
+                        parts.push(String(r));
+                    continue;
+                }
+                if (typeof r === 'object' && r !== null) {
+                    if ('#text' in r)
+                        parts.push(String(r['#text']));
+                    else if ('#cdata' in r)
+                        parts.push(String(r['#cdata']));
+                    else
+                        parts.push(String(r));
+                }
+                else {
+                    parts.push(String(r));
+                }
+            }
+        }
+    };
+    if (val.p) {
+        const sElements = Array.isArray(val.p.s) ? val.p.s : val.p.s ? [val.p.s] : [];
+        sElements.forEach(processS);
+    }
+    else if (val.s) {
+        const sElements = Array.isArray(val.s) ? val.s : [val.s];
+        sElements.forEach(processS);
+    }
+    else if (val.r !== undefined) {
+        processS(val);
+    }
+    return parts.length > 0 ? parts.join('').trim() : undefined;
+}
+function extractParamValue(param) {
+    if (typeof param === 'string')
+        return param;
+    if (param.p || param.s || (param.r !== undefined && typeof param.r !== 'string')) {
+        const structured = textOfStructured(param);
+        if (structured !== undefined)
+            return structured;
+    }
+    const simple = param['#text'] ?? param.text ?? param.value;
+    if (simple !== undefined)
+        return simple;
+    return textOfStructured(param);
+}
 function extractCommandParameters(command) {
     const parameters = {};
     const params = command.Parameter || command.parameter;
@@ -928,7 +983,7 @@ function extractCommandParameters(command) {
     const paramArray = Array.isArray(params) ? params : [params];
     for (const param of paramArray) {
         const key = param['@_Key'] || param.Key || param.key;
-        let value = param['#text'] ?? param.text ?? param.value;
+        let value = extractParamValue(param);
         if (key && value !== undefined) {
             // Try to convert to number if it looks numeric
             if (typeof value === 'string' && /^\d+$/.test(value)) {

package/dist/processors/gridsetProcessor.js

@@ -703,6 +703,13 @@ class GridsetProcessor extends baseProcessor_1.BaseProcessor {
                             }
                         }
                     }
+                    if (pageWordListItems.length > 0) {
+                        page.wordListItems = pageWordListItems.map((item) => ({
+                            text: item.text,
+                            image: item.image,
+                            partOfSpeech: item.partOfSpeech,
+                        }));
+                    }
                     // Track WordList AutoContent cells and their positions for "more" button placement
                     const wordListAutoContentCells = [];
                     let wordListCellIndex = 0;
@@ -1035,6 +1042,15 @@ class GridsetProcessor extends baseProcessor_1.BaseProcessor {
                                     const param = getRawParam(key);
                                     if (param === undefined)
                                         return undefined;
+                                    if (typeof param === 'string')
+                                        return param;
+                                    if (param.p ||
+                                        param.s ||
+                                        (param.r !== undefined && typeof param.r !== 'string')) {
+                                        const structuredValue = this.textOf(param);
+                                        if (structuredValue !== undefined)
+                                            return structuredValue;
+                                    }
                                     const simpleValue = param['#text'] ?? param.text ?? param.value;
                                     if (typeof simpleValue === 'string')
                                         return simpleValue;
@@ -1043,8 +1059,6 @@ class GridsetProcessor extends baseProcessor_1.BaseProcessor {
                                     const structuredValue = this.textOf(param);
                                     if (structuredValue !== undefined)
                                         return structuredValue;
-                                    if (typeof param === 'string')
-                                        return param;
                                     return undefined;
                                 };
                                 // Skip PredictThis in primary action loop as it was handled in pre-pass

package/dist/processors/snapProcessor.js

@@ -336,6 +336,9 @@ class SnapProcessor extends baseProcessor_1.BaseProcessor {
                         buttonColumns.has('BackgroundColor') ? 'b.BackgroundColor' : 'NULL AS BackgroundColor',
                         buttonColumns.has('NavigatePageId') ? 'b.NavigatePageId' : 'NULL AS NavigatePageId',
                         buttonColumns.has('ContentType') ? 'b.ContentType' : 'NULL AS ContentType',
+                        buttonColumns.has('SerializedContentTypeHandler')
+                            ? 'b.SerializedContentTypeHandler'
+                            : 'NULL AS SerializedContentTypeHandler',
                     ];
                     if (this.loadAudio) {
                         selectFields.push(buttonColumns.has('MessageRecordingId')
@@ -551,22 +554,54 @@ class SnapProcessor extends baseProcessor_1.BaseProcessor {
                             },
                         };
                     }
+                    let snapContentType;
+                    let snapContentSubType;
+                    let snapGrammarPos;
+                    let snapGrammarHandler;
+                    if (btnRow.ContentType === 1) {
+                        snapContentType = 'AutoContent';
+                        snapContentSubType = 'Prediction';
+                    }
+                    else if (btnRow.ContentType === 3 && btnRow.SerializedContentTypeHandler) {
+                        snapContentType = 'Inflector';
+                        snapGrammarHandler = String(btnRow.SerializedContentTypeHandler);
+                        const colonIdx = snapGrammarHandler.indexOf(':');
+                        if (colonIdx !== -1) {
+                            const subtype = snapGrammarHandler.substring(colonIdx + 1).split(',')[0];
+                            snapContentSubType = subtype;
+                            const { TDSnapLexiconParser, } = require('../utilities/analytics/morphology/tdsnapLexiconParser');
+                            snapGrammarPos = TDSnapLexiconParser.tagToPos(subtype);
+                        }
+                    }
                     const button = new treeStructure_1.AACButton({
                         id: String(btnRow.Id),
                         label: btnRow.Label || (btnRow.ContentType === 1 ? '[Prediction]' : ''),
                         message: btnRow.Message || (btnRow.ContentType === 1 ? '[Prediction]' : btnRow.Label || ''),
                         targetPageId: targetPageUniqueId,
                         semanticAction: semanticAction,
-                        contentType: btnRow.ContentType === 1 ? 'AutoContent' : undefined,
-                        contentSubType: btnRow.ContentType === 1 ? 'Prediction' : undefined,
+                        contentType: snapContentType,
+                        contentSubType: snapContentSubType,
                         audioRecording: audioRecording,
                         visibility: mapSnapVisibility(btnRow.Visible),
                         semantic_id: btnRow.LibrarySymbolId
                             ? `snap_symbol_${btnRow.LibrarySymbolId}`
-                            : undefined, // Extract semantic_id from LibrarySymbolId
+                            : undefined,
                         image: buttonImage,
                         resolvedImageEntry: buttonImage,
-                        parameters: Object.keys(buttonParameters).length > 0 ? buttonParameters : undefined,
+                        parameters: {
+                            ...(Object.keys(buttonParameters).length > 0 ? buttonParameters : {}),
+                            ...(snapGrammarHandler
+                                ? {
+                                    grammar: {
+                                        handler: snapGrammarHandler,
+                                        category: snapGrammarHandler.substring(0, snapGrammarHandler.indexOf(':') !== -1
+                                            ? snapGrammarHandler.indexOf(':')
+                                            : snapGrammarHandler.length),
+                                        subtype: snapContentSubType,
+                                    },
+                                }
+                                : {}),
+                        },
                         style: {
                             backgroundColor: btnRow.BackgroundColor
                                 ? `#${btnRow.BackgroundColor.toString(16)}`
@@ -579,6 +614,9 @@ class SnapProcessor extends baseProcessor_1.BaseProcessor {
                             fontStyle: btnRow.FontStyle?.toString(),
                         },
                     });
+                    if (snapGrammarPos) {
+                        button.pos = snapGrammarPos;
+                    }
                     // Add to the intended parent page
                     const parentPage = tree.getPage(parentUniqueId);
                     if (parentPage) {

package/dist/types/aac.d.ts

@@ -83,7 +83,7 @@ export interface AACButton {
         identifier?: string;
         metadata?: string;
     };
-    contentType?: 'Normal' | 'AutoContent' | 'Workspace' | 'LiveCell';
+    contentType?: 'Normal' | 'AutoContent' | 'Workspace' | 'LiveCell' | 'Inflector' | 'Prediction';
     contentSubType?: string;
     image?: string;
     resolvedImageEntry?: string;

package/dist/utilities/analytics/metrics/core.d.ts

@@ -42,6 +42,10 @@ export declare class MetricsCalculator {
     /**
      * Quick check whether any button in the tree has a POS tag.
      * Used to auto-enable smart grammar without requiring explicit opt-in.
+     *
+     * IMPORTANT: Only counts POS from non-Inflector and non-Suffix buttons.
+     * TDSnap Inflector buttons and Grid3 Suffix buttons are grammar controls,
+     * not content words — they should NOT auto-enable morphology.
      */
     private treeHasPosTags;
     /**
@@ -53,6 +57,35 @@ export declare class MetricsCalculator {
      * before calculateWordFormMetrics assigns effort to each form.
      */
     private expandMorphologicalPredictions;
+    /**
+     * Expand morphological predictions for Grid3 pagesets.
+     *
+     * Grid3 uses suffix buttons (pos='Suffix') on the same page as content words.
+     * Different pages have different suffix buttons — e.g., topic pages may only
+     * have -s (plural), while the Magic Wand page has -s, -er, -est, -ly, -y, -'s.
+     *
+     * Rules:
+     * 1. Build a suffix→formSlot map (-s → plural, -er → comparative, etc.)
+     * 2. For each page, collect available suffix buttons
+     * 3. Only generate forms for slots that have matching suffix buttons on that page
+     * 4. POS inference is used for untagged content words (Grid3 grids often lack POS)
+     */
+    private expandGrid3Predictions;
+    /**
+     * Expand morphological predictions for TDSnap pagesets.
+     *
+     * TDSnap uses Inflector buttons (ContentType=3) on "Word Forms" pages to
+     * provide morphology. These pages are loaded dynamically by the runtime,
+     * NOT via navigation buttons, so they are unreachable in our tree model.
+     *
+     * Rules:
+     * 1. If the pageset has NO Inflector buttons → no morphology at all
+     * 2. Only generate forms whose grammar tag matches an available Inflector
+     *    (e.g., if there's no -ly Inflector, don't generate "happily")
+     * 3. No POS inference — only the lexicon determines which words get forms
+     */
+    private expandTDSnapPredictions;
+    private filterFormsByAvailableTags;
     /**
      * Calculate metrics for word forms (smart grammar predictions)
      *