@willwade/aac-processors 0.3.1 → 0.3.2

package/dist/utilities/analytics/competence.js

@@ -0,0 +1,520 @@
+"use strict";
+/**
+ * Linguistic Competence Metrics
+ *
+ * Privacy-preserving analysis of AAC *spoken output* (phrase history), based on:
+ *   - Niemeijer, Sheldon & Hillary Zisk (2025), "Measuring AAC user linguistic
+ *     competence: A novel approach", AssistiveWare (Communication Matters handout).
+ *   - Frisch, Wade et al. (2026), "It's Complicated: On the Design and Evaluation
+ *     of AI-Powered AAC Interfaces", arXiv:2606.24854.
+ *
+ * DESIGN (read me):
+ *   - **Source-agnostic.** The only input is `{ text, timestampMs }[]`. It does
+ *     not know or care whether the speech history came from Grid 3, Snap,
+ *     TouchChat, OBF/OBFL logs, or anything else. See `historyEntriesToCompetence*
+ *     Utterances` (in history.ts) to adapt any `HistoryEntry[]` source.
+ *   - **Language-agnostic core.** This module contains NO word lists. Language-
+ *     specific resources (a closed-class word set, an inflection classifier) are
+ *     INJECTED via `LanguageResources`. When a resource is missing for a
+ *     language, the affected measure is reported as `unavailable` with a reason
+ *     and a warning is raised — never silently wrong.
+ *   - **Pure / no I/O.** No filesystem, no platform APIs. Runs anywhere (browser
+ *     included) and emits only aggregate statistics (never the raw text).
+ *
+ * The four dimensions of linguistic competence (Light, 1989) and the measures we
+ * use for each, following the AssistiveWare findings:
+ *
+ *   Semantic      -> MATTR-30 lexical diversity             (always available)
+ *   Syntactic     -> preposition/conjunction diversity        (needs closedClassWords)
+ *   Morphological -> inflected-form diversity                 (needs classifyInflection)
+ *   Phonological  -> proportion of unique words in a dictionary (needs a dictionary)
+ *
+ * All diversity measures use 30-word moving-average windows (Covington & McFall,
+ * 2010), making them sample-length independent and usable for the tiny, highly
+ * variable samples typical of AAC. MLU is intentionally NOT a headline (it
+ * conflates linguistic/operational/strategic/social competence in AAC); it is
+ * reported only as a distribution.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.tokenize = tokenize;
+exports.movingAverageTTR = movingAverageTTR;
+exports.lexicalDiversity = lexicalDiversity;
+exports.syntacticDiversity = syntacticDiversity;
+exports.morphologicalDiversity = morphologicalDiversity;
+exports.spellingValidity = spellingValidity;
+exports.summarizeActivity = summarizeActivity;
+exports.analyzeTimeline = analyzeTimeline;
+/* ------------------------------------------------------------------ *
+ * Small statistics helpers
+ * ------------------------------------------------------------------ */
+function mean(values) {
+    if (values.length === 0)
+        return null;
+    let sum = 0;
+    for (const v of values)
+        sum += v;
+    return sum / values.length;
+}
+function median(values) {
+    if (values.length === 0)
+        return null;
+    const sorted = [...values].sort((a, b) => a - b);
+    const mid = Math.floor(sorted.length / 2);
+    return sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid];
+}
+function quantile(values, q) {
+    if (values.length === 0)
+        return null;
+    const sorted = [...values].sort((a, b) => a - b);
+    const pos = (sorted.length - 1) * q;
+    const base = Math.floor(pos);
+    const rest = pos - base;
+    if (sorted[base + 1] !== undefined) {
+        return sorted[base] + rest * (sorted[base + 1] - sorted[base]);
+    }
+    return sorted[base];
+}
+/* ------------------------------------------------------------------ *
+ * Tokenisation
+ * ------------------------------------------------------------------ */
+const TOKEN_RE = /[\p{L}\p{N}]+(?:['’][\p{L}\p{N}]+)?/gu;
+/**
+ * Tokenise raw text into a lowercased word stream.
+ * Keeps intra-word apostrophes (don't, children's) but drops leading/trailing
+ * punctuation and pure whitespace. Accented characters are preserved (\p{L}).
+ */
+function tokenize(text) {
+    if (!text)
+        return [];
+    const out = [];
+    let m;
+    TOKEN_RE.lastIndex = 0;
+    while ((m = TOKEN_RE.exec(text)) !== null) {
+        let tok = m[0].toLowerCase();
+        tok = tok.replace(/^['’]+|['’]+$/g, '');
+        if (tok.length > 0)
+            out.push(tok);
+    }
+    return out;
+}
+/* ------------------------------------------------------------------ *
+ * Semantic competence: lexical diversity (MATTR-30)  — always available
+ * ------------------------------------------------------------------ */
+function segmentTTR(segment) {
+    if (segment.length === 0)
+        return 0;
+    return new Set(segment).size / segment.length;
+}
+/**
+ * Moving-Average Type-Token Ratio (Covington & McFall, 2010).
+ *
+ * Slides a fixed-size window across the word stream and computes the TTR for
+ * each window. The median across windows is sample-length independent, which is
+ * exactly why it is preferred over plain TTR for highly variable AAC samples.
+ */
+function movingAverageTTR(words, windowSize = 30) {
+    const n = words.length;
+    const w = Math.max(1, Math.floor(windowSize));
+    if (n === 0)
+        return { median: null, mean: null, nWindows: 0, windowSize: w };
+    if (n < w) {
+        const v = segmentTTR(words);
+        return { median: v, mean: v, nWindows: 1, windowSize: w };
+    }
+    const values = [];
+    for (let i = 0; i <= n - w; i++) {
+        values.push(segmentTTR(words.slice(i, i + w)));
+    }
+    return {
+        median: median(values),
+        mean: mean(values),
+        nWindows: values.length,
+        windowSize: w,
+    };
+}
+/** Convenience: MATTR-30 lexical diversity (the semantic headline). */
+function lexicalDiversity(words, windowSize = 30) {
+    return movingAverageTTR(words, windowSize);
+}
+/* ------------------------------------------------------------------ *
+ * Syntactic competence: closed-class diversity (MA-UPC-TWR-30)
+ * ------------------------------------------------------------------ */
+/**
+ * Closed-class diversity (generalises AssistiveWare's MA-UPC-TWR-30).
+ *
+ * For each window, compute the type-token ratio restricted to the supplied
+ * closed-class words (prepositions + conjunctions in the original paper). Windows
+ * containing none are skipped. The caller supplies the set via
+ * `closedClassWords`, so this works for any language without hardcoding here.
+ */
+function syntacticDiversity(words, options = {}) {
+    const w = Math.max(1, Math.floor(options.windowSize ?? 30));
+    const closed = options.closedClassWords;
+    if (!closed || closed.size === 0) {
+        return {
+            median: null,
+            mean: null,
+            nWindows: 0,
+            windowSize: w,
+            unavailable: 'no closed-class word data provided for this language',
+        };
+    }
+    const n = words.length;
+    if (n === 0)
+        return { median: null, mean: null, nWindows: 0, windowSize: w };
+    const values = [];
+    const scan = (segment) => {
+        const cc = segment.filter((tok) => closed.has(tok));
+        if (cc.length > 0) {
+            values.push(new Set(cc).size / cc.length);
+        }
+    };
+    if (n < w) {
+        scan(words);
+    }
+    else {
+        for (let i = 0; i <= n - w; i++) {
+            scan(words.slice(i, i + w));
+        }
+    }
+    if (values.length === 0) {
+        return {
+            median: null,
+            mean: null,
+            nWindows: 0,
+            windowSize: w,
+            unavailable: 'no closed-class words found in the sample',
+        };
+    }
+    return {
+        median: median(values),
+        mean: mean(values),
+        nWindows: values.length,
+        windowSize: w,
+    };
+}
+/* ------------------------------------------------------------------ *
+ * Morphological competence: inflected-form diversity (MA-UMORPH-TLWR-30 proxy)
+ * ------------------------------------------------------------------ */
+/**
+ * Morphological diversity (proxy for MA-UMORPH-TLWR-30).
+ *
+ * Within each window, words the supplied classifier marks as inflected (any
+ * category other than "base") contribute their surface forms to a type-token
+ * ratio. Windows with no inflected words are skipped. The classifier is injected
+ * (`classifyInflection`) so the heuristic lives with the caller, per language.
+ *
+ * Caveat (per AssistiveWare): for symbol-supported AAC, pre-stored morphology
+ * buttons ("finished", "is", ...) heavily affect this measure — interpret trends
+ * rather than absolutes.
+ */
+function morphologicalDiversity(words, options = {}) {
+    const w = Math.max(1, Math.floor(options.windowSize ?? 30));
+    const classify = options.classifyInflection;
+    if (!classify) {
+        return {
+            median: null,
+            mean: null,
+            nWindows: 0,
+            windowSize: w,
+            unavailable: 'no inflection classifier provided for this language',
+        };
+    }
+    const n = words.length;
+    if (n === 0)
+        return { median: null, mean: null, nWindows: 0, windowSize: w };
+    const values = [];
+    const scan = (segment) => {
+        const inflected = segment.filter((tok) => classify(tok) !== 'base');
+        if (inflected.length > 0) {
+            values.push(new Set(inflected).size / inflected.length);
+        }
+    };
+    if (n < w) {
+        scan(words);
+    }
+    else {
+        for (let i = 0; i <= n - w; i++) {
+            scan(words.slice(i, i + w));
+        }
+    }
+    if (values.length === 0) {
+        return {
+            median: null,
+            mean: null,
+            nWindows: 0,
+            windowSize: w,
+            unavailable: 'no inflected word forms found in the sample',
+        };
+    }
+    return {
+        median: median(values),
+        mean: mean(values),
+        nWindows: values.length,
+        windowSize: w,
+    };
+}
+/* ------------------------------------------------------------------ *
+ * Phonological competence: spelling validity (optional, weak)
+ * ------------------------------------------------------------------ */
+/**
+ * Proportion of unique alphabetic words present in the supplied dictionary.
+ * Unique words only, so it is not skewed by repetition or repeated misspellings.
+ * Returns null (unavailable) if no dictionary is provided.
+ */
+function spellingValidity(words, dictionary) {
+    if (!dictionary || dictionary.size === 0)
+        return null;
+    const unique = new Set(words);
+    let checked = 0;
+    let correct = 0;
+    for (const tok of unique) {
+        if (!/^\p{L}{2,}$/u.test(tok))
+            continue;
+        checked++;
+        if (dictionary.has(tok))
+            correct++;
+    }
+    return checked === 0 ? null : correct / checked;
+}
+function distribution(values) {
+    return {
+        median: median(values),
+        mean: mean(values),
+        p25: quantile(values, 0.25),
+        p75: quantile(values, 0.75),
+        n: values.length,
+    };
+}
+/** Compute engagement/activity stats for a set of utterances. */
+function summarizeActivity(utterances) {
+    const days = new Set();
+    let totalWords = 0;
+    const wpu = [];
+    for (const u of utterances) {
+        const toks = tokenize(u.text);
+        totalWords += toks.length;
+        wpu.push(toks.length);
+        const day = Math.floor(u.timestampMs / 86400000);
+        days.add(day);
+    }
+    return {
+        utterances: utterances.length,
+        words: totalWords,
+        uniqueWords: 0,
+        activeDays: days.size,
+        wordsPerUtterance: distribution(wpu),
+    };
+}
+function monthKey(timestampMs) {
+    const d = new Date(timestampMs);
+    const y = d.getFullYear();
+    const m = String(d.getMonth() + 1).padStart(2, '0');
+    return `${y}-${m}`;
+}
+function weightedSlope(points) {
+    const usable = points.filter((p) => p.y !== null && isFinite(p.y));
+    if (usable.length < 2)
+        return null;
+    let sw = 0;
+    let swx = 0;
+    let swy = 0;
+    let swxx = 0;
+    let swxy = 0;
+    for (const p of usable) {
+        const wgt = Math.max(p.w, 1);
+        sw += wgt;
+        swx += wgt * p.x;
+        swy += wgt * p.y;
+        swxx += wgt * p.x * p.x;
+        swxy += wgt * p.x * p.y;
+    }
+    const denom = sw * swxx - swx * swx;
+    if (denom === 0)
+        return null;
+    return (sw * swxy - swx * swy) / denom;
+}
+/**
+ * Analyse a corpus of utterances as a longitudinal competence report.
+ *
+ * Utterances are filtered to the trailing `months` window, binned by calendar
+ * month, and each bin is scored on the four competence dimensions plus activity.
+ * Language-specific measures require matching `resources`; missing resources are
+ * reported under `support` and `warnings` rather than silently dropped.
+ *
+ * No raw text, word list, or fringe-vocabulary frequency is included in the
+ * returned report — only aggregate statistics.
+ */
+function analyzeTimeline(utterances, options = {}) {
+    const months = Math.max(1, Math.floor(options.months ?? 12));
+    const windowSize = Math.max(1, Math.floor(options.windowSize ?? 30));
+    const lang = options.lang ?? 'en';
+    const minWordsPerMonth = Math.max(0, Math.floor(options.minWordsPerMonth ?? 150));
+    const dictionary = options.dictionary;
+    const resources = options.resources ?? {};
+    const now = options.now ?? Date.now();
+    // ---- Filter to the trailing N months ----------------------------------
+    const windowMs = months * 31 * 86400000;
+    const windowStartMs = now - windowMs;
+    const inWindow = utterances.filter((u) => u.timestampMs <= now && u.timestampMs > windowStartMs && u.text && u.text.trim().length > 0);
+    // ---- Bin by month -----------------------------------------------------
+    const bins = new Map();
+    for (const u of inWindow) {
+        const key = monthKey(u.timestampMs);
+        const arr = bins.get(key) ?? [];
+        arr.push(u);
+        bins.set(key, arr);
+    }
+    const sortedKeys = [...bins.keys()].sort();
+    const timeline = [];
+    for (const key of sortedKeys) {
+        const monthUtts = (bins.get(key) ?? []).slice().sort((a, b) => a.timestampMs - b.timestampMs);
+        const stream = [];
+        for (const u of monthUtts)
+            stream.push(...tokenize(u.text));
+        const activity = summarizeActivity(monthUtts);
+        activity.uniqueWords = new Set(stream).size;
+        const suppressed = stream.length < minWordsPerMonth;
+        const suppressReason = suppressed
+            ? `fewer than ${minWordsPerMonth} words (${stream.length})`
+            : null;
+        const lex = suppressed
+            ? { median: null, mean: null, nWindows: 0, windowSize }
+            : lexicalDiversity(stream, windowSize);
+        const syn = suppressed
+            ? { median: null, mean: null, nWindows: 0, windowSize }
+            : syntacticDiversity(stream, { windowSize, closedClassWords: resources.closedClassWords });
+        const mor = suppressed
+            ? { median: null, mean: null, nWindows: 0, windowSize }
+            : morphologicalDiversity(stream, {
+                windowSize,
+                classifyInflection: resources.classifyInflection,
+            });
+        const spell = suppressed ? null : spellingValidity(stream, dictionary);
+        timeline.push({
+            month: key,
+            utterances: activity.utterances,
+            words: activity.words,
+            uniqueWords: activity.uniqueWords,
+            activeDays: activity.activeDays,
+            wordsPerUtterance: activity.wordsPerUtterance,
+            lexicalDiversity: lex,
+            syntacticDiversity: syn,
+            morphologicalDiversity: mor,
+            spellingValidity: spell,
+            suppressed,
+            suppressReason,
+        });
+    }
+    // ---- Support + warnings (no silent language degradation) --------------
+    const warnings = [];
+    const hasCC = !!resources.closedClassWords && resources.closedClassWords.size > 0;
+    const hasMorph = !!resources.classifyInflection;
+    if (!hasCC) {
+        warnings.push(`Syntactic diversity unavailable: no closed-class word data for language '${lang}'. ` +
+            `Provide resources.closedClassWords to enable it.`);
+    }
+    if (!hasMorph) {
+        warnings.push(`Morphological diversity unavailable: no inflection classifier for language '${lang}'. ` +
+            `Provide resources.classifyInflection to enable it.`);
+    }
+    if (!dictionary) {
+        warnings.push('Spelling validity unavailable: no dictionary provided.');
+    }
+    const suppCount = timeline.filter((b) => b.suppressed).length;
+    if (suppCount > 0) {
+        warnings.push(`${suppCount} month(s) suppressed for having fewer than ${minWordsPerMonth} words.`);
+    }
+    // ---- Trend on the headline lexical-diversity median -------------------
+    const points = [];
+    for (let i = 0; i < timeline.length; i++) {
+        const b = timeline[i];
+        if (b.suppressed)
+            continue;
+        const y = b.lexicalDiversity.median;
+        if (y === null)
+            continue;
+        points.push({ x: i, y, w: b.words });
+    }
+    const slope = weightedSlope(points);
+    const validYs = points.map((p) => p.y);
+    const half = Math.floor(validYs.length / 2);
+    let firstHalf = null;
+    let secondHalf = null;
+    if (validYs.length >= 2) {
+        const fh = validYs.slice(0, Math.max(1, half));
+        const sh = validYs.slice(Math.max(1, half));
+        firstHalf = mean(fh);
+        secondHalf = mean(sh);
+    }
+    const delta = firstHalf !== null && secondHalf !== null ? secondHalf - firstHalf : null;
+    let direction = 'unknown';
+    if (slope !== null) {
+        if (Math.abs(slope) < 0.0005)
+            direction = 'flat';
+        else
+            direction = slope > 0 ? 'up' : 'down';
+    }
+    else if (delta !== null) {
+        if (Math.abs(delta) < 0.005)
+            direction = 'flat';
+        else
+            direction = delta > 0 ? 'up' : 'down';
+    }
+    const totalUtts = timeline.reduce((s, b) => s + b.utterances, 0);
+    const totalWords = timeline.reduce((s, b) => s + b.words, 0);
+    const dim = (available, reason) => available ? { available: true } : { available: false, reason };
+    return {
+        schema: 'aac-competence-report/v1',
+        generatedAt: new Date(now).toISOString(),
+        privacy: {
+            rawUtterancesIncluded: false,
+            wordListsIncluded: false,
+            fringeWordFrequencyIncluded: false,
+            minAggregationWindowDays: 31,
+            notes: [
+                'All metrics computed locally; only aggregate statistics are emitted.',
+                'Utterances are binned by calendar month so no pattern can be tied to a specific day or time.',
+                'No word lists or fringe-vocabulary frequencies are included (per AssistiveWare privacy guidance).',
+            ],
+        },
+        source: {
+            platform: options.platform ?? 'Grid3',
+            langCode: options.langCode,
+            userLabel: options.userLabel,
+            dbPathIncluded: options.dbPathIncluded === true,
+        },
+        config: {
+            months,
+            windowSize,
+            lang,
+            minWordsPerMonth,
+            dictionaryProvided: !!dictionary,
+        },
+        overall: {
+            windowStart: sortedKeys[0] ?? monthKey(windowStartMs),
+            windowEnd: sortedKeys[sortedKeys.length - 1] ?? monthKey(now),
+            totalUtterances: totalUtts,
+            totalWords: totalWords,
+            monthsCovered: timeline.length,
+            monthsSuppressed: suppCount,
+        },
+        timeline,
+        trend: {
+            metric: 'lexicalDiversity.median',
+            slopePerMonth: slope,
+            firstHalf,
+            secondHalf,
+            delta,
+            direction,
+        },
+        support: {
+            lang,
+            semantic: dim(true),
+            syntactic: dim(hasCC, hasCC ? undefined : 'no closed-class data for this language'),
+            morphological: dim(hasMorph, hasMorph ? undefined : 'no inflection classifier for this language'),
+            phonological: dim(!!dictionary, dictionary ? undefined : 'no dictionary provided'),
+        },
+        warnings,
+    };
+}

package/dist/utilities/analytics/history.d.ts

@@ -2,6 +2,7 @@ import { dotNetTicksToDate } from '../../utils/dotnetTicks';
 import { Grid3UserPath } from '../../processors/gridset/helpers';
 import { SnapUserInfo } from '../../processors/snap/helpers';
 import { AACSemanticCategory, AACSemanticIntent } from '../../core/treeStructure';
+import type { CompetenceUtterance } from './competence';
 export type HistorySource = 'Grid' | 'Snap' | 'OBL' | string;
 export interface HistoryOccurrence {
     timestamp: Date;
@@ -36,6 +37,14 @@ export interface HistoryEntry {
     platform?: HistoryPlatformExtras;
 }
 export { dotNetTicksToDate };
+/**
+ * Adapt any `HistoryEntry[]` (Grid 3, Snap, OBF/OBFL logs, ...) into the generic
+ * utterance stream consumed by the linguistic-competence engine
+ * (`analyzeTimeline`). Each occurrence of each phrase becomes one utterance,
+ * timestamped by its occurrence time. This keeps the competence metrics fully
+ * source-agnostic: anything the library can read as history can be analysed.
+ */
+export declare function historyEntriesToCompetenceUtterances(entries: HistoryEntry[]): CompetenceUtterance[];
 export interface BatonExportMetadata {
     timestamp: string;
     latitude?: number | null;

package/dist/utilities/analytics/history.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.dotNetTicksToDate = void 0;
+exports.historyEntriesToCompetenceUtterances = historyEntriesToCompetenceUtterances;
 exports.exportHistoryToBaton = exportHistoryToBaton;
 exports.readGrid3History = readGrid3History;
 exports.readGrid3HistoryForUser = readGrid3HistoryForUser;
@@ -14,6 +15,28 @@ const dotnetTicks_1 = require("../../utils/dotnetTicks");
 Object.defineProperty(exports, "dotNetTicksToDate", { enumerable: true, get: function () { return dotnetTicks_1.dotNetTicksToDate; } });
 const helpers_1 = require("../../processors/gridset/helpers");
 const helpers_2 = require("../../processors/snap/helpers");
+/**
+ * Adapt any `HistoryEntry[]` (Grid 3, Snap, OBF/OBFL logs, ...) into the generic
+ * utterance stream consumed by the linguistic-competence engine
+ * (`analyzeTimeline`). Each occurrence of each phrase becomes one utterance,
+ * timestamped by its occurrence time. This keeps the competence metrics fully
+ * source-agnostic: anything the library can read as history can be analysed.
+ */
+function historyEntriesToCompetenceUtterances(entries) {
+    const out = [];
+    for (const e of entries) {
+        const text = e.content;
+        if (!text || text.trim().length === 0)
+            continue;
+        const occs = e.occurrences ?? [];
+        for (const occ of occs) {
+            if (!occ.timestamp)
+                continue;
+            out.push({ text, timestampMs: occ.timestamp.getTime() });
+        }
+    }
+    return out;
+}
 const generateUuid = () => {
     if (typeof globalThis.crypto?.randomUUID === 'function') {
         return globalThis.crypto.randomUUID();

package/dist/utilities/analytics/index.d.ts

@@ -21,6 +21,7 @@ export { VocabularyAnalyzer } from './metrics/vocabulary';
 export { SentenceAnalyzer } from './metrics/sentence';
 export { ComparisonAnalyzer } from './metrics/comparison';
 export { ReferenceLoader } from './reference';
+export * from './competence';
 /**
  * Get the default reference data path
  */

package/dist/utilities/analytics/index.js

@@ -52,6 +52,8 @@ var comparison_1 = require("./metrics/comparison");
 Object.defineProperty(exports, "ComparisonAnalyzer", { enumerable: true, get: function () { return comparison_1.ComparisonAnalyzer; } });
 var reference_1 = require("./reference");
 Object.defineProperty(exports, "ReferenceLoader", { enumerable: true, get: function () { return reference_1.ReferenceLoader; } });
+// Export linguistic-competence measures (privacy-preserving spoken-output analysis)
+__exportStar(require("./competence"), exports);
 /**
  * Get the default reference data path
  */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@willwade/aac-processors",
-  "version": "0.3.1",
+  "version": "0.3.2",
   "description": "A comprehensive TypeScript library for processing AAC (Augmentative and Alternative Communication) file formats with translation support",
   "main": "dist/index.js",
   "browser": "dist/browser/index.browser.js",