@capsiynau/intelligence 0.2.0

package/src/corrections/tracker.js

@@ -0,0 +1,279 @@
+/**
+ * @capsiynau/intelligence/corrections/tracker
+ *
+ * Moved here in Phase A (see docs/intelligence-layer-phase-a.md +
+ * PR #448) so Nodiadau can git-subtree this package rather than
+ * maintain a hand-mirrored copy. Original path: src/lib/correctionTracker.js.
+ *
+ * Ported verbatim (TS → JS) from Nodiadau commit 5d75dde
+ * (aledprysparry/notes-app#239) so Capsiynau Live edits feed the same
+ * closed-loop vocabulary learning channel.
+ *
+ * When a moderator fixes a Whisper mis-recognition in the live transcript
+ * (e.g. "Allard Perry" → "Aled Parry"), we want the model to recognise the
+ * corrected term next time. The Capsiynau live-relay reads
+ * `word_boost_approved` for the session's org + language and injects
+ * matching terms into the Whisper `prompt` parameter (PR #421).
+ *
+ * Flow:
+ *   1. Diff the segment text before/after the edit (whitespace-tokenised).
+ *   2. Find equal-length word swaps — if the lengths differ, skip; structural
+ *      changes are too noisy to feed back into vocabulary biasing.
+ *   3. For each swap, decide whether the corrected token looks like a proper
+ *      noun worth biasing (Title-Case, ≥4 chars, not a Welsh/English
+ *      start-of-sentence stopword, alphabetic-only).
+ *   4. Upsert into word_boost_approved with the session's organisation_id +
+ *      source_language. RLS scopes by org via get_user_org_ids() so a term
+ *      never leaks into another tenant's bias.
+ *
+ * Keep parity with the Nodiadau implementation — any heuristic change here
+ * must mirror in notes-app/src/lib/correctionTracker.ts (or vice versa) or
+ * the two apps' learning behaviour silently diverges.
+ */
+
+// Phase B (2026-05-17, PR #495): the Supabase client is passed in by
+// the caller rather than imported from a Capsiynau-specific path. This
+// removes the package's only coupling to the host app's module alias
+// (`@/api/supabaseClient`) and lets Nodiadau consume captureCorrections
+// without a re-export shim. Breaking API change vs 0.1.0 — bumped to
+// 0.2.0. Callers now pass `supabase` in the options bag alongside
+// `sessionId`.
+
+/**
+ * Words that are *not* worth promoting even if Title-Case. Keep narrow —
+ * this is a deny-list, not a dictionary.
+ */
+const PROPER_NOUN_STOPWORDS = new Set([
+  // English filler
+  'The', 'This', 'That', 'These', 'Those', 'There', 'Their', 'They',
+  'Then', 'When', 'Where', 'What', 'Which', 'While', 'Whose', 'Whom',
+  'With', 'Without', 'About', 'Above', 'After', 'Again', 'Against',
+  'Also', 'Some', 'Such', 'Same', 'Should', 'Would', 'Could', 'Might',
+  'Have', 'Been', 'Were', 'From', 'Into', 'Onto',
+  // Welsh filler (Title-Case start-of-sentence is common in Welsh too)
+  'Mae', "Mae'r", "Mae'n", 'Dyma', 'Dyna', 'Roedd', "Roedd'r",
+  'Rwy', 'Rwyt', 'Bydd', "Bydd'r", 'Bod', 'Cael',
+  'Wedi', 'Wedyn', 'Felly', 'Ond', 'Achos', 'Ar', "Ar y",
+  'Yn', 'Mewn', 'Heb', 'Drwy', 'Drwyddo', 'Drwyddi', 'Drwyddyn',
+])
+
+const MIN_PROPER_NOUN_LENGTH = 4
+const ALPHA_ONLY = /^[\p{L}\p{M}'-]+$/u
+
+/**
+ * Collapse the session's source_language to the two-bucket convention used
+ * everywhere else in this repo for STT bias loading: cy-GB (any Welsh
+ * variant — broadcast, north, south, formal) or en-GB (everything else,
+ * including NULL).
+ *
+ * Pre-this fix, this module defaulted unknown / NULL to 'cy-GB', which
+ * silently routed English users' legacy-session learned vocab into the
+ * Welsh bias bucket. Mirrors notes-app/src/lib/correctionTracker.ts.
+ */
+function normaliseLanguageBucket(source) {
+  // Case-insensitive: legacy / externally-written source_language values
+  // can arrive as 'CY-GB' or 'Cy-GB-north'. Without toLowerCase() those
+  // get classified as en-GB and learned terms land in the wrong bucket
+  // (Codex catch on notes-app#261 / @capsiynau/intelligence 0.2.0).
+  if (typeof source === 'string' && source.toLowerCase().startsWith('cy')) return 'cy-GB'
+  return 'en-GB'
+}
+
+/**
+ * Tokenise on whitespace AND strip trailing punctuation per word. Keeps
+ * in-word punctuation (apostrophes, hyphens) which Welsh names need
+ * ("ap Iolo", "Tŷ'r-pol", etc.).
+ */
+function tokenise(text) {
+  return text
+    .trim()
+    .split(/\s+/)
+    .map((tok) => tok.replace(/^[\p{P}\p{S}]+|[\p{P}\p{S}]+$/gu, ''))
+    .filter(Boolean)
+}
+
+/**
+ * Returns true if the token looks like a name/place/brand we want to bias
+ * toward. Heuristics, not a dictionary.
+ *
+ * ─── PART OF THE LANGUAGE-AGNOSTIC CONTRACT ──────────────────────────
+ * The relay's `word_boost_approved` loader (worker/live-relay.js) reads
+ * rows from this table language-agnostically, trusting that this
+ * heuristic only matches semantically language-invariant terms (Title-
+ * Case proper nouns: personal names, brand names, place names).
+ *
+ * If you ever loosen this heuristic to also match per-language vocabulary
+ * (hyphenated compounds, lowercase acronyms, dialect markers, domain
+ * lexicon, etc.), the loader has to follow. Either:
+ *   (a) re-introduce a per-language filter on the read side, OR
+ *   (b) add a row-level `word_boost_approved.cross_language` discriminator
+ *       so the loader can decide per row (Phase 1 of notes-app#262 queues
+ *       this micro-add).
+ *
+ * Change the heuristic, change the loader.
+ * ─────────────────────────────────────────────────────────────────────
+ */
+function looksLikeProperNoun(token) {
+  if (!token || token.length < MIN_PROPER_NOUN_LENGTH) return false
+  if (!ALPHA_ONLY.test(token)) return false
+  const first = token[0]
+  // Must start with an upper-case letter
+  if (first.toUpperCase() !== first || first.toLowerCase() === first) return false
+  if (PROPER_NOUN_STOPWORDS.has(token)) return false
+  // Avoid promoting ALL-CAPS strings (likely acronyms or shouting)
+  if (token.toUpperCase() === token && token.length > 4) return false
+  return true
+}
+
+/**
+ * Compute the set of proper-noun corrections worth promoting. Exported
+ * for unit tests.
+ *
+ * Returned terms become rows in `word_boost_approved` (per-org, with the
+ * session's source_language tag). The relay's loader treats those rows
+ * as language-agnostic on read - see the contract note on
+ * `looksLikeProperNoun` above for the full invariant and what must
+ * change if the heuristic ever expands beyond proper nouns.
+ */
+export function extractProperNounCorrections(before, after) {
+  const a = tokenise(before)
+  const b = tokenise(after)
+  // Skip structural changes (token count differs) — same heuristic as the
+  // 2am vocabulary-learning cron. Word-for-word swaps are the high-signal class.
+  if (a.length !== b.length || a.length === 0) return []
+  const out = new Set()
+  for (let i = 0; i < a.length; i++) {
+    if (a[i] === b[i]) continue
+    const corrected = b[i]
+    if (looksLikeProperNoun(corrected)) out.add(corrected)
+  }
+  return Array.from(out)
+}
+
+/**
+ * Upsert proper-noun corrections into word_boost_approved for the session's
+ * org + source language. Returns the list of terms persisted (excludes
+ * duplicates already in the table — upsert is silent on conflict).
+ *
+ * Never throws — failures are logged and swallowed so a segment-edit save
+ * is never blocked by a vocabulary-learning hiccup.
+ *
+ * @param {string} before  Original segment text before the edit.
+ * @param {string} after   Text after the edit.
+ * @param {object} opts
+ * @param {string} opts.sessionId  Live session UUID for org / fallback-lang lookup.
+ * @param {string} [opts.segmentId] Phase 1+ (notes-app#262): live_segments
+ *                                 UUID for the edited segment. When provided,
+ *                                 the segment's own `language` column is used
+ *                                 for the new word_boost_approved row (drops
+ *                                 silent rot in mixed-language sessions where
+ *                                 Phase 3's per-chunk detected_language differs
+ *                                 from session.source_language). Falls back to
+ *                                 session.source_language if absent or the
+ *                                 segment's language column is null.
+ * @param {object} opts.supabase   Caller-provided Supabase client (DI).
+ *                                 Must have `.from()` and `.rpc()` methods.
+ *                                 RLS scopes writes by org.
+ */
+export async function captureCorrections(before, after, { sessionId, segmentId, supabase }) {
+  if (!supabase) {
+    console.warn('[correctionTracker] captureCorrections called without a supabase client; skipping.')
+    return []
+  }
+  const terms = extractProperNounCorrections(before, after)
+  if (terms.length === 0) return []
+
+  // Phase 1+ (notes-app#262): prefer the segment's per-row language over
+  // the session's blanket source_language so corrections in a mixed-
+  // language session (Phase 3 per-chunk detected_language) land in the
+  // correct bucket. Best-effort: pre-Phase-1 envs or older relays will
+  // have segment.language = NULL, which falls through to the session
+  // lookup below.
+  //
+  // The lookup is bound to (segmentId, sessionId) together - never id
+  // alone. A stale or mismatched segmentId from a client race or a
+  // tampered payload could otherwise reference a row in a different
+  // session within the same org, contaminating the bucket with that
+  // OTHER segment's language while writing under THIS session's org
+  // (Codex follow-up on PR #511). RLS prevents cross-org leakage but
+  // not cross-session-within-org, so we constrain explicitly.
+  let segmentLanguage = null
+  if (segmentId) {
+    const { data: seg } = await supabase
+      .from('live_segments')
+      .select('language')
+      .eq('id', segmentId)
+      .eq('session_id', sessionId)
+      .maybeSingle()
+    segmentLanguage = seg?.language || null
+  }
+
+  // Look up the session's org + fallback-language at write time so the
+  // caller doesn't need to thread them through. One small SELECT,
+  // RLS-scoped. Always needed (org_id is the bucketing key) even when
+  // segmentLanguage resolved above.
+  const { data: sess, error: sessErr } = await supabase
+    .from('live_sessions')
+    .select('organisation_id, source_language')
+    .eq('id', sessionId)
+    .maybeSingle()
+
+  if (sessErr || !sess || !sess.organisation_id) {
+    console.warn('[correctionTracker] session lookup failed or missing org:', sessErr?.message)
+    return []
+  }
+
+  const language = normaliseLanguageBucket(segmentLanguage || sess.source_language)
+
+  // Phase 4.2 — call the SECURITY DEFINER RPC for an atomic
+  // INSERT ... ON CONFLICT DO UPDATE SET confirmed_by_count =
+  // confirmed_by_count + 1, last_confirmed_at = now(). This is what makes
+  // repeat confirmations of the same proper noun count toward bias
+  // ranking instead of silently no-op'ing under ignoreDuplicates.
+  //
+  // Fallback to plain upsert if the RPC isn't deployed yet (envs that
+  // haven't applied 20260515_word_boost_approved_confidence.sql).
+  // PostgREST returns a structured error with code='PGRST202' or
+  // message includes 'Could not find the function'; we catch broadly
+  // and degrade gracefully.
+  const persisted = []
+  let rpcMissing = false
+  for (const term of terms) {
+    const { error: rpcErr } = await supabase.rpc('bump_word_boost_approved', {
+      p_term:            term,
+      p_language:        language,
+      p_organisation_id: sess.organisation_id,
+      p_auto_approved:   true,
+    })
+    if (rpcErr) {
+      // RPC not deployed on this env → bail out and fall back to upsert.
+      if (rpcErr.code === 'PGRST202' || /Could not find the function/i.test(rpcErr.message || '')) {
+        rpcMissing = true
+        break
+      }
+      // Other errors (forbidden, network) — log and skip this term.
+      console.warn('[correctionTracker] bump_word_boost_approved error:', rpcErr.message)
+      continue
+    }
+    persisted.push(term)
+  }
+
+  if (rpcMissing) {
+    const rows = terms.map((term) => ({
+      term,
+      language,
+      organisation_id: sess.organisation_id,
+      auto_approved:   true,
+    }))
+    const { error } = await supabase
+      .from('word_boost_approved')
+      .upsert(rows, { onConflict: 'term,language,organisation_id', ignoreDuplicates: true })
+    if (error) {
+      console.warn('[correctionTracker] word_boost_approved upsert fallback failed:', error.message)
+      return []
+    }
+    return terms
+  }
+
+  return persisted
+}

package/src/glossary/README.md

@@ -0,0 +1,45 @@
+# `@capsiynau/intelligence/glossary`
+
+Shared engine for the Glossary Builder wizard. Consumed by Capsiynau
+(this repo) and Nodiadau (via git-subtree, mirroring the
+`@capsiynau/intelligence/corrections` arrangement landed in PR #448).
+
+## Exports
+
+| Export | Purpose |
+|---|---|
+| `normaliseUrl(raw)` | Prepend `https://` when scheme is missing; trim whitespace. |
+| `isHttpUrl(url)` | Cheap guard before handing a URL to Anthropic's web_search. |
+| `extractText(message)` | Pull plain text out of an Anthropic Messages response, filtering tool-use / tool-result / thinking blocks. Safe on Claude 4.5+. |
+| `parseTermsJson(raw)` | Defensive parse: strip markdown fences, fall back to greedy `[...]` regex, return `null` on total failure. Never throws. |
+| `normaliseTermRow(t)` | Tolerate `{term}` / `{source_term}` / `{preferred_term}` and `{definition}` / `{description}` / `{meaning}` shape wobbles; return `null` on missing basics. |
+| `RESEARCH_SYSTEM` | System prompt for the research call. |
+| `buildGenerationSystemPrompt({ audience, topics })` | System prompt for the generation call. |
+| `WEB_SEARCH_TOOL` | `{ type: 'web_search_20250305', name: 'web_search' }`. |
+
+## Why this split
+
+The handler at `api/functions/_handlers/glossaryBuilder.js` stays
+thin and lives at the Vercel serverless boundary. The engine is pure
+functions plus prompts - trivially testable, shareable between the
+two apps without dragging the Vercel handler shape into Nodiadau.
+
+## Phase 4 extension points
+
+- `buildGenerationSystemPrompt` will gain `languageMode` (`'en' | 'cy'
+  | 'bilingual'`) and `dialect` (`'north' | 'south' | 'broadcast' |
+  'formal'`) parameters; bilingual mode returns `{term_en, term_cy,
+  definition_en, definition_cy, category}` rows.
+- `normaliseTermRow` will gain `category` + bilingual fields, piping
+  `*_cy` values through `normaliseWelsh()` from
+  `@capsiynau/intelligence/welsh`.
+- Diacritic correctness will be enforced as a hard guard - flag, do
+  not silently strip.
+
+## Phase 5 additions (alongside, same folder)
+
+- `parseImportedCsv(raw)` + `parseImportedText(raw)` for the
+  import-and-merge flow.
+- `mergeRows(existing, additions, { onConflict })` returning conflict
+  records the UI surfaces for human resolution rather than silent
+  merge.

package/src/glossary/import.js

@@ -0,0 +1,199 @@
+/**
+ * Glossary Builder - import + merge helpers.
+ *
+ * The wizard lets a user paste or upload an existing partial glossary
+ * (CSV or plain text). Parse defensively, surface errors per line
+ * rather than crashing the whole import, then split incoming rows
+ * against the existing set so the UI can prompt for conflict
+ * resolution rather than silently overwrite.
+ *
+ * v1 scope: single-language imports. In bilingual wizard mode rows
+ * populate EN slots only; users top up CY via Expand-with-AI or
+ * inline edit. The import dialog documents this.
+ */
+
+const HEADER_TOKENS = ['term', 'definition', 'category', 'name', 'meaning', 'tag']
+
+/**
+ * Parse a CSV-ish string. Accepts (with or without header row):
+ *   term,definition
+ *   term,definition,category
+ *   term,category,definition
+ * Header row (detected by token match) maps columns by name and
+ * tolerates aliases (name, meaning, tag). RFC-4180-lite: quoted
+ * fields + escaped quotes on a single line. Multi-line quoted values
+ * are out of scope - report and skip.
+ */
+export function parseImportedCsv(raw) {
+  if (typeof raw !== 'string' || !raw.trim()) return { rows: [], errors: [] }
+  const text = raw.replace(/^/, '') // strip BOM
+  const lines = text.split(/\r?\n/).filter((l) => l.trim().length > 0)
+  if (lines.length === 0) return { rows: [], errors: [] }
+
+  const firstFields = parseCsvLine(lines[0])
+  const headerLower = firstFields.map((f) => f.toLowerCase())
+  const hasHeader = headerLower.some((h) => HEADER_TOKENS.includes(h))
+
+  let mapping
+  if (hasHeader) {
+    mapping = headerLower.map((h) => {
+      if (h === 'term' || h === 'name') return 'source_term'
+      if (h === 'definition' || h === 'meaning') return 'definition'
+      if (h === 'category' || h === 'tag') return 'category'
+      return null
+    })
+  } else {
+    // Positional fallback: 2 cols = term,definition; 3 cols use
+    // header-aware aliases. If column 2 looks short (<= 25 chars
+    // average) and column 3 is long, assume term,category,definition;
+    // otherwise term,definition,category.
+    if (firstFields.length === 2) {
+      mapping = ['source_term', 'definition']
+    } else if (firstFields.length >= 3) {
+      mapping = guessPositionalThree(lines)
+    } else {
+      mapping = ['source_term']
+    }
+  }
+
+  const rows = []
+  const errors = []
+  const startIdx = hasHeader ? 1 : 0
+  for (let i = startIdx; i < lines.length; i++) {
+    const fields = parseCsvLine(lines[i])
+    const row = mappingToRow(fields, mapping)
+    if (!row) {
+      errors.push({ line: i + 1, reason: 'Missing term or definition' })
+      continue
+    }
+    rows.push(row)
+  }
+  return { rows, errors }
+}
+
+/**
+ * Parse plain text. One term per line. Separator tried in order: tab,
+ * `:` followed by space, ` - ` (hyphen with spaces). Three columns
+ * (term, definition, optional category) match our copy-as-text output
+ * so users can round-trip cleanly.
+ */
+export function parseImportedText(raw) {
+  if (typeof raw !== 'string' || !raw.trim()) return { rows: [], errors: [] }
+  const lines = raw.split(/\r?\n/).map((l) => l.trim())
+  const rows = []
+  const errors = []
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i]
+    if (!line || line.startsWith('#')) continue
+    let parts = line.split('\t')
+    if (parts.length < 2) parts = line.split(/:\s+/)
+    if (parts.length < 2) parts = line.split(/\s-\s/)
+    if (parts.length < 2) {
+      errors.push({ line: i + 1, reason: 'No separator (tab, ": ", or " - ") found' })
+      continue
+    }
+    const [term, definition, category] = parts
+    if (!term?.trim() || !definition?.trim()) {
+      errors.push({ line: i + 1, reason: 'Missing term or definition' })
+      continue
+    }
+    rows.push({
+      source_term: term.trim(),
+      preferred_term: term.trim(),
+      definition: definition.trim(),
+      category: typeof category === 'string' && category.trim() ? category.trim() : null,
+    })
+  }
+  return { rows, errors }
+}
+
+/**
+ * Split incoming rows against the existing set. Matches case-insensitive
+ * on trimmed source_term. Returns {additions, conflicts} where conflicts
+ * is an array of {existing, incoming} pairs the UI surfaces for
+ * resolution. Dedup within the import itself happens against additions
+ * (the second occurrence of a duplicated import term becomes a conflict
+ * with the first occurrence).
+ */
+export function splitImportAgainstExisting(existing, incoming) {
+  const have = new Map()
+  for (const t of existing || []) {
+    const key = (t?.source_term || '').trim().toLowerCase()
+    if (key) have.set(key, t)
+  }
+  const additions = []
+  const conflicts = []
+  for (const t of incoming || []) {
+    const key = (t?.source_term || '').trim().toLowerCase()
+    if (!key) continue
+    if (have.has(key)) {
+      conflicts.push({ existing: have.get(key), incoming: t })
+    } else {
+      additions.push(t)
+      have.set(key, t)
+    }
+  }
+  return { additions, conflicts }
+}
+
+// ── internal helpers ─────────────────────────────────────────────────────
+
+function parseCsvLine(line) {
+  const fields = []
+  let buf = ''
+  let inQuotes = false
+  for (let i = 0; i < line.length; i++) {
+    const c = line[i]
+    if (inQuotes) {
+      if (c === '"' && line[i + 1] === '"') {
+        buf += '"'
+        i++
+      } else if (c === '"') {
+        inQuotes = false
+      } else {
+        buf += c
+      }
+    } else if (c === '"') {
+      inQuotes = true
+    } else if (c === ',') {
+      fields.push(buf)
+      buf = ''
+    } else {
+      buf += c
+    }
+  }
+  fields.push(buf)
+  return fields.map((f) => f.trim())
+}
+
+function mappingToRow(fields, mapping) {
+  const row = { source_term: '', preferred_term: '', definition: '', category: null }
+  for (let i = 0; i < mapping.length && i < fields.length; i++) {
+    const key = mapping[i]
+    if (!key) continue
+    row[key] = fields[i] || ''
+  }
+  if (!row.source_term?.trim() || !row.definition?.trim()) return null
+  row.source_term = row.source_term.trim()
+  row.preferred_term = row.source_term
+  row.definition = row.definition.trim()
+  row.category = typeof row.category === 'string' && row.category.trim() ? row.category.trim() : null
+  return row
+}
+
+function guessPositionalThree(lines) {
+  // Sample a few rows; if column 2 is consistently short (likely a
+  // tag) and column 3 long (likely a definition), it's term,cat,def.
+  // Otherwise default to term,def,cat (matches our CSV export).
+  let shortMid = 0
+  let total = 0
+  for (let i = 0; i < Math.min(lines.length, 5); i++) {
+    const f = parseCsvLine(lines[i])
+    if (f.length < 3) continue
+    total++
+    if ((f[1] || '').length <= 20 && (f[2] || '').length > 20) shortMid++
+  }
+  return total > 0 && shortMid / total >= 0.6
+    ? ['source_term', 'category', 'definition']
+    : ['source_term', 'definition', 'category']
+}

package/src/glossary/index.js

@@ -0,0 +1,41 @@
+/**
+ * @capsiynau/intelligence/glossary
+ *
+ * Shared engine for the Glossary Builder wizard. Consumed by Capsiynau
+ * (this repo) and Nodiadau (via git-subtree, mirroring the
+ * @capsiynau/intelligence/corrections arrangement from PR #448).
+ *
+ * Phase 2 (this PR): URL + parse + prompts + tools extracted from the
+ * legacy api/functions/_handlers/glossaryBuilder.js. Handler now
+ * delegates here.
+ *
+ * Phase 4 will add a bilingual prompt variant + Welsh-aware row
+ * normaliser that pipes definition_cy through normaliseWelsh().
+ * Phase 5 will add CSV/text import parsers alongside.
+ */
+
+export { normaliseUrl, isHttpUrl } from './url.js'
+export {
+  extractText,
+  parseTermsJson,
+  normaliseTermRow,
+  flagDiacriticStrip,
+} from './parse.js'
+export {
+  RESEARCH_SYSTEM,
+  buildGenerationSystemPrompt,
+  buildExpandSystemPrompt,
+} from './prompts.js'
+export { WEB_SEARCH_TOOL } from './tools.js'
+export {
+  parseImportedCsv,
+  parseImportedText,
+  splitImportAgainstExisting,
+} from './import.js'
+export {
+  suggestGlossaryName,
+  languagePairFor,
+  expandRowsForDb,
+  buildGlossaryRow,
+  suggestVersionedName,
+} from './persist.js'