@capsiynau/intelligence 0.2.0

package/src/glossary/parse.js

@@ -0,0 +1,133 @@
+/**
+ * Parsing helpers for the Glossary Builder engine.
+ *
+ * extractText pulls plain text out of an Anthropic Messages response.
+ * With the web_search tool active, content is a mix of
+ * server_tool_use, web_search_tool_result, and text blocks - we want
+ * only the text the model produced AFTER the searches resolved.
+ *
+ * Claude 4.5+ also prepends thinking blocks; .filter / .find is the
+ * safe accessor, never content[0].text. (Capsiynau lesson #16, PR
+ * #375.)
+ */
+
+import { normaliseWelsh } from '../welsh/normaliser.js'
+
+export function extractText(message) {
+  if (!message?.content) return ''
+  return message.content
+    .filter((b) => b.type === 'text')
+    .map((b) => b.text || '')
+    .join('\n')
+    .trim()
+}
+
+/**
+ * Defensive JSON parse for the generation phase. The model is told to
+ * reply with raw JSON only, but markdown fences and conversational
+ * preamble do leak through. Strip what we can, then fall back to the
+ * first [...] match. Return null on total failure - never throw.
+ */
+export function parseTermsJson(raw) {
+  if (!raw) return null
+  let s = String(raw).trim()
+  s = s.replace(/^```(?:json)?\s*/i, '').replace(/```\s*$/, '')
+  try {
+    const parsed = JSON.parse(s)
+    if (Array.isArray(parsed)) return parsed
+  } catch {}
+  const m = s.match(/\[[\s\S]*\]/)
+  if (m) {
+    try {
+      const parsed = JSON.parse(m[0])
+      if (Array.isArray(parsed)) return parsed
+    } catch {}
+  }
+  return null
+}
+
+const WELSH_DIACRITICS = /[âêîôûŵŷÂÊÎÔÛŴŶ]/
+
+/**
+ * Heuristic guard: any Welsh text longer than 25 chars with ZERO
+ * diacritics is suspicious - the model probably stripped them.
+ *
+ * Not a perfect check (some short Welsh text genuinely has none) but
+ * catches the common "model normalised everything to ASCII" failure
+ * mode. Substitution detection (ŵ → w, ŷ → y in specific positions)
+ * requires a dictionary and is deferred to a future Tier-3 spellcheck
+ * integration; not in scope for the wizard.
+ */
+export function flagDiacriticStrip(text) {
+  if (typeof text !== 'string' || text.length < 25) return false
+  return !WELSH_DIACRITICS.test(text)
+}
+
+/**
+ * Normalise one row from the model's JSON output.
+ *
+ * In 'en' mode: tolerate naming wobbles ({term} vs {source_term}; etc).
+ * In 'cy' mode: same shape, but route Welsh text through normaliseWelsh().
+ * In 'bilingual' mode: expect {term_en, term_cy, definition_en,
+ *   definition_cy, category}; route the *_cy fields through normaliseWelsh().
+ *
+ * Returns null for entries missing the required fields so the caller
+ * can `.map(normaliseTermRow).filter(Boolean)`. Adds a transient
+ * `_diacritic_warning` flag the handler can aggregate; it does NOT
+ * persist to the DB.
+ */
+export function normaliseTermRow(t, opts = {}) {
+  const { languageMode = 'en' } = opts
+  const rawCategory = t?.category || t?.tag || t?.group
+  const category = typeof rawCategory === 'string' ? rawCategory.trim() || null : null
+
+  if (languageMode === 'bilingual') {
+    const termEn = t?.term_en || t?.termEn
+    const termCy = t?.term_cy || t?.termCy
+    const defEn = t?.definition_en || t?.definitionEn
+    const defCy = t?.definition_cy || t?.definitionCy
+    const allStrings = [termEn, termCy, defEn, defCy].every((v) => typeof v === 'string' && v.trim())
+    if (!allStrings) return null
+    const cyTerm = normaliseWelsh(termCy.trim())
+    const cyDef = normaliseWelsh(defCy.trim())
+    const enTerm = termEn.trim()
+    const enDef = defEn.trim()
+    return {
+      // Canonical key stays EN for dedup + Phase 7 source_term
+      // persistence; bilingual columns live alongside.
+      source_term: enTerm,
+      preferred_term: enTerm,
+      definition: enDef,
+      term_en: enTerm,
+      term_cy: cyTerm,
+      definition_en: enDef,
+      definition_cy: cyDef,
+      category,
+      _diacritic_warning: flagDiacriticStrip(cyDef) || flagDiacriticStrip(cyTerm),
+    }
+  }
+
+  // 'en' or 'cy' - single-language shape
+  const rawTerm = t?.term || t?.source_term || t?.preferred_term
+  const rawDef = t?.definition || t?.description || t?.meaning
+  if (!rawTerm || !rawDef || typeof rawTerm !== 'string' || typeof rawDef !== 'string') {
+    return null
+  }
+  if (languageMode === 'cy') {
+    const cyTerm = normaliseWelsh(rawTerm.trim())
+    const cyDef = normaliseWelsh(rawDef.trim())
+    return {
+      source_term: cyTerm,
+      preferred_term: cyTerm,
+      definition: cyDef,
+      category,
+      _diacritic_warning: flagDiacriticStrip(cyDef) || flagDiacriticStrip(cyTerm),
+    }
+  }
+  return {
+    source_term: rawTerm.trim(),
+    preferred_term: rawTerm.trim(),
+    definition: rawDef.trim(),
+    category,
+  }
+}

package/src/glossary/persist.js

@@ -0,0 +1,160 @@
+/**
+ * Pure helpers for converting Glossary Builder wizard rows into the
+ * shapes the `glossaries` + `terms` Supabase tables expect.
+ *
+ * No Supabase calls here - those live in the view layer
+ * (src/components/glossaryBuilder/savePersist.js) so this module
+ * stays unit-testable and side-effect-free, mirroring how
+ * @capsiynau/intelligence/welsh/normaliser splits pure logic from
+ * the API handler.
+ */
+
+/**
+ * Suggest a glossary name based on the source URL. Strips
+ * protocol + leading `www.` so the result reads naturally
+ * ("Site terms - example.com").
+ */
+export function suggestGlossaryName(url) {
+  if (typeof url !== 'string' || !url.trim()) return 'Site terms'
+  try {
+    const u = new URL(url)
+    const host = u.hostname.replace(/^www\./, '')
+    return `Site terms - ${host}`
+  } catch {
+    return 'Site terms'
+  }
+}
+
+/**
+ * Map wizard languageMode to glossaries.language_pair. Single-language
+ * modes get 'en-GB' or 'cy-GB'; bilingual is 'en-GB,cy-GB' (the
+ * existing convention in the schema for multi-language glossaries).
+ */
+export function languagePairFor(languageMode) {
+  if (languageMode === 'cy') return 'cy-GB'
+  if (languageMode === 'bilingual') return 'en-GB,cy-GB'
+  return 'en-GB'
+}
+
+/**
+ * Expand wizard rows into terms-table inserts.
+ *
+ * Single-language modes produce one row per wizard row.
+ * Bilingual produces TWO rows per pair (EN + CY), linked by sharing
+ * glossary_id and category, with source_language differing. This uses
+ * the schema as designed and keeps the live-relay engine able to
+ * query by language without a JOIN.
+ *
+ * Drops rows missing term or definition silently (the wizard
+ * shouldn't let them through, but defence in depth at persistence
+ * time means a partial bilingual row contributes only its complete
+ * leg rather than aborting the whole save).
+ */
+export function expandRowsForDb(wizardRows, opts = {}) {
+  if (!Array.isArray(wizardRows)) return []
+  const { languageMode = 'en', glossaryId, organisationId, createdBy } = opts
+  const base = (extra) => ({
+    glossary_id: glossaryId,
+    organisation_id: organisationId,
+    created_by: createdBy,
+    approval_status: 'approved',
+    rule_strength: 'soft',
+    ...extra,
+  })
+  const out = []
+  for (const t of wizardRows) {
+    if (!t) continue
+    if (languageMode === 'bilingual') {
+      const termEn = (t.term_en || t.source_term || '').trim()
+      const termCy = (t.term_cy || '').trim()
+      const defEn = (t.definition_en || t.definition || '').trim()
+      const defCy = (t.definition_cy || '').trim()
+      const category = t.category || null
+      if (termEn && defEn) {
+        out.push(
+          base({
+            source_term: termEn,
+            preferred_term: termEn,
+            definition: defEn,
+            category,
+            source_language: 'en-GB',
+            target_language: 'en-GB',
+          }),
+        )
+      }
+      if (termCy && defCy) {
+        out.push(
+          base({
+            source_term: termCy,
+            preferred_term: termCy,
+            definition: defCy,
+            category,
+            source_language: 'cy-GB',
+            target_language: 'cy-GB',
+          }),
+        )
+      }
+    } else {
+      const term = (t.source_term || t.preferred_term || '').trim()
+      const definition = (t.definition || '').trim()
+      if (!term || !definition) continue
+      const lang = languageMode === 'cy' ? 'cy-GB' : 'en-GB'
+      out.push(
+        base({
+          source_term: term,
+          preferred_term: term,
+          definition,
+          category: t.category || null,
+          source_language: lang,
+          target_language: lang,
+        }),
+      )
+    }
+  }
+  return out
+}
+
+/**
+ * Build the glossaries-table insert payload for a wizard save.
+ * Defaults to visibility='shared' (org-wide, not just creator) - the
+ * wizard is collaborative tooling, not personal scratch space.
+ */
+export function buildGlossaryRow({
+  name,
+  url,
+  languageMode,
+  organisationId,
+  createdBy,
+  visibility = 'shared',
+}) {
+  const trimmedName = (name || '').trim()
+  return {
+    name: trimmedName || suggestGlossaryName(url),
+    description: url
+      ? `Generated by Glossary Builder from ${url}`
+      : 'Generated by Glossary Builder',
+    organisation_id: organisationId,
+    created_by: createdBy,
+    visibility,
+    language_pair: languagePairFor(languageMode),
+    source_url: url || null,
+    is_default: false,
+    is_system: false,
+  }
+}
+
+/**
+ * Suggest a versioned name when "Create new version" is chosen on
+ * conflict. Bumps an existing (vN) suffix or appends (v2) to a name
+ * that lacks one.
+ */
+export function suggestVersionedName(existingName) {
+  const s = String(existingName || '').trim() || 'Site terms'
+  const m = s.match(/^(.*?)\s*\(v(\d+)\)\s*$/)
+  if (m) {
+    const base = m[1].trim()
+    const next = parseInt(m[2], 10) + 1
+    return `${base} (v${next})`
+  }
+  return `${s} (v2)`
+}

package/src/glossary/prompts.js

@@ -0,0 +1,122 @@
+/**
+ * System prompts for the Glossary Builder engine.
+ *
+ * Three exports:
+ *   - RESEARCH_SYSTEM: research call (identify industry/products/audience).
+ *   - buildGenerationSystemPrompt: initial 12-18 term generation.
+ *   - buildExpandSystemPrompt: Phase 6 - 10 ADDITIONAL terms given the
+ *     existing list as first-class context. Replaces the prototype's
+ *     topics-hijack hack where the existing terms were stuffed into
+ *     the `topics` field as a string instruction.
+ *
+ * Generation and Expand share the language-branching helper so the
+ * two calls produce structurally identical JSON in each language mode.
+ */
+
+import { getDialectSystemPrompt } from '../welsh/normaliser.js'
+
+export const RESEARCH_SYSTEM = `You are helping a SaaS user build a starter glossary for their organisation. You have a web_search tool - use it to look up the URL the user provides and identify the industry, the products / services they offer, and the likely audience. Reply conversationally in 2-3 sentences confirming what you found, then ask who the glossary is for (e.g. "clients", "internal team", "general public", "broadcasters"). Keep the tone warm and direct. Do NOT produce a glossary yet - that comes in a follow-up step.`
+
+const DIACRITIC_REMINDER = `Preserve Welsh diacritics (â ê î ô û ŵ ŷ). Fix contractions (e.g. dwi'n, mae'n, wedi'i).`
+
+// Shared body for both generation and expand. Opening sentence is
+// caller-supplied so the two prompts can frame themselves correctly
+// ("producing" vs "extending").
+const SEARCH_GUIDANCE = (audienceLine, topicsLine) => `Use the web_search tool to look up real product names, sector jargon, and technical terms FROM the URL (don't invent generic industry words).
+
+${audienceLine}
+${topicsLine}
+
+Definitions must be short (one or two sentences), no marketing fluff, no circular definitions ("a Foo is a Foo that does Foo-ing").
+
+Each entry also carries a short "category" tag (1-2 words, lower-case) that groups related terms. Use a small consistent vocabulary across the glossary - e.g. "product", "process", "role", "acronym", "system", "concept". Don't invent a new category for every entry; cluster.`
+
+const JSON_TAIL = (schemaExample) => `Respond with a RAW JSON ARRAY ONLY. No markdown fences. No preamble. No trailing commentary. The exact format:
+
+[
+  ${schemaExample}
+]
+
+That's it. Anything else breaks the downstream parser.`
+
+function audienceTopicsLines({ audience, topics }) {
+  return {
+    audienceLine: audience
+      ? `Audience: ${audience}.`
+      : 'Audience: general professional readers.',
+    topicsLine: topics
+      ? `Specific topics or areas to prioritise: ${topics}.`
+      : 'No specific topics requested - cover the broadest set of recurring terms.',
+  }
+}
+
+// Language-specific header + JSON schema. Shared by generation + expand
+// so both calls produce structurally identical rows in each mode.
+function languageBlock(languageMode, dialect) {
+  if (languageMode === 'cy') {
+    return {
+      header: `LANGUAGE: Welsh (Cymraeg). All terms and definitions in Welsh.
+${getDialectSystemPrompt(dialect)}
+${DIACRITIC_REMINDER}`,
+      schema: '{"term": "...", "definition": "...", "category": "..."}',
+    }
+  }
+  if (languageMode === 'bilingual') {
+    return {
+      header: `LANGUAGE: Bilingual - each entry carries both English and Welsh.
+${getDialectSystemPrompt(dialect)}
+${DIACRITIC_REMINDER}
+Definitions in plain language for the stated audience in BOTH languages. Categories stay in English (they are tags, not user-facing copy).`,
+      schema: '{"term_en": "...", "term_cy": "...", "definition_en": "...", "definition_cy": "...", "category": "..."}',
+    }
+  }
+  return {
+    header: 'Definitions in plain English suitable for the stated audience.',
+    schema: '{"term": "...", "definition": "...", "category": "..."}',
+  }
+}
+
+export function buildGenerationSystemPrompt({
+  audience,
+  topics,
+  languageMode = 'en',
+  dialect = 'broadcast',
+} = {}) {
+  const { audienceLine, topicsLine } = audienceTopicsLines({ audience, topics })
+  const lang = languageBlock(languageMode, dialect)
+  return `You are producing a starter glossary tailored to a specific organisation's website. ${SEARCH_GUIDANCE(audienceLine, topicsLine)}
+
+Produce 12 to 18 entries.
+
+${lang.header}
+
+${JSON_TAIL(lang.schema)}`
+}
+
+export function buildExpandSystemPrompt({
+  audience,
+  topics,
+  languageMode = 'en',
+  dialect = 'broadcast',
+  existingTerms = [],
+} = {}) {
+  const { audienceLine, topicsLine } = audienceTopicsLines({ audience, topics })
+  const lang = languageBlock(languageMode, dialect)
+  // Cap at 50 to keep the prompt budget reasonable. If the glossary
+  // is bigger than that the model still gets the most-recent slice
+  // and the client-side APPEND_TERMS dedup catches any repeats.
+  const cappedTerms = (existingTerms || []).slice(0, 50).filter(Boolean)
+  const existingLine = cappedTerms.length
+    ? `EXISTING TERMS IN THE GLOSSARY (do NOT repeat or paraphrase these): ${cappedTerms.join('; ')}.`
+    : 'The glossary is currently empty - treat this as an initial generation.'
+
+  return `You are EXTENDING an existing starter glossary for a specific organisation's website with new, additional terms. ${SEARCH_GUIDANCE(audienceLine, topicsLine)}
+
+${existingLine}
+
+Produce 10 ADDITIONAL entries that the existing list does not already cover. Pick terms that complement the existing set - genuinely new ground, not paraphrases or near-synonyms of what's already there.
+
+${lang.header}
+
+${JSON_TAIL(lang.schema)}`
+}

package/src/glossary/tools.js

@@ -0,0 +1,7 @@
+/**
+ * Anthropic tool descriptors used by the Glossary Builder engine.
+ * web_search runs server-side - the model issues searches and
+ * Anthropic executes them. We never implement the search ourselves.
+ */
+
+export const WEB_SEARCH_TOOL = { type: 'web_search_20250305', name: 'web_search' }

package/src/glossary/url.js

@@ -0,0 +1,19 @@
+/**
+ * URL helpers for the Glossary Builder engine.
+ *
+ * Anthropic's web_search expects a full URL; a bare 'example.com'
+ * returns "no relevant pages" and the whole flow degrades silently.
+ * normaliseUrl prepends https:// when the caller forgot a scheme.
+ */
+
+export function normaliseUrl(raw) {
+  if (typeof raw !== 'string') return ''
+  const trimmed = raw.trim()
+  if (!trimmed) return ''
+  if (/^https?:\/\//i.test(trimmed)) return trimmed
+  return `https://${trimmed}`
+}
+
+export function isHttpUrl(url) {
+  return typeof url === 'string' && /^https?:\/\//i.test(url)
+}

package/src/index.js

@@ -0,0 +1,11 @@
+/**
+ * @capsiynau/intelligence
+ *
+ * Shared bilingual intelligence layer. Re-exports the per-domain modules
+ * so consumers can pull from a single import path.
+ *
+ * Phase A (2026-05-15): extraction-only. Public surface will stabilise
+ * once App #3 lands and informs the contract.
+ */
+export * from './corrections/index.js'
+export * from './welsh/index.js'

package/src/spellcheck/core.js

@@ -0,0 +1,154 @@
+/**
+ * @capsiynau/intelligence/spellcheck/core
+ *
+ * Mutation-aware Welsh + English spell checking, built on top of
+ * Bangor's hunspell-cy + en-GB dictionaries via nspell.
+ *
+ * Three reasons this isn't just a raw nspell wrapper:
+ *   1. Welsh mutations make surface forms diverge from dictionary
+ *      entries — `cath` is in the lexicon, `gath` (soft) and `chath`
+ *      (aspirate) are not. We use possibleRoots() from mutations.js
+ *      to recover every plausible root and accept the word if any
+ *      root is in the lexicon.
+ *   2. Org-specific proper nouns (panellist names, brand terms,
+ *      programme titles) must NOT be flagged. Callers pass a
+ *      `glossary` set: any word appearing there is accepted as-is.
+ *   3. word_boost_approved (per-org learned vocab from corrections)
+ *      is the same shape as `glossary` — passed in the same arg.
+ *
+ * Pure functions; the nspell instance is constructed once per worker
+ * and passed in. Caller manages caching.
+ */
+
+import { possibleRoots } from '../welsh/mutations.js'
+
+/**
+ * Check a single word against the supplied dictionary + accept-list.
+ *
+ * @param {string} word - The word to check (no punctuation, single token).
+ * @param {object} opts
+ * @param {object} opts.nspell - An nspell instance (server side, built
+ *                               from dictionary-cy or dictionary-en-gb).
+ * @param {Set<string>} [opts.accept] - Lowercased proper nouns + glossary
+ *                                       + word_boost_approved terms that
+ *                                       must never be flagged.
+ * @param {boolean} [opts.welsh] - When true, additionally try mutation
+ *                                  reversal before declaring the word
+ *                                  misspelled. Pass false for English.
+ * @returns {boolean} true if the word is considered correctly spelled.
+ */
+export function checkWord(word, { nspell, accept, welsh }) {
+  if (typeof word !== 'string' || word.length === 0) return true
+  const cleaned = word.replace(/[^\p{L}\p{M}'’-]/gu, '')
+  if (cleaned.length === 0) return true
+  const lower = cleaned.toLowerCase()
+  if (accept && accept.has(lower)) return true
+  if (nspell.correct(cleaned)) return true
+  if (welsh) {
+    // Mutation-aware fallback: try every plausible root form.
+    const candidates = possibleRoots(cleaned)
+    for (const { root } of candidates) {
+      if (root === cleaned) continue // already tried
+      if (accept && accept.has(root.toLowerCase())) return true
+      if (nspell.correct(root)) return true
+    }
+  }
+  return false
+}
+
+/**
+ * Tokenise a string into checkable words + their positions, so a UI
+ * can underline only the wrong tokens without re-tokenising itself.
+ *
+ * Position is the offset in the original string. Tokens are word-like
+ * runs of letters/marks/apostrophes/hyphens; punctuation is skipped.
+ *
+ * @param {string} text
+ * @returns {{ word: string, start: number, end: number }[]}
+ */
+export function tokenise(text) {
+  if (typeof text !== 'string') return []
+  const tokens = []
+  // \p{L} + \p{M} covers Welsh accented letters; ' and ’ + - keep
+  // contractions and hyphenated words intact.
+  const re = /[\p{L}\p{M}'’-]+/gu
+  let m
+  while ((m = re.exec(text)) !== null) {
+    tokens.push({ word: m[0], start: m.index, end: m.index + m[0].length })
+  }
+  return tokens
+}
+
+/**
+ * Check an entire string and return the misspelled tokens (if any).
+ * Convenience wrapper around tokenise + checkWord.
+ *
+ * @param {string} text
+ * @param {object} opts - same shape as checkWord opts
+ * @returns {{ word: string, start: number, end: number }[]}
+ */
+export function checkSentence(text, opts) {
+  return tokenise(text).filter(t => !checkWord(t.word, opts))
+}
+
+/**
+ * Generate up to N suggestions for a misspelled word. Wraps
+ * nspell.suggest() but biases toward `accept` (proper nouns are
+ * surfaced first when they're close in edit distance).
+ *
+ * @param {string} word
+ * @param {object} opts
+ * @param {object} opts.nspell
+ * @param {Set<string>} [opts.accept]
+ * @param {number} [opts.max] - default 6
+ * @returns {string[]}
+ */
+export function suggest(word, { nspell, accept, max = 6 }) {
+  if (typeof word !== 'string' || word.length === 0) return []
+  const base = nspell.suggest(word) || []
+  if (!accept || accept.size === 0) return base.slice(0, max)
+  const acceptHits = []
+  const lower = word.toLowerCase()
+  for (const term of accept) {
+    if (term === lower) continue
+    if (editDistance(lower, term, 2) <= 2) acceptHits.push(term)
+  }
+  // De-dupe: glossary hits first, then nspell suggestions that aren't dupes.
+  const seen = new Set(acceptHits.map(t => t.toLowerCase()))
+  const out = [...acceptHits]
+  for (const s of base) {
+    if (out.length >= max) break
+    if (!seen.has(s.toLowerCase())) {
+      out.push(s)
+      seen.add(s.toLowerCase())
+    }
+  }
+  return out
+}
+
+// Tight edit-distance with early abort at maxDist+1. Borrowed shape
+// from worker/livePolish.js so behaviour matches the bias-loader.
+function editDistance(a, b, maxDist) {
+  if (a === b) return 0
+  const al = a.length, bl = b.length
+  if (Math.abs(al - bl) > maxDist) return maxDist + 1
+  let prev = new Array(bl + 1)
+  let curr = new Array(bl + 1)
+  for (let j = 0; j <= bl; j++) prev[j] = j
+  for (let i = 1; i <= al; i++) {
+    curr[0] = i
+    let rowMin = i
+    for (let j = 1; j <= bl; j++) {
+      const cost = a[i - 1] === b[j - 1] ? 0 : 1
+      curr[j] = Math.min(
+        prev[j] + 1,
+        curr[j - 1] + 1,
+        prev[j - 1] + cost,
+      )
+      if (curr[j] < rowMin) rowMin = curr[j]
+    }
+    if (rowMin > maxDist) return maxDist + 1
+    ;[prev, curr] = [curr, prev]
+  }
+  return prev[bl]
+}

package/src/spellcheck/index.js

@@ -0,0 +1,8 @@
+/**
+ * @capsiynau/intelligence/spellcheck
+ *
+ * Mutation-aware Welsh + English spellcheck. See ./core.js for the
+ * pure functions. This module exposes the public surface only.
+ */
+
+export { checkWord, checkSentence, tokenise, suggest } from './core.js'