@capsiynau/intelligence 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +63 -0
- package/README.md +35 -4
- package/package.json +33 -7
- package/src/corrections/index.js +11 -1
- package/src/corrections/pure.js +133 -0
- package/src/corrections/tracker.js +36 -116
- package/src/speakers/index.js +204 -0
- package/src/spellcheck/core.js +73 -6
- package/src/welsh/normaliser.js +11 -0
- package/types/corrections/index.d.ts +2 -0
- package/types/corrections/pure.d.ts +21 -0
- package/types/corrections/tracker.d.ts +36 -0
- package/types/glossary/import.d.ts +55 -0
- package/types/glossary/index.d.ts +6 -0
- package/types/glossary/parse.d.ts +63 -0
- package/types/glossary/persist.d.ts +66 -0
- package/types/glossary/prompts.d.ts +10 -0
- package/types/glossary/tools.d.ts +4 -0
- package/types/glossary/url.d.ts +9 -0
- package/types/index.d.ts +2 -0
- package/types/speakers/index.d.ts +85 -0
- package/types/spellcheck/core.d.ts +74 -0
- package/types/spellcheck/index.d.ts +1 -0
- package/types/welsh/digraphs.d.ts +102 -0
- package/types/welsh/index.d.ts +3 -0
- package/types/welsh/mutations.d.ts +76 -0
- package/types/welsh/normaliser.d.ts +13 -0
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @capsiynau/intelligence/speakers
|
|
3
|
+
*
|
|
4
|
+
* Base-level speaker-handling primitives, shared by every app that consumes
|
|
5
|
+
* diarisation (Capsiynau engine, Nodiadau, future apps). Pure + dependency-
|
|
6
|
+
* free; runs in Node or the browser.
|
|
7
|
+
*
|
|
8
|
+
* This first export is the DIARISATION QUALITY GATE. Acoustic diarisation
|
|
9
|
+
* (AssemblyAI) is language-agnostic but not always reliable - especially on
|
|
10
|
+
* Welsh audio (unofficial support), short memos, or heavy cross-talk. The
|
|
11
|
+
* gate decides whether a diarisation result is trustworthy enough to SHOW
|
|
12
|
+
* speaker labels, or whether the consumer should suppress them. Showing
|
|
13
|
+
* "Speaker 1" on everything when four people spoke is worse than showing
|
|
14
|
+
* nothing - so when in doubt, suppress.
|
|
15
|
+
*
|
|
16
|
+
* Thresholds are documented constants, deliberately conservative. Tune them
|
|
17
|
+
* once real Welsh-meeting telemetry exists (schedule phase C2). The function
|
|
18
|
+
* never throws and never mutates its inputs.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
export const DIARISATION_GATE_DEFAULTS = {
|
|
22
|
+
// A multi-minute recording that diarisation collapses to ONE speaker is the
|
|
23
|
+
// classic false-negative (it failed to separate voices). Below this span a
|
|
24
|
+
// genuine solo memo is plausible, so a sole speaker is allowed; above it a
|
|
25
|
+
// sole speaker is suspicious and we suppress.
|
|
26
|
+
soleSpeakerSuspectAboveMs: 90_000, // 90 seconds
|
|
27
|
+
// Fraction of the transcript span that labelled utterances must cover.
|
|
28
|
+
// Sparse coverage means the speaker timeline barely overlaps the words, so
|
|
29
|
+
// per-segment assignment would be mostly guesswork.
|
|
30
|
+
minCoverage: 0.6,
|
|
31
|
+
// This many distinct speakers (with adequate coverage) is trusted as a
|
|
32
|
+
// clear multi-party separation.
|
|
33
|
+
trustAtSpeakers: 2,
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Total milliseconds the utterances cover WITHIN the transcript span
|
|
38
|
+
* `[spanStart, spanEnd]`, counted as a UNION so overlapping utterances don't
|
|
39
|
+
* double-count. Each utterance is first clipped to the span (speech entirely
|
|
40
|
+
* outside the transcript window contributes 0), then the clipped intervals are
|
|
41
|
+
* merged and their lengths summed. This is what makes coverage measure the
|
|
42
|
+
* fraction of the transcript actually overlapped by the speaker timeline,
|
|
43
|
+
* rather than total diarisation duration (which can sit outside the window
|
|
44
|
+
* when diarisation ran over the full audio but the transcript is a clip).
|
|
45
|
+
*
|
|
46
|
+
* @param {Array<{start?:number,end?:number}>} utts
|
|
47
|
+
* @param {number} spanStart inclusive lower bound (ms)
|
|
48
|
+
* @param {number} spanEnd inclusive upper bound (ms)
|
|
49
|
+
* @returns {number} unioned, clipped duration in ms (>= 0)
|
|
50
|
+
*/
|
|
51
|
+
export function clampedUnionMs(utts, spanStart, spanEnd) {
|
|
52
|
+
const clipped = []
|
|
53
|
+
for (const u of utts) {
|
|
54
|
+
const start = Math.max(u.start ?? 0, spanStart)
|
|
55
|
+
const end = Math.min(u.end ?? 0, spanEnd)
|
|
56
|
+
if (end > start) clipped.push([start, end])
|
|
57
|
+
}
|
|
58
|
+
if (clipped.length === 0) return 0
|
|
59
|
+
|
|
60
|
+
clipped.sort((a, b) => a[0] - b[0])
|
|
61
|
+
let total = 0
|
|
62
|
+
let [curStart, curEnd] = clipped[0]
|
|
63
|
+
for (let i = 1; i < clipped.length; i++) {
|
|
64
|
+
const [s, e] = clipped[i]
|
|
65
|
+
if (s <= curEnd) {
|
|
66
|
+
// Overlapping or contiguous - extend the current merged interval.
|
|
67
|
+
if (e > curEnd) curEnd = e
|
|
68
|
+
} else {
|
|
69
|
+
total += curEnd - curStart
|
|
70
|
+
curStart = s
|
|
71
|
+
curEnd = e
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
total += curEnd - curStart
|
|
75
|
+
return total
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Judge whether a diarisation timeline is reliable enough to label segments.
|
|
80
|
+
*
|
|
81
|
+
* @param {Array<{start:number,end:number,speaker:string}>} utterances
|
|
82
|
+
* diarisation timeline in milliseconds (AssemblyAI `utterances` shape).
|
|
83
|
+
* @param {Array<{start_ms:number,end_ms:number}>} segments
|
|
84
|
+
* transcript segments in milliseconds (used to measure coverage).
|
|
85
|
+
* @param {Partial<typeof DIARISATION_GATE_DEFAULTS>} [opts] threshold overrides.
|
|
86
|
+
* @returns {{ reliable: boolean, speakerCount: number, coverage: number,
|
|
87
|
+
* durationMs: number, reason: string }}
|
|
88
|
+
*/
|
|
89
|
+
export function assessDiarisation(utterances, segments, opts = {}) {
|
|
90
|
+
const t = { ...DIARISATION_GATE_DEFAULTS, ...opts }
|
|
91
|
+
const utts = Array.isArray(utterances)
|
|
92
|
+
? utterances.filter((u) => u && typeof u.speaker === 'string' && u.speaker)
|
|
93
|
+
: []
|
|
94
|
+
|
|
95
|
+
if (utts.length === 0) {
|
|
96
|
+
return { reliable: false, speakerCount: 0, coverage: 0, durationMs: 0, reason: 'no_utterances' }
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
const speakerCount = new Set(utts.map((u) => u.speaker)).size
|
|
100
|
+
|
|
101
|
+
// Total span: prefer the transcript's extent; fall back to the utterances'.
|
|
102
|
+
const segs = Array.isArray(segments) ? segments.filter(Boolean) : []
|
|
103
|
+
const starts = segs.length ? segs.map((s) => s.start_ms ?? 0) : utts.map((u) => u.start)
|
|
104
|
+
const ends = segs.length ? segs.map((s) => s.end_ms ?? 0) : utts.map((u) => u.end)
|
|
105
|
+
const spanStart = Math.min(...starts)
|
|
106
|
+
const spanEnd = Math.max(...ends)
|
|
107
|
+
const durationMs = Math.max(0, spanEnd - spanStart)
|
|
108
|
+
const totalSpan = Math.max(1, durationMs)
|
|
109
|
+
|
|
110
|
+
// Coverage = fraction of the transcript span actually overlapped by the
|
|
111
|
+
// speaker timeline. Clip each utterance to [spanStart, spanEnd] and union the
|
|
112
|
+
// clipped intervals: speech outside the transcript window (diarisation ran
|
|
113
|
+
// over full audio, transcript is a clip) must NOT inflate coverage, and
|
|
114
|
+
// overlapping utterances must NOT double-count past the real covered fraction.
|
|
115
|
+
const labelledMs = clampedUnionMs(utts, spanStart, spanEnd)
|
|
116
|
+
const coverage = Math.min(1, labelledMs / totalSpan)
|
|
117
|
+
|
|
118
|
+
if (coverage < t.minCoverage) {
|
|
119
|
+
return { reliable: false, speakerCount, coverage, durationMs, reason: 'low_coverage' }
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
// Single speaker: trust only on short audio (a real solo memo); on long
|
|
123
|
+
// audio a lone speaker almost always means diarisation failed to separate.
|
|
124
|
+
if (speakerCount < t.trustAtSpeakers) {
|
|
125
|
+
if (durationMs > t.soleSpeakerSuspectAboveMs) {
|
|
126
|
+
return { reliable: false, speakerCount, coverage, durationMs, reason: 'sole_speaker_long_audio' }
|
|
127
|
+
}
|
|
128
|
+
return { reliable: true, speakerCount, coverage, durationMs, reason: 'sole_speaker_short_audio' }
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
return { reliable: true, speakerCount, coverage, durationMs, reason: 'ok' }
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
export const SPEAKER_MAP_DEFAULTS = {
|
|
135
|
+
// The winning name must cover at least this fraction of the acoustic
|
|
136
|
+
// speaker's total speaking time. Below it the overlap is too weak to trust.
|
|
137
|
+
minOverlapFraction: 0.5,
|
|
138
|
+
// The winning name's overlap must beat the runner-up by at least this ratio,
|
|
139
|
+
// else the acoustic speaker straddles two names too ambiguously to name.
|
|
140
|
+
dominanceRatio: 1.5,
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* Map acoustic diarisation speakers ("Speaker 1/2/3") to REAL names from a
|
|
145
|
+
* labelled reference timeline (platform transcript / roster), by greatest
|
|
146
|
+
* total time-overlap. Upgrades anonymous acoustic labels to real names when a
|
|
147
|
+
* labelled source exists (speaker-ID ladder rung A over C). Pure.
|
|
148
|
+
*
|
|
149
|
+
* Only confident matches are returned: the winning name must cover
|
|
150
|
+
* >= minOverlapFraction of the acoustic speaker's speaking time AND beat the
|
|
151
|
+
* runner-up by >= dominanceRatio. Ambiguous / unmatched acoustic speakers are
|
|
152
|
+
* listed in `unmapped`, so the caller keeps their anonymous label.
|
|
153
|
+
*
|
|
154
|
+
* Note: two acoustic speakers can map to the same real name (acoustic over-
|
|
155
|
+
* split one person). That is acceptable - downstream can merge. 1:1 dedup is a
|
|
156
|
+
* future refinement.
|
|
157
|
+
*
|
|
158
|
+
* @param {Array<{start:number,end:number,speaker:string}>} acoustic utterances (ms)
|
|
159
|
+
* @param {Array<{start:number,end:number,name:string}>} labelled real-name turns (ms)
|
|
160
|
+
* @param {Partial<typeof SPEAKER_MAP_DEFAULTS>} [opts]
|
|
161
|
+
* @returns {{ map: Record<string,string>, unmapped: string[] }}
|
|
162
|
+
*/
|
|
163
|
+
export function mapSpeakersToNames(acoustic, labelled, opts = {}) {
|
|
164
|
+
const t = { ...SPEAKER_MAP_DEFAULTS, ...opts }
|
|
165
|
+
const utts = Array.isArray(acoustic) ? acoustic.filter((u) => u && u.speaker) : []
|
|
166
|
+
const refs = Array.isArray(labelled) ? labelled.filter((r) => r && r.name) : []
|
|
167
|
+
|
|
168
|
+
const speakers = [...new Set(utts.map((u) => u.speaker))]
|
|
169
|
+
const map = {}
|
|
170
|
+
const unmapped = []
|
|
171
|
+
if (speakers.length === 0) return { map, unmapped }
|
|
172
|
+
if (refs.length === 0) return { map, unmapped: speakers }
|
|
173
|
+
|
|
174
|
+
const overlapMs = (a, b) => Math.max(0, Math.min(a.end, b.end) - Math.max(a.start, b.start))
|
|
175
|
+
|
|
176
|
+
for (const spk of speakers) {
|
|
177
|
+
const spkUtts = utts.filter((u) => u.speaker === spk)
|
|
178
|
+
const spkTotal = spkUtts.reduce((s, u) => s + Math.max(0, u.end - u.start), 0)
|
|
179
|
+
|
|
180
|
+
const byName = {}
|
|
181
|
+
for (const u of spkUtts) {
|
|
182
|
+
for (const r of refs) {
|
|
183
|
+
const ov = overlapMs(u, r)
|
|
184
|
+
if (ov > 0) byName[r.name] = (byName[r.name] || 0) + ov
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
const ranked = Object.entries(byName).sort((a, b) => b[1] - a[1])
|
|
188
|
+
if (ranked.length === 0) {
|
|
189
|
+
unmapped.push(spk)
|
|
190
|
+
continue
|
|
191
|
+
}
|
|
192
|
+
const [bestName, bestOv] = ranked[0]
|
|
193
|
+
const secondOv = ranked[1] ? ranked[1][1] : 0
|
|
194
|
+
const fraction = spkTotal > 0 ? bestOv / spkTotal : 0
|
|
195
|
+
const dominant = secondOv === 0 || bestOv >= secondOv * t.dominanceRatio
|
|
196
|
+
|
|
197
|
+
if (fraction >= t.minOverlapFraction && dominant) {
|
|
198
|
+
map[spk] = bestName
|
|
199
|
+
} else {
|
|
200
|
+
unmapped.push(spk)
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
return { map, unmapped }
|
|
204
|
+
}
|
package/src/spellcheck/core.js
CHANGED
|
@@ -20,7 +20,16 @@
|
|
|
20
20
|
* and passed in. Caller manages caching.
|
|
21
21
|
*/
|
|
22
22
|
|
|
23
|
-
import { possibleRoots } from '../welsh/mutations.js'
|
|
23
|
+
import { possibleRoots, applyMutation, MUTATION } from '../welsh/mutations.js'
|
|
24
|
+
|
|
25
|
+
// Edit-distance ceiling for the accept-list bias inside suggest().
|
|
26
|
+
// Raised to 3 (was 2 pre-audit 2026-05-20) so proper-noun typos one
|
|
27
|
+
// extra step beyond what nspell catches still surface the right
|
|
28
|
+
// glossary hit. Eg 'Gyfyrddin' -> 'Gaerfyrddin' is distance 3.
|
|
29
|
+
const ACCEPT_BIAS_MAX_DIST = 3
|
|
30
|
+
|
|
31
|
+
// Welsh mutations to try when forward-mutating accept-list entries.
|
|
32
|
+
const ACCEPT_MUTATION_KINDS = [MUTATION.soft, MUTATION.nasal, MUTATION.aspirate]
|
|
24
33
|
|
|
25
34
|
/**
|
|
26
35
|
* Check a single word against the supplied dictionary + accept-list.
|
|
@@ -41,6 +50,17 @@ export function checkWord(word, { nspell, accept, welsh }) {
|
|
|
41
50
|
if (typeof word !== 'string' || word.length === 0) return true
|
|
42
51
|
const cleaned = word.replace(/[^\p{L}\p{M}'’-]/gu, '')
|
|
43
52
|
if (cleaned.length === 0) return true
|
|
53
|
+
// No letter characters at all (e.g. a bare hyphen) -- treat as OK.
|
|
54
|
+
if (!/\p{L}/u.test(cleaned)) return true
|
|
55
|
+
// Hyphenated compounds (e.g. 'cyd-destun', 'ar-ol', 'rhag-brawf'):
|
|
56
|
+
// Welsh dictionaries store roots, not every compound form. Split on
|
|
57
|
+
// hyphen and accept if every component passes individually.
|
|
58
|
+
if (cleaned.includes('-')) {
|
|
59
|
+
const parts = cleaned.split('-').filter(p => /\p{L}/u.test(p))
|
|
60
|
+
if (parts.length > 1) {
|
|
61
|
+
return parts.every(part => checkWord(part, { nspell, accept, welsh }))
|
|
62
|
+
}
|
|
63
|
+
}
|
|
44
64
|
const lower = cleaned.toLowerCase()
|
|
45
65
|
if (accept && accept.has(lower)) return true
|
|
46
66
|
if (nspell.correct(cleaned)) return true
|
|
@@ -74,7 +94,12 @@ export function tokenise(text) {
|
|
|
74
94
|
const re = /[\p{L}\p{M}'’-]+/gu
|
|
75
95
|
let m
|
|
76
96
|
while ((m = re.exec(text)) !== null) {
|
|
77
|
-
|
|
97
|
+
// Guard: skip tokens with no letter at all. A bare '-' or apostrophe-only
|
|
98
|
+
// run is punctuation, not a word. Without this, 'word - word' produces a
|
|
99
|
+
// standalone '-' token that nspell.correct() rejects -> false red squiggle.
|
|
100
|
+
if (/\p{L}/u.test(m[0])) {
|
|
101
|
+
tokens.push({ word: m[0], start: m.index, end: m.index + m[0].length })
|
|
102
|
+
}
|
|
78
103
|
}
|
|
79
104
|
return tokens
|
|
80
105
|
}
|
|
@@ -85,6 +110,9 @@ export function tokenise(text) {
|
|
|
85
110
|
*
|
|
86
111
|
* @param {string} text
|
|
87
112
|
* @param {object} opts - same shape as checkWord opts
|
|
113
|
+
* @param {object} opts.nspell - An nspell instance.
|
|
114
|
+
* @param {Set<string>} [opts.accept] - Terms that must never be flagged.
|
|
115
|
+
* @param {boolean} [opts.welsh] - Try mutation reversal before flagging.
|
|
88
116
|
* @returns {{ word: string, start: number, end: number }[]}
|
|
89
117
|
*/
|
|
90
118
|
export function checkSentence(text, opts) {
|
|
@@ -100,22 +128,45 @@ export function checkSentence(text, opts) {
|
|
|
100
128
|
* @param {object} opts
|
|
101
129
|
* @param {object} opts.nspell
|
|
102
130
|
* @param {Set<string>} [opts.accept]
|
|
131
|
+
* @param {boolean} [opts.welsh] - Bias suggestions for Welsh mutation forms.
|
|
103
132
|
* @param {number} [opts.max] - default 6
|
|
104
133
|
* @returns {string[]}
|
|
105
134
|
*/
|
|
106
|
-
export function suggest(word, { nspell, accept, max = 6 }) {
|
|
135
|
+
export function suggest(word, { nspell, accept, welsh, max = 6 }) {
|
|
107
136
|
if (typeof word !== 'string' || word.length === 0) return []
|
|
108
137
|
const base = nspell.suggest(word) || []
|
|
109
138
|
if (!accept || accept.size === 0) return base.slice(0, max)
|
|
110
|
-
const acceptHits = []
|
|
111
139
|
const lower = word.toLowerCase()
|
|
140
|
+
|
|
141
|
+
// Build candidate surface forms from each accept-list entry.
|
|
142
|
+
// Welsh: try the bare term plus each mutation, and pick whichever form
|
|
143
|
+
// is closest to the misspelling - so 'Caerfyrddin' in the glossary
|
|
144
|
+
// surfaces as 'Gaerfyrddin' when the typo was 'Gyfyrddin', preserving
|
|
145
|
+
// the grammatically-correct mutated surface form.
|
|
146
|
+
// English (welsh falsy): just the bare term, no mutation table.
|
|
147
|
+
const acceptHits = []
|
|
148
|
+
const seenHit = new Set()
|
|
112
149
|
for (const term of accept) {
|
|
113
150
|
if (term === lower) continue
|
|
114
|
-
|
|
151
|
+
const candidates = welsh ? mutatedCandidates(term) : [term]
|
|
152
|
+
let bestForm = null
|
|
153
|
+
let bestDist = ACCEPT_BIAS_MAX_DIST + 1
|
|
154
|
+
for (const cand of candidates) {
|
|
155
|
+
const d = editDistance(lower, cand.toLowerCase(), ACCEPT_BIAS_MAX_DIST)
|
|
156
|
+
if (d <= ACCEPT_BIAS_MAX_DIST && d < bestDist) {
|
|
157
|
+
bestDist = d
|
|
158
|
+
bestForm = cand
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
if (bestForm && !seenHit.has(bestForm.toLowerCase())) {
|
|
162
|
+
acceptHits.push(bestForm)
|
|
163
|
+
seenHit.add(bestForm.toLowerCase())
|
|
164
|
+
}
|
|
115
165
|
}
|
|
166
|
+
|
|
116
167
|
// De-dupe: glossary hits first, then nspell suggestions that aren't dupes.
|
|
117
|
-
const seen = new Set(acceptHits.map(t => t.toLowerCase()))
|
|
118
168
|
const out = [...acceptHits]
|
|
169
|
+
const seen = new Set(acceptHits.map(t => t.toLowerCase()))
|
|
119
170
|
for (const s of base) {
|
|
120
171
|
if (out.length >= max) break
|
|
121
172
|
if (!seen.has(s.toLowerCase())) {
|
|
@@ -123,6 +174,22 @@ export function suggest(word, { nspell, accept, max = 6 }) {
|
|
|
123
174
|
seen.add(s.toLowerCase())
|
|
124
175
|
}
|
|
125
176
|
}
|
|
177
|
+
return out.slice(0, max)
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
// Expand a Welsh accept-list term to its surface forms: bare + every
|
|
181
|
+
// mutation. applyMutation() returns the input unchanged when the initial
|
|
182
|
+
// consonant isn't subject to that mutation, so dedupe before returning.
|
|
183
|
+
function mutatedCandidates(term) {
|
|
184
|
+
const out = [term]
|
|
185
|
+
const seen = new Set([term.toLowerCase()])
|
|
186
|
+
for (const kind of ACCEPT_MUTATION_KINDS) {
|
|
187
|
+
const m = applyMutation(term, kind)
|
|
188
|
+
if (m && !seen.has(m.toLowerCase())) {
|
|
189
|
+
out.push(m)
|
|
190
|
+
seen.add(m.toLowerCase())
|
|
191
|
+
}
|
|
192
|
+
}
|
|
126
193
|
return out
|
|
127
194
|
}
|
|
128
195
|
|
package/src/welsh/normaliser.js
CHANGED
|
@@ -20,6 +20,17 @@ const ZERO_WIDTH = /[\u200B\u200C\u200D\uFEFF]/g
|
|
|
20
20
|
// Deterministic Welsh broken-contraction repairs (ASR artefacts)
|
|
21
21
|
const CONTRACTION_REPAIRS = [
|
|
22
22
|
[/\b(a|o|i|â) r\b/gi, "$1'r"],
|
|
23
|
+
// Verb + 'r: ASR drops the apostrophe before the definite article.
|
|
24
|
+
// e.g. "mae r athro" → "mae'r athro", "yw r plant" → "yw'r plant".
|
|
25
|
+
// Drift fix: these were absent from the drifted frontend copy (#449).
|
|
26
|
+
// VOWEL-ENDING verbs only ('r attaches after vowels).
|
|
27
|
+
[/\b(mae|yw|dyma|dyna) r\b/gi, "$1'r"],
|
|
28
|
+
// CONSONANT-ENDING verbs take the full article, not 'r: "oedd r ffordd"
|
|
29
|
+
// is the ASR mishearing "oedd y ffordd", and "oedd'r" is ungrammatical.
|
|
30
|
+
// Repair to yr before vowels/h, y otherwise (Codex on #1103; Aled's call
|
|
31
|
+
// 2026-06-12: restore the article rather than leave the artefact).
|
|
32
|
+
[/\b(oedd|fydd|bydd|sydd|roedd|caiff|gall) r (?=[aeiouwyhâêîôûŵŷáéíóúàèìòùäëïöü])/gi, '$1 yr '],
|
|
33
|
+
[/\b(oedd|fydd|bydd|sydd|roedd|caiff|gall) r\b/gi, '$1 y'],
|
|
23
34
|
[/\bwedi (u|i|m|n|ch)\b/gi, "wedi'$1"],
|
|
24
35
|
[/\bwedi eu\b/gi, "wedi'u"],
|
|
25
36
|
[/\bwedi ei\b/gi, "wedi'i"],
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Compute the set of proper-noun corrections worth promoting.
|
|
3
|
+
*
|
|
4
|
+
* Diffs `before` against `after` on whitespace tokens and returns the
|
|
5
|
+
* corrected tokens that look like proper nouns. Structural edits (the
|
|
6
|
+
* token count changed) return `[]`: word-for-word swaps are the
|
|
7
|
+
* high-signal class and everything else is too noisy to learn from.
|
|
8
|
+
*
|
|
9
|
+
* In Capsiynau these terms become rows in `word_boost_approved` (per-org,
|
|
10
|
+
* tagged with the session's language) and the relay's loader treats those
|
|
11
|
+
* rows as language-agnostic on read - see the contract note on
|
|
12
|
+
* `looksLikeProperNoun` above for the full invariant and what must change
|
|
13
|
+
* if the heuristic ever expands beyond proper nouns. The function itself
|
|
14
|
+
* knows nothing about that table; a consumer on a different schema can
|
|
15
|
+
* do whatever it likes with the returned terms.
|
|
16
|
+
*
|
|
17
|
+
* @param {string} before Original text before the edit.
|
|
18
|
+
* @param {string} after Text after the edit.
|
|
19
|
+
* @returns {string[]} Deduplicated corrected tokens worth biasing toward.
|
|
20
|
+
*/
|
|
21
|
+
export function extractProperNounCorrections(before: string, after: string): string[];
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Upsert proper-noun corrections into word_boost_approved for the session's
|
|
3
|
+
* org + source language. Returns the list of terms persisted (excludes
|
|
4
|
+
* duplicates already in the table - upsert is silent on conflict).
|
|
5
|
+
*
|
|
6
|
+
* Never throws - failures are logged and swallowed so a segment-edit save
|
|
7
|
+
* is never blocked by a vocabulary-learning hiccup.
|
|
8
|
+
*
|
|
9
|
+
* CAPSIYNAU-SPECIFIC. Requires `live_sessions`, `live_segments` and
|
|
10
|
+
* `word_boost_approved`. If you only want the heuristic, import
|
|
11
|
+
* `extractProperNounCorrections` from
|
|
12
|
+
* `@capsiynau/intelligence/corrections/pure`.
|
|
13
|
+
*
|
|
14
|
+
* @param {string} before Original segment text before the edit.
|
|
15
|
+
* @param {string} after Text after the edit.
|
|
16
|
+
* @param {object} opts
|
|
17
|
+
* @param {string} opts.sessionId Live session UUID for org / fallback-lang lookup.
|
|
18
|
+
* @param {string} [opts.segmentId] Phase 1+ (notes-app#262): live_segments
|
|
19
|
+
* UUID for the edited segment. When provided,
|
|
20
|
+
* the segment's own `language` column is used
|
|
21
|
+
* for the new word_boost_approved row (drops
|
|
22
|
+
* silent rot in mixed-language sessions where
|
|
23
|
+
* Phase 3's per-chunk detected_language differs
|
|
24
|
+
* from session.source_language). Falls back to
|
|
25
|
+
* session.source_language if absent or the
|
|
26
|
+
* segment's language column is null.
|
|
27
|
+
* @param {object} opts.supabase Caller-provided Supabase client (DI).
|
|
28
|
+
* Must have `.from()` and `.rpc()` methods.
|
|
29
|
+
* RLS scopes writes by org.
|
|
30
|
+
* @returns {Promise<string[]>} Terms persisted (empty on any failure).
|
|
31
|
+
*/
|
|
32
|
+
export function captureCorrections(before: string, after: string, { sessionId, segmentId, supabase }: {
|
|
33
|
+
sessionId: string;
|
|
34
|
+
segmentId?: string;
|
|
35
|
+
supabase: object;
|
|
36
|
+
}): Promise<string[]>;
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Parse a CSV-ish string. Accepts (with or without header row):
|
|
3
|
+
* term,definition
|
|
4
|
+
* term,definition,category
|
|
5
|
+
* term,category,definition
|
|
6
|
+
* Header row (detected by token match) maps columns by name and
|
|
7
|
+
* tolerates aliases (name, meaning, tag). RFC-4180-lite: quoted
|
|
8
|
+
* fields + escaped quotes on a single line. Multi-line quoted values
|
|
9
|
+
* are out of scope - report and skip.
|
|
10
|
+
*/
|
|
11
|
+
export function parseImportedCsv(raw: any): {
|
|
12
|
+
rows: {
|
|
13
|
+
source_term: string;
|
|
14
|
+
preferred_term: string;
|
|
15
|
+
definition: string;
|
|
16
|
+
category: any;
|
|
17
|
+
}[];
|
|
18
|
+
errors: {
|
|
19
|
+
line: number;
|
|
20
|
+
reason: string;
|
|
21
|
+
}[];
|
|
22
|
+
};
|
|
23
|
+
/**
|
|
24
|
+
* Parse plain text. One term per line. Separator tried in order: tab,
|
|
25
|
+
* `:` followed by space, ` - ` (hyphen with spaces). Three columns
|
|
26
|
+
* (term, definition, optional category) match our copy-as-text output
|
|
27
|
+
* so users can round-trip cleanly.
|
|
28
|
+
*/
|
|
29
|
+
export function parseImportedText(raw: any): {
|
|
30
|
+
rows: {
|
|
31
|
+
source_term: string;
|
|
32
|
+
preferred_term: string;
|
|
33
|
+
definition: string;
|
|
34
|
+
category: string;
|
|
35
|
+
}[];
|
|
36
|
+
errors: {
|
|
37
|
+
line: number;
|
|
38
|
+
reason: string;
|
|
39
|
+
}[];
|
|
40
|
+
};
|
|
41
|
+
/**
|
|
42
|
+
* Split incoming rows against the existing set. Matches case-insensitive
|
|
43
|
+
* on trimmed source_term. Returns {additions, conflicts} where conflicts
|
|
44
|
+
* is an array of {existing, incoming} pairs the UI surfaces for
|
|
45
|
+
* resolution. Dedup within the import itself happens against additions
|
|
46
|
+
* (the second occurrence of a duplicated import term becomes a conflict
|
|
47
|
+
* with the first occurrence).
|
|
48
|
+
*/
|
|
49
|
+
export function splitImportAgainstExisting(existing: any, incoming: any): {
|
|
50
|
+
additions: any[];
|
|
51
|
+
conflicts: {
|
|
52
|
+
existing: any;
|
|
53
|
+
incoming: any;
|
|
54
|
+
}[];
|
|
55
|
+
};
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
export { WEB_SEARCH_TOOL } from "./tools.js";
|
|
2
|
+
export { normaliseUrl, isHttpUrl } from "./url.js";
|
|
3
|
+
export { extractText, parseTermsJson, normaliseTermRow, flagDiacriticStrip } from "./parse.js";
|
|
4
|
+
export { RESEARCH_SYSTEM, buildGenerationSystemPrompt, buildExpandSystemPrompt } from "./prompts.js";
|
|
5
|
+
export { parseImportedCsv, parseImportedText, splitImportAgainstExisting } from "./import.js";
|
|
6
|
+
export { suggestGlossaryName, languagePairFor, expandRowsForDb, buildGlossaryRow, suggestVersionedName } from "./persist.js";
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
export function extractText(message: any): any;
|
|
2
|
+
/**
|
|
3
|
+
* Defensive JSON parse for the generation phase. The model is told to
|
|
4
|
+
* reply with raw JSON only, but markdown fences and conversational
|
|
5
|
+
* preamble do leak through. Strip what we can, then fall back to the
|
|
6
|
+
* first [...] match. Return null on total failure - never throw.
|
|
7
|
+
*/
|
|
8
|
+
export function parseTermsJson(raw: any): any[];
|
|
9
|
+
/**
|
|
10
|
+
* Heuristic guard: any Welsh text longer than 25 chars with ZERO
|
|
11
|
+
* diacritics is suspicious - the model probably stripped them.
|
|
12
|
+
*
|
|
13
|
+
* Not a perfect check (some short Welsh text genuinely has none) but
|
|
14
|
+
* catches the common "model normalised everything to ASCII" failure
|
|
15
|
+
* mode. Substitution detection (ŵ → w, ŷ → y in specific positions)
|
|
16
|
+
* requires a dictionary and is deferred to a future Tier-3 spellcheck
|
|
17
|
+
* integration; not in scope for the wizard.
|
|
18
|
+
*/
|
|
19
|
+
export function flagDiacriticStrip(text: any): boolean;
|
|
20
|
+
/**
|
|
21
|
+
* Normalise one row from the model's JSON output.
|
|
22
|
+
*
|
|
23
|
+
* In 'en' mode: tolerate naming wobbles ({term} vs {source_term}; etc).
|
|
24
|
+
* In 'cy' mode: same shape, but route Welsh text through normaliseWelsh().
|
|
25
|
+
* In 'bilingual' mode: expect {term_en, term_cy, definition_en,
|
|
26
|
+
* definition_cy, category}; route the *_cy fields through normaliseWelsh().
|
|
27
|
+
*
|
|
28
|
+
* Returns null for entries missing the required fields so the caller
|
|
29
|
+
* can `.map(normaliseTermRow).filter(Boolean)`. Adds a transient
|
|
30
|
+
* `_diacritic_warning` flag the handler can aggregate; it does NOT
|
|
31
|
+
* persist to the DB.
|
|
32
|
+
*/
|
|
33
|
+
export function normaliseTermRow(t: any, opts?: {}): {
|
|
34
|
+
source_term: any;
|
|
35
|
+
preferred_term: any;
|
|
36
|
+
definition: any;
|
|
37
|
+
term_en: any;
|
|
38
|
+
term_cy: any;
|
|
39
|
+
definition_en: any;
|
|
40
|
+
definition_cy: any;
|
|
41
|
+
category: string;
|
|
42
|
+
_diacritic_warning: boolean;
|
|
43
|
+
} | {
|
|
44
|
+
source_term: any;
|
|
45
|
+
preferred_term: any;
|
|
46
|
+
definition: any;
|
|
47
|
+
category: string;
|
|
48
|
+
_diacritic_warning: boolean;
|
|
49
|
+
term_en?: undefined;
|
|
50
|
+
term_cy?: undefined;
|
|
51
|
+
definition_en?: undefined;
|
|
52
|
+
definition_cy?: undefined;
|
|
53
|
+
} | {
|
|
54
|
+
source_term: string;
|
|
55
|
+
preferred_term: string;
|
|
56
|
+
definition: string;
|
|
57
|
+
category: string;
|
|
58
|
+
term_en?: undefined;
|
|
59
|
+
term_cy?: undefined;
|
|
60
|
+
definition_en?: undefined;
|
|
61
|
+
definition_cy?: undefined;
|
|
62
|
+
_diacritic_warning?: undefined;
|
|
63
|
+
};
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pure helpers for converting Glossary Builder wizard rows into the
|
|
3
|
+
* shapes the `glossaries` + `terms` Supabase tables expect.
|
|
4
|
+
*
|
|
5
|
+
* No Supabase calls here - those live in the view layer
|
|
6
|
+
* (src/components/glossaryBuilder/savePersist.js) so this module
|
|
7
|
+
* stays unit-testable and side-effect-free, mirroring how
|
|
8
|
+
* @capsiynau/intelligence/welsh/normaliser splits pure logic from
|
|
9
|
+
* the API handler.
|
|
10
|
+
*/
|
|
11
|
+
/**
|
|
12
|
+
* Suggest a glossary name based on the source URL. Strips
|
|
13
|
+
* protocol + leading `www.` so the result reads naturally
|
|
14
|
+
* ("Site terms - example.com").
|
|
15
|
+
*/
|
|
16
|
+
export function suggestGlossaryName(url: any): string;
|
|
17
|
+
/**
|
|
18
|
+
* Map wizard languageMode to glossaries.language_pair. Single-language
|
|
19
|
+
* modes get 'en-GB' or 'cy-GB'; bilingual is 'en-GB,cy-GB' (the
|
|
20
|
+
* existing convention in the schema for multi-language glossaries).
|
|
21
|
+
*/
|
|
22
|
+
export function languagePairFor(languageMode: any): "cy-GB" | "en-GB,cy-GB" | "en-GB";
|
|
23
|
+
/**
|
|
24
|
+
* Expand wizard rows into terms-table inserts.
|
|
25
|
+
*
|
|
26
|
+
* Single-language modes produce one row per wizard row.
|
|
27
|
+
* Bilingual produces TWO rows per pair (EN + CY), linked by sharing
|
|
28
|
+
* glossary_id and category, with source_language differing. This uses
|
|
29
|
+
* the schema as designed and keeps the live-relay engine able to
|
|
30
|
+
* query by language without a JOIN.
|
|
31
|
+
*
|
|
32
|
+
* Drops rows missing term or definition silently (the wizard
|
|
33
|
+
* shouldn't let them through, but defence in depth at persistence
|
|
34
|
+
* time means a partial bilingual row contributes only its complete
|
|
35
|
+
* leg rather than aborting the whole save).
|
|
36
|
+
*/
|
|
37
|
+
export function expandRowsForDb(wizardRows: any, opts?: {}): any[];
|
|
38
|
+
/**
|
|
39
|
+
* Build the glossaries-table insert payload for a wizard save.
|
|
40
|
+
* Defaults to visibility='shared' (org-wide, not just creator) - the
|
|
41
|
+
* wizard is collaborative tooling, not personal scratch space.
|
|
42
|
+
*/
|
|
43
|
+
export function buildGlossaryRow({ name, url, languageMode, organisationId, createdBy, visibility, }: {
|
|
44
|
+
name: any;
|
|
45
|
+
url: any;
|
|
46
|
+
languageMode: any;
|
|
47
|
+
organisationId: any;
|
|
48
|
+
createdBy: any;
|
|
49
|
+
visibility?: string;
|
|
50
|
+
}): {
|
|
51
|
+
name: any;
|
|
52
|
+
description: string;
|
|
53
|
+
organisation_id: any;
|
|
54
|
+
created_by: any;
|
|
55
|
+
visibility: string;
|
|
56
|
+
language_pair: string;
|
|
57
|
+
source_url: any;
|
|
58
|
+
is_default: boolean;
|
|
59
|
+
is_system: boolean;
|
|
60
|
+
};
|
|
61
|
+
/**
|
|
62
|
+
* Suggest a versioned name when "Create new version" is chosen on
|
|
63
|
+
* conflict. Bumps an existing (vN) suffix or appends (v2) to a name
|
|
64
|
+
* that lacks one.
|
|
65
|
+
*/
|
|
66
|
+
export function suggestVersionedName(existingName: any): string;
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export function buildGenerationSystemPrompt({ audience, topics, languageMode, dialect, }?: {
|
|
2
|
+
languageMode?: string;
|
|
3
|
+
dialect?: string;
|
|
4
|
+
}): string;
|
|
5
|
+
export function buildExpandSystemPrompt({ audience, topics, languageMode, dialect, existingTerms, }?: {
|
|
6
|
+
languageMode?: string;
|
|
7
|
+
dialect?: string;
|
|
8
|
+
existingTerms?: any[];
|
|
9
|
+
}): string;
|
|
10
|
+
export const RESEARCH_SYSTEM: "You are helping a SaaS user build a starter glossary for their organisation. You have a web_search tool - use it to look up the URL the user provides and identify the industry, the products / services they offer, and the likely audience. Reply conversationally in 2-3 sentences confirming what you found, then ask who the glossary is for (e.g. \"clients\", \"internal team\", \"general public\", \"broadcasters\"). Keep the tone warm and direct. Do NOT produce a glossary yet - that comes in a follow-up step.";
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* URL helpers for the Glossary Builder engine.
|
|
3
|
+
*
|
|
4
|
+
* Anthropic's web_search expects a full URL; a bare 'example.com'
|
|
5
|
+
* returns "no relevant pages" and the whole flow degrades silently.
|
|
6
|
+
* normaliseUrl prepends https:// when the caller forgot a scheme.
|
|
7
|
+
*/
|
|
8
|
+
export function normaliseUrl(raw: any): string;
|
|
9
|
+
export function isHttpUrl(url: any): boolean;
|
package/types/index.d.ts
ADDED