@atlaskit/editor-plugin-autocomplete 8.0.0 → 9.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +80 -0
- package/dist/cjs/analytics/ufo.js +2 -1
- package/dist/cjs/pm-plugins/artifact-loader.js +135 -0
- package/dist/cjs/pm-plugins/artifacts-manifest.js +3 -0
- package/dist/cjs/pm-plugins/autocomplete-plugin.js +416 -129
- package/dist/cjs/pm-plugins/canonical-lm-scoring.js +371 -0
- package/dist/cjs/pm-plugins/debug-mode.js +90 -7
- package/dist/cjs/pm-plugins/ghost-text-decoration.js +12 -3
- package/dist/cjs/pm-plugins/local-slow-lane-client.js +1202 -453
- package/dist/cjs/pm-plugins/scoring-pipeline.js +164 -19
- package/dist/cjs/pm-plugins/slow-lane-client.js +100 -37
- package/dist/cjs/pm-plugins/text-predictor.js +1951 -273
- package/dist/es2019/analytics/ufo.js +2 -0
- package/dist/es2019/pm-plugins/artifact-loader.js +70 -0
- package/dist/es2019/pm-plugins/artifacts-manifest.js +3 -0
- package/dist/es2019/pm-plugins/autocomplete-plugin.js +425 -130
- package/dist/es2019/pm-plugins/canonical-lm-scoring.js +270 -0
- package/dist/es2019/pm-plugins/debug-mode.js +87 -7
- package/dist/es2019/pm-plugins/ghost-text-decoration.js +12 -3
- package/dist/es2019/pm-plugins/local-slow-lane-client.js +940 -326
- package/dist/es2019/pm-plugins/scoring-pipeline.js +137 -13
- package/dist/es2019/pm-plugins/slow-lane-client.js +94 -37
- package/dist/es2019/pm-plugins/text-predictor.js +1603 -199
- package/dist/esm/analytics/ufo.js +2 -1
- package/dist/esm/pm-plugins/artifact-loader.js +128 -0
- package/dist/esm/pm-plugins/artifacts-manifest.js +3 -0
- package/dist/esm/pm-plugins/autocomplete-plugin.js +419 -132
- package/dist/esm/pm-plugins/canonical-lm-scoring.js +364 -0
- package/dist/esm/pm-plugins/debug-mode.js +89 -6
- package/dist/esm/pm-plugins/ghost-text-decoration.js +12 -3
- package/dist/esm/pm-plugins/local-slow-lane-client.js +1200 -451
- package/dist/esm/pm-plugins/scoring-pipeline.js +163 -18
- package/dist/esm/pm-plugins/slow-lane-client.js +100 -37
- package/dist/esm/pm-plugins/text-predictor.js +1953 -277
- package/dist/types/analytics/ufo.d.ts +1 -0
- package/dist/types/pm-plugins/artifact-loader.d.ts +44 -0
- package/dist/types/pm-plugins/artifacts-manifest.d.ts +3 -0
- package/dist/types/pm-plugins/autocomplete-plugin.d.ts +39 -0
- package/dist/types/pm-plugins/canonical-lm-scoring.d.ts +158 -0
- package/dist/types/pm-plugins/debug-mode.d.ts +36 -1
- package/dist/types/pm-plugins/ghost-text-decoration.d.ts +5 -2
- package/dist/types/pm-plugins/local-slow-lane-client.d.ts +87 -54
- package/dist/types/pm-plugins/scoring-pipeline.d.ts +33 -1
- package/dist/types/pm-plugins/slow-lane-client.d.ts +51 -3
- package/dist/types/pm-plugins/text-predictor.d.ts +120 -2
- package/package.json +2 -2
- package/src/analytics/ufo.ts +5 -0
- package/src/pm-plugins/artifact-loader.ts +92 -0
- package/src/pm-plugins/artifacts-manifest.ts +3 -0
- package/src/pm-plugins/autocomplete-plugin.ts +517 -133
- package/src/pm-plugins/canonical-lm-scoring.ts +412 -0
- package/src/pm-plugins/debug-mode.ts +109 -7
- package/src/pm-plugins/ghost-text-decoration.ts +13 -4
- package/src/pm-plugins/local-slow-lane-client.ts +1302 -386
- package/src/pm-plugins/scoring-pipeline.ts +167 -17
- package/src/pm-plugins/slow-lane-client.ts +147 -62
- package/src/pm-plugins/text-predictor.ts +2006 -255
- package/CANONICAL_FIX__DO_NOT_USE_ME_A/package.json +0 -8
- package/CANONICAL_FIX__DO_NOT_USE_ME_B/package.json +0 -8
- package/CANONICAL_FIX__DO_NOT_USE_ME_C/package.json +0 -8
- package/dist/cjs/entry-points/src-pm-plugins-autocomplete-plugin.js +0 -18
- package/dist/cjs/entry-points/src-pm-plugins-slow-lane-client.js +0 -36
- package/dist/cjs/entry-points/src-pm-plugins-text-predictor.js +0 -66
- package/dist/es2019/entry-points/src-pm-plugins-autocomplete-plugin.js +0 -2
- package/dist/es2019/entry-points/src-pm-plugins-slow-lane-client.js +0 -2
- package/dist/es2019/entry-points/src-pm-plugins-text-predictor.js +0 -2
- package/dist/esm/entry-points/src-pm-plugins-autocomplete-plugin.js +0 -2
- package/dist/esm/entry-points/src-pm-plugins-slow-lane-client.js +0 -2
- package/dist/esm/entry-points/src-pm-plugins-text-predictor.js +0 -2
- package/dist/types/entry-points/src-pm-plugins-autocomplete-plugin.d.ts +0 -2
- package/dist/types/entry-points/src-pm-plugins-slow-lane-client.d.ts +0 -2
- package/dist/types/entry-points/src-pm-plugins-text-predictor.d.ts +0 -2
- package/scripts/gen_first_token_to_words.py +0 -170
- package/src/entry-points/src-pm-plugins-autocomplete-plugin.ts +0 -7
- package/src/entry-points/src-pm-plugins-slow-lane-client.ts +0 -13
- package/src/entry-points/src-pm-plugins-text-predictor.ts +0 -14
- package/src/pm-plugins/autocomplete-plugin/package.json +0 -8
- package/src/pm-plugins/slow-lane-client/package.json +0 -8
- package/src/pm-plugins/text-predictor/package.json +0 -8
|
@@ -11,23 +11,41 @@
|
|
|
11
11
|
import { EXPERIENCE_NAME, failExp, startExp, succeedExp } from '../analytics/ufo';
|
|
12
12
|
|
|
13
13
|
import { ARTIFACT_NAME, fetchArtifactJson } from './artifacts-manifest';
|
|
14
|
+
import type { CanonicalLmEvidence } from './canonical-lm-scoring';
|
|
14
15
|
import { isAutocompleteDebugEnabled } from './debug-mode';
|
|
15
16
|
|
|
16
17
|
// ─── Types ──────────────────────────────────────────────────
|
|
17
18
|
|
|
19
|
+
/**
|
|
20
|
+
* A completion term is `(term, termType)`. `word` is the v1 default; `bigram`
|
|
21
|
+
* (2 tokens) and `phrase` (3 tokens) are multi-word units mined by the producer.
|
|
22
|
+
*/
|
|
23
|
+
export type TermType = 'word' | 'bigram' | 'phrase';
|
|
24
|
+
|
|
18
25
|
export interface ScoringCandidate {
|
|
26
|
+
/** NPMI-style association strength (bigram/phrase only). */
|
|
27
|
+
association?: number | null;
|
|
19
28
|
authorFreq: number;
|
|
20
29
|
docFreq: number;
|
|
30
|
+
/** Last-token POS of a multi-word term (bigram/phrase only). */
|
|
31
|
+
headPos?: string | null;
|
|
32
|
+
/** Per-token POS sequence of a multi-word term (bigram/phrase only). */
|
|
33
|
+
posSeq?: string[] | null;
|
|
21
34
|
sessionFreq: number;
|
|
22
35
|
tenantFreq: number;
|
|
36
|
+
/** Defaults to `'word'` when omitted (back-compat with word-only callers). */
|
|
37
|
+
termType?: TermType;
|
|
23
38
|
word: string;
|
|
24
39
|
}
|
|
25
40
|
|
|
26
41
|
export interface ScoredCandidate {
|
|
27
42
|
finalScore: number;
|
|
28
43
|
freqScore: number;
|
|
44
|
+
lmEvidence: CanonicalLmEvidence | null;
|
|
29
45
|
lmScore: number;
|
|
30
46
|
semanticScore: number;
|
|
47
|
+
stage1Score: number;
|
|
48
|
+
termType: TermType;
|
|
31
49
|
word: string;
|
|
32
50
|
}
|
|
33
51
|
|
|
@@ -48,14 +66,32 @@ interface GrammarTransitions {
|
|
|
48
66
|
transitions: Record<string, PosTransitionRule>;
|
|
49
67
|
}
|
|
50
68
|
|
|
69
|
+
type PosTagsRecord = Record<string, string[]>;
|
|
70
|
+
|
|
51
71
|
// ─── Scoring Constants ──────────────────────────────────────
|
|
52
72
|
|
|
53
73
|
const ALPHA = 0.5;
|
|
54
74
|
const BETA = 0.5;
|
|
55
75
|
const NEUTRAL_SCORE = 0.5;
|
|
76
|
+
// Multiplicative Stage-1 boost applied to a multi-word term proportional to its
|
|
77
|
+
// (clamped [0,1]) NPMI association, so a strongly-collocated phrase outranks a
|
|
78
|
+
// coincidental word sequence of comparable freq/semantics. Tunable.
|
|
79
|
+
const ASSOCIATION_BOOST = 0.5;
|
|
56
80
|
export const STAGE1_WEIGHT = 0.35;
|
|
57
81
|
export const STAGE2_WEIGHT = 0.65;
|
|
58
82
|
export const MIN_STAGE1_SCORE = 0.35;
|
|
83
|
+
/**
|
|
84
|
+
* How far the leading candidate must sit above the runner-up on the blended
|
|
85
|
+
* score before a ghost is committed.
|
|
86
|
+
*
|
|
87
|
+
* Lives beside the weights it is measured in, because it is only meaningful on
|
|
88
|
+
* that scale — and because both the arbitration that applies it and the
|
|
89
|
+
* scheduler that decides when reading more tokens can no longer change the
|
|
90
|
+
* outcome have to use the same number. When those two drifted apart, the
|
|
91
|
+
* scheduler stopped early on a margin the arbitration did not recognise and the
|
|
92
|
+
* abstention that followed was invisible from either side.
|
|
93
|
+
*/
|
|
94
|
+
export const MIN_WINNER_MARGIN = 0.08;
|
|
59
95
|
const L1_SESSION_CAP = 1.2;
|
|
60
96
|
// Minimum prefix-payload max LM probability before Stage 2 activates.
|
|
61
97
|
// Below this threshold the LM signal is too weak to suppress Stage 1 — finalScore
|
|
@@ -75,13 +111,36 @@ let posTags: Map<string, string[]> = new Map();
|
|
|
75
111
|
*/
|
|
76
112
|
let ghostPosTags: Record<string, string[]> = {};
|
|
77
113
|
|
|
114
|
+
let grammarDataPromise: Promise<void> | undefined;
|
|
115
|
+
|
|
78
116
|
/**
|
|
79
117
|
* Precomputed map from each POS tag to the set of allowed next POS tags, so
|
|
80
118
|
* applyGrammarFilter never re-iterates the transition rules per call.
|
|
81
119
|
*/
|
|
82
120
|
let precomputedAllowedByPos: Map<string, Set<string>> = new Map();
|
|
83
121
|
|
|
84
|
-
|
|
122
|
+
const isPosTagsRecord = (payload: unknown): payload is PosTagsRecord => {
|
|
123
|
+
if (payload == null || typeof payload !== 'object') {
|
|
124
|
+
return false;
|
|
125
|
+
}
|
|
126
|
+
return Object.values(payload as Record<string, unknown>).every(
|
|
127
|
+
(value) => Array.isArray(value) && value.every((entry) => typeof entry === 'string'),
|
|
128
|
+
);
|
|
129
|
+
};
|
|
130
|
+
|
|
131
|
+
const isGrammarTransitions = (payload: unknown): payload is GrammarTransitions => {
|
|
132
|
+
if (payload == null || typeof payload !== 'object') {
|
|
133
|
+
return false;
|
|
134
|
+
}
|
|
135
|
+
const transitions = (payload as { transitions?: unknown }).transitions;
|
|
136
|
+
if (transitions == null || typeof transitions !== 'object') {
|
|
137
|
+
return false;
|
|
138
|
+
}
|
|
139
|
+
return Object.values(transitions as Record<string, unknown>).every((value) => {
|
|
140
|
+
const allowed = (value as { allowed?: unknown })?.allowed;
|
|
141
|
+
return Array.isArray(allowed) && allowed.every((entry) => typeof entry === 'string');
|
|
142
|
+
});
|
|
143
|
+
};
|
|
85
144
|
|
|
86
145
|
const setGrammarData = (data: {
|
|
87
146
|
ghostPosTags: Record<string, string[]>;
|
|
@@ -125,20 +184,35 @@ export const loadGrammarDataAsync = (options?: {
|
|
|
125
184
|
|
|
126
185
|
try {
|
|
127
186
|
const [posTagsData, ghostPosTagsData, grammarTransitionsData] = await Promise.all([
|
|
128
|
-
fetchArtifactJson<
|
|
129
|
-
fetchArtifactJson<
|
|
130
|
-
fetchArtifactJson<
|
|
187
|
+
fetchArtifactJson<unknown>(ARTIFACT_NAME.POS_TAGS),
|
|
188
|
+
fetchArtifactJson<unknown>(ARTIFACT_NAME.GHOST_POS_TAGS),
|
|
189
|
+
fetchArtifactJson<unknown>(ARTIFACT_NAME.GRAMMAR_TRANSITIONS),
|
|
131
190
|
]);
|
|
132
191
|
|
|
133
|
-
|
|
192
|
+
// A missing wrapper and a malformed ruleset are reported separately: the
|
|
193
|
+
// first points at the publishing step, the second at the payload itself.
|
|
194
|
+
if ((grammarTransitionsData as { transitions?: unknown })?.transitions == null) {
|
|
134
195
|
throw new Error(
|
|
135
196
|
`[scoring-pipeline] ${ARTIFACT_NAME.GRAMMAR_TRANSITIONS} is missing its \`transitions\` wrapper`,
|
|
136
197
|
);
|
|
137
198
|
}
|
|
199
|
+
if (!isGrammarTransitions(grammarTransitionsData)) {
|
|
200
|
+
throw new Error(
|
|
201
|
+
`[scoring-pipeline] ${ARTIFACT_NAME.GRAMMAR_TRANSITIONS} payload shape was invalid`,
|
|
202
|
+
);
|
|
203
|
+
}
|
|
204
|
+
if (!isPosTagsRecord(posTagsData)) {
|
|
205
|
+
throw new Error(`[scoring-pipeline] ${ARTIFACT_NAME.POS_TAGS} payload shape was invalid`);
|
|
206
|
+
}
|
|
207
|
+
if (!isPosTagsRecord(ghostPosTagsData)) {
|
|
208
|
+
throw new Error(
|
|
209
|
+
`[scoring-pipeline] ${ARTIFACT_NAME.GHOST_POS_TAGS} payload shape was invalid`,
|
|
210
|
+
);
|
|
211
|
+
}
|
|
138
212
|
|
|
139
213
|
setGrammarData({
|
|
140
|
-
posTags: posTagsData
|
|
141
|
-
ghostPosTags: ghostPosTagsData
|
|
214
|
+
posTags: posTagsData,
|
|
215
|
+
ghostPosTags: ghostPosTagsData,
|
|
142
216
|
grammarTransitions: grammarTransitionsData,
|
|
143
217
|
});
|
|
144
218
|
|
|
@@ -271,7 +345,16 @@ function applyGrammarFilter(
|
|
|
271
345
|
const dropped: string[] = [];
|
|
272
346
|
|
|
273
347
|
for (const entry of candidates) {
|
|
274
|
-
|
|
348
|
+
// The producer validates a multi-word term's *internal* grammar offline
|
|
349
|
+
// (boundary-clean, noun-headed, AUP-passed — spec §9), but nothing judges
|
|
350
|
+
// the transition into it. Exempting multi-word terms therefore let a noun
|
|
351
|
+
// this filter had just rejected re-enter wrapped in a bigram — "be" [AUX]
|
|
352
|
+
// dropped "quality" and then displayed "quality assurance". Only the term's
|
|
353
|
+
// first word takes part in the transition, so gate every term on that.
|
|
354
|
+
const surface = entry.candidate.word;
|
|
355
|
+
const firstSpace = surface.indexOf(' ');
|
|
356
|
+
const transitionWord = firstSpace === -1 ? surface : surface.slice(0, firstSpace);
|
|
357
|
+
const candidateTags = posTags.get(transitionWord.toLowerCase());
|
|
275
358
|
|
|
276
359
|
// If candidate has no tags (unknown word), let it pass to be safe
|
|
277
360
|
if (!candidateTags || candidateTags.length === 0) {
|
|
@@ -326,23 +409,38 @@ export interface RankCandidatesResult {
|
|
|
326
409
|
pipelineDebug: PipelineDebug;
|
|
327
410
|
}
|
|
328
411
|
|
|
412
|
+
/**
|
|
413
|
+
* Apply Stage 1, grammar, and nullable context-local LM evidence to candidates.
|
|
414
|
+
*/
|
|
329
415
|
export function rankCandidates(
|
|
330
416
|
candidates: ScoringCandidate[],
|
|
331
417
|
contextVector: Float32Array | null,
|
|
332
418
|
getWordVector: (word: string) => Float32Array | null,
|
|
333
419
|
lmLogits: Record<string, number> | null,
|
|
334
|
-
|
|
420
|
+
maxFreqByType: Record<TermType, number>,
|
|
335
421
|
previousWord: string,
|
|
422
|
+
getCanonicalLmEvidence?: (surface: string) => CanonicalLmEvidence | null,
|
|
336
423
|
): RankCandidatesResult {
|
|
337
424
|
// Stage 1
|
|
338
425
|
const stage1Results = candidates.map((candidate) => {
|
|
426
|
+
const termType = candidate.termType ?? 'word';
|
|
427
|
+
// Normalize freq within the candidate's own term-type population so a
|
|
428
|
+
// phrase (whose absolute counts are far below top unigrams) isn't crushed
|
|
429
|
+
// by the unigram max.
|
|
430
|
+
const maxFreq = maxFreqByType[termType] || 1;
|
|
339
431
|
const { semanticScore, freqScore, stage1Score } = scoreStage1(
|
|
340
432
|
candidate,
|
|
341
433
|
contextVector,
|
|
342
434
|
getWordVector,
|
|
343
|
-
|
|
435
|
+
maxFreq,
|
|
344
436
|
);
|
|
345
|
-
|
|
437
|
+
// Multi-word units get an NPMI association boost (Stage-1 only).
|
|
438
|
+
let adjustedStage1 = stage1Score;
|
|
439
|
+
if (termType !== 'word' && typeof candidate.association === 'number') {
|
|
440
|
+
const assoc = Math.max(0, Math.min(1, candidate.association));
|
|
441
|
+
adjustedStage1 = stage1Score * (1 + ASSOCIATION_BOOST * assoc);
|
|
442
|
+
}
|
|
443
|
+
return { candidate, semanticScore, freqScore, stage1Score: adjustedStage1 };
|
|
346
444
|
});
|
|
347
445
|
const stage1Survivors: typeof stage1Results = [];
|
|
348
446
|
const stage1Rejected: string[] = [];
|
|
@@ -363,22 +461,71 @@ export function rankCandidates(
|
|
|
363
461
|
const values = Object.values(lmLogits);
|
|
364
462
|
if (values.length > 0) lmMax = Math.max(...values);
|
|
365
463
|
}
|
|
464
|
+
const canonicalEvidenceBySurface = new Map<string, CanonicalLmEvidence>();
|
|
465
|
+
const maxLmRankingValueByTierAndContext = new Map<string, number>();
|
|
466
|
+
for (const entry of filtered) {
|
|
467
|
+
const evidence = getCanonicalLmEvidence?.(entry.candidate.word) ?? null;
|
|
468
|
+
if (!evidence) {
|
|
469
|
+
continue;
|
|
470
|
+
}
|
|
471
|
+
canonicalEvidenceBySurface.set(entry.candidate.word, evidence);
|
|
472
|
+
const rankingValue =
|
|
473
|
+
evidence.source === 'canonical-full-surface'
|
|
474
|
+
? evidence.meanTokenLogProbability
|
|
475
|
+
: evidence.rawLogit;
|
|
476
|
+
if (rankingValue !== undefined && Number.isFinite(rankingValue)) {
|
|
477
|
+
const key = `${evidence.source}\u0000${evidence.contextKey}`;
|
|
478
|
+
const current = maxLmRankingValueByTierAndContext.get(key) ?? -Infinity;
|
|
479
|
+
if (rankingValue > current) {
|
|
480
|
+
maxLmRankingValueByTierAndContext.set(key, rankingValue);
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
}
|
|
366
484
|
|
|
367
485
|
const scored: ScoredCandidate[] = filtered.map((entry) => {
|
|
486
|
+
const termType = entry.candidate.termType ?? 'word';
|
|
487
|
+
const isMultiWord = termType !== 'word';
|
|
368
488
|
let lmScore = 0;
|
|
369
489
|
let finalScore = entry.stage1Score;
|
|
370
|
-
|
|
371
|
-
if (
|
|
490
|
+
let lmEvidence = canonicalEvidenceBySurface.get(entry.candidate.word) ?? null;
|
|
491
|
+
if (lmEvidence) {
|
|
492
|
+
const rankingValue =
|
|
493
|
+
lmEvidence.source === 'canonical-full-surface'
|
|
494
|
+
? lmEvidence.meanTokenLogProbability
|
|
495
|
+
: lmEvidence.rawLogit;
|
|
496
|
+
const maxRankingValue = maxLmRankingValueByTierAndContext.get(
|
|
497
|
+
`${lmEvidence.source}\u0000${lmEvidence.contextKey}`,
|
|
498
|
+
);
|
|
499
|
+
if (rankingValue !== undefined && maxRankingValue !== undefined) {
|
|
500
|
+
lmEvidence = {
|
|
501
|
+
...lmEvidence,
|
|
502
|
+
score: Math.exp(rankingValue - maxRankingValue),
|
|
503
|
+
};
|
|
504
|
+
}
|
|
505
|
+
}
|
|
506
|
+
if (lmEvidence) {
|
|
507
|
+
lmScore = lmEvidence.score;
|
|
508
|
+
finalScore = STAGE1_WEIGHT * entry.stage1Score + STAGE2_WEIGHT * lmScore;
|
|
509
|
+
} else if (!isMultiWord && lmLogits && lmMax >= LM_GATE_THRESHOLD) {
|
|
510
|
+
// Backward-compatible network slow-lane path.
|
|
372
511
|
const rawLm = getLmScore(entry.candidate.word, lmLogits);
|
|
373
512
|
|
|
374
513
|
if (rawLm !== 0) {
|
|
375
514
|
const logitDiff = Math.log(rawLm) - Math.log(lmMax);
|
|
376
515
|
lmScore = Math.exp(logitDiff);
|
|
516
|
+
lmEvidence = {
|
|
517
|
+
contextKey: 'network',
|
|
518
|
+
score: lmScore,
|
|
519
|
+
separatorKind: 'whitespace',
|
|
520
|
+
source: 'network-logit',
|
|
521
|
+
totalSurfaceCharCount: entry.candidate.word.length,
|
|
522
|
+
// The legacy BE word map does not report canonical token depth.
|
|
523
|
+
totalTokenCount: 0,
|
|
524
|
+
verifiedCharCount: 0,
|
|
525
|
+
verifiedTokenCount: 0,
|
|
526
|
+
};
|
|
527
|
+
finalScore = STAGE1_WEIGHT * entry.stage1Score + STAGE2_WEIGHT * lmScore;
|
|
377
528
|
}
|
|
378
|
-
// Words absent from the prefix-filtered payload get lmScore = 0,
|
|
379
|
-
// not 0.05, so they don't outrank genuine LM predictions.
|
|
380
|
-
|
|
381
|
-
finalScore = STAGE1_WEIGHT * entry.stage1Score + STAGE2_WEIGHT * lmScore;
|
|
382
529
|
}
|
|
383
530
|
|
|
384
531
|
return {
|
|
@@ -386,7 +533,10 @@ export function rankCandidates(
|
|
|
386
533
|
freqScore: entry.freqScore,
|
|
387
534
|
semanticScore: entry.semanticScore,
|
|
388
535
|
lmScore,
|
|
536
|
+
lmEvidence,
|
|
389
537
|
finalScore,
|
|
538
|
+
stage1Score: entry.stage1Score,
|
|
539
|
+
termType,
|
|
390
540
|
};
|
|
391
541
|
});
|
|
392
542
|
|
|
@@ -9,7 +9,15 @@
|
|
|
9
9
|
|
|
10
10
|
import { abortExp, EXPERIENCE_NAME, failExp, startExp, succeedExp } from '../analytics/ufo';
|
|
11
11
|
|
|
12
|
-
import {
|
|
12
|
+
import { buildAutocompleteGatewayUrl } from './artifact-loader';
|
|
13
|
+
import type {
|
|
14
|
+
BoundaryLmState,
|
|
15
|
+
BoundaryPrimeRequest,
|
|
16
|
+
ProgressiveSurfaceEvidence,
|
|
17
|
+
SurfaceScore,
|
|
18
|
+
SurfaceScoreRequest,
|
|
19
|
+
} from './canonical-lm-scoring';
|
|
20
|
+
import { CTC_STYLES, ctcTag } from './debug-mode';
|
|
13
21
|
|
|
14
22
|
// ─── Types ───────────────────────────────────────────────────────────────────
|
|
15
23
|
|
|
@@ -29,8 +37,6 @@ export interface TypeaheadEncodingsResponse {
|
|
|
29
37
|
|
|
30
38
|
// eslint-disable-next-line require-unicode-regexp
|
|
31
39
|
const WORD_BOUNDARY_CHARS = /[\s.,;:!?]/;
|
|
32
|
-
// eslint-disable-next-line require-unicode-regexp
|
|
33
|
-
const TRAILING_SLASH_REGEX = /\/$/;
|
|
34
40
|
const DEFAULT_DEBOUNCE_MS = 300;
|
|
35
41
|
|
|
36
42
|
/**
|
|
@@ -57,16 +63,28 @@ export interface SlowLaneClientConfig {
|
|
|
57
63
|
surface?: string;
|
|
58
64
|
}
|
|
59
65
|
|
|
60
|
-
export
|
|
61
|
-
|
|
62
|
-
)
|
|
66
|
+
export interface SlowLaneClient {
|
|
67
|
+
getBoundaryLmState?: (contextKey: string) => BoundaryLmState | null;
|
|
68
|
+
getCanonicalSurfaceCount?: () => number;
|
|
69
|
+
getCanonicalSurfaceTokenIds?: (surface: string) => number[] | null;
|
|
70
|
+
getContextInput?: () => string | null;
|
|
63
71
|
getContextVector: () => Float32Array | null;
|
|
64
72
|
getLmLogits: () => Record<string, number> | null;
|
|
73
|
+
getProgressiveSurfaceEvidence?: (
|
|
74
|
+
contextKey: string,
|
|
75
|
+
surface: string,
|
|
76
|
+
) => ProgressiveSurfaceEvidence | null;
|
|
77
|
+
getSurfaceScore?: (contextKey: string, surface: string) => SurfaceScore | null;
|
|
78
|
+
isReady?: () => boolean;
|
|
65
79
|
isWordBoundary: (text: string) => boolean;
|
|
80
|
+
primeBoundaryLm?: (input: BoundaryPrimeRequest) => void;
|
|
81
|
+
requestProgressiveSurfaceScores?: (input: SurfaceScoreRequest) => void;
|
|
66
82
|
setContextVector: (vector: Float32Array | null) => void;
|
|
67
83
|
setLmLogits: (logits: Record<string, number> | null) => void;
|
|
68
84
|
updateContext: (text: string) => void;
|
|
69
|
-
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
export const createSlowLaneClient = (config: SlowLaneClientConfig): SlowLaneClient => {
|
|
70
88
|
const {
|
|
71
89
|
baseUrl,
|
|
72
90
|
sessionId: configSessionId,
|
|
@@ -88,6 +106,7 @@ export const createSlowLaneClient = (
|
|
|
88
106
|
|
|
89
107
|
let debounceTimer: ReturnType<typeof setTimeout> | null = null;
|
|
90
108
|
let lastRequestedText = '';
|
|
109
|
+
let storedContextInput: string | null = null;
|
|
91
110
|
let storedContextVector: Float32Array | null = null;
|
|
92
111
|
let storedLmLogits: Record<string, number> | null = null;
|
|
93
112
|
let requestSeq = 0;
|
|
@@ -98,7 +117,7 @@ export const createSlowLaneClient = (
|
|
|
98
117
|
return;
|
|
99
118
|
}
|
|
100
119
|
|
|
101
|
-
const url =
|
|
120
|
+
const url = buildAutocompleteGatewayUrl(endpoint, baseUrl);
|
|
102
121
|
const payload: TypeaheadEncodingsRequest = {
|
|
103
122
|
text,
|
|
104
123
|
session_id: sessionId,
|
|
@@ -110,20 +129,7 @@ export const createSlowLaneClient = (
|
|
|
110
129
|
...(surface ? { surface } : {}),
|
|
111
130
|
});
|
|
112
131
|
|
|
113
|
-
|
|
114
|
-
// eslint-disable-next-line no-console
|
|
115
|
-
console.groupCollapsed(
|
|
116
|
-
`%c[SlowLane] %c📤 Sending context | ${text.length} chars`,
|
|
117
|
-
'color: #9c27b0; font-weight: bold;',
|
|
118
|
-
'color: inherit;',
|
|
119
|
-
);
|
|
120
|
-
text.split('\n').forEach((line, i, arr) => {
|
|
121
|
-
// eslint-disable-next-line no-console
|
|
122
|
-
console.log(` ${i === arr.length - 1 ? '▶' : ' '} ${line}`);
|
|
123
|
-
});
|
|
124
|
-
// eslint-disable-next-line no-console
|
|
125
|
-
console.groupEnd();
|
|
126
|
-
}
|
|
132
|
+
ctcTag('signal', `📤 network context sent · ${text.length} chars`, CTC_STYLES.section);
|
|
127
133
|
|
|
128
134
|
try {
|
|
129
135
|
const res = await fetchFn(url, {
|
|
@@ -133,6 +139,7 @@ export const createSlowLaneClient = (
|
|
|
133
139
|
});
|
|
134
140
|
|
|
135
141
|
if (!res.ok) {
|
|
142
|
+
storedContextInput = null;
|
|
136
143
|
storedContextVector = null;
|
|
137
144
|
storedLmLogits = null;
|
|
138
145
|
failExp(EXPERIENCE_NAME.SLOW_LANE_FETCH, requestId, {
|
|
@@ -141,14 +148,7 @@ export const createSlowLaneClient = (
|
|
|
141
148
|
isLocalLLM: false,
|
|
142
149
|
...(surface ? { surface } : {}),
|
|
143
150
|
});
|
|
144
|
-
|
|
145
|
-
// eslint-disable-next-line no-console
|
|
146
|
-
console.log(
|
|
147
|
-
`%c[SlowLane] %c❌ Request failed (${res.status})`,
|
|
148
|
-
'color: #9c27b0; font-weight: bold;',
|
|
149
|
-
'color: #f44336;',
|
|
150
|
-
);
|
|
151
|
-
}
|
|
151
|
+
ctcTag('signal', `❌ network request failed (${res.status})`, CTC_STYLES.bad);
|
|
152
152
|
return;
|
|
153
153
|
}
|
|
154
154
|
|
|
@@ -156,8 +156,10 @@ export const createSlowLaneClient = (
|
|
|
156
156
|
|
|
157
157
|
if (data.semantic_vector && Array.isArray(data.semantic_vector)) {
|
|
158
158
|
storedContextVector = new Float32Array(data.semantic_vector);
|
|
159
|
+
storedContextInput = text;
|
|
159
160
|
} else {
|
|
160
161
|
storedContextVector = null;
|
|
162
|
+
storedContextInput = null;
|
|
161
163
|
}
|
|
162
164
|
|
|
163
165
|
if (data.lm_logits && typeof data.lm_logits === 'object') {
|
|
@@ -166,28 +168,13 @@ export const createSlowLaneClient = (
|
|
|
166
168
|
storedLmLogits = null;
|
|
167
169
|
}
|
|
168
170
|
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
// eslint-disable-next-line no-console
|
|
177
|
-
console.log(
|
|
178
|
-
storedContextVector
|
|
179
|
-
? `✅ semantic_vector: ${storedContextVector.length} dims`
|
|
180
|
-
: '❌ No semantic_vector',
|
|
181
|
-
);
|
|
182
|
-
// eslint-disable-next-line no-console
|
|
183
|
-
console.log(
|
|
184
|
-
storedLmLogits
|
|
185
|
-
? `✅ lm_logits: ${Object.keys(storedLmLogits).length} tokens`
|
|
186
|
-
: '❌ No lm_logits',
|
|
187
|
-
);
|
|
188
|
-
// eslint-disable-next-line no-console
|
|
189
|
-
console.groupEnd();
|
|
190
|
-
}
|
|
171
|
+
ctcTag(
|
|
172
|
+
'signal',
|
|
173
|
+
`📥 network response · ${
|
|
174
|
+
storedContextVector ? `semantic ✅ ${storedContextVector.length}d` : 'semantic ❌'
|
|
175
|
+
} · ${storedLmLogits ? `logits ✅ ${Object.keys(storedLmLogits).length}tok` : 'logits ❌'}`,
|
|
176
|
+
CTC_STYLES.section,
|
|
177
|
+
);
|
|
191
178
|
|
|
192
179
|
succeedExp(EXPERIENCE_NAME.SLOW_LANE_FETCH, requestId, {
|
|
193
180
|
textLength: text.length,
|
|
@@ -204,6 +191,7 @@ export const createSlowLaneClient = (
|
|
|
204
191
|
});
|
|
205
192
|
// eslint-disable-next-line no-unused-vars
|
|
206
193
|
} catch (e) {
|
|
194
|
+
storedContextInput = null;
|
|
207
195
|
storedContextVector = null;
|
|
208
196
|
storedLmLogits = null;
|
|
209
197
|
failExp(EXPERIENCE_NAME.SLOW_LANE_FETCH, requestId, {
|
|
@@ -211,14 +199,7 @@ export const createSlowLaneClient = (
|
|
|
211
199
|
isLocalLLM: false,
|
|
212
200
|
...(surface ? { surface } : {}),
|
|
213
201
|
});
|
|
214
|
-
|
|
215
|
-
// eslint-disable-next-line no-console
|
|
216
|
-
console.log(
|
|
217
|
-
'%c[SlowLane] %c❌ Network error — context cleared',
|
|
218
|
-
'color: #9c27b0; font-weight: bold;',
|
|
219
|
-
'color: #f44336;',
|
|
220
|
-
);
|
|
221
|
-
}
|
|
202
|
+
ctcTag('signal', '❌ network error — context cleared', CTC_STYLES.bad);
|
|
222
203
|
} finally {
|
|
223
204
|
if (inflightRequestId === requestId) {
|
|
224
205
|
inflightRequestId = null;
|
|
@@ -247,9 +228,11 @@ export const createSlowLaneClient = (
|
|
|
247
228
|
|
|
248
229
|
return {
|
|
249
230
|
updateContext: updateContextDebounced,
|
|
231
|
+
getContextInput: () => storedContextInput,
|
|
250
232
|
getContextVector: () => storedContextVector,
|
|
251
233
|
getLmLogits: () => storedLmLogits,
|
|
252
234
|
setContextVector: (vector) => {
|
|
235
|
+
storedContextInput = null;
|
|
253
236
|
storedContextVector = vector;
|
|
254
237
|
},
|
|
255
238
|
setLmLogits: (logits) => {
|
|
@@ -259,16 +242,118 @@ export const createSlowLaneClient = (
|
|
|
259
242
|
};
|
|
260
243
|
};
|
|
261
244
|
|
|
262
|
-
|
|
245
|
+
export type SlowLaneClientKind = 'localLlm' | 'server';
|
|
246
|
+
|
|
247
|
+
export interface SlowLaneClientRegistration {
|
|
248
|
+
clientId: string;
|
|
249
|
+
kind: SlowLaneClientKind;
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
export interface SlowLaneClientStatus {
|
|
253
|
+
canonicalScoringSupported: boolean;
|
|
254
|
+
canonicalSurfaceCount: number | null;
|
|
255
|
+
clientId: string | null;
|
|
256
|
+
clientKind: SlowLaneClientKind | null;
|
|
257
|
+
localModelReady: boolean | null;
|
|
258
|
+
registered: boolean;
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
let defaultSlowLaneClient: SlowLaneClient | null = null;
|
|
262
|
+
let defaultSlowLaneClientRegistration: SlowLaneClientRegistration | null = null;
|
|
263
|
+
|
|
264
|
+
const supportsCanonicalSurfaceScoring = (client: SlowLaneClient | null): boolean =>
|
|
265
|
+
typeof client?.primeBoundaryLm === 'function' &&
|
|
266
|
+
typeof client?.requestProgressiveSurfaceScores === 'function';
|
|
263
267
|
|
|
264
268
|
export const setDefaultSlowLaneClient = (
|
|
265
|
-
client:
|
|
269
|
+
client: SlowLaneClient | null,
|
|
270
|
+
registration?: SlowLaneClientRegistration,
|
|
266
271
|
): void => {
|
|
267
272
|
defaultSlowLaneClient = client;
|
|
273
|
+
defaultSlowLaneClientRegistration = client
|
|
274
|
+
? (registration ?? {
|
|
275
|
+
clientId: 'untracked',
|
|
276
|
+
kind: supportsCanonicalSurfaceScoring(client) ? 'localLlm' : 'server',
|
|
277
|
+
})
|
|
278
|
+
: null;
|
|
279
|
+
};
|
|
280
|
+
|
|
281
|
+
/**
|
|
282
|
+
* Clear the shared client only when the caller still owns the registration.
|
|
283
|
+
*
|
|
284
|
+
* Multiple editor instances can briefly overlap during a React/editor remount.
|
|
285
|
+
* An older instance must not disconnect the newer instance when its delayed
|
|
286
|
+
* teardown runs after the newer client has registered.
|
|
287
|
+
*/
|
|
288
|
+
export const clearDefaultSlowLaneClient = (client: SlowLaneClient): boolean => {
|
|
289
|
+
if (defaultSlowLaneClient !== client) {
|
|
290
|
+
return false;
|
|
291
|
+
}
|
|
292
|
+
defaultSlowLaneClient = null;
|
|
293
|
+
defaultSlowLaneClientRegistration = null;
|
|
294
|
+
return true;
|
|
295
|
+
};
|
|
296
|
+
|
|
297
|
+
export const getDefaultSlowLaneClientStatus = (): SlowLaneClientStatus => {
|
|
298
|
+
const canonicalScoringSupported = supportsCanonicalSurfaceScoring(defaultSlowLaneClient);
|
|
299
|
+
const isLocalClient = defaultSlowLaneClientRegistration?.kind === 'localLlm';
|
|
300
|
+
return {
|
|
301
|
+
registered: defaultSlowLaneClient !== null,
|
|
302
|
+
clientId: defaultSlowLaneClientRegistration?.clientId ?? null,
|
|
303
|
+
clientKind: defaultSlowLaneClientRegistration?.kind ?? null,
|
|
304
|
+
canonicalScoringSupported,
|
|
305
|
+
localModelReady:
|
|
306
|
+
isLocalClient && typeof defaultSlowLaneClient?.isReady === 'function'
|
|
307
|
+
? defaultSlowLaneClient.isReady()
|
|
308
|
+
: null,
|
|
309
|
+
canonicalSurfaceCount:
|
|
310
|
+
canonicalScoringSupported &&
|
|
311
|
+
typeof defaultSlowLaneClient?.getCanonicalSurfaceCount === 'function'
|
|
312
|
+
? defaultSlowLaneClient.getCanonicalSurfaceCount()
|
|
313
|
+
: null,
|
|
314
|
+
};
|
|
268
315
|
};
|
|
269
316
|
|
|
270
317
|
export const getStoredContextVector = (): Float32Array | null =>
|
|
271
318
|
defaultSlowLaneClient?.getContextVector() ?? null;
|
|
272
319
|
|
|
320
|
+
export const getStoredContextInput = (): string | null =>
|
|
321
|
+
defaultSlowLaneClient?.getContextInput?.() ?? null;
|
|
322
|
+
|
|
273
323
|
export const getStoredLmLogits = (): Record<string, number> | null =>
|
|
274
324
|
defaultSlowLaneClient?.getLmLogits() ?? null;
|
|
325
|
+
|
|
326
|
+
export const getBoundaryLmState = (contextKey: string): BoundaryLmState | null =>
|
|
327
|
+
defaultSlowLaneClient?.getBoundaryLmState?.(contextKey) ?? null;
|
|
328
|
+
|
|
329
|
+
export const getCanonicalSurfaceTokenIds = (surface: string): number[] | null =>
|
|
330
|
+
defaultSlowLaneClient?.getCanonicalSurfaceTokenIds?.(surface) ?? null;
|
|
331
|
+
|
|
332
|
+
export const getSurfaceScore = (contextKey: string, surface: string): SurfaceScore | null =>
|
|
333
|
+
defaultSlowLaneClient?.getSurfaceScore?.(contextKey, surface) ?? null;
|
|
334
|
+
|
|
335
|
+
export const getProgressiveSurfaceEvidence = (
|
|
336
|
+
contextKey: string,
|
|
337
|
+
surface: string,
|
|
338
|
+
): ProgressiveSurfaceEvidence | null =>
|
|
339
|
+
defaultSlowLaneClient?.getProgressiveSurfaceEvidence?.(contextKey, surface) ?? null;
|
|
340
|
+
|
|
341
|
+
export const primeBoundaryLm = (input: BoundaryPrimeRequest): void => {
|
|
342
|
+
defaultSlowLaneClient?.primeBoundaryLm?.(input);
|
|
343
|
+
};
|
|
344
|
+
|
|
345
|
+
export const requestProgressiveSurfaceScores = (input: SurfaceScoreRequest): void => {
|
|
346
|
+
defaultSlowLaneClient?.requestProgressiveSurfaceScores?.(input);
|
|
347
|
+
};
|
|
348
|
+
|
|
349
|
+
export const isCanonicalSurfaceScoringSupported = (): boolean =>
|
|
350
|
+
supportsCanonicalSurfaceScoring(defaultSlowLaneClient);
|
|
351
|
+
|
|
352
|
+
/**
|
|
353
|
+
* How many surfaces the canonical token map currently holds. Goes from zero to
|
|
354
|
+
* its final size once the artifact lands, which is the only point at which
|
|
355
|
+
* `getCanonicalSurfaceTokenIds` starts answering — so callers memoising over
|
|
356
|
+
* that lookup can use this to detect the transition.
|
|
357
|
+
*/
|
|
358
|
+
export const getCanonicalSurfaceCount = (): number =>
|
|
359
|
+
defaultSlowLaneClient?.getCanonicalSurfaceCount?.() ?? 0;
|