@flighthq/textsegment 0.5.0-next.1944.cd6b6e7 → 0.5.1-edge.1047.858b9b6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/contract.d.ts +2 -0
- package/dist/contract.d.ts.map +1 -1
- package/dist/contract.js +2 -0
- package/dist/contract.js.map +1 -1
- package/dist/enableTextSegmentGuards.d.ts +5 -0
- package/dist/enableTextSegmentGuards.d.ts.map +1 -0
- package/dist/enableTextSegmentGuards.js +17 -0
- package/dist/enableTextSegmentGuards.js.map +1 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/textSegment.d.ts +4 -4
- package/dist/textSegment.d.ts.map +1 -1
- package/dist/textSegment.js +6 -6
- package/dist/textSegment.js.map +1 -1
- package/dist/textSegmentBoundary.d.ts +8 -6
- package/dist/textSegmentBoundary.d.ts.map +1 -1
- package/dist/textSegmentBoundary.js +90 -11
- package/dist/textSegmentBoundary.js.map +1 -1
- package/dist/textSegmentGuards.d.ts +4 -0
- package/dist/textSegmentGuards.d.ts.map +1 -0
- package/dist/textSegmentGuards.js +8 -0
- package/dist/textSegmentGuards.js.map +1 -0
- package/dist/textSegmenterBackend.d.ts +13 -2
- package/dist/textSegmenterBackend.d.ts.map +1 -1
- package/dist/textSegmenterBackend.js +44 -16
- package/dist/textSegmenterBackend.js.map +1 -1
- package/package.json +4 -2
- package/src/enableTextSegmentGuards.test.ts +47 -0
- package/src/textSegment.test.ts +42 -7
- package/src/textSegmentBoundary.test.ts +59 -7
- package/src/textSegmentConformance.test.ts +18 -0
- package/src/textSegmentGuards.test.ts +21 -0
- package/src/textSegmenterBackend.test.ts +79 -11
package/README.md
CHANGED
|
@@ -13,5 +13,5 @@ Import the supported application-facing API from `@flighthq/textsegment`. The `@
|
|
|
13
13
|
This package is part of the locked-version Flight SDK graph. Applications may instead install and import `@flighthq/sdk` when package-level tree shaking is sufficient.
|
|
14
14
|
|
|
15
15
|
- [Flight project](https://github.com/flighthq/flight)
|
|
16
|
-
- [Source for @flighthq/textsegment@0.5.
|
|
17
|
-
- [License](https://github.com/flighthq/flight/blob/
|
|
16
|
+
- [Source for @flighthq/textsegment@0.5.1-edge.1047.858b9b6](https://github.com/flighthq/flight/tree/858b9b601de814f6b6940cbb1ffff0916b903268/packages/textsegment)
|
|
17
|
+
- [License](https://github.com/flighthq/flight/blob/858b9b601de814f6b6940cbb1ffff0916b903268/LICENSE.md)
|
package/dist/contract.d.ts
CHANGED
package/dist/contract.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"contract.d.ts","sourceRoot":"","sources":["../src/contract.ts"],"names":[],"mappings":"AAAA,cAAc,eAAe,CAAC;AAC9B,cAAc,uBAAuB,CAAC;AACtC,cAAc,wBAAwB,CAAC"}
|
|
1
|
+
{"version":3,"file":"contract.d.ts","sourceRoot":"","sources":["../src/contract.ts"],"names":[],"mappings":"AAAA,cAAc,2BAA2B,CAAC;AAC1C,cAAc,eAAe,CAAC;AAC9B,cAAc,uBAAuB,CAAC;AACtC,cAAc,qBAAqB,CAAC;AACpC,cAAc,wBAAwB,CAAC"}
|
package/dist/contract.js
CHANGED
package/dist/contract.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"contract.js","sourceRoot":"","sources":["../src/contract.ts"],"names":[],"mappings":"AAAA,cAAc,eAAe,CAAC;AAC9B,cAAc,uBAAuB,CAAC;AACtC,cAAc,wBAAwB,CAAC"}
|
|
1
|
+
{"version":3,"file":"contract.js","sourceRoot":"","sources":["../src/contract.ts"],"names":[],"mappings":"AAAA,cAAc,2BAA2B,CAAC;AAC1C,cAAc,eAAe,CAAC;AAC9B,cAAc,uBAAuB,CAAC;AACtC,cAAc,qBAAqB,CAAC;AACpC,cAAc,wBAAwB,CAAC"}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
/** Uninstalls the guard installed by enableTextSegmentGuards. */
|
|
2
|
+
export declare function disableTextSegmentGuards(): void;
|
|
3
|
+
/** Installs an opt-in warning for use of the bundled provider where Intl.Segmenter is absent. */
|
|
4
|
+
export declare function enableTextSegmentGuards(): void;
|
|
5
|
+
//# sourceMappingURL=enableTextSegmentGuards.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"enableTextSegmentGuards.d.ts","sourceRoot":"","sources":["../src/enableTextSegmentGuards.ts"],"names":[],"mappings":"AAKA,iEAAiE;AACjE,wBAAgB,wBAAwB,IAAI,IAAI,CAE/C;AAED,iGAAiG;AACjG,wBAAgB,uBAAuB,IAAI,IAAI,CAE9C"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { logOnce } from '@flighthq/log/contract';
|
|
2
|
+
import { LogLevel } from '@flighthq/types/contract';
|
|
3
|
+
import { setTextSegmentGuard } from './textSegmentGuards';
|
|
4
|
+
/** Uninstalls the guard installed by enableTextSegmentGuards. */
|
|
5
|
+
export function disableTextSegmentGuards() {
|
|
6
|
+
setTextSegmentGuard(null);
|
|
7
|
+
}
|
|
8
|
+
/** Installs an opt-in warning for use of the bundled provider where Intl.Segmenter is absent. */
|
|
9
|
+
export function enableTextSegmentGuards() {
|
|
10
|
+
setTextSegmentGuard(warnOnMissingIntlSegmenter);
|
|
11
|
+
}
|
|
12
|
+
function warnOnMissingIntlSegmenter() {
|
|
13
|
+
logOnce('textsegment:intl-segmenter-unavailable', LogLevel.Warn, {
|
|
14
|
+
message: 'segmentGraphemes: Intl.Segmenter is unavailable and no replacement provider was supplied, so segmentation returned no results. Install a backend with setTextSegmenterBackend(backend), or pass a HasTextSegmenter host directly to the operation.',
|
|
15
|
+
}, 'textsegment');
|
|
16
|
+
}
|
|
17
|
+
//# sourceMappingURL=enableTextSegmentGuards.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"enableTextSegmentGuards.js","sourceRoot":"","sources":["../src/enableTextSegmentGuards.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AACjD,OAAO,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AAEpD,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAE1D,iEAAiE;AACjE,MAAM,UAAU,wBAAwB;IACtC,mBAAmB,CAAC,IAAI,CAAC,CAAC;AAC5B,CAAC;AAED,iGAAiG;AACjG,MAAM,UAAU,uBAAuB;IACrC,mBAAmB,CAAC,0BAA0B,CAAC,CAAC;AAClD,CAAC;AAED,SAAS,0BAA0B;IACjC,OAAO,CACL,wCAAwC,EACxC,QAAQ,CAAC,IAAI,EACb;QACE,OAAO,EACL,oPAAoP;KACvP,EACD,aAAa,CACd,CAAC;AACJ,CAAC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export { getNextGraphemeBoundary, getNextWordBoundary, getPreviousGraphemeBoundary, getPreviousWordBoundary, getWordRangeAt, segmentGraphemes, segmentSentences, segmentWords, } from './contract';
|
|
1
|
+
export { disableTextSegmentGuards, enableTextSegmentGuards, explainTextSegmenterBackend, getNextGraphemeBoundary, getNextSentenceBoundary, getNextWordBoundary, getPreviousGraphemeBoundary, getPreviousSentenceBoundary, getPreviousWordBoundary, getWordRangeAt, segmentGraphemes, segmentSentences, segmentWords, webTextSegmenterBackend, } from './contract';
|
|
2
2
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,uBAAuB,EACvB,mBAAmB,EACnB,2BAA2B,EAC3B,uBAAuB,EACvB,cAAc,EACd,gBAAgB,EAChB,gBAAgB,EAChB,YAAY,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,wBAAwB,EACxB,uBAAuB,EACvB,2BAA2B,EAC3B,uBAAuB,EACvB,uBAAuB,EACvB,mBAAmB,EACnB,2BAA2B,EAC3B,2BAA2B,EAC3B,uBAAuB,EACvB,cAAc,EACd,gBAAgB,EAChB,gBAAgB,EAChB,YAAY,EACZ,uBAAuB,GACxB,MAAM,YAAY,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export { getNextGraphemeBoundary, getNextWordBoundary, getPreviousGraphemeBoundary, getPreviousWordBoundary, getWordRangeAt, segmentGraphemes, segmentSentences, segmentWords, } from './contract';
|
|
1
|
+
export { disableTextSegmentGuards, enableTextSegmentGuards, explainTextSegmenterBackend, getNextGraphemeBoundary, getNextSentenceBoundary, getNextWordBoundary, getPreviousGraphemeBoundary, getPreviousSentenceBoundary, getPreviousWordBoundary, getWordRangeAt, segmentGraphemes, segmentSentences, segmentWords, webTextSegmenterBackend, } from './contract';
|
|
2
2
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,uBAAuB,EACvB,mBAAmB,EACnB,2BAA2B,EAC3B,uBAAuB,EACvB,cAAc,EACd,gBAAgB,EAChB,gBAAgB,EAChB,YAAY,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,wBAAwB,EACxB,uBAAuB,EACvB,2BAA2B,EAC3B,uBAAuB,EACvB,uBAAuB,EACvB,mBAAmB,EACnB,2BAA2B,EAC3B,2BAA2B,EAC3B,uBAAuB,EACvB,cAAc,EACd,gBAAgB,EAChB,gBAAgB,EAChB,YAAY,EACZ,uBAAuB,GACxB,MAAM,YAAY,CAAC"}
|
package/dist/textSegment.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import type { TextSegment } from '@flighthq/types/contract';
|
|
2
|
-
export declare function segmentGraphemes(text: string, locale?: string): readonly TextSegment[];
|
|
3
|
-
export declare function segmentSentences(text: string, locale?: string): readonly TextSegment[];
|
|
4
|
-
export declare function segmentWords(text: string, locale?: string): readonly TextSegment[];
|
|
1
|
+
import type { HasTextSegmenter, TextSegment } from '@flighthq/types/contract';
|
|
2
|
+
export declare function segmentGraphemes(text: string, locale?: string, host?: HasTextSegmenter): readonly TextSegment[];
|
|
3
|
+
export declare function segmentSentences(text: string, locale?: string, host?: HasTextSegmenter): readonly TextSegment[];
|
|
4
|
+
export declare function segmentWords(text: string, locale?: string, host?: HasTextSegmenter): readonly TextSegment[];
|
|
5
5
|
//# sourceMappingURL=textSegment.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"textSegment.d.ts","sourceRoot":"","sources":["../src/textSegment.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;
|
|
1
|
+
{"version":3,"file":"textSegment.d.ts","sourceRoot":"","sources":["../src/textSegment.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAQ9E,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,gBAAgB,GAAG,SAAS,WAAW,EAAE,CAE/G;AAID,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,gBAAgB,GAAG,SAAS,WAAW,EAAE,CAE/G;AAKD,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,gBAAgB,GAAG,SAAS,WAAW,EAAE,CAE3G"}
|
package/dist/textSegment.js
CHANGED
|
@@ -3,18 +3,18 @@ import { getTextSegmenterBackend } from './textSegmenterBackend';
|
|
|
3
3
|
// user-perceived character: an emoji ZWJ sequence or a base-plus-combining-mark cluster is a single
|
|
4
4
|
// segment, not its constituent code points. `locale` is threaded to the backend for the rare
|
|
5
5
|
// locale-tailored cases. isWordLike is absent on grapheme segments.
|
|
6
|
-
export function segmentGraphemes(text, locale) {
|
|
7
|
-
return getTextSegmenterBackend().segment(text, 'grapheme', locale);
|
|
6
|
+
export function segmentGraphemes(text, locale, host) {
|
|
7
|
+
return getTextSegmenterBackend(host).segment(text, 'grapheme', locale);
|
|
8
8
|
}
|
|
9
9
|
// Enumerates the sentences of `text` (UAX #29) via the active backend, in order and covering the
|
|
10
10
|
// whole string. isWordLike is absent on sentence segments.
|
|
11
|
-
export function segmentSentences(text, locale) {
|
|
12
|
-
return getTextSegmenterBackend().segment(text, 'sentence', locale);
|
|
11
|
+
export function segmentSentences(text, locale, host) {
|
|
12
|
+
return getTextSegmenterBackend(host).segment(text, 'sentence', locale);
|
|
13
13
|
}
|
|
14
14
|
// Enumerates the words of `text` (UAX #29) via the active backend. Word segments carry isWordLike:
|
|
15
15
|
// true for letters/numbers (a real word), false for the punctuation and whitespace runs between
|
|
16
16
|
// them. Filter on isWordLike to keep only words. `locale` is threaded to the backend.
|
|
17
|
-
export function segmentWords(text, locale) {
|
|
18
|
-
return getTextSegmenterBackend().segment(text, 'word', locale);
|
|
17
|
+
export function segmentWords(text, locale, host) {
|
|
18
|
+
return getTextSegmenterBackend(host).segment(text, 'word', locale);
|
|
19
19
|
}
|
|
20
20
|
//# sourceMappingURL=textSegment.js.map
|
package/dist/textSegment.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"textSegment.js","sourceRoot":"","sources":["../src/textSegment.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AAEjE,mGAAmG;AACnG,oGAAoG;AACpG,6FAA6F;AAC7F,oEAAoE;AACpE,MAAM,UAAU,gBAAgB,CAAC,IAAY,EAAE,MAAe;
|
|
1
|
+
{"version":3,"file":"textSegment.js","sourceRoot":"","sources":["../src/textSegment.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AAEjE,mGAAmG;AACnG,oGAAoG;AACpG,6FAA6F;AAC7F,oEAAoE;AACpE,MAAM,UAAU,gBAAgB,CAAC,IAAY,EAAE,MAAe,EAAE,IAAuB;IACrF,OAAO,uBAAuB,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;AACzE,CAAC;AAED,iGAAiG;AACjG,2DAA2D;AAC3D,MAAM,UAAU,gBAAgB,CAAC,IAAY,EAAE,MAAe,EAAE,IAAuB;IACrF,OAAO,uBAAuB,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;AACzE,CAAC;AAED,mGAAmG;AACnG,gGAAgG;AAChG,sFAAsF;AACtF,MAAM,UAAU,YAAY,CAAC,IAAY,EAAE,MAAe,EAAE,IAAuB;IACjF,OAAO,uBAAuB,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AACrE,CAAC"}
|
|
@@ -1,7 +1,9 @@
|
|
|
1
|
-
import type { TextSegmentRange } from '@flighthq/types/contract';
|
|
2
|
-
export declare function getNextGraphemeBoundary(text: string, index: number, locale?: string): number;
|
|
3
|
-
export declare function
|
|
4
|
-
export declare function
|
|
5
|
-
export declare function
|
|
6
|
-
export declare function
|
|
1
|
+
import type { HasTextSegmenter, TextSegmentRange } from '@flighthq/types/contract';
|
|
2
|
+
export declare function getNextGraphemeBoundary(text: string, index: number, locale?: string, host?: HasTextSegmenter): number;
|
|
3
|
+
export declare function getNextSentenceBoundary(text: string, index: number, locale?: string, host?: HasTextSegmenter): number;
|
|
4
|
+
export declare function getNextWordBoundary(text: string, index: number, locale?: string, host?: HasTextSegmenter): number;
|
|
5
|
+
export declare function getPreviousGraphemeBoundary(text: string, index: number, locale?: string, host?: HasTextSegmenter): number;
|
|
6
|
+
export declare function getPreviousSentenceBoundary(text: string, index: number, locale?: string, host?: HasTextSegmenter): number;
|
|
7
|
+
export declare function getPreviousWordBoundary(text: string, index: number, locale?: string, host?: HasTextSegmenter): number;
|
|
8
|
+
export declare function getWordRangeAt(text: string, index: number, locale?: string, host?: HasTextSegmenter): TextSegmentRange | null;
|
|
7
9
|
//# sourceMappingURL=textSegmentBoundary.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"textSegmentBoundary.d.ts","sourceRoot":"","sources":["../src/textSegmentBoundary.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"textSegmentBoundary.d.ts","sourceRoot":"","sources":["../src/textSegmentBoundary.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAuC,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAQxH,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,gBAAgB,GAAG,MAAM,CAErH;AAID,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,gBAAgB,GAAG,MAAM,CAErH;AAKD,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,gBAAgB,GAAG,MAAM,CAEjH;AAKD,wBAAgB,2BAA2B,CACzC,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,EACb,MAAM,CAAC,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,gBAAgB,GACtB,MAAM,CAER;AAID,wBAAgB,2BAA2B,CACzC,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,EACb,MAAM,CAAC,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,gBAAgB,GACtB,MAAM,CAER;AAID,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,gBAAgB,GAAG,MAAM,CAErH;AAMD,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,EACb,MAAM,CAAC,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,gBAAgB,GACtB,gBAAgB,GAAG,IAAI,CAyBzB"}
|
|
@@ -1,39 +1,63 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { getTextSegmenterBackend, webTextSegmenterBackend } from './textSegmenterBackend';
|
|
2
|
+
import { reportTextSegmenterUnavailable } from './textSegmentGuards';
|
|
2
3
|
// Returns the next grapheme-cluster boundary at or after `index` — the offset a caret lands on when
|
|
3
4
|
// stepping right by one user-perceived character, so an emoji or combining sequence is crossed in a
|
|
4
5
|
// single step. `index` is clamped into [0, text.length]; at or past the end it returns text.length.
|
|
5
|
-
export function getNextGraphemeBoundary(text, index, locale) {
|
|
6
|
-
return
|
|
6
|
+
export function getNextGraphemeBoundary(text, index, locale, host) {
|
|
7
|
+
return getNextTextSegmentBoundary(text, index, 'grapheme', locale, host);
|
|
8
|
+
}
|
|
9
|
+
// Returns the next sentence boundary at or after `index`. The bundled Intl path stops its iterator as
|
|
10
|
+
// soon as the answer is known and does not materialize the sentence strings or a segment array.
|
|
11
|
+
export function getNextSentenceBoundary(text, index, locale, host) {
|
|
12
|
+
return getNextTextSegmentBoundary(text, index, 'sentence', locale, host);
|
|
7
13
|
}
|
|
8
14
|
// Returns the next word boundary at or after `index` — the offset a caret lands on when jumping right
|
|
9
15
|
// by one word (Ctrl/Alt-Right). `index` is clamped into [0, text.length]; at or past the end it
|
|
10
16
|
// returns text.length.
|
|
11
|
-
export function getNextWordBoundary(text, index, locale) {
|
|
12
|
-
return
|
|
17
|
+
export function getNextWordBoundary(text, index, locale, host) {
|
|
18
|
+
return getNextTextSegmentBoundary(text, index, 'word', locale, host);
|
|
13
19
|
}
|
|
14
20
|
// Returns the previous grapheme-cluster boundary at or before `index` — the offset a caret lands on
|
|
15
21
|
// when stepping left by one user-perceived character. `index` is clamped into [0, text.length]; at or
|
|
16
22
|
// before the start it returns 0.
|
|
17
|
-
export function getPreviousGraphemeBoundary(text, index, locale) {
|
|
18
|
-
return
|
|
23
|
+
export function getPreviousGraphemeBoundary(text, index, locale, host) {
|
|
24
|
+
return getPreviousTextSegmentBoundary(text, index, 'grapheme', locale, host);
|
|
25
|
+
}
|
|
26
|
+
// Returns the previous sentence boundary at or before `index`. The bundled Intl path walks offsets
|
|
27
|
+
// directly and avoids building TextSegment records; custom backends retain their existing array seam.
|
|
28
|
+
export function getPreviousSentenceBoundary(text, index, locale, host) {
|
|
29
|
+
return getPreviousTextSegmentBoundary(text, index, 'sentence', locale, host);
|
|
19
30
|
}
|
|
20
31
|
// Returns the previous word boundary at or before `index` — the offset a caret lands on when jumping
|
|
21
32
|
// left by one word. `index` is clamped into [0, text.length]; at or before the start it returns 0.
|
|
22
|
-
export function getPreviousWordBoundary(text, index, locale) {
|
|
23
|
-
return
|
|
33
|
+
export function getPreviousWordBoundary(text, index, locale, host) {
|
|
34
|
+
return getPreviousTextSegmentBoundary(text, index, 'word', locale, host);
|
|
24
35
|
}
|
|
25
36
|
// Returns the word-like segment's range covering `index`, for double-click / word-select. `index` is
|
|
26
37
|
// clamped into [0, text.length]; at the very end it resolves against the last character. Returns null
|
|
27
38
|
// when the covered segment is not word-like (whitespace or punctuation) or the text is empty — the
|
|
28
39
|
// caller should then select nothing, mirroring how a double-click on a space selects no word.
|
|
29
|
-
export function getWordRangeAt(text, index, locale) {
|
|
40
|
+
export function getWordRangeAt(text, index, locale, host) {
|
|
30
41
|
if (text.length === 0)
|
|
31
42
|
return null;
|
|
32
43
|
const clamped = clampIndex(index, text.length);
|
|
33
44
|
// At the trailing boundary there is no segment starting at text.length; resolve against the last
|
|
34
45
|
// character so a double-click at the end still selects the final word.
|
|
35
46
|
const lookup = clamped === text.length ? text.length - 1 : clamped;
|
|
36
|
-
const
|
|
47
|
+
const backend = getTextSegmenterBackend(host);
|
|
48
|
+
if (backend === webTextSegmenterBackend) {
|
|
49
|
+
const segmenter = getBoundarySegmenter(locale, 'word');
|
|
50
|
+
if (segmenter === null)
|
|
51
|
+
return null;
|
|
52
|
+
for (const segment of segmenter.segment(text)) {
|
|
53
|
+
const end = segment.index + segment.segment.length;
|
|
54
|
+
if (lookup >= segment.index && lookup < end) {
|
|
55
|
+
return segment.isWordLike === true ? { start: segment.index, end } : null;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
return null;
|
|
59
|
+
}
|
|
60
|
+
const segments = backend.segment(text, 'word', locale);
|
|
37
61
|
for (const segment of segments) {
|
|
38
62
|
if (lookup >= segment.start && lookup < segment.end) {
|
|
39
63
|
return segment.isWordLike === true ? { start: segment.start, end: segment.end } : null;
|
|
@@ -41,6 +65,59 @@ export function getWordRangeAt(text, index, locale) {
|
|
|
41
65
|
}
|
|
42
66
|
return null;
|
|
43
67
|
}
|
|
68
|
+
function getBoundarySegmenter(locale, granularity) {
|
|
69
|
+
if (typeof Intl === 'undefined' || typeof Intl.Segmenter === 'undefined') {
|
|
70
|
+
reportTextSegmenterUnavailable();
|
|
71
|
+
return null;
|
|
72
|
+
}
|
|
73
|
+
const key = `${locale ?? ''}|${granularity}`;
|
|
74
|
+
const existing = boundarySegmenterCache.get(key);
|
|
75
|
+
if (existing !== undefined)
|
|
76
|
+
return existing;
|
|
77
|
+
const segmenter = new Intl.Segmenter(locale, { granularity });
|
|
78
|
+
if (boundarySegmenterCache.size >= BOUNDARY_SEGMENTER_CACHE_CAPACITY) {
|
|
79
|
+
const oldest = boundarySegmenterCache.keys().next().value;
|
|
80
|
+
if (oldest !== undefined)
|
|
81
|
+
boundarySegmenterCache.delete(oldest);
|
|
82
|
+
}
|
|
83
|
+
boundarySegmenterCache.set(key, segmenter);
|
|
84
|
+
return segmenter;
|
|
85
|
+
}
|
|
86
|
+
function getNextTextSegmentBoundary(text, index, granularity, locale, host) {
|
|
87
|
+
const from = clampIndex(index, text.length);
|
|
88
|
+
if (from >= text.length)
|
|
89
|
+
return text.length;
|
|
90
|
+
const backend = getTextSegmenterBackend(host);
|
|
91
|
+
if (backend !== webTextSegmenterBackend) {
|
|
92
|
+
return nextSegmentBoundary(backend.segment(text, granularity, locale), from, text.length);
|
|
93
|
+
}
|
|
94
|
+
const segmenter = getBoundarySegmenter(locale, granularity);
|
|
95
|
+
if (segmenter === null)
|
|
96
|
+
return text.length;
|
|
97
|
+
for (const segment of segmenter.segment(text)) {
|
|
98
|
+
if (segment.index > from)
|
|
99
|
+
return segment.index;
|
|
100
|
+
}
|
|
101
|
+
return text.length;
|
|
102
|
+
}
|
|
103
|
+
function getPreviousTextSegmentBoundary(text, index, granularity, locale, host) {
|
|
104
|
+
const from = clampIndex(index, text.length);
|
|
105
|
+
if (from <= 0)
|
|
106
|
+
return 0;
|
|
107
|
+
const backend = getTextSegmenterBackend(host);
|
|
108
|
+
if (backend !== webTextSegmenterBackend)
|
|
109
|
+
return previousSegmentBoundary(backend.segment(text, granularity, locale), from);
|
|
110
|
+
const segmenter = getBoundarySegmenter(locale, granularity);
|
|
111
|
+
if (segmenter === null)
|
|
112
|
+
return 0;
|
|
113
|
+
let previous = 0;
|
|
114
|
+
for (const segment of segmenter.segment(text)) {
|
|
115
|
+
if (segment.index >= from)
|
|
116
|
+
break;
|
|
117
|
+
previous = segment.index;
|
|
118
|
+
}
|
|
119
|
+
return previous;
|
|
120
|
+
}
|
|
44
121
|
// The boundary offsets are every segment start plus text.length (segments cover the string in order
|
|
45
122
|
// with strictly increasing starts). Clamping keeps out-of-range indices from throwing per house rules.
|
|
46
123
|
function clampIndex(index, length) {
|
|
@@ -73,4 +150,6 @@ function previousSegmentBoundary(segments, index) {
|
|
|
73
150
|
}
|
|
74
151
|
return previous;
|
|
75
152
|
}
|
|
153
|
+
const boundarySegmenterCache = new Map();
|
|
154
|
+
const BOUNDARY_SEGMENTER_CACHE_CAPACITY = 64;
|
|
76
155
|
//# sourceMappingURL=textSegmentBoundary.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"textSegmentBoundary.js","sourceRoot":"","sources":["../src/textSegmentBoundary.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"textSegmentBoundary.js","sourceRoot":"","sources":["../src/textSegmentBoundary.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,uBAAuB,EAAE,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AAC1F,OAAO,EAAE,8BAA8B,EAAE,MAAM,qBAAqB,CAAC;AAErE,oGAAoG;AACpG,oGAAoG;AACpG,oGAAoG;AACpG,MAAM,UAAU,uBAAuB,CAAC,IAAY,EAAE,KAAa,EAAE,MAAe,EAAE,IAAuB;IAC3G,OAAO,0BAA0B,CAAC,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;AAC3E,CAAC;AAED,sGAAsG;AACtG,gGAAgG;AAChG,MAAM,UAAU,uBAAuB,CAAC,IAAY,EAAE,KAAa,EAAE,MAAe,EAAE,IAAuB;IAC3G,OAAO,0BAA0B,CAAC,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;AAC3E,CAAC;AAED,sGAAsG;AACtG,gGAAgG;AAChG,uBAAuB;AACvB,MAAM,UAAU,mBAAmB,CAAC,IAAY,EAAE,KAAa,EAAE,MAAe,EAAE,IAAuB;IACvG,OAAO,0BAA0B,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;AACvE,CAAC;AAED,oGAAoG;AACpG,sGAAsG;AACtG,iCAAiC;AACjC,MAAM,UAAU,2BAA2B,CACzC,IAAY,EACZ,KAAa,EACb,MAAe,EACf,IAAuB;IAEvB,OAAO,8BAA8B,CAAC,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;AAC/E,CAAC;AAED,mGAAmG;AACnG,sGAAsG;AACtG,MAAM,UAAU,2BAA2B,CACzC,IAAY,EACZ,KAAa,EACb,MAAe,EACf,IAAuB;IAEvB,OAAO,8BAA8B,CAAC,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;AAC/E,CAAC;AAED,qGAAqG;AACrG,mGAAmG;AACnG,MAAM,UAAU,uBAAuB,CAAC,IAAY,EAAE,KAAa,EAAE,MAAe,EAAE,IAAuB;IAC3G,OAAO,8BAA8B,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;AAC3E,CAAC;AAED,qGAAqG;AACrG,sGAAsG;AACtG,mGAAmG;AACnG,8FAA8F;AAC9F,MAAM,UAAU,cAAc,CAC5B,IAAY,EACZ,KAAa,EACb,MAAe,EACf,IAAuB;IAEvB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACnC,MAAM,OAAO,GAAG,UAAU,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IAC/C,iGAAiG;IACjG,uEAAuE;IACvE,MAAM,MAAM,GAAG,OAAO,KAAK,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;IACnE,MAAM,OAAO,GAAG,uBAAuB,CAAC,IAAI,CAAC,CAAC;IAC9C,IAAI,OAAO,KAAK,uBAAuB,EAAE,CAAC;QACxC,MAAM,SAAS,GAAG,oBAAoB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACvD,IAAI,SAAS,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;QACpC,KAAK,MAAM,OAAO,IAAI,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YAC9C,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC;YACnD,IAAI,MAAM,IAAI,OAAO,CAAC,KAAK,IAAI,MAAM,GAAG,GAAG,EAAE,CAAC;gBAC5C,OAAO,OAAO,CAAC,UAAU,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;YAC5E,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IACvD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,IAAI,MAAM,IAAI,OAAO,CAAC,KAAK,IAAI,MAAM,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;YACpD,OAAO,OAAO,CAAC,UAAU,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QACzF,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,oBAAoB,CAAC,MAA0B,EAAE,WAAmC;IAC3F,IAAI,OAAO,IAAI,KAAK,WAAW,IAAI,OAAO,IAAI,CAAC,SAAS,KAAK,WAAW,EAAE,CAAC;QACzE,8BAA8B,EAAE,CAAC;QACjC,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,GAAG,GAAG,GAAG,MAAM,IAAI,EAAE,IAAI,WAAW,EAAE,CAAC;IAC7C,MAAM,QAAQ,GAAG,sBAAsB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACjD,IAAI,QAAQ,KAAK,SAAS;QAAE,OAAO,QAAQ,CAAC;IAC5C,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;IAC9D,IAAI,sBAAsB,CAAC,IAAI,IAAI,iCAAiC,EAAE,CAAC;QACrE,MAAM,MAAM,GAAG,sBAAsB,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;QAC1D,IAAI,MAAM,KAAK,SAAS;YAAE,sBAAsB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAClE,CAAC;IACD,sBAAsB,CAAC,GAAG,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;IAC3C,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,0BAA0B,CACjC,IAAY,EACZ,KAAa,EACb,WAAmC,EACnC,MAA0B,EAC1B,IAAkC;IAElC,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IAC5C,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC,MAAM,CAAC;IAC5C,MAAM,OAAO,GAAG,uBAAuB,CAAC,IAAI,CAAC,CAAC;IAC9C,IAAI,OAAO,KAAK,uBAAuB,EAAE,CAAC;QACxC,OAAO,mBAAmB,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,WAAW,EAAE,MAAM,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IAC5F,CAAC;IACD,MAAM,SAAS,GAAG,oBAAoB,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IAC5D,IAAI,SAAS,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC,MAAM,CAAC;IAC3C,KAAK,MAAM,OAAO,IAAI,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9C,IAAI,OAAO,CAAC,KAAK,GAAG,IAAI;YAAE,OAAO,OAAO,CAAC,KAAK,CAAC;IACjD,CAAC;IACD,OAAO,IAAI,CAAC,MAAM,CAAC;AACrB,CAAC;AAED,SAAS,8BAA8B,CACrC,IAAY,EACZ,KAAa,EACb,WAAmC,EACnC,MAA0B,EAC1B,IAAkC;IAElC,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IAC5C,IAAI,IAAI,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IACxB,MAAM,OAAO,GAAG,uBAAuB,CAAC,IAAI,CAAC,CAAC;IAC9C,IAAI,OAAO,KAAK,uBAAuB;QACrC,OAAO,uBAAuB,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,WAAW,EAAE,MAAM,CAAC,EAAE,IAAI,CAAC,CAAC;IACnF,MAAM,SAAS,GAAG,oBAAoB,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IAC5D,IAAI,SAAS,KAAK,IAAI;QAAE,OAAO,CAAC,CAAC;IACjC,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,KAAK,MAAM,OAAO,IAAI,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9C,IAAI,OAAO,CAAC,KAAK,IAAI,IAAI;YAAE,MAAM;QACjC,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC;IAC3B,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,oGAAoG;AACpG,uGAAuG;AACvG,SAAS,UAAU,CAAC,KAAa,EAAE,MAAc;IAC/C,IAAI,KAAK,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IACxB,IAAI,KAAK,GAAG,MAAM;QAAE,OAAO,MAAM,CAAC;IAClC,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,mBAAmB,CAAC,QAAgC,EAAE,KAAa,EAAE,MAAc;IAC1F,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACvC,IAAI,IAAI,IAAI,MAAM;QAAE,OAAO,MAAM,CAAC;IAClC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,IAAI,OAAO,CAAC,KAAK,GAAG,IAAI;YAAE,OAAO,OAAO,CAAC,KAAK,CAAC;IACjD,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,uBAAuB,CAAC,QAAgC,EAAE,KAAa;IAC9E,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IAC7E,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACvC,IAAI,IAAI,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IACxB,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,IAAI,OAAO,CAAC,KAAK,IAAI,IAAI;YAAE,MAAM;QACjC,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC;IAC3B,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,sBAAsB,GAAG,IAAI,GAAG,EAA0B,CAAC;AACjE,MAAM,iCAAiC,GAAG,EAAE,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"textSegmentGuards.d.ts","sourceRoot":"","sources":["../src/textSegmentGuards.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAEjE,wBAAgB,8BAA8B,IAAI,IAAI,CAErD;AAED,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,gBAAgB,GAAG,IAAI,GAAG,IAAI,CAExE"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"textSegmentGuards.js","sourceRoot":"","sources":["../src/textSegmentGuards.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,8BAA8B;IAC5C,MAAM,EAAE,EAAE,CAAC;AACb,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,KAA8B;IAChE,MAAM,GAAG,KAAK,CAAC;AACjB,CAAC;AAED,IAAI,MAAM,GAA4B,IAAI,CAAC"}
|
|
@@ -1,5 +1,16 @@
|
|
|
1
|
-
import type { TextSegmenterBackend } from '@flighthq/types/contract';
|
|
1
|
+
import type { EntityConstruction, HasTextSegmenter, TextSegmenterBackend, TextSegmenterBackendExplanation } from '@flighthq/types/contract';
|
|
2
2
|
export declare function createWebTextSegmenterBackend(): TextSegmenterBackend;
|
|
3
|
-
export declare
|
|
3
|
+
export declare const webTextSegmenterBackend: TextSegmenterBackend;
|
|
4
|
+
/** Describes which provider an operation would use and whether Intl.Segmenter is present. */
|
|
5
|
+
export declare function explainTextSegmenterBackend(host?: HasTextSegmenter): TextSegmenterBackendExplanation;
|
|
6
|
+
export declare function getTextSegmenterBackend(host?: HasTextSegmenter): TextSegmenterBackend;
|
|
7
|
+
export declare function initializeWebTextSegmenterBackend(out: EntityConstruction<TextSegmenterBackend>): void;
|
|
8
|
+
/**
|
|
9
|
+
* Installs the backend used by calls that omit an explicit host; pass null to restore the stable
|
|
10
|
+
* bundled web default.
|
|
11
|
+
*
|
|
12
|
+
* @deprecated Pass a HasTextSegmenter to the segmentation or boundary operation. Retained for
|
|
13
|
+
* source compatibility until the legacy global path is removed.
|
|
14
|
+
*/
|
|
4
15
|
export declare function setTextSegmenterBackend(backend: TextSegmenterBackend | null): void;
|
|
5
16
|
//# sourceMappingURL=textSegmenterBackend.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"textSegmenterBackend.d.ts","sourceRoot":"","sources":["../src/textSegmenterBackend.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"textSegmenterBackend.d.ts","sourceRoot":"","sources":["../src/textSegmenterBackend.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,kBAAkB,EAClB,gBAAgB,EAGhB,oBAAoB,EACpB,+BAA+B,EAChC,MAAM,0BAA0B,CAAC;AAUlC,wBAAgB,6BAA6B,IAAI,oBAAoB,CAIpE;AAID,eAAO,MAAM,uBAAuB,EAAE,oBAAsD,CAAC;AAE7F,6FAA6F;AAC7F,wBAAgB,2BAA2B,CAAC,IAAI,CAAC,EAAE,gBAAgB,GAAG,+BAA+B,CASpG;AAKD,wBAAgB,uBAAuB,CAAC,IAAI,CAAC,EAAE,gBAAgB,GAAG,oBAAoB,CAErF;AAED,wBAAgB,iCAAiC,CAAC,GAAG,EAAE,kBAAkB,CAAC,oBAAoB,CAAC,GAAG,IAAI,CAErG;AAED;;;;;;GAMG;AACH,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,oBAAoB,GAAG,IAAI,GAAG,IAAI,CAElF"}
|
|
@@ -1,23 +1,46 @@
|
|
|
1
|
+
import { allocateEntity, finishEntity } from '@flighthq/entity/contract';
|
|
2
|
+
import { reportTextSegmenterUnavailable } from './textSegmentGuards';
|
|
1
3
|
// Builds the default web backend: a wrapper over the browser-native Intl.Segmenter. It ships no
|
|
2
4
|
// Unicode tables — the engine already carries them — so the common path costs nothing in bundle
|
|
3
5
|
// weight. Intl.Segmenter instances are cached by (locale, granularity) because constructing one is
|
|
4
6
|
// expensive relative to a single segment() call. Where Intl.Segmenter is absent (an old or headless
|
|
5
|
-
// engine), segment() returns [] rather than throwing;
|
|
6
|
-
//
|
|
7
|
+
// engine), segment() returns [] rather than throwing; compose a from-scratch UAX #29 backend into
|
|
8
|
+
// the host for those environments.
|
|
7
9
|
export function createWebTextSegmenterBackend() {
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
//
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
10
|
+
const out = allocateEntity();
|
|
11
|
+
initializeWebTextSegmenterBackend(out);
|
|
12
|
+
return finishEntity(out);
|
|
13
|
+
}
|
|
14
|
+
// Stable bundled web provider. Hosts can import this directly when composing their explicit
|
|
15
|
+
// capability object; callers that omit a host receive it as the final fallback.
|
|
16
|
+
export const webTextSegmenterBackend = createWebTextSegmenterBackend();
|
|
17
|
+
/** Describes which provider an operation would use and whether Intl.Segmenter is present. */
|
|
18
|
+
export function explainTextSegmenterBackend(host) {
|
|
19
|
+
const backend = getTextSegmenterBackend(host);
|
|
20
|
+
const web = backend === webTextSegmenterBackend;
|
|
21
|
+
const intlSegmenterAvailable = hasIntlSegmenter();
|
|
22
|
+
return {
|
|
23
|
+
available: !web || intlSegmenterAvailable,
|
|
24
|
+
backend: web ? 'web-intl' : 'custom',
|
|
25
|
+
intlSegmenterAvailable,
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
// Returns the explicit host's provider when supplied, then the legacy installed backend, and
|
|
29
|
+
// finally the bundled web provider. The explicit dependency always wins, so independent callers
|
|
30
|
+
// can interleave different hosts without mutating shared capability state.
|
|
31
|
+
export function getTextSegmenterBackend(host) {
|
|
32
|
+
return host?.text.segmenter ?? _backend ?? webTextSegmenterBackend;
|
|
33
|
+
}
|
|
34
|
+
export function initializeWebTextSegmenterBackend(out) {
|
|
35
|
+
out.segment = segmentWithIntlSegmenter;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Installs the backend used by calls that omit an explicit host; pass null to restore the stable
|
|
39
|
+
* bundled web default.
|
|
40
|
+
*
|
|
41
|
+
* @deprecated Pass a HasTextSegmenter to the segmentation or boundary operation. Retained for
|
|
42
|
+
* source compatibility until the legacy global path is removed.
|
|
43
|
+
*/
|
|
21
44
|
export function setTextSegmenterBackend(backend) {
|
|
22
45
|
_backend = backend;
|
|
23
46
|
}
|
|
@@ -28,8 +51,10 @@ let _backend = null;
|
|
|
28
51
|
const _segmenterCache = new Map();
|
|
29
52
|
const _segmenterCacheCapacity = 64;
|
|
30
53
|
function getCachedSegmenter(locale, granularity) {
|
|
31
|
-
if (
|
|
54
|
+
if (!hasIntlSegmenter()) {
|
|
55
|
+
reportTextSegmenterUnavailable();
|
|
32
56
|
return null;
|
|
57
|
+
}
|
|
33
58
|
const key = `${locale ?? ''}|${granularity}`;
|
|
34
59
|
const existing = _segmenterCache.get(key);
|
|
35
60
|
if (existing !== undefined)
|
|
@@ -43,6 +68,9 @@ function getCachedSegmenter(locale, granularity) {
|
|
|
43
68
|
_segmenterCache.set(key, built);
|
|
44
69
|
return built;
|
|
45
70
|
}
|
|
71
|
+
function hasIntlSegmenter() {
|
|
72
|
+
return typeof Intl !== 'undefined' && typeof Intl.Segmenter !== 'undefined';
|
|
73
|
+
}
|
|
46
74
|
function segmentWithIntlSegmenter(text, granularity, locale) {
|
|
47
75
|
const segmenter = getCachedSegmenter(locale, granularity);
|
|
48
76
|
if (segmenter === null)
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"textSegmenterBackend.js","sourceRoot":"","sources":["../src/textSegmenterBackend.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"textSegmenterBackend.js","sourceRoot":"","sources":["../src/textSegmenterBackend.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAUzE,OAAO,EAAE,8BAA8B,EAAE,MAAM,qBAAqB,CAAC;AAErE,gGAAgG;AAChG,gGAAgG;AAChG,mGAAmG;AACnG,oGAAoG;AACpG,kGAAkG;AAClG,mCAAmC;AACnC,MAAM,UAAU,6BAA6B;IAC3C,MAAM,GAAG,GAAG,cAAc,EAAwB,CAAC;IACnD,iCAAiC,CAAC,GAAG,CAAC,CAAC;IACvC,OAAO,YAAY,CAAC,GAAG,CAAC,CAAC;AAC3B,CAAC;AAED,4FAA4F;AAC5F,gFAAgF;AAChF,MAAM,CAAC,MAAM,uBAAuB,GAAyB,6BAA6B,EAAE,CAAC;AAE7F,6FAA6F;AAC7F,MAAM,UAAU,2BAA2B,CAAC,IAAuB;IACjE,MAAM,OAAO,GAAG,uBAAuB,CAAC,IAAI,CAAC,CAAC;IAC9C,MAAM,GAAG,GAAG,OAAO,KAAK,uBAAuB,CAAC;IAChD,MAAM,sBAAsB,GAAG,gBAAgB,EAAE,CAAC;IAClD,OAAO;QACL,SAAS,EAAE,CAAC,GAAG,IAAI,sBAAsB;QACzC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ;QACpC,sBAAsB;KACvB,CAAC;AACJ,CAAC;AAED,6FAA6F;AAC7F,gGAAgG;AAChG,2EAA2E;AAC3E,MAAM,UAAU,uBAAuB,CAAC,IAAuB;IAC7D,OAAO,IAAI,EAAE,IAAI,CAAC,SAAS,IAAI,QAAQ,IAAI,uBAAuB,CAAC;AACrE,CAAC;AAED,MAAM,UAAU,iCAAiC,CAAC,GAA6C;IAC7F,GAAG,CAAC,OAAO,GAAG,wBAAwB,CAAC;AACzC,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,uBAAuB,CAAC,OAAoC;IAC1E,QAAQ,GAAG,OAAO,CAAC;AACrB,CAAC;AAED,IAAI,QAAQ,GAAgC,IAAI,CAAC;AAEjD,qGAAqG;AACrG,oGAAoG;AACpG,mDAAmD;AACnD,MAAM,eAAe,GAAG,IAAI,GAAG,EAA0B,CAAC;AAC1D,MAAM,uBAAuB,GAAG,EAAE,CAAC;AAEnC,SAAS,kBAAkB,CAAC,MAA0B,EAAE,WAAmC;IACzF,IAAI,CAAC,gBAAgB,EAAE,EAAE,CAAC;QACxB,8BAA8B,EAAE,CAAC;QACjC,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,GAAG,GAAG,GAAG,MAAM,IAAI,EAAE,IAAI,WAAW,EAAE,CAAC;IAC7C,MAAM,QAAQ,GAAG,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC1C,IAAI,QAAQ,KAAK,SAAS;QAAE,OAAO,QAAQ,CAAC;IAE5C,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;IAC1D,IAAI,eAAe,CAAC,IAAI,IAAI,uBAAuB,EAAE,CAAC;QACpD,MAAM,MAAM,GAAG,eAAe,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;QACnD,IAAI,MAAM,KAAK,SAAS;YAAE,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC3D,CAAC;IACD,eAAe,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IAChC,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,gBAAgB;IACvB,OAAO,OAAO,IAAI,KAAK,WAAW,IAAI,OAAO,IAAI,CAAC,SAAS,KAAK,WAAW,CAAC;AAC9E,CAAC;AAED,SAAS,wBAAwB,CAC/B,IAAY,EACZ,WAAmC,EACnC,MAAe;IAEf,MAAM,SAAS,GAAG,kBAAkB,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IAC1D,IAAI,SAAS,KAAK,IAAI;QAAE,OAAO,EAAE,CAAC;IAElC,MAAM,GAAG,GAAkB,EAAE,CAAC;IAC9B,MAAM,iBAAiB,GAAG,WAAW,KAAK,MAAM,CAAC;IACjD,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QAC3C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACzB,MAAM,MAAM,GAAgB,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC;QAC5F,4EAA4E;QAC5E,IAAI,iBAAiB;YAAE,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,KAAK,CAAC;QACpE,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACnB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@flighthq/textsegment",
|
|
3
|
-
"version": "0.5.
|
|
3
|
+
"version": "0.5.1-edge.1047.858b9b6",
|
|
4
4
|
"author": "Joshua Granick and other contributors",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -38,7 +38,9 @@
|
|
|
38
38
|
"clean:dist": "tsx ../../scripts/clean-package-dist.ts"
|
|
39
39
|
},
|
|
40
40
|
"dependencies": {
|
|
41
|
-
"@flighthq/
|
|
41
|
+
"@flighthq/entity": "0.5.1-edge.1047.858b9b6",
|
|
42
|
+
"@flighthq/log": "0.5.1-edge.1047.858b9b6",
|
|
43
|
+
"@flighthq/types": "0.5.1-edge.1047.858b9b6"
|
|
42
44
|
},
|
|
43
45
|
"devDependencies": {
|
|
44
46
|
"typescript": "^5.3.0"
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { clearLogOnceKeys, setLogSink } from '@flighthq/log/contract';
|
|
2
|
+
import type { LogEntry } from '@flighthq/types/contract';
|
|
3
|
+
import { vi } from 'vitest';
|
|
4
|
+
|
|
5
|
+
import { disableTextSegmentGuards, enableTextSegmentGuards } from './enableTextSegmentGuards';
|
|
6
|
+
import { segmentGraphemes } from './textSegment';
|
|
7
|
+
|
|
8
|
+
let entries: LogEntry[];
|
|
9
|
+
|
|
10
|
+
beforeEach(() => {
|
|
11
|
+
clearLogOnceKeys();
|
|
12
|
+
entries = [];
|
|
13
|
+
setLogSink((entry) => entries.push(entry));
|
|
14
|
+
});
|
|
15
|
+
|
|
16
|
+
afterEach(() => {
|
|
17
|
+
disableTextSegmentGuards();
|
|
18
|
+
setLogSink(null);
|
|
19
|
+
vi.unstubAllGlobals();
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
describe('disableTextSegmentGuards', () => {
|
|
23
|
+
it('restores silent missing-engine recovery', () => {
|
|
24
|
+
enableTextSegmentGuards();
|
|
25
|
+
disableTextSegmentGuards();
|
|
26
|
+
vi.stubGlobal('Intl', { Segmenter: undefined });
|
|
27
|
+
segmentGraphemes('text');
|
|
28
|
+
expect(entries).toEqual([]);
|
|
29
|
+
});
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
describe('enableTextSegmentGuards', () => {
|
|
33
|
+
it('warns once and names the backend fixing call when Intl.Segmenter is absent', () => {
|
|
34
|
+
enableTextSegmentGuards();
|
|
35
|
+
vi.stubGlobal('Intl', { Segmenter: undefined });
|
|
36
|
+
segmentGraphemes('first');
|
|
37
|
+
segmentGraphemes('second');
|
|
38
|
+
expect(entries).toHaveLength(1);
|
|
39
|
+
expect(String((entries[0].data as { message?: unknown }).message)).toContain('setTextSegmenterBackend');
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
it('stays silent when the bundled Intl provider is available', () => {
|
|
43
|
+
enableTextSegmentGuards();
|
|
44
|
+
segmentGraphemes('text');
|
|
45
|
+
expect(entries).toEqual([]);
|
|
46
|
+
});
|
|
47
|
+
});
|
package/src/textSegment.test.ts
CHANGED
|
@@ -1,4 +1,11 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { allocateEntity, finishEntity } from '@flighthq/entity/contract';
|
|
2
|
+
import type { EntityRuntimeKey } from '@flighthq/types/contract';
|
|
3
|
+
import type {
|
|
4
|
+
HasTextSegmenter,
|
|
5
|
+
TextSegment,
|
|
6
|
+
TextSegmentGranularity,
|
|
7
|
+
TextSegmenterBackend,
|
|
8
|
+
} from '@flighthq/types/contract';
|
|
2
9
|
|
|
3
10
|
import { segmentGraphemes, segmentSentences, segmentWords } from './textSegment';
|
|
4
11
|
import { setTextSegmenterBackend } from './textSegmenterBackend';
|
|
@@ -6,6 +13,25 @@ import { setTextSegmenterBackend } from './textSegmenterBackend';
|
|
|
6
13
|
afterEach(() => setTextSegmenterBackend(null));
|
|
7
14
|
|
|
8
15
|
describe('segmentGraphemes', () => {
|
|
16
|
+
it('accepts an optional explicit segmenter host without breaking the existing signature', () => {
|
|
17
|
+
expectTypeOf(segmentGraphemes).toEqualTypeOf<
|
|
18
|
+
(text: string, locale?: string, host?: HasTextSegmenter) => readonly TextSegment[]
|
|
19
|
+
>();
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
it('gives an explicit host precedence over the legacy installed backend', () => {
|
|
23
|
+
setTextSegmenterBackend(taggingBackend('legacy'));
|
|
24
|
+
expect(segmentGraphemes('hi', undefined, segmenterHost(taggingBackend('explicit')))[0].text).toBe('explicit');
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
it('isolates callers that interleave different explicit hosts', () => {
|
|
28
|
+
const first = segmenterHost(taggingBackend('first'));
|
|
29
|
+
const second = segmenterHost(taggingBackend('second'));
|
|
30
|
+
expect(segmentGraphemes('hi', undefined, first)[0].text).toBe('first');
|
|
31
|
+
expect(segmentGraphemes('hi', undefined, second)[0].text).toBe('second');
|
|
32
|
+
expect(segmentGraphemes('hi', undefined, first)[0].text).toBe('first');
|
|
33
|
+
});
|
|
34
|
+
|
|
9
35
|
it('treats a ZWJ family emoji as one grapheme', () => {
|
|
10
36
|
const segments = segmentGraphemes('a👨👩👧b');
|
|
11
37
|
expect(segments.map((s) => s.text)).toEqual(['a', '👨👩👧', 'b']);
|
|
@@ -21,18 +47,27 @@ describe('segmentGraphemes', () => {
|
|
|
21
47
|
|
|
22
48
|
it('threads the locale argument to the active backend', () => {
|
|
23
49
|
let seenLocale: string | undefined = 'unset';
|
|
24
|
-
const fake
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
},
|
|
50
|
+
const fake = allocateEntity<TextSegmenterBackend>();
|
|
51
|
+
fake.segment = (text: string, _granularity: TextSegmentGranularity, locale?: string): readonly TextSegment[] => {
|
|
52
|
+
seenLocale = locale;
|
|
53
|
+
return [{ start: 0, end: text.length, text }];
|
|
29
54
|
};
|
|
30
|
-
setTextSegmenterBackend(fake);
|
|
55
|
+
setTextSegmenterBackend(finishEntity(fake));
|
|
31
56
|
segmentGraphemes('hi', 'de-DE');
|
|
32
57
|
expect(seenLocale).toBe('de-DE');
|
|
33
58
|
});
|
|
34
59
|
});
|
|
35
60
|
|
|
61
|
+
function segmenterHost(segmenter: TextSegmenterBackend): HasTextSegmenter {
|
|
62
|
+
return { text: { segmenter } };
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function taggingBackend(tag: string): TextSegmenterBackend {
|
|
66
|
+
const out = allocateEntity<TextSegmenterBackend>();
|
|
67
|
+
out.segment = () => [{ start: 0, end: tag.length, text: tag }];
|
|
68
|
+
return finishEntity(out);
|
|
69
|
+
}
|
|
70
|
+
|
|
36
71
|
describe('segmentSentences', () => {
|
|
37
72
|
it('splits into sentences', () => {
|
|
38
73
|
const segments = segmentSentences('Hi. Bye.');
|
|
@@ -1,11 +1,18 @@
|
|
|
1
|
+
import { allocateEntity, finishEntity } from '@flighthq/entity/contract';
|
|
2
|
+
import type { EntityRuntimeKey } from '@flighthq/types/contract';
|
|
3
|
+
import type { HasTextSegmenter, TextSegmenterBackend } from '@flighthq/types/contract';
|
|
4
|
+
import { vi } from 'vitest';
|
|
5
|
+
|
|
1
6
|
import {
|
|
2
7
|
getNextGraphemeBoundary,
|
|
8
|
+
getNextSentenceBoundary,
|
|
3
9
|
getNextWordBoundary,
|
|
4
10
|
getPreviousGraphemeBoundary,
|
|
11
|
+
getPreviousSentenceBoundary,
|
|
5
12
|
getPreviousWordBoundary,
|
|
6
13
|
getWordRangeAt,
|
|
7
14
|
} from './textSegmentBoundary';
|
|
8
|
-
import { setTextSegmenterBackend } from './textSegmenterBackend';
|
|
15
|
+
import { setTextSegmenterBackend, webTextSegmenterBackend } from './textSegmenterBackend';
|
|
9
16
|
|
|
10
17
|
afterEach(() => setTextSegmenterBackend(null));
|
|
11
18
|
|
|
@@ -31,6 +38,23 @@ describe('getNextGraphemeBoundary', () => {
|
|
|
31
38
|
});
|
|
32
39
|
});
|
|
33
40
|
|
|
41
|
+
describe('getNextSentenceBoundary', () => {
|
|
42
|
+
it('advances to the next sentence start', () => {
|
|
43
|
+
expect(getNextSentenceBoundary('First. Second!', 0)).toBe(7);
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
it('clamps at text.length', () => {
|
|
47
|
+
expect(getNextSentenceBoundary('First.', 99)).toBe(6);
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
it('stops the bundled iterator without materializing backend TextSegment records', () => {
|
|
51
|
+
const segment = vi.spyOn(webTextSegmenterBackend, 'segment');
|
|
52
|
+
expect(getNextSentenceBoundary('First. Second!', 0)).toBe(7);
|
|
53
|
+
expect(segment).not.toHaveBeenCalled();
|
|
54
|
+
segment.mockRestore();
|
|
55
|
+
});
|
|
56
|
+
});
|
|
57
|
+
|
|
34
58
|
describe('getNextWordBoundary', () => {
|
|
35
59
|
it('jumps to the end of the current word', () => {
|
|
36
60
|
expect(getNextWordBoundary('foo bar', 0)).toBe(3);
|
|
@@ -57,6 +81,16 @@ describe('getPreviousGraphemeBoundary', () => {
|
|
|
57
81
|
});
|
|
58
82
|
});
|
|
59
83
|
|
|
84
|
+
describe('getPreviousSentenceBoundary', () => {
|
|
85
|
+
it('steps back to the current sentence start', () => {
|
|
86
|
+
expect(getPreviousSentenceBoundary('First. Second!', 14)).toBe(7);
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
it('clamps at zero', () => {
|
|
90
|
+
expect(getPreviousSentenceBoundary('First.', -1)).toBe(0);
|
|
91
|
+
});
|
|
92
|
+
});
|
|
93
|
+
|
|
60
94
|
describe('getPreviousWordBoundary', () => {
|
|
61
95
|
it('jumps to the start of the current word', () => {
|
|
62
96
|
expect(getPreviousWordBoundary('foo bar', 7)).toBe(4);
|
|
@@ -68,6 +102,20 @@ describe('getPreviousWordBoundary', () => {
|
|
|
68
102
|
});
|
|
69
103
|
|
|
70
104
|
describe('getWordRangeAt', () => {
|
|
105
|
+
it('threads an explicit host through the boundary helper', () => {
|
|
106
|
+
const explicit = allocateEntity<TextSegmenterBackend>();
|
|
107
|
+
explicit.segment = (text: string) => [{ start: 0, end: text.length, text, isWordLike: true }];
|
|
108
|
+
const host: HasTextSegmenter = { text: { segmenter: finishEntity(explicit) } };
|
|
109
|
+
setTextSegmenterBackend(
|
|
110
|
+
(() => {
|
|
111
|
+
const out = allocateEntity<TextSegmenterBackend>();
|
|
112
|
+
out.segment = () => [];
|
|
113
|
+
return finishEntity(out);
|
|
114
|
+
})(),
|
|
115
|
+
);
|
|
116
|
+
expect(getWordRangeAt('word', 1, undefined, host)).toEqual({ start: 0, end: 4 });
|
|
117
|
+
});
|
|
118
|
+
|
|
71
119
|
it('returns the word range under an index', () => {
|
|
72
120
|
expect(getWordRangeAt('foo bar', 1)).toEqual({ start: 0, end: 3 });
|
|
73
121
|
});
|
|
@@ -86,12 +134,16 @@ describe('getWordRangeAt', () => {
|
|
|
86
134
|
|
|
87
135
|
it('threads the locale to the active backend', () => {
|
|
88
136
|
let seenLocale: string | undefined = 'unset';
|
|
89
|
-
setTextSegmenterBackend(
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
137
|
+
setTextSegmenterBackend(
|
|
138
|
+
(() => {
|
|
139
|
+
const out = allocateEntity<TextSegmenterBackend>();
|
|
140
|
+
out.segment = (text: string, _granularity: string, locale: string) => {
|
|
141
|
+
seenLocale = locale;
|
|
142
|
+
return [{ start: 0, end: text.length, text, isWordLike: true }];
|
|
143
|
+
};
|
|
144
|
+
return finishEntity(out);
|
|
145
|
+
})(),
|
|
146
|
+
);
|
|
95
147
|
getWordRangeAt('word', 1, 'ja-JP');
|
|
96
148
|
expect(seenLocale).toBe('ja-JP');
|
|
97
149
|
});
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { describe, expect, it } from 'vitest';
|
|
2
|
+
|
|
3
|
+
import { segmentGraphemes } from './textSegment';
|
|
4
|
+
|
|
5
|
+
const graphemeFixtures: ReadonlyArray<{ clusters: string[]; text: string }> = [
|
|
6
|
+
{ text: '🇺🇸🇨🇦', clusters: ['🇺🇸', '🇨🇦'] },
|
|
7
|
+
{ text: '👨👩👧', clusters: ['👨👩👧'] },
|
|
8
|
+
{ text: '\r\n', clusters: ['\r\n'] },
|
|
9
|
+
{ text: '각', clusters: ['각'] },
|
|
10
|
+
];
|
|
11
|
+
|
|
12
|
+
describe('text segment conformance fixtures', () => {
|
|
13
|
+
it('covers regional indicators, ZWJ emoji, CRLF, and Hangul jamo', () => {
|
|
14
|
+
for (const fixture of graphemeFixtures) {
|
|
15
|
+
expect(segmentGraphemes(fixture.text).map((segment) => segment.text)).toEqual(fixture.clusters);
|
|
16
|
+
}
|
|
17
|
+
});
|
|
18
|
+
});
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { reportTextSegmenterUnavailable, setTextSegmentGuard } from './textSegmentGuards';
|
|
2
|
+
|
|
3
|
+
afterEach(() => setTextSegmentGuard(null));
|
|
4
|
+
|
|
5
|
+
describe('reportTextSegmenterUnavailable', () => {
|
|
6
|
+
it('is silent when no guard is installed', () => {
|
|
7
|
+
expect(() => reportTextSegmenterUnavailable()).not.toThrow();
|
|
8
|
+
});
|
|
9
|
+
});
|
|
10
|
+
|
|
11
|
+
describe('setTextSegmentGuard', () => {
|
|
12
|
+
it('replaces the active callback', () => {
|
|
13
|
+
let first = 0;
|
|
14
|
+
let second = 0;
|
|
15
|
+
setTextSegmentGuard(() => first++);
|
|
16
|
+
setTextSegmentGuard(() => second++);
|
|
17
|
+
reportTextSegmenterUnavailable();
|
|
18
|
+
expect(first).toBe(0);
|
|
19
|
+
expect(second).toBe(1);
|
|
20
|
+
});
|
|
21
|
+
});
|
|
@@ -1,9 +1,20 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { allocateEntity, finishEntity } from '@flighthq/entity/contract';
|
|
2
|
+
import type { EntityRuntimeKey } from '@flighthq/types/contract';
|
|
3
|
+
import type {
|
|
4
|
+
HasTextSegmenter,
|
|
5
|
+
TextSegment,
|
|
6
|
+
TextSegmentGranularity,
|
|
7
|
+
TextSegmenterBackend,
|
|
8
|
+
} from '@flighthq/types/contract';
|
|
9
|
+
import { vi } from 'vitest';
|
|
2
10
|
|
|
3
11
|
import {
|
|
4
12
|
createWebTextSegmenterBackend,
|
|
13
|
+
explainTextSegmenterBackend,
|
|
5
14
|
getTextSegmenterBackend,
|
|
15
|
+
initializeWebTextSegmenterBackend,
|
|
6
16
|
setTextSegmenterBackend,
|
|
17
|
+
webTextSegmenterBackend,
|
|
7
18
|
} from './textSegmenterBackend';
|
|
8
19
|
|
|
9
20
|
interface RecordingBackend extends TextSegmenterBackend {
|
|
@@ -12,16 +23,19 @@ interface RecordingBackend extends TextSegmenterBackend {
|
|
|
12
23
|
|
|
13
24
|
function recordingBackend(): RecordingBackend {
|
|
14
25
|
const calls: RecordingBackend['calls'] = [];
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
},
|
|
26
|
+
const out = allocateEntity<any>();
|
|
27
|
+
out.calls = calls;
|
|
28
|
+
out.segment = (text: string, granularity: TextSegmentGranularity, locale?: string): readonly TextSegment[] => {
|
|
29
|
+
calls.push({ text, granularity, locale });
|
|
30
|
+
return [{ start: 0, end: text.length, text }];
|
|
21
31
|
};
|
|
32
|
+
return finishEntity(out);
|
|
22
33
|
}
|
|
23
34
|
|
|
24
|
-
afterEach(() =>
|
|
35
|
+
afterEach(() => {
|
|
36
|
+
setTextSegmenterBackend(null);
|
|
37
|
+
vi.unstubAllGlobals();
|
|
38
|
+
});
|
|
25
39
|
|
|
26
40
|
describe('createWebTextSegmenterBackend', () => {
|
|
27
41
|
it('segments a ZWJ family emoji as a single grapheme', () => {
|
|
@@ -47,10 +61,45 @@ describe('createWebTextSegmenterBackend', () => {
|
|
|
47
61
|
});
|
|
48
62
|
});
|
|
49
63
|
|
|
64
|
+
describe('explainTextSegmenterBackend', () => {
|
|
65
|
+
it('reports the bundled Intl provider as available', () => {
|
|
66
|
+
expect(explainTextSegmenterBackend()).toEqual({
|
|
67
|
+
available: true,
|
|
68
|
+
backend: 'web-intl',
|
|
69
|
+
intlSegmenterAvailable: true,
|
|
70
|
+
});
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
it('identifies a custom provider independently of Intl availability', () => {
|
|
74
|
+
expect(explainTextSegmenterBackend(segmenterHost(recordingBackend()))).toEqual({
|
|
75
|
+
available: true,
|
|
76
|
+
backend: 'custom',
|
|
77
|
+
intlSegmenterAvailable: true,
|
|
78
|
+
});
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
it('reports the bundled provider unavailable when the runtime primitive is absent', () => {
|
|
82
|
+
vi.stubGlobal('Intl', { Segmenter: undefined });
|
|
83
|
+
expect(explainTextSegmenterBackend()).toEqual({
|
|
84
|
+
available: false,
|
|
85
|
+
backend: 'web-intl',
|
|
86
|
+
intlSegmenterAvailable: false,
|
|
87
|
+
});
|
|
88
|
+
});
|
|
89
|
+
});
|
|
90
|
+
|
|
50
91
|
describe('getTextSegmenterBackend', () => {
|
|
51
|
-
it('
|
|
92
|
+
it('returns an explicit host provider ahead of the legacy backend', () => {
|
|
93
|
+
const legacy = recordingBackend();
|
|
94
|
+
const explicit = recordingBackend();
|
|
95
|
+
const host: HasTextSegmenter = { text: { segmenter: explicit } };
|
|
96
|
+
setTextSegmenterBackend(legacy);
|
|
97
|
+
expect(getTextSegmenterBackend(host)).toBe(explicit);
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
it('falls back to the stable bundled web backend when none is registered', () => {
|
|
52
101
|
const backend = getTextSegmenterBackend();
|
|
53
|
-
expect(backend).
|
|
102
|
+
expect(backend).toBe(webTextSegmenterBackend);
|
|
54
103
|
expect(backend.segment('ab', 'grapheme').length).toBe(2);
|
|
55
104
|
});
|
|
56
105
|
|
|
@@ -61,6 +110,12 @@ describe('getTextSegmenterBackend', () => {
|
|
|
61
110
|
});
|
|
62
111
|
});
|
|
63
112
|
|
|
113
|
+
describe('initializeWebTextSegmenterBackend', () => {
|
|
114
|
+
it('is the construction initializer of createWebTextSegmenterBackend', () => {
|
|
115
|
+
expect(typeof initializeWebTextSegmenterBackend).toBe('function');
|
|
116
|
+
});
|
|
117
|
+
});
|
|
118
|
+
|
|
64
119
|
describe('setTextSegmenterBackend', () => {
|
|
65
120
|
it('routes segmentation through the installed backend', () => {
|
|
66
121
|
const backend = recordingBackend();
|
|
@@ -69,10 +124,23 @@ describe('setTextSegmenterBackend', () => {
|
|
|
69
124
|
expect(backend.calls).toEqual([{ text: 'hello', granularity: 'sentence', locale: 'fr' }]);
|
|
70
125
|
});
|
|
71
126
|
|
|
72
|
-
it('clears back to the
|
|
127
|
+
it('clears back to the stable bundled web default when passed null', () => {
|
|
73
128
|
setTextSegmenterBackend(recordingBackend());
|
|
74
129
|
setTextSegmenterBackend(null);
|
|
75
130
|
// The web default segments a ZWJ cluster as one grapheme; a recording backend would not.
|
|
76
131
|
expect(getTextSegmenterBackend().segment('a👨👩👧b', 'grapheme').length).toBe(3);
|
|
77
132
|
});
|
|
78
133
|
});
|
|
134
|
+
describe('webTextSegmenterBackend', () => {
|
|
135
|
+
it('provides the stable bundled Intl fallback', () => {
|
|
136
|
+
expect(webTextSegmenterBackend.segment('a👨👩👧b', 'grapheme').map((segment) => segment.text)).toEqual([
|
|
137
|
+
'a',
|
|
138
|
+
'👨👩👧',
|
|
139
|
+
'b',
|
|
140
|
+
]);
|
|
141
|
+
});
|
|
142
|
+
});
|
|
143
|
+
|
|
144
|
+
function segmenterHost(segmenter: TextSegmenterBackend): HasTextSegmenter {
|
|
145
|
+
return { text: { segmenter } };
|
|
146
|
+
}
|