@flighthq/textsegment 0.5.1-next.1350.8ce488b β 0.5.1-next.1404.457d1da
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/enableTextSegmentGuards.js +1 -1
- package/dist/enableTextSegmentGuards.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/textSegment.d.ts +4 -4
- package/dist/textSegment.d.ts.map +1 -1
- package/dist/textSegment.js +6 -16
- package/dist/textSegment.js.map +1 -1
- package/dist/textSegmentBoundary.d.ts +8 -8
- package/dist/textSegmentBoundary.d.ts.map +1 -1
- package/dist/textSegmentBoundary.js +22 -46
- package/dist/textSegmentBoundary.js.map +1 -1
- package/dist/textSegmenterBackend.d.ts +6 -14
- package/dist/textSegmenterBackend.d.ts.map +1 -1
- package/dist/textSegmenterBackend.js +8 -23
- package/dist/textSegmenterBackend.js.map +1 -1
- package/package.json +4 -4
- package/src/enableTextSegmentGuards.test.ts +9 -6
- package/src/textSegment.test.ts +17 -39
- package/src/textSegmentBoundary.test.ts +33 -49
- package/src/textSegmentConformance.test.ts +5 -2
- package/src/textSegmenterBackend.test.ts +13 -53
package/README.md
CHANGED
|
@@ -13,5 +13,5 @@ Import the supported application-facing API from `@flighthq/textsegment`. The `@
|
|
|
13
13
|
This package is part of the locked-version Flight SDK graph. Applications may instead install and import `@flighthq/sdk` when package-level tree shaking is sufficient.
|
|
14
14
|
|
|
15
15
|
- [Flight project](https://github.com/flighthq/flight)
|
|
16
|
-
- [Source for @flighthq/textsegment@0.5.1-next.
|
|
17
|
-
- [License](https://github.com/flighthq/flight/blob/
|
|
16
|
+
- [Source for @flighthq/textsegment@0.5.1-next.1404.457d1da](https://github.com/flighthq/flight/tree/457d1da918a60be5853b84eb6c25530a66662cef/packages/textsegment)
|
|
17
|
+
- [License](https://github.com/flighthq/flight/blob/457d1da918a60be5853b84eb6c25530a66662cef/LICENSE.md)
|
|
@@ -11,7 +11,7 @@ export function enableTextSegmentGuards() {
|
|
|
11
11
|
}
|
|
12
12
|
function warnOnMissingIntlSegmenter() {
|
|
13
13
|
logOnce('textsegment:intl-segmenter-unavailable', LogLevel.Warn, {
|
|
14
|
-
message: 'segmentGraphemes: Intl.Segmenter is unavailable and no replacement provider was supplied, so segmentation returned no results.
|
|
14
|
+
message: 'segmentGraphemes: Intl.Segmenter is unavailable and no replacement provider was supplied, so segmentation returned no results. Pass a HostTextSegmenterCapability with full Unicode coverage to the operation.',
|
|
15
15
|
}, 'textsegment');
|
|
16
16
|
}
|
|
17
17
|
//# sourceMappingURL=enableTextSegmentGuards.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"enableTextSegmentGuards.js","sourceRoot":"","sources":["../src/enableTextSegmentGuards.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AACjD,OAAO,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AAEpD,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAE1D,iEAAiE;AACjE,MAAM,UAAU,wBAAwB;IACtC,mBAAmB,CAAC,IAAI,CAAC,CAAC;AAC5B,CAAC;AAED,iGAAiG;AACjG,MAAM,UAAU,uBAAuB;IACrC,mBAAmB,CAAC,0BAA0B,CAAC,CAAC;AAClD,CAAC;AAED,SAAS,0BAA0B;IACjC,OAAO,CACL,wCAAwC,EACxC,QAAQ,CAAC,IAAI,EACb;QACE,OAAO,EACL,
|
|
1
|
+
{"version":3,"file":"enableTextSegmentGuards.js","sourceRoot":"","sources":["../src/enableTextSegmentGuards.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AACjD,OAAO,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AAEpD,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAE1D,iEAAiE;AACjE,MAAM,UAAU,wBAAwB;IACtC,mBAAmB,CAAC,IAAI,CAAC,CAAC;AAC5B,CAAC;AAED,iGAAiG;AACjG,MAAM,UAAU,uBAAuB;IACrC,mBAAmB,CAAC,0BAA0B,CAAC,CAAC;AAClD,CAAC;AAED,SAAS,0BAA0B;IACjC,OAAO,CACL,wCAAwC,EACxC,QAAQ,CAAC,IAAI,EACb;QACE,OAAO,EACL,gNAAgN;KACnN,EACD,aAAa,CACd,CAAC;AACJ,CAAC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export { disableTextSegmentGuards, enableTextSegmentGuards, explainTextSegmenterBackend, getNextGraphemeBoundary, getNextSentenceBoundary, getNextWordBoundary, getPreviousGraphemeBoundary, getPreviousSentenceBoundary, getPreviousWordBoundary, getWordRangeAt, segmentGraphemes, segmentSentences, segmentWords, webTextSegmenterBackend, } from './contract';
|
|
1
|
+
export { createDefaultTextSegmenterBackend, disableTextSegmentGuards, enableTextSegmentGuards, explainTextSegmenterBackend, getNextGraphemeBoundary, getNextSentenceBoundary, getNextWordBoundary, getPreviousGraphemeBoundary, getPreviousSentenceBoundary, getPreviousWordBoundary, getWordRangeAt, segmentGraphemes, segmentSentences, segmentWords, webTextSegmenterBackend, } from './contract';
|
|
2
2
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,wBAAwB,EACxB,uBAAuB,EACvB,2BAA2B,EAC3B,uBAAuB,EACvB,uBAAuB,EACvB,mBAAmB,EACnB,2BAA2B,EAC3B,2BAA2B,EAC3B,uBAAuB,EACvB,cAAc,EACd,gBAAgB,EAChB,gBAAgB,EAChB,YAAY,EACZ,uBAAuB,GACxB,MAAM,YAAY,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,iCAAiC,EACjC,wBAAwB,EACxB,uBAAuB,EACvB,2BAA2B,EAC3B,uBAAuB,EACvB,uBAAuB,EACvB,mBAAmB,EACnB,2BAA2B,EAC3B,2BAA2B,EAC3B,uBAAuB,EACvB,cAAc,EACd,gBAAgB,EAChB,gBAAgB,EAChB,YAAY,EACZ,uBAAuB,GACxB,MAAM,YAAY,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export { disableTextSegmentGuards, enableTextSegmentGuards, explainTextSegmenterBackend, getNextGraphemeBoundary, getNextSentenceBoundary, getNextWordBoundary, getPreviousGraphemeBoundary, getPreviousSentenceBoundary, getPreviousWordBoundary, getWordRangeAt, segmentGraphemes, segmentSentences, segmentWords, webTextSegmenterBackend, } from './contract';
|
|
1
|
+
export { createDefaultTextSegmenterBackend, disableTextSegmentGuards, enableTextSegmentGuards, explainTextSegmenterBackend, getNextGraphemeBoundary, getNextSentenceBoundary, getNextWordBoundary, getPreviousGraphemeBoundary, getPreviousSentenceBoundary, getPreviousWordBoundary, getWordRangeAt, segmentGraphemes, segmentSentences, segmentWords, webTextSegmenterBackend, } from './contract';
|
|
2
2
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,wBAAwB,EACxB,uBAAuB,EACvB,2BAA2B,EAC3B,uBAAuB,EACvB,uBAAuB,EACvB,mBAAmB,EACnB,2BAA2B,EAC3B,2BAA2B,EAC3B,uBAAuB,EACvB,cAAc,EACd,gBAAgB,EAChB,gBAAgB,EAChB,YAAY,EACZ,uBAAuB,GACxB,MAAM,YAAY,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,iCAAiC,EACjC,wBAAwB,EACxB,uBAAuB,EACvB,2BAA2B,EAC3B,uBAAuB,EACvB,uBAAuB,EACvB,mBAAmB,EACnB,2BAA2B,EAC3B,2BAA2B,EAC3B,uBAAuB,EACvB,cAAc,EACd,gBAAgB,EAChB,gBAAgB,EAChB,YAAY,EACZ,uBAAuB,GACxB,MAAM,YAAY,CAAC"}
|
package/dist/textSegment.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
export declare function segmentGraphemes(text: string, locale?: string
|
|
3
|
-
export declare function segmentSentences(text: string, locale?: string
|
|
4
|
-
export declare function segmentWords(text: string, locale?: string
|
|
1
|
+
import type { HostTextSegmenterCapability, TextSegment } from '@flighthq/types/contract';
|
|
2
|
+
export declare function segmentGraphemes(textSegmenter: Readonly<HostTextSegmenterCapability>, text: string, locale?: string): readonly TextSegment[];
|
|
3
|
+
export declare function segmentSentences(textSegmenter: Readonly<HostTextSegmenterCapability>, text: string, locale?: string): readonly TextSegment[];
|
|
4
|
+
export declare function segmentWords(textSegmenter: Readonly<HostTextSegmenterCapability>, text: string, locale?: string): readonly TextSegment[];
|
|
5
5
|
//# sourceMappingURL=textSegment.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"textSegment.d.ts","sourceRoot":"","sources":["../src/textSegment.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,
|
|
1
|
+
{"version":3,"file":"textSegment.d.ts","sourceRoot":"","sources":["../src/textSegment.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,2BAA2B,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAEzF,wBAAgB,gBAAgB,CAC9B,aAAa,EAAE,QAAQ,CAAC,2BAA2B,CAAC,EACpD,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,MAAM,GACd,SAAS,WAAW,EAAE,CAExB;AAED,wBAAgB,gBAAgB,CAC9B,aAAa,EAAE,QAAQ,CAAC,2BAA2B,CAAC,EACpD,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,MAAM,GACd,SAAS,WAAW,EAAE,CAExB;AAED,wBAAgB,YAAY,CAC1B,aAAa,EAAE,QAAQ,CAAC,2BAA2B,CAAC,EACpD,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,MAAM,GACd,SAAS,WAAW,EAAE,CAExB"}
|
package/dist/textSegment.js
CHANGED
|
@@ -1,20 +1,10 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
// user-perceived character: an emoji ZWJ sequence or a base-plus-combining-mark cluster is a single
|
|
4
|
-
// segment, not its constituent code points. `locale` is threaded to the backend for the rare
|
|
5
|
-
// locale-tailored cases. isWordLike is absent on grapheme segments.
|
|
6
|
-
export function segmentGraphemes(text, locale, hostTextSegmenter) {
|
|
7
|
-
return getTextSegmenterBackend(hostTextSegmenter).segment(text, 'grapheme', locale);
|
|
1
|
+
export function segmentGraphemes(textSegmenter, text, locale) {
|
|
2
|
+
return textSegmenter.segment(text, 'grapheme', locale);
|
|
8
3
|
}
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
export function segmentSentences(text, locale, hostTextSegmenter) {
|
|
12
|
-
return getTextSegmenterBackend(hostTextSegmenter).segment(text, 'sentence', locale);
|
|
4
|
+
export function segmentSentences(textSegmenter, text, locale) {
|
|
5
|
+
return textSegmenter.segment(text, 'sentence', locale);
|
|
13
6
|
}
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
// them. Filter on isWordLike to keep only words. `locale` is threaded to the backend.
|
|
17
|
-
export function segmentWords(text, locale, hostTextSegmenter) {
|
|
18
|
-
return getTextSegmenterBackend(hostTextSegmenter).segment(text, 'word', locale);
|
|
7
|
+
export function segmentWords(textSegmenter, text, locale) {
|
|
8
|
+
return textSegmenter.segment(text, 'word', locale);
|
|
19
9
|
}
|
|
20
10
|
//# sourceMappingURL=textSegment.js.map
|
package/dist/textSegment.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"textSegment.js","sourceRoot":"","sources":["../src/textSegment.ts"],"names":[],"mappings":"AAEA,
|
|
1
|
+
{"version":3,"file":"textSegment.js","sourceRoot":"","sources":["../src/textSegment.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,gBAAgB,CAC9B,aAAoD,EACpD,IAAY,EACZ,MAAe;IAEf,OAAO,aAAa,CAAC,OAAO,CAAC,IAAI,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;AACzD,CAAC;AAED,MAAM,UAAU,gBAAgB,CAC9B,aAAoD,EACpD,IAAY,EACZ,MAAe;IAEf,OAAO,aAAa,CAAC,OAAO,CAAC,IAAI,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;AACzD,CAAC;AAED,MAAM,UAAU,YAAY,CAC1B,aAAoD,EACpD,IAAY,EACZ,MAAe;IAEf,OAAO,aAAa,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AACrD,CAAC"}
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
export declare function getNextGraphemeBoundary(text: string, index: number, locale?: string
|
|
3
|
-
export declare function getNextSentenceBoundary(text: string, index: number, locale?: string
|
|
4
|
-
export declare function getNextWordBoundary(text: string, index: number, locale?: string
|
|
5
|
-
export declare function getPreviousGraphemeBoundary(text: string, index: number, locale?: string
|
|
6
|
-
export declare function getPreviousSentenceBoundary(text: string, index: number, locale?: string
|
|
7
|
-
export declare function getPreviousWordBoundary(text: string, index: number, locale?: string
|
|
8
|
-
export declare function getWordRangeAt(text: string, index: number, locale?: string
|
|
1
|
+
import type { HostTextSegmenterCapability, TextSegmentRange } from '@flighthq/types/contract';
|
|
2
|
+
export declare function getNextGraphemeBoundary(textSegmenter: Readonly<HostTextSegmenterCapability>, text: string, index: number, locale?: string): number;
|
|
3
|
+
export declare function getNextSentenceBoundary(textSegmenter: Readonly<HostTextSegmenterCapability>, text: string, index: number, locale?: string): number;
|
|
4
|
+
export declare function getNextWordBoundary(textSegmenter: Readonly<HostTextSegmenterCapability>, text: string, index: number, locale?: string): number;
|
|
5
|
+
export declare function getPreviousGraphemeBoundary(textSegmenter: Readonly<HostTextSegmenterCapability>, text: string, index: number, locale?: string): number;
|
|
6
|
+
export declare function getPreviousSentenceBoundary(textSegmenter: Readonly<HostTextSegmenterCapability>, text: string, index: number, locale?: string): number;
|
|
7
|
+
export declare function getPreviousWordBoundary(textSegmenter: Readonly<HostTextSegmenterCapability>, text: string, index: number, locale?: string): number;
|
|
8
|
+
export declare function getWordRangeAt(textSegmenter: Readonly<HostTextSegmenterCapability>, text: string, index: number, locale?: string): TextSegmentRange | null;
|
|
9
9
|
//# sourceMappingURL=textSegmentBoundary.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"textSegmentBoundary.d.ts","sourceRoot":"","sources":["../src/textSegmentBoundary.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,
|
|
1
|
+
{"version":3,"file":"textSegmentBoundary.d.ts","sourceRoot":"","sources":["../src/textSegmentBoundary.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,2BAA2B,EAG3B,gBAAgB,EACjB,MAAM,0BAA0B,CAAC;AAKlC,wBAAgB,uBAAuB,CACrC,aAAa,EAAE,QAAQ,CAAC,2BAA2B,CAAC,EACpD,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,EACb,MAAM,CAAC,EAAE,MAAM,GACd,MAAM,CAER;AAED,wBAAgB,uBAAuB,CACrC,aAAa,EAAE,QAAQ,CAAC,2BAA2B,CAAC,EACpD,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,EACb,MAAM,CAAC,EAAE,MAAM,GACd,MAAM,CAER;AAED,wBAAgB,mBAAmB,CACjC,aAAa,EAAE,QAAQ,CAAC,2BAA2B,CAAC,EACpD,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,EACb,MAAM,CAAC,EAAE,MAAM,GACd,MAAM,CAER;AAED,wBAAgB,2BAA2B,CACzC,aAAa,EAAE,QAAQ,CAAC,2BAA2B,CAAC,EACpD,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,EACb,MAAM,CAAC,EAAE,MAAM,GACd,MAAM,CAER;AAED,wBAAgB,2BAA2B,CACzC,aAAa,EAAE,QAAQ,CAAC,2BAA2B,CAAC,EACpD,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,EACb,MAAM,CAAC,EAAE,MAAM,GACd,MAAM,CAER;AAED,wBAAgB,uBAAuB,CACrC,aAAa,EAAE,QAAQ,CAAC,2BAA2B,CAAC,EACpD,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,EACb,MAAM,CAAC,EAAE,MAAM,GACd,MAAM,CAER;AAED,wBAAgB,cAAc,CAC5B,aAAa,EAAE,QAAQ,CAAC,2BAA2B,CAAC,EACpD,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,EACb,MAAM,CAAC,EAAE,MAAM,GACd,gBAAgB,GAAG,IAAI,CAwBzB"}
|
|
@@ -1,51 +1,31 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { webTextSegmenterBackend } from './textSegmenterBackend';
|
|
2
2
|
import { reportTextSegmenterUnavailable } from './textSegmentGuards';
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
// single step. `index` is clamped into [0, text.length]; at or past the end it returns text.length.
|
|
6
|
-
export function getNextGraphemeBoundary(text, index, locale, hostTextSegmenter) {
|
|
7
|
-
return getNextTextSegmentBoundary(text, index, 'grapheme', locale, hostTextSegmenter);
|
|
3
|
+
export function getNextGraphemeBoundary(textSegmenter, text, index, locale) {
|
|
4
|
+
return getNextTextSegmentBoundary(textSegmenter, text, index, 'grapheme', locale);
|
|
8
5
|
}
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
export function getNextSentenceBoundary(text, index, locale, hostTextSegmenter) {
|
|
12
|
-
return getNextTextSegmentBoundary(text, index, 'sentence', locale, hostTextSegmenter);
|
|
6
|
+
export function getNextSentenceBoundary(textSegmenter, text, index, locale) {
|
|
7
|
+
return getNextTextSegmentBoundary(textSegmenter, text, index, 'sentence', locale);
|
|
13
8
|
}
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
// returns text.length.
|
|
17
|
-
export function getNextWordBoundary(text, index, locale, hostTextSegmenter) {
|
|
18
|
-
return getNextTextSegmentBoundary(text, index, 'word', locale, hostTextSegmenter);
|
|
9
|
+
export function getNextWordBoundary(textSegmenter, text, index, locale) {
|
|
10
|
+
return getNextTextSegmentBoundary(textSegmenter, text, index, 'word', locale);
|
|
19
11
|
}
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
// before the start it returns 0.
|
|
23
|
-
export function getPreviousGraphemeBoundary(text, index, locale, hostTextSegmenter) {
|
|
24
|
-
return getPreviousTextSegmentBoundary(text, index, 'grapheme', locale, hostTextSegmenter);
|
|
12
|
+
export function getPreviousGraphemeBoundary(textSegmenter, text, index, locale) {
|
|
13
|
+
return getPreviousTextSegmentBoundary(textSegmenter, text, index, 'grapheme', locale);
|
|
25
14
|
}
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
export function getPreviousSentenceBoundary(text, index, locale, hostTextSegmenter) {
|
|
29
|
-
return getPreviousTextSegmentBoundary(text, index, 'sentence', locale, hostTextSegmenter);
|
|
15
|
+
export function getPreviousSentenceBoundary(textSegmenter, text, index, locale) {
|
|
16
|
+
return getPreviousTextSegmentBoundary(textSegmenter, text, index, 'sentence', locale);
|
|
30
17
|
}
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
export function getPreviousWordBoundary(text, index, locale, hostTextSegmenter) {
|
|
34
|
-
return getPreviousTextSegmentBoundary(text, index, 'word', locale, hostTextSegmenter);
|
|
18
|
+
export function getPreviousWordBoundary(textSegmenter, text, index, locale) {
|
|
19
|
+
return getPreviousTextSegmentBoundary(textSegmenter, text, index, 'word', locale);
|
|
35
20
|
}
|
|
36
|
-
|
|
37
|
-
// clamped into [0, text.length]; at the very end it resolves against the last character. Returns null
|
|
38
|
-
// when the covered segment is not word-like (whitespace or punctuation) or the text is empty β the
|
|
39
|
-
// caller should then select nothing, mirroring how a double-click on a space selects no word.
|
|
40
|
-
export function getWordRangeAt(text, index, locale, hostTextSegmenter) {
|
|
21
|
+
export function getWordRangeAt(textSegmenter, text, index, locale) {
|
|
41
22
|
if (text.length === 0)
|
|
42
23
|
return null;
|
|
43
24
|
const clamped = clampIndex(index, text.length);
|
|
44
25
|
// At the trailing boundary there is no segment starting at text.length; resolve against the last
|
|
45
26
|
// character so a double-click at the end still selects the final word.
|
|
46
27
|
const lookup = clamped === text.length ? text.length - 1 : clamped;
|
|
47
|
-
|
|
48
|
-
if (backend === webTextSegmenterBackend) {
|
|
28
|
+
if (textSegmenter === webTextSegmenterBackend) {
|
|
49
29
|
const segmenter = getBoundarySegmenter(locale, 'word');
|
|
50
30
|
if (segmenter === null)
|
|
51
31
|
return null;
|
|
@@ -57,7 +37,7 @@ export function getWordRangeAt(text, index, locale, hostTextSegmenter) {
|
|
|
57
37
|
}
|
|
58
38
|
return null;
|
|
59
39
|
}
|
|
60
|
-
const segments =
|
|
40
|
+
const segments = textSegmenter.segment(text, 'word', locale);
|
|
61
41
|
for (const segment of segments) {
|
|
62
42
|
if (lookup >= segment.start && lookup < segment.end) {
|
|
63
43
|
return segment.isWordLike === true ? { start: segment.start, end: segment.end } : null;
|
|
@@ -83,13 +63,12 @@ function getBoundarySegmenter(locale, granularity) {
|
|
|
83
63
|
boundarySegmenterCache.set(key, segmenter);
|
|
84
64
|
return segmenter;
|
|
85
65
|
}
|
|
86
|
-
function getNextTextSegmentBoundary(text, index, granularity, locale
|
|
66
|
+
function getNextTextSegmentBoundary(textSegmenter, text, index, granularity, locale) {
|
|
87
67
|
const from = clampIndex(index, text.length);
|
|
88
68
|
if (from >= text.length)
|
|
89
69
|
return text.length;
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
return nextSegmentBoundary(backend.segment(text, granularity, locale), from, text.length);
|
|
70
|
+
if (textSegmenter !== webTextSegmenterBackend) {
|
|
71
|
+
return nextSegmentBoundary(textSegmenter.segment(text, granularity, locale), from, text.length);
|
|
93
72
|
}
|
|
94
73
|
const segmenter = getBoundarySegmenter(locale, granularity);
|
|
95
74
|
if (segmenter === null)
|
|
@@ -100,13 +79,12 @@ function getNextTextSegmentBoundary(text, index, granularity, locale, hostTextSe
|
|
|
100
79
|
}
|
|
101
80
|
return text.length;
|
|
102
81
|
}
|
|
103
|
-
function getPreviousTextSegmentBoundary(text, index, granularity, locale
|
|
82
|
+
function getPreviousTextSegmentBoundary(textSegmenter, text, index, granularity, locale) {
|
|
104
83
|
const from = clampIndex(index, text.length);
|
|
105
84
|
if (from <= 0)
|
|
106
85
|
return 0;
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
return previousSegmentBoundary(backend.segment(text, granularity, locale), from);
|
|
86
|
+
if (textSegmenter !== webTextSegmenterBackend)
|
|
87
|
+
return previousSegmentBoundary(textSegmenter.segment(text, granularity, locale), from);
|
|
110
88
|
const segmenter = getBoundarySegmenter(locale, granularity);
|
|
111
89
|
if (segmenter === null)
|
|
112
90
|
return 0;
|
|
@@ -118,8 +96,6 @@ function getPreviousTextSegmentBoundary(text, index, granularity, locale, hostTe
|
|
|
118
96
|
}
|
|
119
97
|
return previous;
|
|
120
98
|
}
|
|
121
|
-
// The boundary offsets are every segment start plus text.length (segments cover the string in order
|
|
122
|
-
// with strictly increasing starts). Clamping keeps out-of-range indices from throwing per house rules.
|
|
123
99
|
function clampIndex(index, length) {
|
|
124
100
|
if (index < 0)
|
|
125
101
|
return 0;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"textSegmentBoundary.js","sourceRoot":"","sources":["../src/textSegmentBoundary.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,uBAAuB,EAAE,
|
|
1
|
+
{"version":3,"file":"textSegmentBoundary.js","sourceRoot":"","sources":["../src/textSegmentBoundary.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AACjE,OAAO,EAAE,8BAA8B,EAAE,MAAM,qBAAqB,CAAC;AAErE,MAAM,UAAU,uBAAuB,CACrC,aAAoD,EACpD,IAAY,EACZ,KAAa,EACb,MAAe;IAEf,OAAO,0BAA0B,CAAC,aAAa,EAAE,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;AACpF,CAAC;AAED,MAAM,UAAU,uBAAuB,CACrC,aAAoD,EACpD,IAAY,EACZ,KAAa,EACb,MAAe;IAEf,OAAO,0BAA0B,CAAC,aAAa,EAAE,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;AACpF,CAAC;AAED,MAAM,UAAU,mBAAmB,CACjC,aAAoD,EACpD,IAAY,EACZ,KAAa,EACb,MAAe;IAEf,OAAO,0BAA0B,CAAC,aAAa,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AAChF,CAAC;AAED,MAAM,UAAU,2BAA2B,CACzC,aAAoD,EACpD,IAAY,EACZ,KAAa,EACb,MAAe;IAEf,OAAO,8BAA8B,CAAC,aAAa,EAAE,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;AACxF,CAAC;AAED,MAAM,UAAU,2BAA2B,CACzC,aAAoD,EACpD,IAAY,EACZ,KAAa,EACb,MAAe;IAEf,OAAO,8BAA8B,CAAC,aAAa,EAAE,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;AACxF,CAAC;AAED,MAAM,UAAU,uBAAuB,CACrC,aAAoD,EACpD,IAAY,EACZ,KAAa,EACb,MAAe;IAEf,OAAO,8BAA8B,CAAC,aAAa,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AACpF,CAAC;AAED,MAAM,UAAU,cAAc,CAC5B,aAAoD,EACpD,IAAY,EACZ,KAAa,EACb,MAAe;IAEf,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACnC,MAAM,OAAO,GAAG,UAAU,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IAC/C,iGAAiG;IACjG,uEAAuE;IACvE,MAAM,MAAM,GAAG,OAAO,KAAK,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;IACnE,IAAI,aAAa,KAAK,uBAAuB,EAAE,CAAC;QAC9C,MAAM,SAAS,GAAG,oBAAoB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACvD,IAAI,SAAS,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;QACpC,KAAK,MAAM,OAAO,IAAI,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YAC9C,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC;YACnD,IAAI,MAAM,IAAI,OAAO,CAAC,KAAK,IAAI,MAAM,GAAG,GAAG,EAAE,CAAC;gBAC5C,OAAO,OAAO,CAAC,UAAU,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;YAC5E,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,QAAQ,GAAG,aAAa,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7D,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,IAAI,MAAM,IAAI,OAAO,CAAC,KAAK,IAAI,MAAM,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;YACpD,OAAO,OAAO,CAAC,UAAU,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QACzF,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,oBAAoB,CAAC,MAA0B,EAAE,WAAmC;IAC3F,IAAI,OAAO,IAAI,KAAK,WAAW,IAAI,OAAO,IAAI,CAAC,SAAS,KAAK,WAAW,EAAE,CAAC;QACzE,8BAA8B,EAAE,CAAC;QACjC,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,GAAG,GAAG,GAAG,MAAM,IAAI,EAAE,IAAI,WAAW,EAAE,CAAC;IAC7C,MAAM,QAAQ,GAAG,sBAAsB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACjD,IAAI,QAAQ,KAAK,SAAS;QAAE,OAAO,QAAQ,CAAC;IAC5C,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;IAC9D,IAAI,sBAAsB,CAAC,IAAI,IAAI,iCAAiC,EAAE,CAAC;QACrE,MAAM,MAAM,GAAG,sBAAsB,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;QAC1D,IAAI,MAAM,KAAK,SAAS;YAAE,sBAAsB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAClE,CAAC;IACD,sBAAsB,CAAC,GAAG,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;IAC3C,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,0BAA0B,CACjC,aAAoD,EACpD,IAAY,EACZ,KAAa,EACb,WAAmC,EACnC,MAA0B;IAE1B,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IAC5C,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC,MAAM,CAAC;IAC5C,IAAI,aAAa,KAAK,uBAAuB,EAAE,CAAC;QAC9C,OAAO,mBAAmB,CAAC,aAAa,CAAC,OAAO,CAAC,IAAI,EAAE,WAAW,EAAE,MAAM,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IAClG,CAAC;IACD,MAAM,SAAS,GAAG,oBAAoB,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IAC5D,IAAI,SAAS,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC,MAAM,CAAC;IAC3C,KAAK,MAAM,OAAO,IAAI,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9C,IAAI,OAAO,CAAC,KAAK,GAAG,IAAI;YAAE,OAAO,OAAO,CAAC,KAAK,CAAC;IACjD,CAAC;IACD,OAAO,IAAI,CAAC,MAAM,CAAC;AACrB,CAAC;AAED,SAAS,8BAA8B,CACrC,aAAoD,EACpD,IAAY,EACZ,KAAa,EACb,WAAmC,EACnC,MAA0B;IAE1B,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IAC5C,IAAI,IAAI,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IACxB,IAAI,aAAa,KAAK,uBAAuB;QAC3C,OAAO,uBAAuB,CAAC,aAAa,CAAC,OAAO,CAAC,IAAI,EAAE,WAAW,EAAE,MAAM,CAAC,EAAE,IAAI,CAAC,CAAC;IACzF,MAAM,SAAS,GAAG,oBAAoB,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IAC5D,IAAI,SAAS,KAAK,IAAI;QAAE,OAAO,CAAC,CAAC;IACjC,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,KAAK,MAAM,OAAO,IAAI,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9C,IAAI,OAAO,CAAC,KAAK,IAAI,IAAI;YAAE,MAAM;QACjC,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC;IAC3B,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,UAAU,CAAC,KAAa,EAAE,MAAc;IAC/C,IAAI,KAAK,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IACxB,IAAI,KAAK,GAAG,MAAM;QAAE,OAAO,MAAM,CAAC;IAClC,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,mBAAmB,CAAC,QAAgC,EAAE,KAAa,EAAE,MAAc;IAC1F,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACvC,IAAI,IAAI,IAAI,MAAM;QAAE,OAAO,MAAM,CAAC;IAClC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,IAAI,OAAO,CAAC,KAAK,GAAG,IAAI;YAAE,OAAO,OAAO,CAAC,KAAK,CAAC;IACjD,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,uBAAuB,CAAC,QAAgC,EAAE,KAAa;IAC9E,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IAC7E,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACvC,IAAI,IAAI,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IACxB,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,IAAI,OAAO,CAAC,KAAK,IAAI,IAAI;YAAE,MAAM;QACjC,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC;IAC3B,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,sBAAsB,GAAG,IAAI,GAAG,EAA0B,CAAC;AACjE,MAAM,iCAAiC,GAAG,EAAE,CAAC"}
|
|
@@ -1,16 +1,8 @@
|
|
|
1
|
-
import type { EntityConstruction,
|
|
2
|
-
export declare function
|
|
3
|
-
export declare const webTextSegmenterBackend:
|
|
1
|
+
import type { EntityConstruction, HostTextSegmenterCapability, TextSegmenterBackendExplanation } from '@flighthq/types/contract';
|
|
2
|
+
export declare function createDefaultTextSegmenterBackend(): HostTextSegmenterCapability;
|
|
3
|
+
export declare const webTextSegmenterBackend: HostTextSegmenterCapability;
|
|
4
|
+
export declare function createWebTextSegmenterBackend(): HostTextSegmenterCapability;
|
|
4
5
|
/** Describes which provider an operation would use and whether Intl.Segmenter is present. */
|
|
5
|
-
export declare function explainTextSegmenterBackend(
|
|
6
|
-
export declare function
|
|
7
|
-
export declare function initializeWebTextSegmenterBackend(out: EntityConstruction<HostTextSegmenterProvider>): void;
|
|
8
|
-
/**
|
|
9
|
-
* Installs the backend used by calls that omit an explicit host; pass null to restore the stable
|
|
10
|
-
* bundled web default.
|
|
11
|
-
*
|
|
12
|
-
* @deprecated Pass a HostTextSegmenterProvider to the segmentation or boundary operation. Retained
|
|
13
|
-
* for source compatibility until the legacy global path is removed.
|
|
14
|
-
*/
|
|
15
|
-
export declare function setTextSegmenterBackend(backend: HostTextSegmenterProvider | null): void;
|
|
6
|
+
export declare function explainTextSegmenterBackend(textSegmenter: Readonly<HostTextSegmenterCapability>): TextSegmenterBackendExplanation;
|
|
7
|
+
export declare function initializeWebTextSegmenterBackend(out: EntityConstruction<HostTextSegmenterCapability>): void;
|
|
16
8
|
//# sourceMappingURL=textSegmenterBackend.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"textSegmenterBackend.d.ts","sourceRoot":"","sources":["../src/textSegmenterBackend.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,kBAAkB,EAClB,
|
|
1
|
+
{"version":3,"file":"textSegmenterBackend.d.ts","sourceRoot":"","sources":["../src/textSegmenterBackend.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,kBAAkB,EAClB,2BAA2B,EAG3B,+BAA+B,EAChC,MAAM,0BAA0B,CAAC;AAIlC,wBAAgB,iCAAiC,IAAI,2BAA2B,CAE/E;AAID,eAAO,MAAM,uBAAuB,EAAE,2BAA6D,CAAC;AAQpG,wBAAgB,6BAA6B,IAAI,2BAA2B,CAI3E;AAED,6FAA6F;AAC7F,wBAAgB,2BAA2B,CACzC,aAAa,EAAE,QAAQ,CAAC,2BAA2B,CAAC,GACnD,+BAA+B,CAQjC;AAED,wBAAgB,iCAAiC,CAAC,GAAG,EAAE,kBAAkB,CAAC,2BAA2B,CAAC,GAAG,IAAI,CAE5G"}
|
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
import { allocateEntity, finishEntity } from '@flighthq/entity/contract';
|
|
2
2
|
import { reportTextSegmenterUnavailable } from './textSegmentGuards';
|
|
3
|
+
export function createDefaultTextSegmenterBackend() {
|
|
4
|
+
return createWebTextSegmenterBackend();
|
|
5
|
+
}
|
|
6
|
+
// Stable bundled web provider. Hosts can import this directly when composing their explicit
|
|
7
|
+
// capability object.
|
|
8
|
+
export const webTextSegmenterBackend = createWebTextSegmenterBackend();
|
|
3
9
|
// Builds the default web backend: a wrapper over the browser-native Intl.Segmenter. It ships no
|
|
4
10
|
// Unicode tables β the engine already carries them β so the common path costs nothing in bundle
|
|
5
11
|
// weight. Intl.Segmenter instances are cached by (locale, granularity) because constructing one is
|
|
@@ -11,13 +17,9 @@ export function createWebTextSegmenterBackend() {
|
|
|
11
17
|
initializeWebTextSegmenterBackend(out);
|
|
12
18
|
return finishEntity(out);
|
|
13
19
|
}
|
|
14
|
-
// Stable bundled web provider. Hosts can import this directly when composing their explicit
|
|
15
|
-
// capability object; callers that omit a host receive it as the final fallback.
|
|
16
|
-
export const webTextSegmenterBackend = createWebTextSegmenterBackend();
|
|
17
20
|
/** Describes which provider an operation would use and whether Intl.Segmenter is present. */
|
|
18
|
-
export function explainTextSegmenterBackend(
|
|
19
|
-
const
|
|
20
|
-
const web = backend === webTextSegmenterBackend;
|
|
21
|
+
export function explainTextSegmenterBackend(textSegmenter) {
|
|
22
|
+
const web = textSegmenter === webTextSegmenterBackend;
|
|
21
23
|
const intlSegmenterAvailable = hasIntlSegmenter();
|
|
22
24
|
return {
|
|
23
25
|
available: !web || intlSegmenterAvailable,
|
|
@@ -25,26 +27,9 @@ export function explainTextSegmenterBackend(hostTextSegmenter) {
|
|
|
25
27
|
intlSegmenterAvailable,
|
|
26
28
|
};
|
|
27
29
|
}
|
|
28
|
-
// Returns the explicit host's provider when supplied, then the legacy installed backend, and
|
|
29
|
-
// finally the bundled web provider. The explicit dependency always wins, so independent callers
|
|
30
|
-
// can interleave different hosts without mutating shared capability state.
|
|
31
|
-
export function getTextSegmenterBackend(hostTextSegmenter) {
|
|
32
|
-
return hostTextSegmenter ?? _backend ?? webTextSegmenterBackend;
|
|
33
|
-
}
|
|
34
30
|
export function initializeWebTextSegmenterBackend(out) {
|
|
35
31
|
out.segment = segmentWithIntlSegmenter;
|
|
36
32
|
}
|
|
37
|
-
/**
|
|
38
|
-
* Installs the backend used by calls that omit an explicit host; pass null to restore the stable
|
|
39
|
-
* bundled web default.
|
|
40
|
-
*
|
|
41
|
-
* @deprecated Pass a HostTextSegmenterProvider to the segmentation or boundary operation. Retained
|
|
42
|
-
* for source compatibility until the legacy global path is removed.
|
|
43
|
-
*/
|
|
44
|
-
export function setTextSegmenterBackend(backend) {
|
|
45
|
-
_backend = backend;
|
|
46
|
-
}
|
|
47
|
-
let _backend = null;
|
|
48
33
|
// Cached Intl.Segmenter instances keyed by `locale|granularity`. A Map preserves insertion order, so
|
|
49
34
|
// the first key is the oldest and drives simple FIFO eviction once the cache is full. Instances are
|
|
50
35
|
// immutable, so sharing them across calls is safe.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"textSegmenterBackend.js","sourceRoot":"","sources":["../src/textSegmenterBackend.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AASzE,OAAO,EAAE,8BAA8B,EAAE,MAAM,qBAAqB,CAAC;AAErE,gGAAgG;AAChG,gGAAgG;AAChG,mGAAmG;AACnG,oGAAoG;AACpG,kGAAkG;AAClG,mCAAmC;AACnC,MAAM,UAAU,6BAA6B;IAC3C,MAAM,GAAG,GAAG,cAAc,
|
|
1
|
+
{"version":3,"file":"textSegmenterBackend.js","sourceRoot":"","sources":["../src/textSegmenterBackend.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AASzE,OAAO,EAAE,8BAA8B,EAAE,MAAM,qBAAqB,CAAC;AAErE,MAAM,UAAU,iCAAiC;IAC/C,OAAO,6BAA6B,EAAE,CAAC;AACzC,CAAC;AAED,4FAA4F;AAC5F,qBAAqB;AACrB,MAAM,CAAC,MAAM,uBAAuB,GAAgC,6BAA6B,EAAE,CAAC;AAEpG,gGAAgG;AAChG,gGAAgG;AAChG,mGAAmG;AACnG,oGAAoG;AACpG,kGAAkG;AAClG,mCAAmC;AACnC,MAAM,UAAU,6BAA6B;IAC3C,MAAM,GAAG,GAAG,cAAc,EAA+B,CAAC;IAC1D,iCAAiC,CAAC,GAAG,CAAC,CAAC;IACvC,OAAO,YAAY,CAAC,GAAG,CAAC,CAAC;AAC3B,CAAC;AAED,6FAA6F;AAC7F,MAAM,UAAU,2BAA2B,CACzC,aAAoD;IAEpD,MAAM,GAAG,GAAG,aAAa,KAAK,uBAAuB,CAAC;IACtD,MAAM,sBAAsB,GAAG,gBAAgB,EAAE,CAAC;IAClD,OAAO;QACL,SAAS,EAAE,CAAC,GAAG,IAAI,sBAAsB;QACzC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ;QACpC,sBAAsB;KACvB,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,iCAAiC,CAAC,GAAoD;IACpG,GAAG,CAAC,OAAO,GAAG,wBAAwB,CAAC;AACzC,CAAC;AAED,qGAAqG;AACrG,oGAAoG;AACpG,mDAAmD;AACnD,MAAM,eAAe,GAAG,IAAI,GAAG,EAA0B,CAAC;AAC1D,MAAM,uBAAuB,GAAG,EAAE,CAAC;AAEnC,SAAS,kBAAkB,CAAC,MAA0B,EAAE,WAAmC;IACzF,IAAI,CAAC,gBAAgB,EAAE,EAAE,CAAC;QACxB,8BAA8B,EAAE,CAAC;QACjC,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,GAAG,GAAG,GAAG,MAAM,IAAI,EAAE,IAAI,WAAW,EAAE,CAAC;IAC7C,MAAM,QAAQ,GAAG,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC1C,IAAI,QAAQ,KAAK,SAAS;QAAE,OAAO,QAAQ,CAAC;IAE5C,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;IAC1D,IAAI,eAAe,CAAC,IAAI,IAAI,uBAAuB,EAAE,CAAC;QACpD,MAAM,MAAM,GAAG,eAAe,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;QACnD,IAAI,MAAM,KAAK,SAAS;YAAE,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC3D,CAAC;IACD,eAAe,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IAChC,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,gBAAgB;IACvB,OAAO,OAAO,IAAI,KAAK,WAAW,IAAI,OAAO,IAAI,CAAC,SAAS,KAAK,WAAW,CAAC;AAC9E,CAAC;AAED,SAAS,wBAAwB,CAC/B,IAAY,EACZ,WAAmC,EACnC,MAAe;IAEf,MAAM,SAAS,GAAG,kBAAkB,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IAC1D,IAAI,SAAS,KAAK,IAAI;QAAE,OAAO,EAAE,CAAC;IAElC,MAAM,GAAG,GAAkB,EAAE,CAAC;IAC9B,MAAM,iBAAiB,GAAG,WAAW,KAAK,MAAM,CAAC;IACjD,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QAC3C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACzB,MAAM,MAAM,GAAgB,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC;QAC5F,4EAA4E;QAC5E,IAAI,iBAAiB;YAAE,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,KAAK,CAAC;QACpE,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACnB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@flighthq/textsegment",
|
|
3
|
-
"version": "0.5.1-next.
|
|
3
|
+
"version": "0.5.1-next.1404.457d1da",
|
|
4
4
|
"author": "Joshua Granick and other contributors",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -38,9 +38,9 @@
|
|
|
38
38
|
"clean:dist": "tsx ../../scripts/clean-package-dist.ts"
|
|
39
39
|
},
|
|
40
40
|
"dependencies": {
|
|
41
|
-
"@flighthq/entity": "0.5.1-next.
|
|
42
|
-
"@flighthq/log": "0.5.1-next.
|
|
43
|
-
"@flighthq/types": "0.5.1-next.
|
|
41
|
+
"@flighthq/entity": "0.5.1-next.1404.457d1da",
|
|
42
|
+
"@flighthq/log": "0.5.1-next.1404.457d1da",
|
|
43
|
+
"@flighthq/types": "0.5.1-next.1404.457d1da"
|
|
44
44
|
},
|
|
45
45
|
"devDependencies": {
|
|
46
46
|
"typescript": "^5.3.0"
|
|
@@ -4,6 +4,9 @@ import { vi } from 'vitest';
|
|
|
4
4
|
|
|
5
5
|
import { disableTextSegmentGuards, enableTextSegmentGuards } from './enableTextSegmentGuards';
|
|
6
6
|
import { segmentGraphemes } from './textSegment';
|
|
7
|
+
import { createDefaultTextSegmenterBackend } from './textSegmenterBackend';
|
|
8
|
+
|
|
9
|
+
const backend = createDefaultTextSegmenterBackend();
|
|
7
10
|
|
|
8
11
|
let entries: LogEntry[];
|
|
9
12
|
|
|
@@ -24,24 +27,24 @@ describe('disableTextSegmentGuards', () => {
|
|
|
24
27
|
enableTextSegmentGuards();
|
|
25
28
|
disableTextSegmentGuards();
|
|
26
29
|
vi.stubGlobal('Intl', { Segmenter: undefined });
|
|
27
|
-
segmentGraphemes('text');
|
|
30
|
+
segmentGraphemes(backend, 'text');
|
|
28
31
|
expect(entries).toEqual([]);
|
|
29
32
|
});
|
|
30
33
|
});
|
|
31
34
|
|
|
32
35
|
describe('enableTextSegmentGuards', () => {
|
|
33
|
-
it('warns once and names the
|
|
36
|
+
it('warns once and names the provider type when Intl.Segmenter is absent', () => {
|
|
34
37
|
enableTextSegmentGuards();
|
|
35
38
|
vi.stubGlobal('Intl', { Segmenter: undefined });
|
|
36
|
-
segmentGraphemes('first');
|
|
37
|
-
segmentGraphemes('second');
|
|
39
|
+
segmentGraphemes(backend, 'first');
|
|
40
|
+
segmentGraphemes(backend, 'second');
|
|
38
41
|
expect(entries).toHaveLength(1);
|
|
39
|
-
expect(String((entries[0].data as { message?: unknown }).message)).toContain('
|
|
42
|
+
expect(String((entries[0].data as { message?: unknown }).message)).toContain('HostTextSegmenterCapability');
|
|
40
43
|
});
|
|
41
44
|
|
|
42
45
|
it('stays silent when the bundled Intl provider is available', () => {
|
|
43
46
|
enableTextSegmentGuards();
|
|
44
|
-
segmentGraphemes('text');
|
|
47
|
+
segmentGraphemes(backend, 'text');
|
|
45
48
|
expect(entries).toEqual([]);
|
|
46
49
|
});
|
|
47
50
|
});
|
package/src/textSegment.test.ts
CHANGED
|
@@ -1,82 +1,60 @@
|
|
|
1
1
|
import { allocateEntity, finishEntity } from '@flighthq/entity/contract';
|
|
2
|
-
import type {
|
|
3
|
-
import type { HostTextSegmenterProvider, TextSegment, TextSegmentGranularity } from '@flighthq/types/contract';
|
|
2
|
+
import type { HostTextSegmenterCapability, TextSegment, TextSegmentGranularity } from '@flighthq/types/contract';
|
|
4
3
|
|
|
5
4
|
import { segmentGraphemes, segmentSentences, segmentWords } from './textSegment';
|
|
6
|
-
import {
|
|
5
|
+
import { createDefaultTextSegmenterBackend } from './textSegmenterBackend';
|
|
7
6
|
|
|
8
|
-
|
|
7
|
+
const backend = createDefaultTextSegmenterBackend();
|
|
9
8
|
|
|
10
9
|
describe('segmentGraphemes', () => {
|
|
11
|
-
it('accepts an optional explicit segmenter host without breaking the existing signature', () => {
|
|
12
|
-
expectTypeOf(segmentGraphemes).toEqualTypeOf<
|
|
13
|
-
(text: string, locale?: string, hostTextSegmenter?: Readonly<HostTextSegmenterProvider>) => readonly TextSegment[]
|
|
14
|
-
>();
|
|
15
|
-
});
|
|
16
|
-
|
|
17
|
-
it('gives an explicit host precedence over the legacy installed backend', () => {
|
|
18
|
-
setTextSegmenterBackend(taggingBackend('legacy'));
|
|
19
|
-
expect(segmentGraphemes('hi', undefined, segmenterHost(taggingBackend('explicit')).text.segmenter)[0].text).toBe(
|
|
20
|
-
'explicit',
|
|
21
|
-
);
|
|
22
|
-
});
|
|
23
|
-
|
|
24
10
|
it('isolates callers that interleave different explicit hosts', () => {
|
|
25
|
-
const first =
|
|
26
|
-
const second =
|
|
27
|
-
expect(segmentGraphemes('hi'
|
|
28
|
-
expect(segmentGraphemes('hi'
|
|
29
|
-
expect(segmentGraphemes('hi'
|
|
11
|
+
const first = taggingBackend('first');
|
|
12
|
+
const second = taggingBackend('second');
|
|
13
|
+
expect(segmentGraphemes(first, 'hi')[0].text).toBe('first');
|
|
14
|
+
expect(segmentGraphemes(second, 'hi')[0].text).toBe('second');
|
|
15
|
+
expect(segmentGraphemes(first, 'hi')[0].text).toBe('first');
|
|
30
16
|
});
|
|
31
17
|
|
|
32
18
|
it('treats a ZWJ family emoji as one grapheme', () => {
|
|
33
|
-
const segments = segmentGraphemes('aπ¨βπ©βπ§b');
|
|
19
|
+
const segments = segmentGraphemes(backend, 'aπ¨βπ©βπ§b');
|
|
34
20
|
expect(segments.map((s) => s.text)).toEqual(['a', 'π¨βπ©βπ§', 'b']);
|
|
35
21
|
expect(segments[1].start).toBe(1);
|
|
36
22
|
expect(segments[1].end).toBe('aπ¨βπ©βπ§'.length);
|
|
37
23
|
});
|
|
38
24
|
|
|
39
25
|
it('treats a base plus combining mark as one grapheme', () => {
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
expect(segments.map((s) => s.text)).toEqual(['eΜ', 'x']);
|
|
26
|
+
const segments = segmentGraphemes(backend, 'Γ©x');
|
|
27
|
+
expect(segments.map((s) => s.text)).toEqual(['Γ©', 'x']);
|
|
43
28
|
});
|
|
44
29
|
|
|
45
30
|
it('threads the locale argument to the active backend', () => {
|
|
46
31
|
let seenLocale: string | undefined = 'unset';
|
|
47
|
-
const fake = allocateEntity<
|
|
32
|
+
const fake = allocateEntity<HostTextSegmenterCapability>();
|
|
48
33
|
fake.segment = (text: string, _granularity: TextSegmentGranularity, locale?: string): readonly TextSegment[] => {
|
|
49
34
|
seenLocale = locale;
|
|
50
35
|
return [{ start: 0, end: text.length, text }];
|
|
51
36
|
};
|
|
52
|
-
|
|
53
|
-
segmentGraphemes('hi', 'de-DE');
|
|
37
|
+
segmentGraphemes(finishEntity(fake), 'hi', 'de-DE');
|
|
54
38
|
expect(seenLocale).toBe('de-DE');
|
|
55
39
|
});
|
|
56
40
|
});
|
|
57
41
|
|
|
58
|
-
function
|
|
59
|
-
|
|
60
|
-
} {
|
|
61
|
-
return { text: { segmenter } };
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
function taggingBackend(tag: string): HostTextSegmenterProvider {
|
|
65
|
-
const out = allocateEntity<HostTextSegmenterProvider>();
|
|
42
|
+
function taggingBackend(tag: string): HostTextSegmenterCapability {
|
|
43
|
+
const out = allocateEntity<HostTextSegmenterCapability>();
|
|
66
44
|
out.segment = () => [{ start: 0, end: tag.length, text: tag }];
|
|
67
45
|
return finishEntity(out);
|
|
68
46
|
}
|
|
69
47
|
|
|
70
48
|
describe('segmentSentences', () => {
|
|
71
49
|
it('splits into sentences', () => {
|
|
72
|
-
const segments = segmentSentences('Hi. Bye.');
|
|
50
|
+
const segments = segmentSentences(backend, 'Hi. Bye.');
|
|
73
51
|
expect(segments.map((s) => s.text.trim())).toEqual(['Hi.', 'Bye.']);
|
|
74
52
|
});
|
|
75
53
|
});
|
|
76
54
|
|
|
77
55
|
describe('segmentWords', () => {
|
|
78
56
|
it('marks words as word-like and punctuation/whitespace as not', () => {
|
|
79
|
-
const segments = segmentWords('Hello, world.');
|
|
57
|
+
const segments = segmentWords(backend, 'Hello, world.');
|
|
80
58
|
const words = segments.filter((s) => s.isWordLike === true).map((s) => s.text);
|
|
81
59
|
expect(words).toEqual(['Hello', 'world']);
|
|
82
60
|
const comma = segments.find((s) => s.text === ',');
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { allocateEntity, finishEntity } from '@flighthq/entity/contract';
|
|
2
|
-
import type {
|
|
3
|
-
import type { HostTextSegmenterProvider } from '@flighthq/types/contract';
|
|
2
|
+
import type { HostTextSegmenterCapability } from '@flighthq/types/contract';
|
|
4
3
|
import { vi } from 'vitest';
|
|
5
4
|
|
|
6
5
|
import {
|
|
@@ -12,9 +11,9 @@ import {
|
|
|
12
11
|
getPreviousWordBoundary,
|
|
13
12
|
getWordRangeAt,
|
|
14
13
|
} from './textSegmentBoundary';
|
|
15
|
-
import {
|
|
14
|
+
import { createDefaultTextSegmenterBackend, webTextSegmenterBackend } from './textSegmenterBackend';
|
|
16
15
|
|
|
17
|
-
|
|
16
|
+
const backend = createDefaultTextSegmenterBackend();
|
|
18
17
|
|
|
19
18
|
// A ZWJ family emoji is a single grapheme cluster spanning several UTF-16 code units.
|
|
20
19
|
const FAMILY = 'π¨βπ©βπ§';
|
|
@@ -25,31 +24,31 @@ const EMOJI_LEN = EMOJI_TEXT.length;
|
|
|
25
24
|
|
|
26
25
|
describe('getNextGraphemeBoundary', () => {
|
|
27
26
|
it('steps over an emoji cluster in a single move', () => {
|
|
28
|
-
expect(getNextGraphemeBoundary(EMOJI_TEXT, FAMILY_START)).toBe(B_START);
|
|
27
|
+
expect(getNextGraphemeBoundary(backend, EMOJI_TEXT, FAMILY_START)).toBe(B_START);
|
|
29
28
|
});
|
|
30
29
|
|
|
31
30
|
it('advances one plain grapheme from the start', () => {
|
|
32
|
-
expect(getNextGraphemeBoundary(EMOJI_TEXT, 0)).toBe(FAMILY_START);
|
|
31
|
+
expect(getNextGraphemeBoundary(backend, EMOJI_TEXT, 0)).toBe(FAMILY_START);
|
|
33
32
|
});
|
|
34
33
|
|
|
35
34
|
it('clamps at text.length', () => {
|
|
36
|
-
expect(getNextGraphemeBoundary(EMOJI_TEXT, EMOJI_LEN)).toBe(EMOJI_LEN);
|
|
37
|
-
expect(getNextGraphemeBoundary(EMOJI_TEXT, 999)).toBe(EMOJI_LEN);
|
|
35
|
+
expect(getNextGraphemeBoundary(backend, EMOJI_TEXT, EMOJI_LEN)).toBe(EMOJI_LEN);
|
|
36
|
+
expect(getNextGraphemeBoundary(backend, EMOJI_TEXT, 999)).toBe(EMOJI_LEN);
|
|
38
37
|
});
|
|
39
38
|
});
|
|
40
39
|
|
|
41
40
|
describe('getNextSentenceBoundary', () => {
|
|
42
41
|
it('advances to the next sentence start', () => {
|
|
43
|
-
expect(getNextSentenceBoundary('First. Second!', 0)).toBe(7);
|
|
42
|
+
expect(getNextSentenceBoundary(backend, 'First. Second!', 0)).toBe(7);
|
|
44
43
|
});
|
|
45
44
|
|
|
46
45
|
it('clamps at text.length', () => {
|
|
47
|
-
expect(getNextSentenceBoundary('First.', 99)).toBe(6);
|
|
46
|
+
expect(getNextSentenceBoundary(backend, 'First.', 99)).toBe(6);
|
|
48
47
|
});
|
|
49
48
|
|
|
50
49
|
it('stops the bundled iterator without materializing backend TextSegment records', () => {
|
|
51
50
|
const segment = vi.spyOn(webTextSegmenterBackend, 'segment');
|
|
52
|
-
expect(getNextSentenceBoundary('First. Second!', 0)).toBe(7);
|
|
51
|
+
expect(getNextSentenceBoundary(backend, 'First. Second!', 0)).toBe(7);
|
|
53
52
|
expect(segment).not.toHaveBeenCalled();
|
|
54
53
|
segment.mockRestore();
|
|
55
54
|
});
|
|
@@ -57,96 +56,81 @@ describe('getNextSentenceBoundary', () => {
|
|
|
57
56
|
|
|
58
57
|
describe('getNextWordBoundary', () => {
|
|
59
58
|
it('jumps to the end of the current word', () => {
|
|
60
|
-
expect(getNextWordBoundary('foo bar', 0)).toBe(3);
|
|
59
|
+
expect(getNextWordBoundary(backend, 'foo bar', 0)).toBe(3);
|
|
61
60
|
});
|
|
62
61
|
|
|
63
62
|
it('clamps at text.length', () => {
|
|
64
|
-
expect(getNextWordBoundary('foo bar', 7)).toBe(7);
|
|
65
|
-
expect(getNextWordBoundary('foo bar', 42)).toBe(7);
|
|
63
|
+
expect(getNextWordBoundary(backend, 'foo bar', 7)).toBe(7);
|
|
64
|
+
expect(getNextWordBoundary(backend, 'foo bar', 42)).toBe(7);
|
|
66
65
|
});
|
|
67
66
|
});
|
|
68
67
|
|
|
69
68
|
describe('getPreviousGraphemeBoundary', () => {
|
|
70
69
|
it('inverts a step over an emoji cluster', () => {
|
|
71
|
-
expect(getPreviousGraphemeBoundary(EMOJI_TEXT, B_START)).toBe(FAMILY_START);
|
|
70
|
+
expect(getPreviousGraphemeBoundary(backend, EMOJI_TEXT, B_START)).toBe(FAMILY_START);
|
|
72
71
|
});
|
|
73
72
|
|
|
74
73
|
it('clamps at 0', () => {
|
|
75
|
-
expect(getPreviousGraphemeBoundary(EMOJI_TEXT, 0)).toBe(0);
|
|
76
|
-
expect(getPreviousGraphemeBoundary(EMOJI_TEXT, -5)).toBe(0);
|
|
74
|
+
expect(getPreviousGraphemeBoundary(backend, EMOJI_TEXT, 0)).toBe(0);
|
|
75
|
+
expect(getPreviousGraphemeBoundary(backend, EMOJI_TEXT, -5)).toBe(0);
|
|
77
76
|
});
|
|
78
77
|
|
|
79
78
|
it('steps back from the end onto the last cluster start', () => {
|
|
80
|
-
expect(getPreviousGraphemeBoundary(EMOJI_TEXT, EMOJI_LEN)).toBe(B_START);
|
|
79
|
+
expect(getPreviousGraphemeBoundary(backend, EMOJI_TEXT, EMOJI_LEN)).toBe(B_START);
|
|
81
80
|
});
|
|
82
81
|
});
|
|
83
82
|
|
|
84
83
|
describe('getPreviousSentenceBoundary', () => {
|
|
85
84
|
it('steps back to the current sentence start', () => {
|
|
86
|
-
expect(getPreviousSentenceBoundary('First. Second!', 14)).toBe(7);
|
|
85
|
+
expect(getPreviousSentenceBoundary(backend, 'First. Second!', 14)).toBe(7);
|
|
87
86
|
});
|
|
88
87
|
|
|
89
88
|
it('clamps at zero', () => {
|
|
90
|
-
expect(getPreviousSentenceBoundary('First.', -1)).toBe(0);
|
|
89
|
+
expect(getPreviousSentenceBoundary(backend, 'First.', -1)).toBe(0);
|
|
91
90
|
});
|
|
92
91
|
});
|
|
93
92
|
|
|
94
93
|
describe('getPreviousWordBoundary', () => {
|
|
95
94
|
it('jumps to the start of the current word', () => {
|
|
96
|
-
expect(getPreviousWordBoundary('foo bar', 7)).toBe(4);
|
|
95
|
+
expect(getPreviousWordBoundary(backend, 'foo bar', 7)).toBe(4);
|
|
97
96
|
});
|
|
98
97
|
|
|
99
98
|
it('clamps at 0', () => {
|
|
100
|
-
expect(getPreviousWordBoundary('foo bar', 0)).toBe(0);
|
|
99
|
+
expect(getPreviousWordBoundary(backend, 'foo bar', 0)).toBe(0);
|
|
101
100
|
});
|
|
102
101
|
});
|
|
103
102
|
|
|
104
103
|
describe('getWordRangeAt', () => {
|
|
105
104
|
it('threads an explicit host through the boundary helper', () => {
|
|
106
|
-
const explicit = allocateEntity<
|
|
105
|
+
const explicit = allocateEntity<HostTextSegmenterCapability>();
|
|
107
106
|
explicit.segment = (text: string) => [{ start: 0, end: text.length, text, isWordLike: true }];
|
|
108
|
-
|
|
109
|
-
text: { segmenter: finishEntity(explicit) },
|
|
110
|
-
};
|
|
111
|
-
setTextSegmenterBackend(
|
|
112
|
-
(() => {
|
|
113
|
-
const out = allocateEntity<HostTextSegmenterProvider>();
|
|
114
|
-
out.segment = () => [];
|
|
115
|
-
return finishEntity(out);
|
|
116
|
-
})(),
|
|
117
|
-
);
|
|
118
|
-
expect(getWordRangeAt('word', 1, undefined, host.text.segmenter)).toEqual({ start: 0, end: 4 });
|
|
107
|
+
expect(getWordRangeAt(finishEntity(explicit), 'word', 1)).toEqual({ start: 0, end: 4 });
|
|
119
108
|
});
|
|
120
109
|
|
|
121
110
|
it('returns the word range under an index', () => {
|
|
122
|
-
expect(getWordRangeAt('foo bar', 1)).toEqual({ start: 0, end: 3 });
|
|
111
|
+
expect(getWordRangeAt(backend, 'foo bar', 1)).toEqual({ start: 0, end: 3 });
|
|
123
112
|
});
|
|
124
113
|
|
|
125
114
|
it('returns null in whitespace (double-click on a space selects no word)', () => {
|
|
126
|
-
expect(getWordRangeAt('foo bar', 3)).toBeNull();
|
|
115
|
+
expect(getWordRangeAt(backend, 'foo bar', 3)).toBeNull();
|
|
127
116
|
});
|
|
128
117
|
|
|
129
118
|
it('resolves the final word from the trailing boundary', () => {
|
|
130
|
-
expect(getWordRangeAt('foo bar', 7)).toEqual({ start: 4, end: 7 });
|
|
119
|
+
expect(getWordRangeAt(backend, 'foo bar', 7)).toEqual({ start: 4, end: 7 });
|
|
131
120
|
});
|
|
132
121
|
|
|
133
122
|
it('returns null for empty text', () => {
|
|
134
|
-
expect(getWordRangeAt('', 0)).toBeNull();
|
|
123
|
+
expect(getWordRangeAt(backend, '', 0)).toBeNull();
|
|
135
124
|
});
|
|
136
125
|
|
|
137
126
|
it('threads the locale to the active backend', () => {
|
|
138
127
|
let seenLocale: string | undefined = 'unset';
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
};
|
|
146
|
-
return finishEntity(out);
|
|
147
|
-
})(),
|
|
148
|
-
);
|
|
149
|
-
getWordRangeAt('word', 1, 'ja-JP');
|
|
128
|
+
const localeBackend = allocateEntity<HostTextSegmenterCapability>();
|
|
129
|
+
localeBackend.segment = (text: string, _granularity: string, locale: string) => {
|
|
130
|
+
seenLocale = locale;
|
|
131
|
+
return [{ start: 0, end: text.length, text, isWordLike: true }];
|
|
132
|
+
};
|
|
133
|
+
getWordRangeAt(finishEntity(localeBackend), 'word', 1, 'ja-JP');
|
|
150
134
|
expect(seenLocale).toBe('ja-JP');
|
|
151
135
|
});
|
|
152
136
|
});
|
|
@@ -1,18 +1,21 @@
|
|
|
1
1
|
import { describe, expect, it } from 'vitest';
|
|
2
2
|
|
|
3
3
|
import { segmentGraphemes } from './textSegment';
|
|
4
|
+
import { createDefaultTextSegmenterBackend } from './textSegmenterBackend';
|
|
5
|
+
|
|
6
|
+
const backend = createDefaultTextSegmenterBackend();
|
|
4
7
|
|
|
5
8
|
const graphemeFixtures: ReadonlyArray<{ clusters: string[]; text: string }> = [
|
|
6
9
|
{ text: 'πΊπΈπ¨π¦', clusters: ['πΊπΈ', 'π¨π¦'] },
|
|
7
10
|
{ text: 'π¨βπ©βπ§', clusters: ['π¨βπ©βπ§'] },
|
|
8
11
|
{ text: '\r\n', clusters: ['\r\n'] },
|
|
9
|
-
{ text: '
|
|
12
|
+
{ text: 'κ°', clusters: ['κ°'] },
|
|
10
13
|
];
|
|
11
14
|
|
|
12
15
|
describe('text segment conformance fixtures', () => {
|
|
13
16
|
it('covers regional indicators, ZWJ emoji, CRLF, and Hangul jamo', () => {
|
|
14
17
|
for (const fixture of graphemeFixtures) {
|
|
15
|
-
expect(segmentGraphemes(fixture.text).map((segment) => segment.text)).toEqual(fixture.clusters);
|
|
18
|
+
expect(segmentGraphemes(backend, fixture.text).map((segment) => segment.text)).toEqual(fixture.clusters);
|
|
16
19
|
}
|
|
17
20
|
});
|
|
18
21
|
});
|
|
@@ -1,18 +1,17 @@
|
|
|
1
1
|
import { allocateEntity, finishEntity } from '@flighthq/entity/contract';
|
|
2
2
|
import type { EntityRuntimeKey } from '@flighthq/types/contract';
|
|
3
|
-
import type {
|
|
3
|
+
import type { HostTextSegmenterCapability, TextSegment, TextSegmentGranularity } from '@flighthq/types/contract';
|
|
4
4
|
import { vi } from 'vitest';
|
|
5
5
|
|
|
6
6
|
import {
|
|
7
|
+
createDefaultTextSegmenterBackend,
|
|
7
8
|
createWebTextSegmenterBackend,
|
|
8
9
|
explainTextSegmenterBackend,
|
|
9
|
-
getTextSegmenterBackend,
|
|
10
10
|
initializeWebTextSegmenterBackend,
|
|
11
|
-
setTextSegmenterBackend,
|
|
12
11
|
webTextSegmenterBackend,
|
|
13
12
|
} from './textSegmenterBackend';
|
|
14
13
|
|
|
15
|
-
interface RecordingBackend extends
|
|
14
|
+
interface RecordingBackend extends HostTextSegmenterCapability {
|
|
16
15
|
calls: Array<{ text: string; granularity: TextSegmentGranularity; locale: string | undefined }>;
|
|
17
16
|
}
|
|
18
17
|
|
|
@@ -28,10 +27,16 @@ function recordingBackend(): RecordingBackend {
|
|
|
28
27
|
}
|
|
29
28
|
|
|
30
29
|
afterEach(() => {
|
|
31
|
-
setTextSegmenterBackend(null);
|
|
32
30
|
vi.unstubAllGlobals();
|
|
33
31
|
});
|
|
34
32
|
|
|
33
|
+
describe('createDefaultTextSegmenterBackend', () => {
|
|
34
|
+
it('returns a web segmenter backend', () => {
|
|
35
|
+
const backend = createDefaultTextSegmenterBackend();
|
|
36
|
+
expect(backend.segment('ab', 'grapheme').length).toBe(2);
|
|
37
|
+
});
|
|
38
|
+
});
|
|
39
|
+
|
|
35
40
|
describe('createWebTextSegmenterBackend', () => {
|
|
36
41
|
it('segments a ZWJ family emoji as a single grapheme', () => {
|
|
37
42
|
const segments = createWebTextSegmenterBackend().segment('aπ¨βπ©βπ§b', 'grapheme');
|
|
@@ -58,7 +63,7 @@ describe('createWebTextSegmenterBackend', () => {
|
|
|
58
63
|
|
|
59
64
|
describe('explainTextSegmenterBackend', () => {
|
|
60
65
|
it('reports the bundled Intl provider as available', () => {
|
|
61
|
-
expect(explainTextSegmenterBackend()).toEqual({
|
|
66
|
+
expect(explainTextSegmenterBackend(webTextSegmenterBackend)).toEqual({
|
|
62
67
|
available: true,
|
|
63
68
|
backend: 'web-intl',
|
|
64
69
|
intlSegmenterAvailable: true,
|
|
@@ -66,7 +71,7 @@ describe('explainTextSegmenterBackend', () => {
|
|
|
66
71
|
});
|
|
67
72
|
|
|
68
73
|
it('identifies a custom provider independently of Intl availability', () => {
|
|
69
|
-
expect(explainTextSegmenterBackend(
|
|
74
|
+
expect(explainTextSegmenterBackend(recordingBackend())).toEqual({
|
|
70
75
|
available: true,
|
|
71
76
|
backend: 'custom',
|
|
72
77
|
intlSegmenterAvailable: true,
|
|
@@ -75,7 +80,7 @@ describe('explainTextSegmenterBackend', () => {
|
|
|
75
80
|
|
|
76
81
|
it('reports the bundled provider unavailable when the runtime primitive is absent', () => {
|
|
77
82
|
vi.stubGlobal('Intl', { Segmenter: undefined });
|
|
78
|
-
expect(explainTextSegmenterBackend()).toEqual({
|
|
83
|
+
expect(explainTextSegmenterBackend(webTextSegmenterBackend)).toEqual({
|
|
79
84
|
available: false,
|
|
80
85
|
backend: 'web-intl',
|
|
81
86
|
intlSegmenterAvailable: false,
|
|
@@ -83,51 +88,12 @@ describe('explainTextSegmenterBackend', () => {
|
|
|
83
88
|
});
|
|
84
89
|
});
|
|
85
90
|
|
|
86
|
-
describe('getTextSegmenterBackend', () => {
|
|
87
|
-
it('returns an explicit host provider ahead of the legacy backend', () => {
|
|
88
|
-
const legacy = recordingBackend();
|
|
89
|
-
const explicit = recordingBackend();
|
|
90
|
-
const host: { readonly text: { readonly segmenter: HostTextSegmenterProvider } } = {
|
|
91
|
-
text: { segmenter: explicit },
|
|
92
|
-
};
|
|
93
|
-
setTextSegmenterBackend(legacy);
|
|
94
|
-
expect(getTextSegmenterBackend(host.text.segmenter)).toBe(explicit);
|
|
95
|
-
});
|
|
96
|
-
|
|
97
|
-
it('falls back to the stable bundled web backend when none is registered', () => {
|
|
98
|
-
const backend = getTextSegmenterBackend();
|
|
99
|
-
expect(backend).toBe(webTextSegmenterBackend);
|
|
100
|
-
expect(backend.segment('ab', 'grapheme').length).toBe(2);
|
|
101
|
-
});
|
|
102
|
-
|
|
103
|
-
it('returns the registered backend', () => {
|
|
104
|
-
const backend = recordingBackend();
|
|
105
|
-
setTextSegmenterBackend(backend);
|
|
106
|
-
expect(getTextSegmenterBackend()).toBe(backend);
|
|
107
|
-
});
|
|
108
|
-
});
|
|
109
|
-
|
|
110
91
|
describe('initializeWebTextSegmenterBackend', () => {
|
|
111
92
|
it('is the construction initializer of createWebTextSegmenterBackend', () => {
|
|
112
93
|
expect(typeof initializeWebTextSegmenterBackend).toBe('function');
|
|
113
94
|
});
|
|
114
95
|
});
|
|
115
96
|
|
|
116
|
-
describe('setTextSegmenterBackend', () => {
|
|
117
|
-
it('routes segmentation through the installed backend', () => {
|
|
118
|
-
const backend = recordingBackend();
|
|
119
|
-
setTextSegmenterBackend(backend);
|
|
120
|
-
getTextSegmenterBackend().segment('hello', 'sentence', 'fr');
|
|
121
|
-
expect(backend.calls).toEqual([{ text: 'hello', granularity: 'sentence', locale: 'fr' }]);
|
|
122
|
-
});
|
|
123
|
-
|
|
124
|
-
it('clears back to the stable bundled web default when passed null', () => {
|
|
125
|
-
setTextSegmenterBackend(recordingBackend());
|
|
126
|
-
setTextSegmenterBackend(null);
|
|
127
|
-
// The web default segments a ZWJ cluster as one grapheme; a recording backend would not.
|
|
128
|
-
expect(getTextSegmenterBackend().segment('aπ¨βπ©βπ§b', 'grapheme').length).toBe(3);
|
|
129
|
-
});
|
|
130
|
-
});
|
|
131
97
|
describe('webTextSegmenterBackend', () => {
|
|
132
98
|
it('provides the stable bundled Intl fallback', () => {
|
|
133
99
|
expect(webTextSegmenterBackend.segment('aπ¨βπ©βπ§b', 'grapheme').map((segment) => segment.text)).toEqual([
|
|
@@ -137,9 +103,3 @@ describe('webTextSegmenterBackend', () => {
|
|
|
137
103
|
]);
|
|
138
104
|
});
|
|
139
105
|
});
|
|
140
|
-
|
|
141
|
-
function segmenterHost(segmenter: HostTextSegmenterProvider): {
|
|
142
|
-
readonly text: { readonly segmenter: HostTextSegmenterProvider };
|
|
143
|
-
} {
|
|
144
|
-
return { text: { segmenter } };
|
|
145
|
-
}
|