@flighthq/textsegment 0.5.1-next.704.19c91ec → 0.5.1-next.729.2aeb707
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/textSegment.d.ts +4 -4
- package/dist/textSegment.d.ts.map +1 -1
- package/dist/textSegment.js +6 -6
- package/dist/textSegment.js.map +1 -1
- package/dist/textSegmentBoundary.d.ts +6 -6
- package/dist/textSegmentBoundary.d.ts.map +1 -1
- package/dist/textSegmentBoundary.js +10 -10
- package/dist/textSegmentBoundary.js.map +1 -1
- package/dist/textSegmenterBackend.d.ts +10 -2
- package/dist/textSegmenterBackend.d.ts.map +1 -1
- package/dist/textSegmenterBackend.js +17 -12
- package/dist/textSegmenterBackend.js.map +1 -1
- package/package.json +2 -2
- package/src/textSegment.test.ts +35 -1
- package/src/textSegmentBoundary.test.ts +11 -0
- package/src/textSegmenterBackend.test.ts +28 -4
package/README.md
CHANGED
|
@@ -13,5 +13,5 @@ Import the supported application-facing API from `@flighthq/textsegment`. The `@
|
|
|
13
13
|
This package is part of the locked-version Flight SDK graph. Applications may instead install and import `@flighthq/sdk` when package-level tree shaking is sufficient.
|
|
14
14
|
|
|
15
15
|
- [Flight project](https://github.com/flighthq/flight)
|
|
16
|
-
- [Source for @flighthq/textsegment@0.5.1-next.
|
|
17
|
-
- [License](https://github.com/flighthq/flight/blob/
|
|
16
|
+
- [Source for @flighthq/textsegment@0.5.1-next.729.2aeb707](https://github.com/flighthq/flight/tree/2aeb707259007d61c5aaf2e31fe47447a260fe2a/packages/textsegment)
|
|
17
|
+
- [License](https://github.com/flighthq/flight/blob/2aeb707259007d61c5aaf2e31fe47447a260fe2a/LICENSE.md)
|
package/dist/index.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export { getNextGraphemeBoundary, getNextWordBoundary, getPreviousGraphemeBoundary, getPreviousWordBoundary, getWordRangeAt, segmentGraphemes, segmentSentences, segmentWords, } from './contract';
|
|
1
|
+
export { getNextGraphemeBoundary, getNextWordBoundary, getPreviousGraphemeBoundary, getPreviousWordBoundary, getWordRangeAt, segmentGraphemes, segmentSentences, segmentWords, webTextSegmenterBackend, } from './contract';
|
|
2
2
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,uBAAuB,EACvB,mBAAmB,EACnB,2BAA2B,EAC3B,uBAAuB,EACvB,cAAc,EACd,gBAAgB,EAChB,gBAAgB,EAChB,YAAY,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,uBAAuB,EACvB,mBAAmB,EACnB,2BAA2B,EAC3B,uBAAuB,EACvB,cAAc,EACd,gBAAgB,EAChB,gBAAgB,EAChB,YAAY,EACZ,uBAAuB,GACxB,MAAM,YAAY,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export { getNextGraphemeBoundary, getNextWordBoundary, getPreviousGraphemeBoundary, getPreviousWordBoundary, getWordRangeAt, segmentGraphemes, segmentSentences, segmentWords, } from './contract';
|
|
1
|
+
export { getNextGraphemeBoundary, getNextWordBoundary, getPreviousGraphemeBoundary, getPreviousWordBoundary, getWordRangeAt, segmentGraphemes, segmentSentences, segmentWords, webTextSegmenterBackend, } from './contract';
|
|
2
2
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,uBAAuB,EACvB,mBAAmB,EACnB,2BAA2B,EAC3B,uBAAuB,EACvB,cAAc,EACd,gBAAgB,EAChB,gBAAgB,EAChB,YAAY,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,uBAAuB,EACvB,mBAAmB,EACnB,2BAA2B,EAC3B,uBAAuB,EACvB,cAAc,EACd,gBAAgB,EAChB,gBAAgB,EAChB,YAAY,EACZ,uBAAuB,GACxB,MAAM,YAAY,CAAC"}
|
package/dist/textSegment.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import type { TextSegment } from '@flighthq/types/contract';
|
|
2
|
-
export declare function segmentGraphemes(text: string, locale?: string): readonly TextSegment[];
|
|
3
|
-
export declare function segmentSentences(text: string, locale?: string): readonly TextSegment[];
|
|
4
|
-
export declare function segmentWords(text: string, locale?: string): readonly TextSegment[];
|
|
1
|
+
import type { HasTextSegmenter, TextSegment } from '@flighthq/types/contract';
|
|
2
|
+
export declare function segmentGraphemes(text: string, locale?: string, host?: HasTextSegmenter): readonly TextSegment[];
|
|
3
|
+
export declare function segmentSentences(text: string, locale?: string, host?: HasTextSegmenter): readonly TextSegment[];
|
|
4
|
+
export declare function segmentWords(text: string, locale?: string, host?: HasTextSegmenter): readonly TextSegment[];
|
|
5
5
|
//# sourceMappingURL=textSegment.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"textSegment.d.ts","sourceRoot":"","sources":["../src/textSegment.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;
|
|
1
|
+
{"version":3,"file":"textSegment.d.ts","sourceRoot":"","sources":["../src/textSegment.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAQ9E,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,gBAAgB,GAAG,SAAS,WAAW,EAAE,CAE/G;AAID,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,gBAAgB,GAAG,SAAS,WAAW,EAAE,CAE/G;AAKD,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,gBAAgB,GAAG,SAAS,WAAW,EAAE,CAE3G"}
|
package/dist/textSegment.js
CHANGED
|
@@ -3,18 +3,18 @@ import { getTextSegmenterBackend } from './textSegmenterBackend';
|
|
|
3
3
|
// user-perceived character: an emoji ZWJ sequence or a base-plus-combining-mark cluster is a single
|
|
4
4
|
// segment, not its constituent code points. `locale` is threaded to the backend for the rare
|
|
5
5
|
// locale-tailored cases. isWordLike is absent on grapheme segments.
|
|
6
|
-
export function segmentGraphemes(text, locale) {
|
|
7
|
-
return getTextSegmenterBackend().segment(text, 'grapheme', locale);
|
|
6
|
+
export function segmentGraphemes(text, locale, host) {
|
|
7
|
+
return getTextSegmenterBackend(host).segment(text, 'grapheme', locale);
|
|
8
8
|
}
|
|
9
9
|
// Enumerates the sentences of `text` (UAX #29) via the active backend, in order and covering the
|
|
10
10
|
// whole string. isWordLike is absent on sentence segments.
|
|
11
|
-
export function segmentSentences(text, locale) {
|
|
12
|
-
return getTextSegmenterBackend().segment(text, 'sentence', locale);
|
|
11
|
+
export function segmentSentences(text, locale, host) {
|
|
12
|
+
return getTextSegmenterBackend(host).segment(text, 'sentence', locale);
|
|
13
13
|
}
|
|
14
14
|
// Enumerates the words of `text` (UAX #29) via the active backend. Word segments carry isWordLike:
|
|
15
15
|
// true for letters/numbers (a real word), false for the punctuation and whitespace runs between
|
|
16
16
|
// them. Filter on isWordLike to keep only words. `locale` is threaded to the backend.
|
|
17
|
-
export function segmentWords(text, locale) {
|
|
18
|
-
return getTextSegmenterBackend().segment(text, 'word', locale);
|
|
17
|
+
export function segmentWords(text, locale, host) {
|
|
18
|
+
return getTextSegmenterBackend(host).segment(text, 'word', locale);
|
|
19
19
|
}
|
|
20
20
|
//# sourceMappingURL=textSegment.js.map
|
package/dist/textSegment.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"textSegment.js","sourceRoot":"","sources":["../src/textSegment.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AAEjE,mGAAmG;AACnG,oGAAoG;AACpG,6FAA6F;AAC7F,oEAAoE;AACpE,MAAM,UAAU,gBAAgB,CAAC,IAAY,EAAE,MAAe;
|
|
1
|
+
{"version":3,"file":"textSegment.js","sourceRoot":"","sources":["../src/textSegment.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AAEjE,mGAAmG;AACnG,oGAAoG;AACpG,6FAA6F;AAC7F,oEAAoE;AACpE,MAAM,UAAU,gBAAgB,CAAC,IAAY,EAAE,MAAe,EAAE,IAAuB;IACrF,OAAO,uBAAuB,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;AACzE,CAAC;AAED,iGAAiG;AACjG,2DAA2D;AAC3D,MAAM,UAAU,gBAAgB,CAAC,IAAY,EAAE,MAAe,EAAE,IAAuB;IACrF,OAAO,uBAAuB,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;AACzE,CAAC;AAED,mGAAmG;AACnG,gGAAgG;AAChG,sFAAsF;AACtF,MAAM,UAAU,YAAY,CAAC,IAAY,EAAE,MAAe,EAAE,IAAuB;IACjF,OAAO,uBAAuB,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AACrE,CAAC"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import type { TextSegmentRange } from '@flighthq/types/contract';
|
|
2
|
-
export declare function getNextGraphemeBoundary(text: string, index: number, locale?: string): number;
|
|
3
|
-
export declare function getNextWordBoundary(text: string, index: number, locale?: string): number;
|
|
4
|
-
export declare function getPreviousGraphemeBoundary(text: string, index: number, locale?: string): number;
|
|
5
|
-
export declare function getPreviousWordBoundary(text: string, index: number, locale?: string): number;
|
|
6
|
-
export declare function getWordRangeAt(text: string, index: number, locale?: string): TextSegmentRange | null;
|
|
1
|
+
import type { HasTextSegmenter, TextSegmentRange } from '@flighthq/types/contract';
|
|
2
|
+
export declare function getNextGraphemeBoundary(text: string, index: number, locale?: string, host?: HasTextSegmenter): number;
|
|
3
|
+
export declare function getNextWordBoundary(text: string, index: number, locale?: string, host?: HasTextSegmenter): number;
|
|
4
|
+
export declare function getPreviousGraphemeBoundary(text: string, index: number, locale?: string, host?: HasTextSegmenter): number;
|
|
5
|
+
export declare function getPreviousWordBoundary(text: string, index: number, locale?: string, host?: HasTextSegmenter): number;
|
|
6
|
+
export declare function getWordRangeAt(text: string, index: number, locale?: string, host?: HasTextSegmenter): TextSegmentRange | null;
|
|
7
7
|
//# sourceMappingURL=textSegmentBoundary.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"textSegmentBoundary.d.ts","sourceRoot":"","sources":["../src/textSegmentBoundary.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAe,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;
|
|
1
|
+
{"version":3,"file":"textSegmentBoundary.d.ts","sourceRoot":"","sources":["../src/textSegmentBoundary.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAe,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAOhG,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,gBAAgB,GAAG,MAAM,CAErH;AAKD,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,gBAAgB,GAAG,MAAM,CAEjH;AAKD,wBAAgB,2BAA2B,CACzC,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,EACb,MAAM,CAAC,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,gBAAgB,GACtB,MAAM,CAER;AAID,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,gBAAgB,GAAG,MAAM,CAErH;AAMD,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,EACb,MAAM,CAAC,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,gBAAgB,GACtB,gBAAgB,GAAG,IAAI,CAazB"}
|
|
@@ -2,38 +2,38 @@ import { segmentGraphemes, segmentWords } from './textSegment';
|
|
|
2
2
|
// Returns the next grapheme-cluster boundary at or after `index` — the offset a caret lands on when
|
|
3
3
|
// stepping right by one user-perceived character, so an emoji or combining sequence is crossed in a
|
|
4
4
|
// single step. `index` is clamped into [0, text.length]; at or past the end it returns text.length.
|
|
5
|
-
export function getNextGraphemeBoundary(text, index, locale) {
|
|
6
|
-
return nextSegmentBoundary(segmentGraphemes(text, locale), index, text.length);
|
|
5
|
+
export function getNextGraphemeBoundary(text, index, locale, host) {
|
|
6
|
+
return nextSegmentBoundary(segmentGraphemes(text, locale, host), index, text.length);
|
|
7
7
|
}
|
|
8
8
|
// Returns the next word boundary at or after `index` — the offset a caret lands on when jumping right
|
|
9
9
|
// by one word (Ctrl/Alt-Right). `index` is clamped into [0, text.length]; at or past the end it
|
|
10
10
|
// returns text.length.
|
|
11
|
-
export function getNextWordBoundary(text, index, locale) {
|
|
12
|
-
return nextSegmentBoundary(segmentWords(text, locale), index, text.length);
|
|
11
|
+
export function getNextWordBoundary(text, index, locale, host) {
|
|
12
|
+
return nextSegmentBoundary(segmentWords(text, locale, host), index, text.length);
|
|
13
13
|
}
|
|
14
14
|
// Returns the previous grapheme-cluster boundary at or before `index` — the offset a caret lands on
|
|
15
15
|
// when stepping left by one user-perceived character. `index` is clamped into [0, text.length]; at or
|
|
16
16
|
// before the start it returns 0.
|
|
17
|
-
export function getPreviousGraphemeBoundary(text, index, locale) {
|
|
18
|
-
return previousSegmentBoundary(segmentGraphemes(text, locale), index);
|
|
17
|
+
export function getPreviousGraphemeBoundary(text, index, locale, host) {
|
|
18
|
+
return previousSegmentBoundary(segmentGraphemes(text, locale, host), index);
|
|
19
19
|
}
|
|
20
20
|
// Returns the previous word boundary at or before `index` — the offset a caret lands on when jumping
|
|
21
21
|
// left by one word. `index` is clamped into [0, text.length]; at or before the start it returns 0.
|
|
22
|
-
export function getPreviousWordBoundary(text, index, locale) {
|
|
23
|
-
return previousSegmentBoundary(segmentWords(text, locale), index);
|
|
22
|
+
export function getPreviousWordBoundary(text, index, locale, host) {
|
|
23
|
+
return previousSegmentBoundary(segmentWords(text, locale, host), index);
|
|
24
24
|
}
|
|
25
25
|
// Returns the word-like segment's range covering `index`, for double-click / word-select. `index` is
|
|
26
26
|
// clamped into [0, text.length]; at the very end it resolves against the last character. Returns null
|
|
27
27
|
// when the covered segment is not word-like (whitespace or punctuation) or the text is empty — the
|
|
28
28
|
// caller should then select nothing, mirroring how a double-click on a space selects no word.
|
|
29
|
-
export function getWordRangeAt(text, index, locale) {
|
|
29
|
+
export function getWordRangeAt(text, index, locale, host) {
|
|
30
30
|
if (text.length === 0)
|
|
31
31
|
return null;
|
|
32
32
|
const clamped = clampIndex(index, text.length);
|
|
33
33
|
// At the trailing boundary there is no segment starting at text.length; resolve against the last
|
|
34
34
|
// character so a double-click at the end still selects the final word.
|
|
35
35
|
const lookup = clamped === text.length ? text.length - 1 : clamped;
|
|
36
|
-
const segments = segmentWords(text, locale);
|
|
36
|
+
const segments = segmentWords(text, locale, host);
|
|
37
37
|
for (const segment of segments) {
|
|
38
38
|
if (lookup >= segment.start && lookup < segment.end) {
|
|
39
39
|
return segment.isWordLike === true ? { start: segment.start, end: segment.end } : null;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"textSegmentBoundary.js","sourceRoot":"","sources":["../src/textSegmentBoundary.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAE/D,oGAAoG;AACpG,oGAAoG;AACpG,oGAAoG;AACpG,MAAM,UAAU,uBAAuB,CAAC,IAAY,EAAE,KAAa,EAAE,MAAe;
|
|
1
|
+
{"version":3,"file":"textSegmentBoundary.js","sourceRoot":"","sources":["../src/textSegmentBoundary.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAE/D,oGAAoG;AACpG,oGAAoG;AACpG,oGAAoG;AACpG,MAAM,UAAU,uBAAuB,CAAC,IAAY,EAAE,KAAa,EAAE,MAAe,EAAE,IAAuB;IAC3G,OAAO,mBAAmB,CAAC,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;AACvF,CAAC;AAED,sGAAsG;AACtG,gGAAgG;AAChG,uBAAuB;AACvB,MAAM,UAAU,mBAAmB,CAAC,IAAY,EAAE,KAAa,EAAE,MAAe,EAAE,IAAuB;IACvG,OAAO,mBAAmB,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;AACnF,CAAC;AAED,oGAAoG;AACpG,sGAAsG;AACtG,iCAAiC;AACjC,MAAM,UAAU,2BAA2B,CACzC,IAAY,EACZ,KAAa,EACb,MAAe,EACf,IAAuB;IAEvB,OAAO,uBAAuB,CAAC,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,KAAK,CAAC,CAAC;AAC9E,CAAC;AAED,qGAAqG;AACrG,mGAAmG;AACnG,MAAM,UAAU,uBAAuB,CAAC,IAAY,EAAE,KAAa,EAAE,MAAe,EAAE,IAAuB;IAC3G,OAAO,uBAAuB,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,KAAK,CAAC,CAAC;AAC1E,CAAC;AAED,qGAAqG;AACrG,sGAAsG;AACtG,mGAAmG;AACnG,8FAA8F;AAC9F,MAAM,UAAU,cAAc,CAC5B,IAAY,EACZ,KAAa,EACb,MAAe,EACf,IAAuB;IAEvB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACnC,MAAM,OAAO,GAAG,UAAU,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IAC/C,iGAAiG;IACjG,uEAAuE;IACvE,MAAM,MAAM,GAAG,OAAO,KAAK,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;IACnE,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;IAClD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,IAAI,MAAM,IAAI,OAAO,CAAC,KAAK,IAAI,MAAM,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;YACpD,OAAO,OAAO,CAAC,UAAU,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QACzF,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,oGAAoG;AACpG,uGAAuG;AACvG,SAAS,UAAU,CAAC,KAAa,EAAE,MAAc;IAC/C,IAAI,KAAK,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IACxB,IAAI,KAAK,GAAG,MAAM;QAAE,OAAO,MAAM,CAAC;IAClC,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,mBAAmB,CAAC,QAAgC,EAAE,KAAa,EAAE,MAAc;IAC1F,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACvC,IAAI,IAAI,IAAI,MAAM;QAAE,OAAO,MAAM,CAAC;IAClC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,IAAI,OAAO,CAAC,KAAK,GAAG,IAAI;YAAE,OAAO,OAAO,CAAC,KAAK,CAAC;IACjD,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,uBAAuB,CAAC,QAAgC,EAAE,KAAa;IAC9E,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IAC7E,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACvC,IAAI,IAAI,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IACxB,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,IAAI,OAAO,CAAC,KAAK,IAAI,IAAI;YAAE,MAAM;QACjC,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC;IAC3B,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC"}
|
|
@@ -1,5 +1,13 @@
|
|
|
1
|
-
import type { TextSegmenterBackend } from '@flighthq/types/contract';
|
|
1
|
+
import type { HasTextSegmenter, TextSegmenterBackend } from '@flighthq/types/contract';
|
|
2
2
|
export declare function createWebTextSegmenterBackend(): TextSegmenterBackend;
|
|
3
|
-
export declare
|
|
3
|
+
export declare const webTextSegmenterBackend: TextSegmenterBackend;
|
|
4
|
+
export declare function getTextSegmenterBackend(host?: HasTextSegmenter): TextSegmenterBackend;
|
|
5
|
+
/**
|
|
6
|
+
* Installs the backend used by calls that omit an explicit host; pass null to restore the stable
|
|
7
|
+
* bundled web default.
|
|
8
|
+
*
|
|
9
|
+
* @deprecated Pass a HasTextSegmenter to the segmentation or boundary operation. Retained for
|
|
10
|
+
* source compatibility until the legacy global path is removed.
|
|
11
|
+
*/
|
|
4
12
|
export declare function setTextSegmenterBackend(backend: TextSegmenterBackend | null): void;
|
|
5
13
|
//# sourceMappingURL=textSegmenterBackend.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"textSegmenterBackend.d.ts","sourceRoot":"","sources":["../src/textSegmenterBackend.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"textSegmenterBackend.d.ts","sourceRoot":"","sources":["../src/textSegmenterBackend.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,gBAAgB,EAGhB,oBAAoB,EACrB,MAAM,0BAA0B,CAAC;AAQlC,wBAAgB,6BAA6B,IAAI,oBAAoB,CAEpE;AAID,eAAO,MAAM,uBAAuB,EAAE,oBAAsD,CAAC;AAK7F,wBAAgB,uBAAuB,CAAC,IAAI,CAAC,EAAE,gBAAgB,GAAG,oBAAoB,CAErF;AAED;;;;;;GAMG;AACH,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,oBAAoB,GAAG,IAAI,GAAG,IAAI,CAElF"}
|
|
@@ -2,22 +2,27 @@
|
|
|
2
2
|
// Unicode tables — the engine already carries them — so the common path costs nothing in bundle
|
|
3
3
|
// weight. Intl.Segmenter instances are cached by (locale, granularity) because constructing one is
|
|
4
4
|
// expensive relative to a single segment() call. Where Intl.Segmenter is absent (an old or headless
|
|
5
|
-
// engine), segment() returns [] rather than throwing;
|
|
6
|
-
//
|
|
5
|
+
// engine), segment() returns [] rather than throwing; compose a from-scratch UAX #29 backend into
|
|
6
|
+
// the host for those environments.
|
|
7
7
|
export function createWebTextSegmenterBackend() {
|
|
8
8
|
return { segment: segmentWithIntlSegmenter };
|
|
9
9
|
}
|
|
10
|
-
//
|
|
11
|
-
//
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
10
|
+
// Stable bundled web provider. Hosts can import this directly when composing their explicit
|
|
11
|
+
// capability object; callers that omit a host receive it as the final fallback.
|
|
12
|
+
export const webTextSegmenterBackend = createWebTextSegmenterBackend();
|
|
13
|
+
// Returns the explicit host's provider when supplied, then the legacy installed backend, and
|
|
14
|
+
// finally the bundled web provider. The explicit dependency always wins, so independent callers
|
|
15
|
+
// can interleave different hosts without mutating shared capability state.
|
|
16
|
+
export function getTextSegmenterBackend(host) {
|
|
17
|
+
return host?.text.segmenter ?? _backend ?? webTextSegmenterBackend;
|
|
17
18
|
}
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
19
|
+
/**
|
|
20
|
+
* Installs the backend used by calls that omit an explicit host; pass null to restore the stable
|
|
21
|
+
* bundled web default.
|
|
22
|
+
*
|
|
23
|
+
* @deprecated Pass a HasTextSegmenter to the segmentation or boundary operation. Retained for
|
|
24
|
+
* source compatibility until the legacy global path is removed.
|
|
25
|
+
*/
|
|
21
26
|
export function setTextSegmenterBackend(backend) {
|
|
22
27
|
_backend = backend;
|
|
23
28
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"textSegmenterBackend.js","sourceRoot":"","sources":["../src/textSegmenterBackend.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"textSegmenterBackend.js","sourceRoot":"","sources":["../src/textSegmenterBackend.ts"],"names":[],"mappings":"AAOA,gGAAgG;AAChG,gGAAgG;AAChG,mGAAmG;AACnG,oGAAoG;AACpG,kGAAkG;AAClG,mCAAmC;AACnC,MAAM,UAAU,6BAA6B;IAC3C,OAAO,EAAE,OAAO,EAAE,wBAAwB,EAAE,CAAC;AAC/C,CAAC;AAED,4FAA4F;AAC5F,gFAAgF;AAChF,MAAM,CAAC,MAAM,uBAAuB,GAAyB,6BAA6B,EAAE,CAAC;AAE7F,6FAA6F;AAC7F,gGAAgG;AAChG,2EAA2E;AAC3E,MAAM,UAAU,uBAAuB,CAAC,IAAuB;IAC7D,OAAO,IAAI,EAAE,IAAI,CAAC,SAAS,IAAI,QAAQ,IAAI,uBAAuB,CAAC;AACrE,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,uBAAuB,CAAC,OAAoC;IAC1E,QAAQ,GAAG,OAAO,CAAC;AACrB,CAAC;AAED,IAAI,QAAQ,GAAgC,IAAI,CAAC;AAEjD,qGAAqG;AACrG,oGAAoG;AACpG,mDAAmD;AACnD,MAAM,eAAe,GAAG,IAAI,GAAG,EAA0B,CAAC;AAC1D,MAAM,uBAAuB,GAAG,EAAE,CAAC;AAEnC,SAAS,kBAAkB,CAAC,MAA0B,EAAE,WAAmC;IACzF,IAAI,OAAO,IAAI,KAAK,WAAW,IAAI,OAAO,IAAI,CAAC,SAAS,KAAK,WAAW;QAAE,OAAO,IAAI,CAAC;IACtF,MAAM,GAAG,GAAG,GAAG,MAAM,IAAI,EAAE,IAAI,WAAW,EAAE,CAAC;IAC7C,MAAM,QAAQ,GAAG,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC1C,IAAI,QAAQ,KAAK,SAAS;QAAE,OAAO,QAAQ,CAAC;IAE5C,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;IAC1D,IAAI,eAAe,CAAC,IAAI,IAAI,uBAAuB,EAAE,CAAC;QACpD,MAAM,MAAM,GAAG,eAAe,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;QACnD,IAAI,MAAM,KAAK,SAAS;YAAE,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC3D,CAAC;IACD,eAAe,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IAChC,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,wBAAwB,CAC/B,IAAY,EACZ,WAAmC,EACnC,MAAe;IAEf,MAAM,SAAS,GAAG,kBAAkB,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IAC1D,IAAI,SAAS,KAAK,IAAI;QAAE,OAAO,EAAE,CAAC;IAElC,MAAM,GAAG,GAAkB,EAAE,CAAC;IAC9B,MAAM,iBAAiB,GAAG,WAAW,KAAK,MAAM,CAAC;IACjD,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QAC3C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACzB,MAAM,MAAM,GAAgB,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC;QAC5F,4EAA4E;QAC5E,IAAI,iBAAiB;YAAE,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,KAAK,CAAC;QACpE,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACnB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@flighthq/textsegment",
|
|
3
|
-
"version": "0.5.1-next.
|
|
3
|
+
"version": "0.5.1-next.729.2aeb707",
|
|
4
4
|
"author": "Joshua Granick and other contributors",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -38,7 +38,7 @@
|
|
|
38
38
|
"clean:dist": "tsx ../../scripts/clean-package-dist.ts"
|
|
39
39
|
},
|
|
40
40
|
"dependencies": {
|
|
41
|
-
"@flighthq/types": "0.5.1-next.
|
|
41
|
+
"@flighthq/types": "0.5.1-next.729.2aeb707"
|
|
42
42
|
},
|
|
43
43
|
"devDependencies": {
|
|
44
44
|
"typescript": "^5.3.0"
|
package/src/textSegment.test.ts
CHANGED
|
@@ -1,4 +1,9 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type {
|
|
2
|
+
HasTextSegmenter,
|
|
3
|
+
TextSegment,
|
|
4
|
+
TextSegmentGranularity,
|
|
5
|
+
TextSegmenterBackend,
|
|
6
|
+
} from '@flighthq/types/contract';
|
|
2
7
|
|
|
3
8
|
import { segmentGraphemes, segmentSentences, segmentWords } from './textSegment';
|
|
4
9
|
import { setTextSegmenterBackend } from './textSegmenterBackend';
|
|
@@ -6,6 +11,25 @@ import { setTextSegmenterBackend } from './textSegmenterBackend';
|
|
|
6
11
|
afterEach(() => setTextSegmenterBackend(null));
|
|
7
12
|
|
|
8
13
|
describe('segmentGraphemes', () => {
|
|
14
|
+
it('accepts an optional explicit segmenter host without breaking the existing signature', () => {
|
|
15
|
+
expectTypeOf(segmentGraphemes).toEqualTypeOf<
|
|
16
|
+
(text: string, locale?: string, host?: HasTextSegmenter) => readonly TextSegment[]
|
|
17
|
+
>();
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
it('gives an explicit host precedence over the legacy installed backend', () => {
|
|
21
|
+
setTextSegmenterBackend(taggingBackend('legacy'));
|
|
22
|
+
expect(segmentGraphemes('hi', undefined, segmenterHost(taggingBackend('explicit')))[0].text).toBe('explicit');
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
it('isolates callers that interleave different explicit hosts', () => {
|
|
26
|
+
const first = segmenterHost(taggingBackend('first'));
|
|
27
|
+
const second = segmenterHost(taggingBackend('second'));
|
|
28
|
+
expect(segmentGraphemes('hi', undefined, first)[0].text).toBe('first');
|
|
29
|
+
expect(segmentGraphemes('hi', undefined, second)[0].text).toBe('second');
|
|
30
|
+
expect(segmentGraphemes('hi', undefined, first)[0].text).toBe('first');
|
|
31
|
+
});
|
|
32
|
+
|
|
9
33
|
it('treats a ZWJ family emoji as one grapheme', () => {
|
|
10
34
|
const segments = segmentGraphemes('a👨👩👧b');
|
|
11
35
|
expect(segments.map((s) => s.text)).toEqual(['a', '👨👩👧', 'b']);
|
|
@@ -33,6 +57,16 @@ describe('segmentGraphemes', () => {
|
|
|
33
57
|
});
|
|
34
58
|
});
|
|
35
59
|
|
|
60
|
+
function segmenterHost(segmenter: TextSegmenterBackend): HasTextSegmenter {
|
|
61
|
+
return { text: { segmenter } };
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function taggingBackend(tag: string): TextSegmenterBackend {
|
|
65
|
+
return {
|
|
66
|
+
segment: () => [{ start: 0, end: tag.length, text: tag }],
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
|
|
36
70
|
describe('segmentSentences', () => {
|
|
37
71
|
it('splits into sentences', () => {
|
|
38
72
|
const segments = segmentSentences('Hi. Bye.');
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import type { HasTextSegmenter, TextSegmenterBackend } from '@flighthq/types/contract';
|
|
2
|
+
|
|
1
3
|
import {
|
|
2
4
|
getNextGraphemeBoundary,
|
|
3
5
|
getNextWordBoundary,
|
|
@@ -68,6 +70,15 @@ describe('getPreviousWordBoundary', () => {
|
|
|
68
70
|
});
|
|
69
71
|
|
|
70
72
|
describe('getWordRangeAt', () => {
|
|
73
|
+
it('threads an explicit host through the boundary helper', () => {
|
|
74
|
+
const explicit: TextSegmenterBackend = {
|
|
75
|
+
segment: (text) => [{ start: 0, end: text.length, text, isWordLike: true }],
|
|
76
|
+
};
|
|
77
|
+
const host: HasTextSegmenter = { text: { segmenter: explicit } };
|
|
78
|
+
setTextSegmenterBackend({ segment: () => [] });
|
|
79
|
+
expect(getWordRangeAt('word', 1, undefined, host)).toEqual({ start: 0, end: 4 });
|
|
80
|
+
});
|
|
81
|
+
|
|
71
82
|
it('returns the word range under an index', () => {
|
|
72
83
|
expect(getWordRangeAt('foo bar', 1)).toEqual({ start: 0, end: 3 });
|
|
73
84
|
});
|
|
@@ -1,9 +1,15 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type {
|
|
2
|
+
HasTextSegmenter,
|
|
3
|
+
TextSegment,
|
|
4
|
+
TextSegmentGranularity,
|
|
5
|
+
TextSegmenterBackend,
|
|
6
|
+
} from '@flighthq/types/contract';
|
|
2
7
|
|
|
3
8
|
import {
|
|
4
9
|
createWebTextSegmenterBackend,
|
|
5
10
|
getTextSegmenterBackend,
|
|
6
11
|
setTextSegmenterBackend,
|
|
12
|
+
webTextSegmenterBackend,
|
|
7
13
|
} from './textSegmenterBackend';
|
|
8
14
|
|
|
9
15
|
interface RecordingBackend extends TextSegmenterBackend {
|
|
@@ -48,9 +54,17 @@ describe('createWebTextSegmenterBackend', () => {
|
|
|
48
54
|
});
|
|
49
55
|
|
|
50
56
|
describe('getTextSegmenterBackend', () => {
|
|
51
|
-
it('
|
|
57
|
+
it('returns an explicit host provider ahead of the legacy backend', () => {
|
|
58
|
+
const legacy = recordingBackend();
|
|
59
|
+
const explicit = recordingBackend();
|
|
60
|
+
const host: HasTextSegmenter = { text: { segmenter: explicit } };
|
|
61
|
+
setTextSegmenterBackend(legacy);
|
|
62
|
+
expect(getTextSegmenterBackend(host)).toBe(explicit);
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
it('falls back to the stable bundled web backend when none is registered', () => {
|
|
52
66
|
const backend = getTextSegmenterBackend();
|
|
53
|
-
expect(backend).
|
|
67
|
+
expect(backend).toBe(webTextSegmenterBackend);
|
|
54
68
|
expect(backend.segment('ab', 'grapheme').length).toBe(2);
|
|
55
69
|
});
|
|
56
70
|
|
|
@@ -69,10 +83,20 @@ describe('setTextSegmenterBackend', () => {
|
|
|
69
83
|
expect(backend.calls).toEqual([{ text: 'hello', granularity: 'sentence', locale: 'fr' }]);
|
|
70
84
|
});
|
|
71
85
|
|
|
72
|
-
it('clears back to the
|
|
86
|
+
it('clears back to the stable bundled web default when passed null', () => {
|
|
73
87
|
setTextSegmenterBackend(recordingBackend());
|
|
74
88
|
setTextSegmenterBackend(null);
|
|
75
89
|
// The web default segments a ZWJ cluster as one grapheme; a recording backend would not.
|
|
76
90
|
expect(getTextSegmenterBackend().segment('a👨👩👧b', 'grapheme').length).toBe(3);
|
|
77
91
|
});
|
|
78
92
|
});
|
|
93
|
+
|
|
94
|
+
describe('webTextSegmenterBackend', () => {
|
|
95
|
+
it('provides the stable bundled Intl fallback', () => {
|
|
96
|
+
expect(webTextSegmenterBackend.segment('a👨👩👧b', 'grapheme').map((segment) => segment.text)).toEqual([
|
|
97
|
+
'a',
|
|
98
|
+
'👨👩👧',
|
|
99
|
+
'b',
|
|
100
|
+
]);
|
|
101
|
+
});
|
|
102
|
+
});
|