@echogarden/text-segmentation 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { WordSequence } from './WordSequence.js';
|
|
2
2
|
export { cldrSuppressions } from './Suppressions.js';
|
|
3
|
+
export { WordSequence, type WordEntry } from './WordSequence.js';
|
|
3
4
|
export declare function segmentText(text: string, options?: SegmentationOptions): Promise<SegmentationResult>;
|
|
4
5
|
export declare function segmentWordSequence(wordSequence: WordSequence): Promise<SegmentationResult>;
|
|
5
6
|
export declare function splitToWords(text: string, options?: SegmentationOptions): Promise<WordSequence>;
|
package/dist/TextSegmentation.js
CHANGED
|
@@ -4,6 +4,7 @@ import { cldrSuppressions, leadingApostropheContractionSuppressions, nounSuppres
|
|
|
4
4
|
import { eastAsianCharRangesRegExp } from './EastAsianCharacterPatterns.js';
|
|
5
5
|
import { WordSequence } from './WordSequence.js';
|
|
6
6
|
export { cldrSuppressions } from './Suppressions.js';
|
|
7
|
+
export { WordSequence } from './WordSequence.js';
|
|
7
8
|
export async function segmentText(text, options) {
|
|
8
9
|
options = { ...defaultSegmentationOptions, ...(options ?? {}) };
|
|
9
10
|
const wordSequence = await splitToWords(text, options);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"TextSegmentation.js","sourceRoot":"","sources":["../src/TextSegmentation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAC7C,OAAO,EAAE,wBAAwB,IAAI,wBAAwB,EAAE,qBAAqB,EAAE,0CAA0C,EAAE,uBAAuB,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAA;AACzM,OAAO,EAAE,gBAAgB,EAAE,wCAAwC,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAA;AACjI,OAAO,EAAE,yBAAyB,EAAE,MAAM,iCAAiC,CAAA;AAC3E,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AAEhD,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAA;
|
|
1
|
+
{"version":3,"file":"TextSegmentation.js","sourceRoot":"","sources":["../src/TextSegmentation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAC7C,OAAO,EAAE,wBAAwB,IAAI,wBAAwB,EAAE,qBAAqB,EAAE,0CAA0C,EAAE,uBAAuB,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAA;AACzM,OAAO,EAAE,gBAAgB,EAAE,wCAAwC,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAA;AACjI,OAAO,EAAE,yBAAyB,EAAE,MAAM,iCAAiC,CAAA;AAC3E,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AAEhD,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAA;AACpD,OAAO,EAAE,YAAY,EAAkB,MAAM,mBAAmB,CAAA;AAEhE,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAY,EAAE,OAA6B;IAC5E,OAAO,GAAG,EAAE,GAAG,0BAA0B,EAAE,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC,EAAE,CAAA;IAE/D,MAAM,YAAY,GAAG,MAAM,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;IAEtD,OAAO,mBAAmB,CAAC,YAAY,CAAC,CAAA;AACzC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,YAA0B;IACnE,MAAM,kBAAkB,GAAY,EAAE,CAAA;IAEtC,IAAI,uBAAuB,GAAG,CAAC,CAAA;IAE/B,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,YAAY,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;QACtE,MAAM,IAAI,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;QAE9C,IAAI,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACxC,OAAO,SAAS,GAAG,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5C,MAAM,QAAQ,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,GAAG,CAAC,CAAC,CAAA;gBAEtD,IAAI,CAAC,0CAA0C,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAChE,MAAK;gBACN,CAAC;gBAED,SAAS,IAAI,CAAC,CAAA;YACf,CAAC;YAED,kBAAkB,CAAC,IAAI,CAAC;gBACvB,KAAK,EAAE,uBAAuB;gBAC9B,GAAG,EAAE,SAAS,GAAG,CAAC;aAClB,CAAC,CAAA;YAEF,uBAAuB,GAAG,SAAS,GAAG,CAAC,CAAA;QACxC,CAAC;IACF,CAAC;IAED,IAAI,uBAAuB,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC;QACnD,kBAAkB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,uBAAuB,EAAE,GAAG,EAAE,YAAY,CAAC,MAAM,EAAE,CAAC,CAAA;IACtF,CAAC;IAED,MAAM,SAAS,GAAe,EAAE,CAAA;IAEhC,KAAK,MAAM,SAAS,IAAI,kBAAkB,EAAE,CAAC;QAC5C,MAAM,oBAAoB,GAAG,YAAY,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,SAAS,CAAC,GAAG,CAAC,CAAA;QAE/E,SAAS,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC,SAAS,EAAE,oBAAoB,CAAC,CAAC,CAAA;IAC9D,CAAC;IAED,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QAClC,MAAM,gBAAgB,GAAY,EAAE,CAAA;QAEpC,IAAI,qBAAqB,GAAG,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAA;QAClD,IAAI,qBAAqB,GAAG,QAAQ,CAAC,SAAS,CAAC,KAAK,CAAA;QAEpD,KAAK,IAAI,SAAS,GAAG,qBAAqB,EAAE,SAAS,GAAG,qBAAqB,EAAE,SAAS,EAAE,EAAE,CAAC;YAC5F,MAAM,WAAW,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;YAErD,IAAI,qBAAqB,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC7C,IAAI,kBAAkB,GAAG,KAAK,CAAA;gBAE9B,OAAO,SAAS,GAAG,qBAAqB,GAAG,CAAC,EAAE,CAAC;oBAC9C,MAAM,QAAQ,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,GAAG,CAAC,CAAC,CAAA;oBAEtD,IAAI,CAAC,0CAA0C,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;wBAChE,MAAK;oBACN,CAAC;oBAED,IAAI,uBAAuB,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;wBAC5C,kBAAkB,GAAG,IAAI,CAAA;oBAC1B,CAAC;oBAED,IAAI,QAAQ,KAAK,GAAG,IAAI,kBAAkB,EAAE,CAAC;wBAC5C,MAAK;oBACN,CAAC;oBAED,SAAS,IAAI,CAAC,CAAA;gBACf,CAAC;gBAED,gBAAgB,CAAC,IAAI,CAAC;oBACrB,KAAK,EAAE,qBAAqB;oBAC5B,GAAG,EAAE,SAAS,GAAG,CAAC;iBAClB,CAAC,CAAA;gBAEF,qBAAqB,GAAG,SAAS,GAAG,CAAC,CAAA;YACtC,CAAC;QACF,CAAC;QAED,IAAI,qBAAqB,GAAG,qBAAqB,EAAE,CAAC;YACnD,gBAAgB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,qBAAqB,EAAE,GAAG,EAAE,qBAAqB,EAAE,CAAC,CAAA;QACpF,CAAC;QAED,KAAK,MAAM,SAAS,IAAI,gBAAgB,EAAE,CAAC;YAC1C,MAAM,kBAAkB,GAAG,YAAY,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,SAAS,CAAC,GAAG,CAAC,CAAA;YAE7E,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,SAAS,EAAE,kBAAkB,CAAC,CAAC,CAAA;QACjE,CAAC;IACF,CAAC;IAED,MAAM,MAAM,GAAuB;QAClC,YAAY;QACZ,SAAS;KACT,CAAA;IAED,OAAO,MAAM,CAAA;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAY,EAAE,OAA6B;IAC7E,OAAO,GAAG,EAAE,GAAG,0BAA0B,EAAE,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC,EAAE,CAAA;IAE/D,MAAM,uBAAuB,GAAG,gBAAgB,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,IAAI,EAAE,CAAA;IAC9E,MAAM,8BAA8B,GAAG,wCAAwC,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,IAAI,EAAE,CAAA;IAC7G,MAAM,6CAA6C,GAAG,8BAA8B,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAA;IACzH,MAAM,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,IAAI,EAAE,CAAA;IAE3D,IAAI,YAAY,GAAG;QAClB,GAAG,kBAAkB;QACrB,GAAG,uBAAuB;QAC1B,GAAG,8BAA8B;QACjC,GAAG,6CAA6C;QAChD,GAAG,gBAAgB;QACnB,GAAG,eAAe;KAClB,CAAA;IAED,MAAM,WAAW,GAAG,wBAAwB,CAAC;QAC5C,GAAG,YAAY;QACf,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACrD,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;KACrD,CAAC,CAAA;IAEF,MAAM,kBAAkB,GAAG,WAAW,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;IAErE,gEAAgE;IAEhE,IAAI,YAAY,GAAG,IAAI,YAAY,EAAE,CAAA;IAErC,SAAS,kBAAkB,CAAC,WAAmB,EAAE,SAAiB;QACjE,MAAM,gBAAgB,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,SAAS,CAAC,CAAA;QAE/D,IAAI,UAAU,GAAG,WAAW,CAAA;QAC5B,IAAI,kBAAkB,GAAG,WAAW,CAAA;QAEpC,SAAS,kBAAkB;YAC1B,IAAI,UAAU,GAAG,kBAAkB,EAAE,CAAC;gBACrC,YAAY,CAAC,YAAY,CAAC,IAAI,EAAE,kBAAkB,EAAE,UAAU,EAAE,IAAI,CAAC,CAAA;gBACrE,kBAAkB,GAAG,UAAU,CAAA;YAChC,CAAC;QACF,CAAC;QAED,KAAK,MAAM,IAAI,IAAI,gBAAgB,EAAE,CAAC;YACrC,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;gBAClB,UAAU,IAAI,CAAC,CAAA;gBAEf,SAAQ;YACT,CAAC;YAED,kBAAkB,EAAE,CAAA;YAEpB,UAAU,IAAI,IAAI,CAAC,MAAM,CAAA;YAEzB,kBAAkB,EAAE,CAAA;QACrB,CAAC;QAED,kBAAkB,EAAE,CAAA;IACrB,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAA;IAErD,IAAI,kBAAkB,GAAG,CAAC,CAAA;IAE1B,IAAI,WAAW,EAAE,CAAC;QACjB,KAAK,MAAM,KAAK,IAAI,WAAW,EAAE,CAAC;YACjC,MAAM,OAAO,GAAG,KAAK,CAAC,OAAQ,CAAC,CAAC,CAAC,CAAA;YACjC,MAAM,gBAAgB,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;YACnC,MAAM,cAAc,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;YAEjC,IAAI,gBAAgB,GAAG,kBAAkB,EAAE,CAAC;gBAC3C,kBAAkB,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,CAAA;YACzD,CAAC;YAED,YAAY,CAAC,YAAY,CAAC,IAAI,EAAE,gBAAgB,EAAE,cAAc,EAAE,KAAK,CAAC,CAAA;YAExE,kBAAkB,GAAG,cAAc,CAAA;QACpC,CAAC;QAED,kBAAkB,CAAC,kBAAkB,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;IACpD,CAAC;IAED,IAAI,OAAO,CAAC,6BAA6B,EAAE,CAAC;QAC3C,YAAY,GAAG,MAAM,yBAAyB,CAAC,IAAI,EAAE,YAAY,CAAC,CAAA;IACnE,CAAC;IAED,OAAO,YAAY,CAAA;AACpB,CAAC;AAED,KAAK,UAAU,yBAAyB,CAAC,cAAsB,EAAE,YAA0B;IAC1F,MAAM,eAAe,GAAG,MAAM,kBAAkB,EAAE,CAAA;IAElD,IAAI,eAAe,KAAK,SAAS,EAAE,CAAC;QACnC,OAAO,YAAY,CAAA;IACpB,CAAC;IAED,IAAI,cAAc,GAAG,KAAK,CAAA;IAE1B,MAAM,eAAe,GAAG,IAAI,YAAY,EAAE,CAAA;IAE1C,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,YAAY,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;QACtE,MAAM,SAAS,GAAG,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;QACjD,MAAM,eAAe,GAAG,SAAS,CAAC,WAAW,CAAA;QAE7C,MAAM,IAAI,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;QAE9C,IAAI,yBAAyB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC1C,IAAI,CAAC,cAAc,EAAE,CAAC;gBACrB,MAAM,eAAe,CAAC,UAAU,EAAE,CAAA;gBAElC,cAAc,GAAG,IAAI,CAAA;YACtB,CAAC;YAED,MAAM,UAAU,GAAG,CAAC,GAAG,eAAe,CAAC,uBAAuB,CAAC,IAAI,CAAC,CAAC,CAAA;YAErE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAChD,eAAe,CAAC,YAAY,CAC3B,cAAc,EACd,eAAe,GAAG,UAAU,CAAC,CAAC,CAAC,EAC/B,eAAe,GAAG,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,EACnC,KAAK,CACL,CAAA;YACF,CAAC;QACF,CAAC;aAAM,CAAC;YACP,eAAe,CAAC,YAAY,CAAC,cAAc,EAAE,SAAS,CAAC,WAAW,EAAE,SAAS,CAAC,SAAS,EAAE,SAAS,CAAC,aAAa,CAAC,CAAA;QAClH,CAAC;IACF,CAAC;IAED,OAAO,eAAe,CAAA;AACvB,CAAC;AAED,KAAK,UAAU,kBAAkB;IAChC,IAAI,CAAC;QACJ,MAAM,eAAe,GAAG,MAAM,MAAM,CAAC,mCAAmC,CAAC,CAAA;QAEzE,OAAO,eAAe,CAAA;IACvB,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAA;IACjB,CAAC;AACF,CAAC;AAOD,MAAM,OAAO,YAAY;IACxB,SAAS,CAAO;IAChB,YAAY,CAAc;IAE1B,YAAY,SAAgB,EAAE,YAA0B;QACvD,IAAI,CAAC,SAAS,GAAG,SAAS,CAAA;QAC1B,IAAI,CAAC,YAAY,GAAG,YAAY,CAAA;IACjC,CAAC;IAED,IAAI,IAAI;QACP,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAA;IAC9B,CAAC;IAED,IAAI,SAAS;QACZ,OAAO;YACN,KAAK,EAAE,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,WAAW;YAC/C,GAAG,EAAE,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,SAAS;SAC1C,CAAA;IACF,CAAC;CACD;AAED,MAAM,OAAO,QAAS,SAAQ,YAAY;IACzC,OAAO,GAAa,EAAE,CAAA;CACtB;AAED,MAAM,OAAO,MAAO,SAAQ,YAAY;CACvC;AAaD,MAAM,CAAC,MAAM,0BAA0B,GAAwB;IAC9D,QAAQ,EAAE,EAAE;IACZ,kBAAkB,EAAE,EAAE;IACtB,6BAA6B,EAAE,IAAI;CACnC,CAAA"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@echogarden/text-segmentation",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.1",
|
|
4
4
|
"description": "A library for multilingual word, phrase and sentence segmentation.",
|
|
5
5
|
"author": "Rotem Dan",
|
|
6
6
|
"license": "MIT",
|
|
@@ -39,7 +39,7 @@
|
|
|
39
39
|
"regexp-composer": "../../regexp-composer"
|
|
40
40
|
},
|
|
41
41
|
"dependencies": {
|
|
42
|
-
"regexp-composer": "^0.
|
|
42
|
+
"regexp-composer": "^0.2.2"
|
|
43
43
|
},
|
|
44
44
|
"peerDependencies": {
|
|
45
45
|
"@echogarden/icu-segmentation-wasm": "^0.1.0"
|
|
@@ -50,6 +50,6 @@
|
|
|
50
50
|
}
|
|
51
51
|
},
|
|
52
52
|
"devDependencies": {
|
|
53
|
-
"@types/node": "^22.
|
|
53
|
+
"@types/node": "^22.15.12"
|
|
54
54
|
}
|
|
55
55
|
}
|
package/src/TextSegmentation.ts
CHANGED
|
@@ -5,6 +5,7 @@ import { eastAsianCharRangesRegExp } from './EastAsianCharacterPatterns.js'
|
|
|
5
5
|
import { WordSequence } from './WordSequence.js'
|
|
6
6
|
|
|
7
7
|
export { cldrSuppressions } from './Suppressions.js'
|
|
8
|
+
export { WordSequence, type WordEntry } from './WordSequence.js'
|
|
8
9
|
|
|
9
10
|
export async function segmentText(text: string, options?: SegmentationOptions) {
|
|
10
11
|
options = { ...defaultSegmentationOptions, ...(options ?? {}) }
|