echogarden 2.9.1 → 2.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/nlp/Segmentation.js
CHANGED
|
@@ -74,8 +74,8 @@ export async function splitToWords(text, langCode) {
|
|
|
74
74
|
for (const wordText of wordArray) {
|
|
75
75
|
const startOffset = offset;
|
|
76
76
|
const endOffset = startOffset + wordText.length;
|
|
77
|
-
const
|
|
78
|
-
wordSequence.addWord(wordText, startOffset,
|
|
77
|
+
const isPunctuation = !isWordOrSymbolWord(wordText);
|
|
78
|
+
wordSequence.addWord(wordText, startOffset, isPunctuation);
|
|
79
79
|
offset = endOffset;
|
|
80
80
|
}
|
|
81
81
|
return wordSequence;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Segmentation.js","sourceRoot":"","sources":["../../src/nlp/Segmentation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAA;AACjE,OAAO,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAA;AAE7D,OAAO,EAAE,aAAa,EAAE,0BAA0B,EAAE,MAAM,iCAAiC,CAAA;AAE3F,OAAO,KAAK,gBAAgB,MAAM,+BAA+B,CAAA;AACjE,OAAO,EAAE,6BAA6B,EAAE,MAAM,0BAA0B,CAAA;AACxE,OAAO,EAAE,iCAAiC,EAAE,MAAM,2BAA2B,CAAA;AAE7E,MAAM,GAAG,GAAG,WAAW,CAAA;AAEvB,MAAM,CAAC,MAAM,mBAAmB,GAAG,yBAAyB,CAAA;AAE5D,kEAAkE;AAClE,MAAM,CAAC,MAAM,mBAAmB,GAAG,oFAAoF,CAAA;AAEvH,MAAM,CAAC,MAAM,iBAAiB,GAAG,oBAAoB,CAAA;AAErD,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAA;AAC7E,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAA;AAEzJ,+FAA+F;AAC/F,aAAa;AACb,+FAA+F;AAC/F,MAAM,UAAU,kBAAkB,CAAC,GAAW;IAC7C,OAAO,MAAM,CAAC,GAAG,CAAC,IAAI,aAAa,CAAC,GAAG,CAAC,IAAI,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAA;AACtE,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,GAAW;IACvC,OAAO,WAAW,CAAC,QAAQ,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAA;AACzC,CAAC;AAED,MAAM,UAAU,MAAM,CAAC,GAAW;IACjC,OAAO,mBAAmB,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAA;AAC7C,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,GAAW;IAC9C,OAAO,iBAAiB,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAA;AAC3C,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,GAAW;IACxC,OAAO,mBAAmB,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAA;AAC7C,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,GAAW;IACvC,OAAO,GAAG,EAAE,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,CAAA;AAChC,CAAC;AAED,+FAA+F;AAC/F,2DAA2D;AAC3D,+FAA+F;AAC/F,MAAM,UAAU,iBAAiB,CAAC,IAAY,EAAE,kBAAsC,EAAE,0BAAgD;IACvI,IAAI,UAAU,GAAa,EAAE,CAAA;IAE7B,IAAI,kBAAkB,KAAK,QAAQ,EAAE,CAAC;QACrC,UAAU,GAAG,0BAA0B,CAAC,IAAI,EAAE,WAAW,CAAC,CAAA;IAC3D,CAAC;SAAM,IAAI,kBAAkB,KAAK,QAAQ,EAAE,CAAC;QAC5C,UAAU,GAAG,0BAA0B,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAA;IAClE,CAAC;SAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,kCAAkC,kBAAkB,GAAG,CAAC,CAAA;IACzE,CAAC;IAED,UAAU,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,yBAAyB,CAAC,CAAC,EAAE,0BAA0B,CAAC,CAAC,CAAA;IAC1F,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IAEjD,OAAO,UAAU,CAAA;AAClB,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,IAAY;IACxC,OAAO,0BAA0B,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAA;AAClD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAY,EAAE,QAAgB;IAC7D,MAAM,aAAa,GAAG,oBAAoB,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAA;IAE1D,MAAM,YAAY,GAAG,MAAM,YAAY,CAAC,IAAI,EAAE,aAAa,CAAC,CAAA;IAE5D,MAAM,UAAU,GAAG,MAAM,gBAAgB,CAAC,mBAAmB,CAAC,YAAY,CAAC,CAAA;IAE3E,OAAO,UAAU,CAAA;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAY,EAAE,QAAgB;IAChE,MAAM,aAAa,GAAG,oBAAoB,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAA;IAE1D,IAAI,aAAa,KAAK,IAAI,IAAI,aAAa,KAAK,KAAK,IAAI,aAAa,KAAK,IAAI,EAAE,CAAC;QACjF,IAAI,SAAS,GAAa,EAAE,CAAA;QAE5B,IAAI,aAAa,KAAK,IAAI,IAAI,aAAa,KAAK,KAAK,EAAE,CAAC;YACvD,SAAS,GAAG,MAAM,6BAA6B,CAAC,IAAI,CAAC,CAAA;QACtD,CAAC;aAAM,CAAC;YACP,SAAS,GAAG,MAAM,iCAAiC,CAAC,IAAI,CAAC,CAAA;QAC1D,CAAC;QAED,MAAM,YAAY,GAAG,IAAI,gBAAgB,CAAC,YAAY,EAAE,CAAA;QAExD,IAAI,MAAM,GAAG,CAAC,CAAA;QAEd,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YAClC,MAAM,WAAW,GAAG,MAAM,CAAA;YAC1B,MAAM,SAAS,GAAG,WAAW,GAAG,QAAQ,CAAC,MAAM,CAAA;YAC/C,MAAM,
|
|
1
|
+
{"version":3,"file":"Segmentation.js","sourceRoot":"","sources":["../../src/nlp/Segmentation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAA;AACjE,OAAO,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAA;AAE7D,OAAO,EAAE,aAAa,EAAE,0BAA0B,EAAE,MAAM,iCAAiC,CAAA;AAE3F,OAAO,KAAK,gBAAgB,MAAM,+BAA+B,CAAA;AACjE,OAAO,EAAE,6BAA6B,EAAE,MAAM,0BAA0B,CAAA;AACxE,OAAO,EAAE,iCAAiC,EAAE,MAAM,2BAA2B,CAAA;AAE7E,MAAM,GAAG,GAAG,WAAW,CAAA;AAEvB,MAAM,CAAC,MAAM,mBAAmB,GAAG,yBAAyB,CAAA;AAE5D,kEAAkE;AAClE,MAAM,CAAC,MAAM,mBAAmB,GAAG,oFAAoF,CAAA;AAEvH,MAAM,CAAC,MAAM,iBAAiB,GAAG,oBAAoB,CAAA;AAErD,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAA;AAC7E,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAA;AAEzJ,+FAA+F;AAC/F,aAAa;AACb,+FAA+F;AAC/F,MAAM,UAAU,kBAAkB,CAAC,GAAW;IAC7C,OAAO,MAAM,CAAC,GAAG,CAAC,IAAI,aAAa,CAAC,GAAG,CAAC,IAAI,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAA;AACtE,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,GAAW;IACvC,OAAO,WAAW,CAAC,QAAQ,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAA;AACzC,CAAC;AAED,MAAM,UAAU,MAAM,CAAC,GAAW;IACjC,OAAO,mBAAmB,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAA;AAC7C,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,GAAW;IAC9C,OAAO,iBAAiB,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAA;AAC3C,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,GAAW;IACxC,OAAO,mBAAmB,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAA;AAC7C,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,GAAW;IACvC,OAAO,GAAG,EAAE,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,CAAA;AAChC,CAAC;AAED,+FAA+F;AAC/F,2DAA2D;AAC3D,+FAA+F;AAC/F,MAAM,UAAU,iBAAiB,CAAC,IAAY,EAAE,kBAAsC,EAAE,0BAAgD;IACvI,IAAI,UAAU,GAAa,EAAE,CAAA;IAE7B,IAAI,kBAAkB,KAAK,QAAQ,EAAE,CAAC;QACrC,UAAU,GAAG,0BAA0B,CAAC,IAAI,EAAE,WAAW,CAAC,CAAA;IAC3D,CAAC;SAAM,IAAI,kBAAkB,KAAK,QAAQ,EAAE,CAAC;QAC5C,UAAU,GAAG,0BAA0B,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAA;IAClE,CAAC;SAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,kCAAkC,kBAAkB,GAAG,CAAC,CAAA;IACzE,CAAC;IAED,UAAU,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,yBAAyB,CAAC,CAAC,EAAE,0BAA0B,CAAC,CAAC,CAAA;IAC1F,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IAEjD,OAAO,UAAU,CAAA;AAClB,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,IAAY;IACxC,OAAO,0BAA0B,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAA;AAClD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAY,EAAE,QAAgB;IAC7D,MAAM,aAAa,GAAG,oBAAoB,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAA;IAE1D,MAAM,YAAY,GAAG,MAAM,YAAY,CAAC,IAAI,EAAE,aAAa,CAAC,CAAA;IAE5D,MAAM,UAAU,GAAG,MAAM,gBAAgB,CAAC,mBAAmB,CAAC,YAAY,CAAC,CAAA;IAE3E,OAAO,UAAU,CAAA;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAY,EAAE,QAAgB;IAChE,MAAM,aAAa,GAAG,oBAAoB,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAA;IAE1D,IAAI,aAAa,KAAK,IAAI,IAAI,aAAa,KAAK,KAAK,IAAI,aAAa,KAAK,IAAI,EAAE,CAAC;QACjF,IAAI,SAAS,GAAa,EAAE,CAAA;QAE5B,IAAI,aAAa,KAAK,IAAI,IAAI,aAAa,KAAK,KAAK,EAAE,CAAC;YACvD,SAAS,GAAG,MAAM,6BAA6B,CAAC,IAAI,CAAC,CAAA;QACtD,CAAC;aAAM,CAAC;YACP,SAAS,GAAG,MAAM,iCAAiC,CAAC,IAAI,CAAC,CAAA;QAC1D,CAAC;QAED,MAAM,YAAY,GAAG,IAAI,gBAAgB,CAAC,YAAY,EAAE,CAAA;QAExD,IAAI,MAAM,GAAG,CAAC,CAAA;QAEd,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YAClC,MAAM,WAAW,GAAG,MAAM,CAAA;YAC1B,MAAM,SAAS,GAAG,WAAW,GAAG,QAAQ,CAAC,MAAM,CAAA;YAC/C,MAAM,aAAa,GAAG,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAA;YAEnD,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,WAAW,EAAE,aAAa,CAAC,CAAA;YAE1D,MAAM,GAAG,SAAS,CAAA;QACnB,CAAC;QAED,OAAO,YAAY,CAAA;IACpB,CAAC;SAAM,CAAC;QACP,OAAO,gBAAgB,CAAC,YAAY,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAA;IACnE,CAAC;AACF,CAAC;AAED,MAAM,UAAU,yBAAyB,CAAC,IAAY,EAAE,0BAAgD;IACvG,IAAI,0BAA0B,KAAK,kBAAkB,EAAE,CAAC;QACvD,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,WAAW,EAAE,GAAG,CAAC,CAAA;IAChD,CAAC;SAAM,IAAI,0BAA0B,KAAK,UAAU,EAAE,CAAC;QACtD,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;IAC3C,CAAC;SAAM,IAAI,0BAA0B,KAAK,UAAU,EAAE,CAAC;QACtD,OAAO,IAAI,CAAA;IACZ,CAAC;SAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,0CAA0C,0BAA0B,GAAG,CAAC,CAAA;IACzF,CAAC;AACF,CAAC;AAED,+FAA+F;AAC/F,wBAAwB;AACxB,EAAE;AACF,uEAAuE;AACvE,+FAA+F;AAC/F,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,IAAY,EAAE,iBAAyB,EAAE,QAAgB,EAAE,iBAAiB,GAAG,IAAI,EAAE,eAAe,GAAG,IAAI;IACjJ,MAAM,UAAU,GAAG,MAAM,oCAAoC,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAA;IAE7E,MAAM,SAAS,GAAe,EAAE,CAAA;IAChC,IAAI,eAAe,GAAG,IAAI,QAAQ,EAAE,CAAA;IAEpC,MAAM,oCAAoC,GAAG,GAAG,EAAE,CAAC,iBAAiB,GAAG,eAAe,CAAC,MAAM,CAAA;IAC7F,MAAM,yBAAyB,GAAG,GAAG,EAAE;QACtC,IAAI,eAAe,CAAC,UAAU,EAAE,CAAC;YAChC,SAAS,CAAC,IAAI,CAAC,eAAe,CAAC,CAAA;YAC/B,eAAe,GAAG,IAAI,QAAQ,EAAE,CAAA;QACjC,CAAC;IACF,CAAC,CAAA;IAED,MAAM,mBAAmB,GAAG,CAAC,OAAgB,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,IAAI,oCAAoC,EAAE,CAAA;IAE1G,KAAK,MAAM,QAAQ,IAAI,UAAU,EAAE,CAAC;QACnC,IAAI,mBAAmB,CAAC,QAAQ,CAAC,EAAE,CAAC;YACnC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;YACvC,SAAQ;QACT,CAAC;QAED,IAAI,iBAAiB,EAAE,CAAC;YACvB,yBAAyB,EAAE,CAAA;YAE3B,IAAI,mBAAmB,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACnC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;gBACvC,SAAQ;YACT,CAAC;QACF,CAAC;QAED,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;YACvC,IAAI,mBAAmB,CAAC,MAAM,CAAC,EAAE,CAAC;gBACjC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;gBACrC,SAAQ;YACT,CAAC;YAGD,IAAI,eAAe,EAAE,CAAC;gBACrB,yBAAyB,EAAE,CAAA;gBAE3B,IAAI,mBAAmB,CAAC,MAAM,CAAC,EAAE,CAAC;oBACjC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;oBACrC,SAAQ;gBACT,CAAC;YACF,CAAC;YAED,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;gBACjC,IAAI,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC/B,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;oBACnC,SAAQ;gBACT,CAAC;gBAED,yBAAyB,EAAE,CAAA;gBAE3B,IAAI,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC/B,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;oBACnC,SAAQ;gBACT,CAAC;gBAED,MAAM,IAAI,KAAK,CAAC,gCAAgC,IAAI,CAAC,MAAM,mDAAmD,iBAAiB,EAAE,CAAC,CAAA;YACnI,CAAC;QACF,CAAC;IACF,CAAC;IAED,yBAAyB,EAAE,CAAA;IAE3B,OAAO,SAAS,CAAA;AACjB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oCAAoC,CAAC,IAAY,EAAE,QAAgB;IACxF,MAAM,aAAa,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAA;IAErD,MAAM,SAAS,GAAe,EAAE,CAAA;IAEhC,KAAK,MAAM,aAAa,IAAI,aAAa,CAAC,SAAS,EAAE,CAAC;QACrD,MAAM,QAAQ,GAAG,IAAI,QAAQ,EAAE,CAAA;QAE/B,KAAK,MAAM,WAAW,IAAI,aAAa,CAAC,OAAO,EAAE,CAAC;YACjD,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAA;YAE3B,KAAK,MAAM,SAAS,IAAI,WAAW,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;gBACnD,MAAM,mBAAmB,GAAG,SAAS,KAAK,aAAa,CAAC,KAAK,CAAC,SAAS,CAAA;gBAEvE,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,mBAAmB,CAAC,CAAA;gBAE1D,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YACxB,CAAC;YAED,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC7B,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;YAC9B,CAAC;QACF,CAAC;QAED,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;IACzB,CAAC;IAED,OAAO,SAAS,CAAA;AACjB,CAAC;AAED,MAAM,OAAO,QAAQ;IACpB,OAAO,GAAa,EAAE,CAAA;IAEb,mBAAmB,GAAG,IAAI,CAAA;IAEnC,IAAI,MAAM,KAAK,OAAO,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA,CAAC,CAAC;IAEzE,IAAI,IAAI,KAAK,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAS,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC,MAAM,GAAG,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA,CAAC,CAAC;CAC/F;AAED,MAAM,OAAO,MAAM;IAClB,KAAK,GAAW,EAAE,CAAA;IAElB,IAAI,MAAM,KAAK,OAAO,QAAQ,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA,CAAC,CAAC;IAEnE,IAAI,IAAI,KAAK,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAS,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC,MAAM,GAAG,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA,CAAC,CAAC;IAEzF,IAAI,QAAQ;QACX,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YAC5B,OAAO,SAAS,CAAA;QACjB,CAAC;QAED,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IACzC,CAAC;IAED,IAAI,mBAAmB,KAAK,OAAO,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC,CAAC,KAAK,CAAA,CAAC,CAAC;CACtG;AAED,MAAM,OAAO,IAAI;IACP,IAAI,CAAQ;IACrB,mBAAmB,CAAS;IAE5B,YAAY,IAAY,EAAE,mBAA4B;QACrD,IAAI,CAAC,IAAI,GAAG,IAAI,CAAA;QAChB,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAA;IAC/C,CAAC;IAED,IAAI,uBAAuB,KAAK,OAAO,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAA,CAAC,CAAC;IAEnG,IAAI,YAAY,KAAK,OAAO,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA,CAAC,CAAC;IAE7D,IAAI,iBAAiB,KAAK,OAAO,IAAI,CAAC,uBAAuB,IAAI,aAAa,CAAC,IAAI,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAA,CAAC,CAAC;IAE7G,IAAI,MAAM,KAAK,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAA,CAAC,CAAC;CACxC;AAID,MAAM,OAAO,QAAQ;IACpB,QAAQ,GAAc,EAAE,CAAA;IAExB,IAAI,MAAM,KAAK,OAAO,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA,CAAC,CAAC;IAE1E,IAAI,IAAI,KAAK,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAS,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,MAAM,GAAG,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA,CAAC,CAAC;IAElG,IAAI,OAAO,KAAK,OAAO,IAAI,CAAC,MAAM,IAAI,CAAC,CAAA,CAAC,CAAC;IAEzC,IAAI,UAAU,KAAK,OAAO,CAAC,IAAI,CAAC,OAAO,CAAA,CAAC,CAAC;IAEzC,IAAI,WAAW;QACd,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,OAAO,SAAS,CAAA;QACjB,CAAC;QAED,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IAC/C,CAAC;CACD"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { addMissingPunctuationWordsToWordSequence, segmentWordSequence, WordSequence } from '@echogarden/text-segmentation';
|
|
2
|
-
import { isWord, isWordOrSymbolWord, splitToParagraphs } from '../nlp/Segmentation.js';
|
|
2
|
+
import { applyWhitespaceProcessing, isWord, isWordOrSymbolWord, splitToParagraphs } from '../nlp/Segmentation.js';
|
|
3
3
|
import { deepClone } from './ObjectUtilities.js';
|
|
4
4
|
import { getUTF32Chars } from './StringUtilities.js';
|
|
5
5
|
import { roundToDigits } from './Utilities.js';
|
|
@@ -56,7 +56,7 @@ export async function wordTimelineToSegmentSentenceTimeline(wordTimelineWithOffs
|
|
|
56
56
|
}
|
|
57
57
|
const { wordSequenceWithPunctuation, originalWordsReverseMapping } = addMissingPunctuationWordsToWordSequence(wordSequence, transcript);
|
|
58
58
|
const segmentedWordSequence = await segmentWordSequence(wordSequenceWithPunctuation);
|
|
59
|
-
const paragraphs = splitToParagraphs(transcript, paragraphBreaks,
|
|
59
|
+
const paragraphs = splitToParagraphs(transcript, paragraphBreaks, 'preserve');
|
|
60
60
|
const sentenceIndexesForParagraph = paragraphs.map(_ => []);
|
|
61
61
|
{
|
|
62
62
|
let sentenceIndex = 0;
|
|
@@ -111,7 +111,7 @@ export async function wordTimelineToSegmentSentenceTimeline(wordTimelineWithOffs
|
|
|
111
111
|
}
|
|
112
112
|
const segmentTimelineEntry = {
|
|
113
113
|
type: 'segment',
|
|
114
|
-
text: paragraph,
|
|
114
|
+
text: applyWhitespaceProcessing(paragraph, whitespace),
|
|
115
115
|
startTime: sentenceTimeline[0]?.startTime,
|
|
116
116
|
endTime: sentenceTimeline[sentenceTimeline.length - 1]?.endTime,
|
|
117
117
|
timeline: sentenceTimeline,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Timeline.js","sourceRoot":"","sources":["../../src/utilities/Timeline.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wCAAwC,EAAE,mBAAmB,EAAE,YAAY,EAAE,MAAM,+BAA+B,CAAA;AAE3H,OAAO,EAAE,MAAM,EAAE,kBAAkB,EAAE,iBAAiB,
|
|
1
|
+
{"version":3,"file":"Timeline.js","sourceRoot":"","sources":["../../src/utilities/Timeline.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wCAAwC,EAAE,mBAAmB,EAAE,YAAY,EAAE,MAAM,+BAA+B,CAAA;AAE3H,OAAO,EAAE,yBAAyB,EAAE,MAAM,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAA;AACjH,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAA;AAChD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAA;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAA;AAE9C,MAAM,UAAU,uBAAuB,CAAC,cAAwB,EAAE,UAAkB;IACnF,IAAI,CAAC,cAAc,EAAE,CAAC;QACrB,OAAO,cAAc,CAAA;IACtB,CAAC;IAED,MAAM,WAAW,GAAG,SAAS,CAAC,cAAc,CAAC,CAAA;IAE7C,KAAK,MAAM,oBAAoB,IAAI,WAAW,EAAE,CAAC;QAChD,oBAAoB,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,oBAAoB,CAAC,SAAS,GAAG,UAAU,EAAE,CAAC,CAAC,CAAA;QACzF,oBAAoB,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,oBAAoB,CAAC,OAAO,GAAG,UAAU,EAAE,CAAC,CAAC,CAAA;QAErF,IAAI,oBAAoB,CAAC,QAAQ,EAAE,CAAC;YACnC,oBAAoB,CAAC,QAAQ,GAAG,uBAAuB,CAAC,oBAAoB,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAA;QACnG,CAAC;IACF,CAAC;IAED,OAAO,WAAW,CAAA;AACnB,CAAC;AAED,MAAM,UAAU,wBAAwB,CAAC,cAAwB,EAAE,MAAc;IAChF,MAAM,WAAW,GAAG,SAAS,CAAC,cAAc,CAAC,CAAA;IAE7C,KAAK,MAAM,oBAAoB,IAAI,WAAW,EAAE,CAAC;QAChD,oBAAoB,CAAC,SAAS,GAAG,oBAAoB,CAAC,SAAS,GAAG,MAAM,CAAA;QACxE,oBAAoB,CAAC,OAAO,GAAG,oBAAoB,CAAC,OAAO,GAAG,MAAM,CAAA;QAEpE,IAAI,oBAAoB,CAAC,QAAQ,EAAE,CAAC;YACnC,oBAAoB,CAAC,QAAQ,GAAG,wBAAwB,CAAC,oBAAoB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;QAChG,CAAC;IACF,CAAC;IAED,OAAO,WAAW,CAAA;AACnB,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,cAAwB,EAAE,aAAa,GAAG,CAAC;IAClF,MAAM,eAAe,GAAG,SAAS,CAAC,cAAc,CAAC,CAAA;IAEjD,KAAK,MAAM,KAAK,IAAI,eAAe,EAAE,CAAC;QACrC,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;YACrB,KAAK,CAAC,SAAS,GAAG,aAAa,CAAC,KAAK,CAAC,SAAS,EAAE,aAAa,CAAC,CAAA;QAChE,CAAC;QAED,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;YACnB,KAAK,CAAC,OAAO,GAAG,aAAa,CAAC,KAAK,CAAC,OAAO,EAAE,aAAa,CAAC,CAAA;QAC5D,CAAC;QAED,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;YACtB,KAAK,CAAC,UAAU,GAAG,aAAa,CAAC,KAAK,CAAC,UAAU,EAAE,aAAa,CAAC,CAAA;QAClE,CAAC;QAED,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;YACpB,KAAK,CAAC,QAAQ,GAAG,uBAAuB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;QACzD,CAAC;IACF,CAAC;IAED,OAAO,eAAe,CAAA;AACvB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qCAAqC,CAAC,uBAAiC,EAAE,UAAkB,EAAE,QAAgB,EAAE,kBAAsC,QAAQ,EAAE,aAAmC,UAAU;IACjO,MAAM,YAAY,GAAG,IAAI,YAAY,EAAE,CAAA;IAEvC,KAAK,MAAM,SAAS,IAAI,uBAAuB,EAAE,CAAC;QACjD,MAAM,eAAe,GAAG,SAAS,CAAC,gBAAiB,CAAA;QACnD,MAAM,aAAa,GAAG,SAAS,CAAC,cAAe,CAAA;QAE/C,MAAM,aAAa,GAAG,CAAC,kBAAkB,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;QAEzD,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,EAAE,eAAe,EAAE,aAAa,CAAC,CAAA;IACrE,CAAC;IAED,MAAM,EAAE,2BAA2B,EAAE,2BAA2B,EAAE,GAAG,wCAAwC,CAAC,YAAY,EAAE,UAAU,CAAC,CAAA;IAEvI,MAAM,qBAAqB,GAAG,MAAM,mBAAmB,CAAC,2BAA2B,CAAC,CAAA;IAEpF,MAAM,UAAU,GAAG,iBAAiB,CAAC,UAAU,EAAE,eAAe,EAAE,UAAU,CAAC,CAAA;IAE7E,MAAM,2BAA2B,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAc,CAAC,CAAA;IAEvE,CAAC;QACA,IAAI,aAAa,GAAG,CAAC,CAAA;QACrB,IAAI,UAAU,GAAG,CAAC,CAAA;QAElB,KAAK,IAAI,cAAc,GAAG,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,MAAM,EAAE,cAAc,EAAE,EAAE,CAAC;YACnF,MAAM,SAAS,GAAG,UAAU,CAAC,cAAc,CAAC,CAAA;YAE5C,MAAM,oBAAoB,GAAG,UAAU,CAAA;YACvC,MAAM,kBAAkB,GAAG,oBAAoB,GAAG,SAAS,CAAC,MAAM,CAAA;YAElE,OAAO,aAAa,GAAG,qBAAqB,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;gBAC/D,MAAM,aAAa,GAAG,qBAAqB,CAAC,SAAS,CAAC,aAAa,CAAC,CAAA;gBACpE,MAAM,mBAAmB,GAAG,aAAa,CAAC,SAAS,CAAC,KAAK,CAAA;gBACzD,MAAM,iBAAiB,GAAG,aAAa,CAAC,SAAS,CAAC,GAAG,CAAA;gBAErD,IAAI,mBAAmB,GAAG,kBAAkB,EAAE,CAAC;oBAC9C,2BAA2B,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAA;oBAC/D,aAAa,EAAE,CAAA;gBAChB,CAAC;qBAAM,CAAC;oBACP,MAAK;gBACN,CAAC;YACF,CAAC;YAED,IAAI,aAAa,KAAK,qBAAqB,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;gBAC9D,MAAK;YACN,CAAC;YAED,UAAU,IAAI,SAAS,CAAC,MAAM,CAAA;QAC/B,CAAC;IACF,CAAC;IAED,MAAM,eAAe,GAAa,EAAE,CAAA;IACpC,IAAI,SAAS,GAAG,CAAC,CAAA;IAEjB,KAAK,IAAI,cAAc,GAAG,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,MAAM,EAAE,cAAc,EAAE,EAAE,CAAC;QACnF,MAAM,SAAS,GAAG,UAAU,CAAC,cAAc,CAAC,CAAA;QAC5C,MAAM,gBAAgB,GAAG,2BAA2B,CAAC,cAAc,CAAC,CAAA;QACpE,MAAM,eAAe,GAAG,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,qBAAqB,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAA;QAE7F,MAAM,gBAAgB,GAAa,EAAE,CAAA;QAErC,KAAK,MAAM,aAAa,IAAI,eAAe,EAAE,CAAC;YAC7C,MAAM,YAAY,GAAa,EAAE,CAAA;YAEjC,KAAK,MAAM,CAAC,IAAI,aAAa,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;gBAC7C,MAAM,iBAAiB,GAAG,2BAA2B,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;gBAEpE,IAAI,iBAAiB,KAAK,SAAS,EAAE,CAAC;oBACrC,MAAM,iBAAiB,GAAG,uBAAuB,CAAC,iBAAiB,CAAC,CAAA;oBAEpE,YAAY,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAA;gBACrC,CAAC;gBAED,SAAS,IAAI,CAAC,CAAA;YACf,CAAC;YAED,MAAM,qBAAqB,GAAkB;gBAC5C,IAAI,EAAE,UAAU;gBAChB,IAAI,EAAE,aAAa,CAAC,IAAI;gBACxB,SAAS,EAAE,YAAY,CAAC,CAAC,CAAC,EAAE,SAAS;gBACrC,OAAO,EAAE,YAAY,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,OAAO;gBAEvD,QAAQ,EAAE,YAAY;aACtB,CAAA;YAED,gBAAgB,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAA;QAC7C,CAAC;QAED,MAAM,oBAAoB,GAAkB;YAC3C,IAAI,EAAE,SAAS;YACf,IAAI,EAAE,yBAAyB,CAAC,SAAS,EAAE,UAAU,CAAC;YACtD,SAAS,EAAE,gBAAgB,CAAC,CAAC,CAAC,EAAE,SAAS;YACzC,OAAO,EAAE,gBAAgB,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,OAAO;YAE/D,QAAQ,EAAE,gBAAgB;SAC1B,CAAA;QAED,eAAe,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAA;IAC3C,CAAC;IAED,OAAO,EAAE,eAAe,EAAE,CAAA;AAC3B,CAAC;AAED,MAAM,UAAU,mCAAmC,CAAC,QAAkB,EAAE,IAAY;IACnF,MAAM,EAAE,gBAAgB,EAAE,GAAG,aAAa,CAAC,IAAI,CAAC,CAAA;IAEhD,IAAI,aAAa,GAAG,CAAC,CAAA;IAErB,SAAS,eAAe,CAAC,QAAkB;QAC1C,IAAI,aAAa,GAAG,CAAC,CAAA;QAErB,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;YAC9B,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBAC3B,IAAI,IAAI,GAAG,KAAK,CAAC,IAAI,CAAA;gBAErB,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;gBAE1C,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;gBAEjC,IAAI,WAA+B,CAAA;gBACnC,IAAI,SAA6B,CAAA;gBAEjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC3C,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,CAAA;oBAE7B,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAA;oBAE5D,IAAI,cAAc,KAAK,CAAC,CAAC,EAAE,CAAC;wBAC3B,SAAQ;oBACT,CAAC;oBAED,aAAa,GAAG,cAAc,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,CAAA;oBAEpD,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;wBACb,WAAW,GAAG,cAAc,CAAA;oBAC7B,CAAC;oBAED,SAAS,GAAG,aAAa,CAAA;gBAC1B,CAAC;gBAED,KAAK,CAAC,gBAAgB,GAAG,WAAW,IAAI,aAAa,CAAA;gBACrD,KAAK,CAAC,cAAc,GAAG,SAAS,IAAI,aAAa,CAAA;gBAEjD,KAAK,CAAC,gBAAgB,GAAG,gBAAgB,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAA;gBACjE,KAAK,CAAC,cAAc,GAAG,gBAAgB,CAAC,KAAK,CAAC,cAAc,CAAC,CAAA;gBAE7D,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;oBAC7B,aAAa,GAAG,SAAS,CAAA;gBAC1B,CAAC;YACF,CAAC;iBAAM,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;gBAC3B,eAAe,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;YAChC,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,eAAe,CAAC,QAAQ,CAAC,CAAA;AACjC,CAAC;AAED,SAAS,oDAAoD,CAAC,UAAkB,EAAE,YAAsB,EAAE,gBAAwB;IACjI,IAAI,gBAAgB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnC,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAA;IACzD,CAAC;IAED,IAAI,kBAAkB,GAAG,UAAU,CAAA;IAEnC,MAAM,cAAc,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAA;IAE1D,KAAK,MAAM,SAAS,IAAI,YAAY,EAAE,CAAC;QACtC,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,CAAA;QAE/B,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;YACvB,SAAQ;QACT,CAAC;QAED,IAAI,WAAW,GAAG,EAAE,CAAA;QACpB,IAAI,SAAS,GAAG,CAAC,CAAA;QAEjB,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC7B,MAAM,WAAW,GAAG,SAAS,KAAK,CAAC,CAAA;YACnC,MAAM,UAAU,GAAG,SAAS,GAAG,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM,CAAA;YAC9D,MAAM,iBAAiB,GAAG,WAAW,IAAI,UAAU,CAAA;YAEnD,IAAI,CAAC,UAAU,IAAI,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;gBAClD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBACtC,WAAW,IAAI,gBAAgB,CAAA;gBAChC,CAAC;YACF,CAAC;iBAAM,CAAC;gBACP,WAAW,IAAI,IAAI,CAAA;YACpB,CAAC;YAED,SAAS,IAAI,IAAI,CAAC,MAAM,CAAA;QACzB,CAAC;QAED,IAAI,WAAW,KAAK,QAAQ,EAAE,CAAC;YAC9B,MAAM,eAAe,GAAG,SAAS,CAAC,gBAAiB,CAAA;YACnD,MAAM,aAAa,GAAG,SAAS,CAAC,cAAe,CAAA;YAE/C,kBAAkB;gBACjB,kBAAkB,CAAC,SAAS,CAAC,CAAC,EAAE,eAAe,CAAC;oBAChD,WAAW;oBACX,kBAAkB,CAAC,SAAS,CAAC,aAAa,CAAC,CAAA;QAC7C,CAAC;IACF,CAAC;IAED,OAAO,kBAAkB,CAAA;AAC1B,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,QAAkB,EAAE,SAA4C;IAC9F,MAAM,mBAAmB,GAAoB,EAAE,CAAA;IAE/C,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC9B,IAAI,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;YACtB,mBAAmB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAChC,CAAC;aAAM,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;YAC3B,mBAAmB,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,KAAK,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAA;QACvE,CAAC;IACF,CAAC;IAED,OAAO,mBAAmB,CAAA;AAC3B,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "echogarden",
|
|
3
|
-
"version": "2.9.
|
|
3
|
+
"version": "2.9.3",
|
|
4
4
|
"description": "An easy-to-use speech toolset. Includes tools for synthesis, recognition, alignment, speech translation, language detection, source separation and more.",
|
|
5
5
|
"author": "Rotem Dan",
|
|
6
6
|
"license": "GPL-3.0",
|
|
@@ -55,7 +55,7 @@
|
|
|
55
55
|
"@aws-sdk/client-polly": "~3.817.0",
|
|
56
56
|
"@aws-sdk/client-transcribe-streaming": "~3.817.0",
|
|
57
57
|
"@echogarden/audio-io": "~0.4.2",
|
|
58
|
-
"@echogarden/espeak-ng-emscripten": "~0.3.
|
|
58
|
+
"@echogarden/espeak-ng-emscripten": "~0.3.5",
|
|
59
59
|
"@echogarden/fasttext-wasm": "~0.1.0",
|
|
60
60
|
"@echogarden/flite-wasi": "~0.1.1",
|
|
61
61
|
"@echogarden/fvad-wasm": "~0.2.0",
|
package/src/nlp/Segmentation.ts
CHANGED
|
@@ -99,9 +99,9 @@ export async function splitToWords(text: string, langCode: string): Promise<Text
|
|
|
99
99
|
for (const wordText of wordArray) {
|
|
100
100
|
const startOffset = offset
|
|
101
101
|
const endOffset = startOffset + wordText.length
|
|
102
|
-
const
|
|
102
|
+
const isPunctuation = !isWordOrSymbolWord(wordText)
|
|
103
103
|
|
|
104
|
-
wordSequence.addWord(wordText, startOffset,
|
|
104
|
+
wordSequence.addWord(wordText, startOffset, isPunctuation)
|
|
105
105
|
|
|
106
106
|
offset = endOffset
|
|
107
107
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { addMissingPunctuationWordsToWordSequence, segmentWordSequence, WordSequence } from '@echogarden/text-segmentation'
|
|
2
2
|
import { ParagraphBreakType, WhitespaceProcessing } from '../api/Common.js'
|
|
3
|
-
import { isWord, isWordOrSymbolWord, splitToParagraphs
|
|
3
|
+
import { applyWhitespaceProcessing, isWord, isWordOrSymbolWord, splitToParagraphs } from '../nlp/Segmentation.js'
|
|
4
4
|
import { deepClone } from './ObjectUtilities.js'
|
|
5
5
|
import { getUTF32Chars } from './StringUtilities.js'
|
|
6
6
|
import { roundToDigits } from './Utilities.js'
|
|
@@ -79,7 +79,8 @@ export async function wordTimelineToSegmentSentenceTimeline(wordTimelineWithOffs
|
|
|
79
79
|
|
|
80
80
|
const segmentedWordSequence = await segmentWordSequence(wordSequenceWithPunctuation)
|
|
81
81
|
|
|
82
|
-
const paragraphs = splitToParagraphs(transcript, paragraphBreaks,
|
|
82
|
+
const paragraphs = splitToParagraphs(transcript, paragraphBreaks, 'preserve')
|
|
83
|
+
|
|
83
84
|
const sentenceIndexesForParagraph = paragraphs.map(_ => [] as number[])
|
|
84
85
|
|
|
85
86
|
{
|
|
@@ -152,7 +153,7 @@ export async function wordTimelineToSegmentSentenceTimeline(wordTimelineWithOffs
|
|
|
152
153
|
|
|
153
154
|
const segmentTimelineEntry: TimelineEntry = {
|
|
154
155
|
type: 'segment',
|
|
155
|
-
text: paragraph,
|
|
156
|
+
text: applyWhitespaceProcessing(paragraph, whitespace),
|
|
156
157
|
startTime: sentenceTimeline[0]?.startTime,
|
|
157
158
|
endTime: sentenceTimeline[sentenceTimeline.length - 1]?.endTime,
|
|
158
159
|
|