echogarden 1.0.1 → 1.0.2

package/docs/Releases.md

@@ -1,6 +1,6 @@
 # Release notes
 
-## `1.0.0` (April 4, 2024)
+## `1.0.0` (April 12, 2024)
 
 **New features**:
 * Add support for [`whisper.cpp`](https://github.com/ggerganov/whisper.cpp), a C++ port of OpenAI's Whisper speech recognition architecture. It is faster than the integrated `whisper` engine, supports large models, and GPU processing. It can now be used for recognition, speech translation and alignment (via `dtw-ra`), though its word timestamps are less accurate than the integrated `whisper` engine.
@@ -13,6 +13,11 @@
 * Expose several new configuration options for the Whisper engine: a settings for maximum tokens per part, and a setting to enable/disable repetition suppression, set custom random seed, disable/enable decoding of timestamp tokens
 * Expose more options for the Elevenlabs engine
 
+**Enhancements**:
+* `detect-voice-activity` now also outputs the cropped voice when an audio file is given as output, with `.cropped` suffix added
+* Improved word segmentation of whisper engine. Words spanning multiple recognized time segments are now split. Word timestamp ranges don't overlap non-speech sections.
+* Whisper and Whisper.cpp outputs now include timestamps for individual recognized tokens
+
 **Behavioral and breaking changes**:
 * Minimal required node version changed to `18.0.0`
 * All recognition, alignment, translation and language recognition operations first apply the new adaptive gate VAD (can be changed to any other VAD engine via the `vad.` option prefix) and remove any sections that are not identified as containing voice, before starting processing. This should improve results in most cases, and reduce processing time
@@ -30,11 +35,6 @@
 * `maxIncreaseDb` post-processing and denoising option renamed to `maxGainIncrease`
 * `dryMixGainDb` denoising option renamed to `dryMixGain`
 
-**Enhancements**:
-* `detect-voice-activity` now also outputs the cropped voice when an audio file is given as output, with `.cropped` suffix added
-* Improved word segmentation of whisper engine. Words spanning multiple recognized time segments are now split. Word timestamp ranges don't overlap non-speech sections.
-* Whisper and Whisper.cpp outputs now include timestamps for individual recognized tokens
-
 **Fixes**:
 * Integrated `whisper` engine now uses `tiktoken` to tokenize text, which produces near identical tokenization compared to the official Python implementation. This fixes issues with decoding Chinese characters, correctly encoding prompts, and several others languages with characters that span multiple tokens
 * Fix voice language lists for multilingual voices in Elevenlabs TTS engine
@@ -43,14 +43,15 @@
 * Fix issue with identifying legacy IEEE Float wave format
 * Fix issue with language detection failing with empty inputs
 * Fix issue with Whisper model failing when no tokens are detected in a part
-* Fix issue with phone alignment not getting the right DTW window size, producing warnings like `all cost directions are equal to infinity
+* Fix issue with phone alignment not getting the right DTW window size, producing warnings like `all cost directions are equal to infinity`
 * Fix phone timelines in DTW-RA
 * Add more workarounds for eSpeak tokenization bugs
 * Various other fixes
 
 **Documentation**:
-* Add new 'Releases' page. Releases before `1.0.x` were retroactively documented based on commit history (may not exactly detail all historical changes)
-* Options reference page restructured and updated with some missing information
+* Add new [release notes](./Releases.md) page. Releases before `1.0.x` were retroactively documented based on commit history (may not exactly detail all historical changes)
+* Add new [development page](./Development.md) with a quick guide for setting up a development environment to make changes to the Echogarden codebase
+* Options reference page reorganized and updated with some missing information
 
 ## `0.12.x` (March 16, 2024)
 
@@ -67,6 +68,7 @@
 **Other**:
 * Include `package-lock.json` in repository
 
+
 ## `0.11.x` (August 17, 2023)
 
 Many features, enhancements, and fixes were incrementally added over the span of 7 months, up to March 2024.
@@ -87,6 +89,7 @@ Many features, enhancements, and fixes were incrementally added over the span of
 * Fix audio playback in macOS
 * Many other fixes
 
+
 ## `0.10.x` (August 2, 2023)
 
 **New features**:
@@ -100,6 +103,7 @@ Many features, enhancements, and fixes were incrementally added over the span of
 **Fixes**:
 * Many fixes
 
+
 ## `0.9.x` (July 29, 2023)
 
 **New features**:
@@ -114,6 +118,7 @@ Many features, enhancements, and fixes were incrementally added over the span of
 * Work around several eSpeak bugs
 * Many fixes
 
+
 ## `0.8.x` (July 25, 2023)
 
 **New features**:
@@ -122,26 +127,25 @@ Many features, enhancements, and fixes were incrementally added over the span of
 * Add punctuation thresholds to Whisper decoder
 * Expose subtitle configuration options to CLI
 
-**Behavioral changes**:
-* Rename `subtitles.minWords` to `subtitles.minWordsInLine`
-
 **Enhancements**:
 * Improve speech language detection to work with arbitrary length audio. Split audio to overlapping parts and detect each part individually. Then average the results.
 * Add more heteronyms
 
+**Behavioral changes**:
+* Rename `subtitles.minWords` to `subtitles.minWordsInLine`
+
 **Fixes**:
 * Improve numerical stability of softmax function
 * Work around eSpeak bug with markers in long inputs
 * Various fixes
 
+
 ## `0.7.x` (July 23, 2023)
 
 **New features**:
 * Alignment: Add option to accept custom lexicons
 * Implement and add decoder temperature option to Whisper model
 
-**Behavioral changes**:
-
 **Enhancements**:
 * Alignment: use preprocessing and lexicons
 * Add language detection to speech translation
@@ -151,10 +155,13 @@ Many features, enhancements, and fixes were incrementally added over the span of
 * Add colors to log messages
 * Warn when maximum DTW window duration is smaller than source audio duration.
 
+**Behavioral changes**:
+
 **Fixes**:
 * Fix support for SSML input in eSpeak engine
 * Various fixes
 
+
 ## `0.6.x` (July 20, 2023)
 
 **New features**:
@@ -165,16 +172,16 @@ Many features, enhancements, and fixes were incrementally added over the span of
 * Add arguments to customize paragraph parsing
 * Change whitespace option to include option for collapsing all whitespace
 
-**Behavioral changes**:
-* Set Elevenlabs defaults to mid-values
-* Split plain text to paragraphs using double line breaks by default.
-* Rename `awsPolly` options to `amazonPolly` to be more consistent with documentation
-
 **Enhancements**:
 * Change whitespace option to include option for collapsing all whitespace
 * Add awareness of guillemets
 * Log full language of selected voice
 
+**Behavioral changes**:
+* Set Elevenlabs defaults to mid-values
+* Split plain text to paragraphs using double line breaks by default.
+* Rename `awsPolly` options to `amazonPolly` to be more consistent with documentation
+
 **Fixes**:
 * Fix and update Elevenlabs engine
 * Fix incorrect gender properties for some VITS voices
@@ -184,26 +191,25 @@ Many features, enhancements, and fixes were incrementally added over the span of
 **Other**:
 * Remove `package-lock.json` from the repository
 
+
 ## `0.5.x` (July, 19 2023)
 
 **New features**:
 
-**Behavioral changes**:
-* Move to new package system supporting version tags, and hosted in a Hugging Face repository
-
 **Enhancements**:
 * Add large numbers of VITS voices
 
+**Behavioral changes**:
+* Move to new package system supporting version tags, and hosted in a Hugging Face repository
+
 **Fixes**:
 * Various fixes
 
+
 ## `0.4.x` (July 9, 2023)
 
 **New features**:
 
-**Behavioral changes**:
-* Remove support for `afplay` and `aplay` for playback. Only SoX is used now.
-
 **Enhancements**:
 * Improve text normalization and add support for currencies
 * Improve heteronym lexicons
@@ -212,32 +218,34 @@ Many features, enhancements, and fixes were incrementally added over the span of
 * Add some British English (RP) pronunciations to heteronym lexicon
 * Add large numbers of VITS voices
 
+**Behavioral changes**:
+* Remove support for `afplay` and `aplay` for playback. Only SoX is used now.
+
 **Fixes**:
 * Various fixes
 
+
 ## `0.3.x` (July 1, 2023)
 
 **New features**:
 
-**Behavioral changes**:
-
 **Enhancements**:
 * Upgrade heteronym disambiguation to an improved, rule-based approach, which doesn't use POS tagging
 * Extend decade normalization
 * Improve text normalization and add support to currencies
 * Update the heteronym lexicon
 
+**Behavioral changes**:
+
 **Fixes**:
 * Remove sentences containing only whitespace when synthesizing
 * Many fixes
 
+
 ## `0.2.x` (May 10, 2023)
 
 **New features**:
 
-**Behavioral changes**:
-* Remove dependency on `xregexp` package. Use Unicode RegExp instead.
-
 **Enhancements**:
 * Show current sentence and segment in synthesis log.
 * Add check for cancellation flag.
@@ -245,6 +253,9 @@ Many features, enhancements, and fixes were incrementally added over the span of
 * Improve auto TTS engine selection
 * Many Enhancements
 
+**Behavioral changes**:
+* Remove dependency on `xregexp` package. Use Unicode RegExp instead.
+
 **Fixes**:
 * Many fixes
 

package/docs/Tasklist.md

@@ -2,7 +2,10 @@
 
 ## Bugs
 
+### Alignment
+
 * In DTW-RA, recognition transcript including something like "Question 2.What does Juan", where "2.What" has a point in the middle, is breaking playback of the timeline.
+* DTW-RA will not work correctly with Polish language texts, due to issues with the eSpeak engine pronouncing `|` characters, which are intended to be used as separators and ignored by all other eSpeak languages.
 
 ### Synthesis
 
@@ -108,8 +111,8 @@
 * See if it's possible to reliably use eSpeak as a segmentation engine
 
 ### Subtitles
+* Split long words if needed. This is especially important for Chinese
 * If a subtitle is too short and at the end of the audio, try to extend it back if possible (for example, if the previous subtitle is already extended, take back from it)
-* Split long words if needed
 * Decide how many punctuation characters to allow before breaking to a new line (currently it's infinite)
 * Add more clause separators, for even more special cases
 * Add option to output usable word or phoneme-level caption files (investigate how it's done on YouTube auto-captions)
@@ -155,10 +158,10 @@
 * Currently, when input is set to be SSML, it is wrapped in a `<speak>` tag. Handle the case where the user made their own SSML document wrapped with a `<speak>` tag as well. Currently, it may send invalid input to Azure
 
 ### Recognition
+* Recognized word entries that span VAD segment boundaries can be split
 * Show alternatives when playing in the CLI. Clear current line and rewrite already printed text for alternatives during the speech recognition process
 * Option to split recognized audio to segments or sentences, as is done with synthesized audio
 * Try to exclude the timing for trailing punctuation tokens in words that contain them. This can help narrow down the end timestamp to cover the word more tightly
-* Recognized word entries that span VAD boundaries may be split
 
 ### Recognition / Whisper
 * May get stuck in a token repeat loop when silence or non-speech segment encountered in audio. Decide what to do

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "echogarden",
-	"version": "1.0.1",
+	"version": "1.0.2",
 	"description": "An integrated speech toolset, providing a variety of synthesis, recognition, alignment, and other processing tools, designed to be directly accessible to end-users.",
 	"author": "Rotem Dan",
 	"license": "GPL-3.0",
@@ -90,11 +90,11 @@
 		"ndarray": "^1.0.19",
 		"ndarray-ops": "^1.2.2",
 		"onnxruntime-node": "^1.17.0",
-		"openai": "^4.33.0",
+		"openai": "^4.33.1",
 		"sam-js": "^0.2.1",
 		"strip-ansi": "^7.1.0",
 		"tar": "^7.0.0",
-		"tiktoken": "^1.0.13",
+		"tiktoken": "^1.0.14",
 		"tinyld": "^1.3.4",
 		"ws": "^8.16.0",
 		"wtf_wikipedia": "^10.3.0"
@@ -127,7 +127,7 @@
 		"@types/recursive-readdir": "^2.2.4",
 		"@types/tar": "^6.1.12",
 		"@types/ws": "^8.5.10",
-		"ts-json-schema-generator": "^1.5.1",
+		"ts-json-schema-generator": "^2.0.0",
 		"typescript": "^5.4.5"
 	}
 }

package/src/alignment/SpeechAlignment.ts

@@ -151,7 +151,7 @@ export async function alignUsingDtw(
 	return mappedTimeline
 }
 
-export async function alignUsingDtwWithRecognitionReference(
+export async function alignUsingDtwWithRecognition(
 	sourceRawAudio: RawAudio,
 	referenceRawAudio: RawAudio,
 	referenceTimeline: Timeline,
@@ -272,6 +272,7 @@ export async function alignUsingDtwWithRecognitionReference(
 
 	let currentSynthesizedToRecognizedMappingIndex = 0
 
+	// Map from synthesized reference timestamps to the recognition timestamps
 	function mapSynthesizedToRecognizedTimeAndAdvance(synthesizedTime: number) {
 		for (; ; currentSynthesizedToRecognizedMappingIndex += 1) {
 			const left = synthesizedToRecognizedTimeMapping[currentSynthesizedToRecognizedMappingIndex].synthesized

package/src/api/Alignment.ts

@@ -12,6 +12,7 @@ import chalk from 'chalk'
 import { DtwGranularity, createAlignmentReferenceUsingEspeak } from '../alignment/SpeechAlignment.js'
 import { SubtitlesConfig, defaultSubtitlesBaseConfig } from '../subtitles/Subtitles.js'
 import { EspeakOptions, defaultEspeakOptions } from '../synthesis/EspeakTTS.js'
+import { isWord } from '../nlp/Segmentation.js'
 
 const log = logToStderr
 
@@ -85,7 +86,7 @@ export async function align(input: AudioSourceParam, transcript: string, options
 
 	logger.start('Load alignment module')
 
-	const { alignUsingDtwWithRecognitionReference, alignUsingDtw } = await import('../alignment/SpeechAlignment.js')
+	const { alignUsingDtwWithRecognition: alignUsingDtwWithRecognitionReference, alignUsingDtw } = await import('../alignment/SpeechAlignment.js')
 
 	function getDtwWindowDurationsAndGranularities() {
 		let granularities: DtwGranularity[]
@@ -145,7 +146,12 @@ export async function align(input: AudioSourceParam, transcript: string, options
 			logger.end()
 
 			// Recognize source audio
-			const { transcript: recognizedTranscript, wordTimeline: recognitionTimeline } = await API.recognize(sourceRawAudio, recognitionOptions)
+			let { wordTimeline: recognitionTimeline } = await API.recognize(sourceRawAudio, recognitionOptions)
+
+			logger.log('')
+
+			// Remove non-word entries from recognition timeline
+			recognitionTimeline = recognitionTimeline.filter(entry => isWord(entry.text))
 
 			// Synthesize the ground-truth transcript and get its timeline
 			logger.start('Synthesize ground-truth transcript with eSpeak')
@@ -154,14 +160,14 @@ export async function align(input: AudioSourceParam, transcript: string, options
 				referenceRawAudio,
 				referenceTimeline,
 				espeakVoice,
-			} = await createAlignmentReferenceUsingEspeak(transcript, language, options.plainText, options.customLexiconPaths, true)
+			} = await createAlignmentReferenceUsingEspeak(transcript, language, options.plainText, options.customLexiconPaths, false)
 
 			logger.end()
 
 			const { windowDurations, granularities } = getDtwWindowDurationsAndGranularities()
 
 			const phoneAlignmentMethod = options.dtw!.phoneAlignmentMethod!
-			
+
 			const espeakOptions: EspeakOptions = {
 				...defaultEspeakOptions,
 				voice: espeakVoice,
@@ -345,10 +351,10 @@ export const defaultAlignmentOptions: AlignmentOptions = {
 			topCandidateCount: 5,
 			punctuationThreshold: 0.2,
 			maxTokensPerPart: 250,
-			autoPromptParts: true,
+			autoPromptParts: false,
 			suppressRepetition: true,
+			decodeTimestampTokens: true,
 			seed: undefined,
-			decodeTimestampTokens: false,
 		}
 	},
 

package/src/dsp/FFT.ts

@@ -122,7 +122,7 @@ export async function stiftr(binsForFrames: Float32Array[], fftOrder: number, wi
 
 	wasmMemory.freeAll()
 
-	// Divide by sum of weight squares for each samples
+	// Divide each output sample by the sum of squared weights
 	for (let i = 0; i < outSamples.length; i++) {
 		outSamples[i] /= sumOfSquaredWeightsForSample[i] + 1e-8
 	}

package/src/nlp/Segmentation.ts

@@ -8,8 +8,9 @@ import { ParagraphBreakType, WhitespaceProcessing } from '../api/Common.js'
 
 const log = logToStderr
 
-export const wordCharacterPattern = /[\p{Letter}\p{Number}]/u
-export const punctuationPattern = /[\p{Punctuation}]/u
+export const wordCharacterPattern = /[\p{Letter}\p{Number}]+/u
+export const punctuationPattern = /[\p{Punctuation}]+/u
+
 export const phraseSeparators = [',', ';', ':']
 export const sentenceSeparators = ['.', '?', '!']
 export const symbolWords = ['$', '€', '¢', '£', '¥', '©', '®', '™', '%', '&', '#', '~', '@', '+', '±', '÷', '/', '*', '=', '¼', '½', '¾']
@@ -18,13 +19,20 @@ export function isWordOrSymbolWord(str: string) {
 	return isWord(str) || symbolWords.includes(str)
 }
 
+export function isSymbolWord(str: string) {
+	return symbolWords.includes(str.trim())
+}
+
 export function isWord(str: string) {
-	str = str.trim()
-	return wordCharacterPattern.test(str) || symbolWords.includes(str)
+	return wordCharacterPattern.test(str.trim())
 }
 
 export function isPunctuation(str: string) {
-	return punctuationPattern.test(str)
+	return punctuationPattern.test(str.trim())
+}
+
+export function isWhitespace(str: string) {
+	return str.trim().length === 0
 }
 
 export class Sentence {

package/src/recognition/WhisperCppSTT.ts

@@ -288,13 +288,14 @@ async function parseResultObject(resultObject: WhisperCppVerboseResult, modelNam
 
 	const allTokenIds = tokenTimeline.map(entry => entry.id!)
 	const transcript = whisper.tokensToText(allTokenIds).trim()
+	const language = resultObject.result.language
 
-	let timeline = whisper.tokenTimelineToWordTimeline(tokenTimeline)
+	const timeline = whisper.tokenTimelineToWordTimeline(tokenTimeline, language)
 
 	return {
 		transcript,
 		timeline,
-		language: resultObject.result.language
+		language
 	}
 }
 

package/src/recognition/WhisperSTT.ts

@@ -19,6 +19,7 @@ import chalk from 'chalk'
 import { XorShift32RNG } from '../utilities/RandomGenerator.js'
 import { detectSpeechLanguageByParts } from '../api/LanguageDetection.js'
 import { type Tiktoken } from 'tiktoken/lite'
+import { isPunctuation, isWhitespace } from '../nlp/Segmentation.js'
 
 export async function recognize(sourceRawAudio: RawAudio, modelName: WhisperModelName, modelDir: string, task: WhisperTask, sourceLanguage: string, options: WhisperOptions) {
 	if (sourceRawAudio.sampleRate != 16000) {
@@ -398,7 +399,7 @@ export class Whisper {
 			logger.end()
 		}
 
-		timeline = this.tokenTimelineToWordTimeline(timeline)
+		timeline = this.tokenTimelineToWordTimeline(timeline, language)
 
 		const transcript = this.tokensToText(allDecodedTokens).trim()
 
@@ -438,7 +439,7 @@ export class Whisper {
 		const alignmentPath = await this.findAlignmentPathFromQKs(crossAttentionQKs, tokens, 0, audioFrameCount)//, this.getAlignmentHeadIndexes())
 		let timeline = await this.getTokenTimelineFromAlignmentPath(alignmentPath, tokens, 0, audioDuration)
 
-		timeline = this.tokenTimelineToWordTimeline(timeline)
+		timeline = this.tokenTimelineToWordTimeline(timeline, language)
 
 		logger.end()
 
@@ -1022,33 +1023,40 @@ export class Whisper {
 		}
 	}
 
-	tokenTimelineToWordTimeline(tokenTimeline: Timeline) {
-		const separatorChars =
-			[' ', '–', '一', '，', '、', '|', '/', '\\', ';', '"', '“', '”', '…', '(', ')', '[', ']', '{', '}']
+	tokenTimelineToWordTimeline(tokenTimeline: Timeline, language: string): Timeline {
+		function isSeparatorCharacter(char: string) {
+			const nonSeparatingPunctuation = [`'`, `-`, `.`, `·`, `•`]
 
-		function startsWithSeparatingPunctuation(text: string) {
-			return separatorChars.some(char => text.startsWith(char))
+			if (nonSeparatingPunctuation.includes(char)) {
+				return false
+			}
+
+			return isWhitespace(char) || isPunctuation(char)
+		}
+
+		function startsWithSeparatorCharacter(text: string) {
+			return isSeparatorCharacter(text[0])
 		}
 
-		function isSeparatorPunctuation(text: string) {
-			return separatorChars.includes(text)
+		function endsWithSeparatorCharacter(text: string) {
+			return isSeparatorCharacter(text[text.length - 1])
 		}
 
 		const resultTimeline: Timeline = []
 
-		const groups: TimelineEntry[][] = []
+		let groups: TimelineEntry[][] = []
 
-		for (let i = 0; i < tokenTimeline.length; i++) {
-			const entry = tokenTimeline[i]
-			const previousEntry = i > 0 ? tokenTimeline[i - 1] : undefined
+		for (let tokenIndex = 0; tokenIndex < tokenTimeline.length; tokenIndex++) {
+			const entry = tokenTimeline[tokenIndex]
+			const previousEntry = tokenIndex > 0 ? tokenTimeline[tokenIndex - 1] : undefined
 
 			const text = entry.text
 			const previousEntryText = previousEntry?.text
 
 			if (groups.length == 0 ||
 				text === '' ||
-				startsWithSeparatingPunctuation(text) ||
-				(previousEntryText != null && isSeparatorPunctuation(previousEntryText))) {
+				startsWithSeparatorCharacter(text) ||
+				(previousEntryText != null && endsWithSeparatorCharacter(previousEntryText))) {
 
 				groups.push([entry])
 			} else {
@@ -1056,8 +1064,28 @@ export class Whisper {
 			}
 		}
 
+		const newGroups: TimelineEntry[][] = []
+
+		for (let groupIndex = 0; groupIndex < groups.length - 1; groupIndex++) {
+			const group = groups[groupIndex]
+			const nextGroup = groups[groupIndex + 1]
+
+			if (
+				group.length > 1 &&
+				group[group.length - 1].text === '.' &&
+				[' ', '['].includes(nextGroup[0].text[0])) {
+
+				newGroups.push(group.slice(0, group.length - 1))
+				newGroups.push(group.slice(group.length - 1))
+			} else {
+				newGroups.push(group)
+			}
+		}
+
+		groups = newGroups
+
 		for (const group of groups) {
-			const groupText = this.tokensToText(group.map(entry => entry.id!))
+			let groupText = this.tokensToText(group.map(entry => entry.id!))
 
 			if (groupText === '') {
 				continue
@@ -1755,8 +1783,8 @@ export interface WhisperOptions {
 	autoPromptParts?: boolean
 	maxTokensPerPart?: number
 	suppressRepetition?: boolean
-	seed?: number
 	decodeTimestampTokens?: boolean
+	seed?: number
 }
 
 export const defaultWhisperOptions: WhisperOptions = {
@@ -1768,6 +1796,6 @@ export const defaultWhisperOptions: WhisperOptions = {
 	autoPromptParts: true,
 	maxTokensPerPart: 250,
 	suppressRepetition: true,
+	decodeTimestampTokens: true,
 	seed: undefined,
-	decodeTimestampTokens: false,
 }

package/src/subtitles/Subtitles.ts

@@ -200,7 +200,7 @@ function getCuesFromTimeline_IsolateSegmentSentence(timeline: Timeline, config:
 			continue
 		}
 
-		const wordTimeline = entry.timeline!.filter(entry => isWord(entry.text))
+		const wordTimeline = entry.timeline!.filter(entry => isWordOrSymbolWord(entry.text))
 
 		// First, add word start and end offsets for all word entries
 		let lastWordEndOffset = 0

package/src/tests/Test.ts

@@ -1,6 +1,7 @@
-import { getRepetitionScoreRelativeToFirstSubstring, logToStderr, setupProgramTerminationListeners } from '../utilities/Utilities.js'
+import { getRepetitionScoreRelativeToFirstSubstring, logToStderr, setupProgramTerminationListeners, writeToStderr } from '../utilities/Utilities.js'
 import { makeTarballsForInstalledPackages } from '../utilities/TarballMaker.js'
 import { testEspeakSynthesisWithPrePhonemizedInputs, testKirshenbaumPhonemization } from '../synthesis/EspeakTTS.js'
+import { isPunctuation } from '../nlp/Segmentation.js'
 
 const log = logToStderr
 
@@ -21,4 +22,19 @@ setupProgramTerminationListeners()
 //getRepetitionScoreRelativeToFirstSubstring(['a', 'b', 'a', 'c', 'a', 'b', 'a', 'c', 'a'])
 //getRepetitionScoreRelativeToFirstSubstring(['a', 'a', 'a', 'b', 'b', 'a', 'a', 'a', 'b'])
 
+/*
+const allPunctuationChars: string[] = []
+
+for (let i = 0; i < 65536; i++) {
+	const char = String.fromCodePoint(i)
+
+	if (isPunctuation(char)) {
+		allPunctuationChars.push(char)
+
+		writeToStderr(`${char} `)
+	}
+}
+*/
+
 process.exit(0)
+