echogarden 3.0.5 → 3.2.0

package/src/utilities/FileSystem.ts

@@ -1,6 +1,5 @@
 import * as fsExtra from 'fs-extra/esm'
 import gracefulFS from 'graceful-fs'
-import * as os from 'node:os'
 
 import { promisify } from 'node:util'
 import { getRandomHexString, parseJson, stringifyAndFormatJson } from './Utilities.js'
@@ -8,7 +7,7 @@ import { appName } from '../api/Common.js'
 import { getAppTempDir, getDirName, joinPath, normalizePath, parsePath } from './PathUtilities.js'
 
 import { createDynamicUint8Array } from '../data-structures/DynamicTypedArray.js'
-import { ChunkedUtf8Decoder } from '../encodings/Utf8.js'
+import { ChunkedUtf8Decoder, ChunkedUtf8Encoder } from '../encodings/Utf8.js'
 import { FileWriter } from './FileWriter.js'
 import { FileReader } from './FileReader.js'
 
@@ -26,9 +25,10 @@ export const access = promisify(gracefulFS.access)
 
 export const readdir = promisify(gracefulFS.readdir)
 export const copyFile = promisify(gracefulFS.copyFile)
+export const rename = promisify(gracefulFS.rename)
+export const fsync = promisify(gracefulFS.fsync)
 
 export const remove = fsExtra.remove
-export const copy = fsExtra.copy
 
 ///////////////////////////////////////////////////////////////////////////////////////////
 // File read operations
@@ -41,32 +41,35 @@ export async function readFileAsBinary(filePath: string) {
 
 	const fileReader = new FileReader(filePath)
 	const buffer = new Uint8Array(chunkSize)
-	const result = createDynamicUint8Array(fileSize)
+	const fileContent = createDynamicUint8Array(fileSize)
 
 	while (!fileReader.isFinished) {
 		const chunk = await fileReader.readChunk(buffer)
 
-		result.addArray(chunk)
+		fileContent.addArray(chunk)
 	}
 
-	return result.toTypedArray()
+	return fileContent.toTypedArray()
 }
 
 export async function readFileAsUtf8(filePath: string) {
-	const chunkSize = 2 ** 20
+	const maxChunkSize = 2 ** 20
 
 	const fileReader = new FileReader(filePath)
-	const buffer = new Uint8Array(chunkSize)
+	const buffer = new Uint8Array(maxChunkSize)
+
+	const chunkedUtf8Decoder = new ChunkedUtf8Decoder()
 
-	const result = new ChunkedUtf8Decoder()
+	let fileContent = ''
 
 	while (!fileReader.isFinished) {
-		const chunk = await fileReader.readChunk(buffer)
+		const utf8Chunk = await fileReader.readChunk(buffer)
 
-		result.writeChunk(chunk)
+		const stringChunk = chunkedUtf8Decoder.writeChunk(utf8Chunk)
+		fileContent += stringChunk
 	}
 
-	return result.toString()
+	return fileContent
 }
 
 export async function readAndParseJsonFile(jsonFilePath: string, useJson5 = false) {
@@ -78,17 +81,17 @@ export async function readAndParseJsonFile(jsonFilePath: string, useJson5 = fals
 ///////////////////////////////////////////////////////////////////////////////////////////
 // File write operations
 ///////////////////////////////////////////////////////////////////////////////////////////
-export async function writeFile(filePath: string, content: Uint8Array | string) {
+export async function writeFile(filePath: string, content: Uint8Array | string, options?: WriteFileOptions) {
 	if (content instanceof Uint8Array) {
-		return writeBinaryFile(filePath, content)
+		return writeBinaryFile(filePath, content, options)
 	} else if (typeof content === 'string') {
-		return writeUtf8File(filePath, content)
+		return writeUtf8File(filePath, content, options)
 	} else {
-		throw new Error(`Content can only be a Uint8Array or string`)
+		throw new Error(`Content can only be a Uint8Array or string.`)
 	}
 }
 
-export async function writeBinaryFile(filePath: string, content: Uint8Array) {
+export async function writeBinaryFile(filePath: string, content: Uint8Array, options?: WriteFileOptions) {
 	const maxChunkSize = 2 ** 20
 
 	const fileDir = getDirName(filePath)
@@ -97,24 +100,28 @@ export async function writeBinaryFile(filePath: string, content: Uint8Array) {
 
 	const fileWriter = new FileWriter(filePath)
 
-	let readOffset = 0
+	try {
+		// The do-while body runs at least once, even for empty content, so the
+		// file is always created (or truncated) via FileWriter's first write.
+		let readOffset = 0
 
-	while (true) {
-		const chunk = content.subarray(readOffset, readOffset + maxChunkSize)
+		do {
+			const chunk = content.subarray(readOffset, readOffset + maxChunkSize)
 
-		if (chunk.length === 0) {
-			break
-		}
+			readOffset += chunk.length
 
-		await fileWriter.write(chunk)
+			await fileWriter.write(chunk)
+		} while (readOffset < content.length)
 
-		readOffset += chunk.length
+		if (options?.fsync === true) {
+			await fileWriter.fsync()
+		}
+	} finally {
+		await fileWriter.dispose()
 	}
-
-	await fileWriter.dispose()
 }
 
-export async function writeUtf8File(filePath: string, content: string) {
+export async function writeUtf8File(filePath: string, content: string, options?: WriteFileOptions) {
 	const maxChunkSize = 2 ** 20
 
 	const fileDir = getDirName(filePath)
@@ -123,25 +130,36 @@ export async function writeUtf8File(filePath: string, content: string) {
 
 	const fileWriter = new FileWriter(filePath)
 
-	const textEncoder = new TextEncoder()
+	const chunkedUtf8Encoder = new ChunkedUtf8Encoder()
 
-	let readOffset = 0
+	try {
+		// The do-while body runs at least once, even for empty content, so the
+		// file is always created (or truncated) via FileWriter's first write.
+		let readOffset = 0
 
-	while (true) {
-		const stringChunk = content.substring(readOffset, readOffset + maxChunkSize)
+		do {
+			const stringChunk = content.substring(readOffset, readOffset + maxChunkSize)
+
+			readOffset += stringChunk.length
 
-		const chunk = textEncoder.encode(stringChunk)
+			const utf8Chunk = chunkedUtf8Encoder.writeChunk(stringChunk)
 
-		await fileWriter.write(chunk)
+			await fileWriter.write(utf8Chunk)
+		} while (readOffset < content.length)
 
-		readOffset += stringChunk.length
+		const finalChunk = chunkedUtf8Encoder.finalize()
 
-		if (stringChunk.length === 0) {
-			break
+		if (finalChunk.length > 0) {
+			await fileWriter.write(finalChunk)
 		}
-	}
 
-	await fileWriter.dispose()
+		if (options?.fsync === true) {
+			await fileWriter.fsync()
+		}
+	} finally {
+		// Always release the file handle, even if an error interrupted the write.
+		await fileWriter.dispose()
+	}
 }
 
 export async function writeJsonFile(filePath: string, content: any, useJson5 = false) {
@@ -151,12 +169,25 @@ export async function writeJsonFile(filePath: string, content: any, useJson5 = f
 }
 
 export async function writeFileSafe(filePath: string, content: Uint8Array | string) {
-	const tempDir = getAppTempDir(appName)
+	const destDir = getDirName(filePath)
+
+	const tempDir = await getTemporaryDirectory(destDir)
 	const tempFilePath = joinPath(tempDir, `${getRandomHexString(16)}.partial`)
 
-	await writeFile(tempFilePath, content)
+	try {
+		await writeFile(tempFilePath, content, { fsync: true })
+
+		await move(tempFilePath, filePath)
+	} catch (e) {
+		// Best-effort cleanup so failed writes don't leave orphaned .partial files behind.
+		try {
+			await remove(tempFilePath)
+		} catch {
+			// Ignore cleanup errors so the original error is not masked.
+		}
 
-	await move(tempFilePath, filePath)
+		throw e
+	}
 }
 
 ///////////////////////////////////////////////////////////////////////////////////////////
@@ -176,19 +207,6 @@ export async function ensureDir(dirPath: string) {
 	}
 }
 
-export async function testDirectoryIsWritable(dir: string) {
-	const testFileName = joinPath(dir, getRandomHexString(16))
-
-	try {
-		await fsExtra.createFile(testFileName)
-		await remove(testFileName)
-	} catch (e) {
-		return false
-	}
-
-	return true
-}
-
 export async function readDirRecursive(dir: string, pathFilter?: (filePath: string) => boolean) {
 	if (!(await stat(dir)).isDirectory()) {
 		throw new Error(`'${dir}' is not a directory`)
@@ -220,33 +238,20 @@ export async function readDirRecursive(dir: string, pathFilter?: (filePath: stri
 	return result
 }
 
-export function getAppDataDir(appName: string) {
-	let dataDir: string
-
-	const platform = process.platform
-	const homeDir = os.homedir()
-
-	if (platform == 'win32') {
-		dataDir = joinPath(homeDir, 'AppData', 'Local', appName)
-	} else if (platform == 'darwin') {
-		dataDir = joinPath(homeDir, 'Library', 'Application Support', appName)
-	} else if (platform == 'linux') {
-		dataDir = joinPath(homeDir, '.local', 'share', appName)
-	} else {
-		throw new Error(`Unsupport platform ${platform}`)
-	}
-
-	return dataDir
-}
-
 ///////////////////////////////////////////////////////////////////////////////////////////
-// Copy operations
+// Copy and move operations
 ///////////////////////////////////////////////////////////////////////////////////////////
 export async function move(source: string, dest: string) {
 	source = normalizePath(source)
 	dest = normalizePath(dest)
 
-	if (existsSync(dest)) {
+	const destStats = existsSync(dest) ? await stat(dest) : undefined
+
+	if (destStats?.isDirectory()) {
+		throw new Error(`The destination path '${dest}' exists and is a directory. A file cannot be moved over a directory.`)
+	}
+
+	if (destStats) {
 		const destPathExistsAndIsWritable = await existsAndIsWritable(dest)
 
 		if (!destPathExistsAndIsWritable) {
@@ -254,29 +259,88 @@ export async function move(source: string, dest: string) {
 		}
 	} else {
 		const destDir = parsePath(dest).dir
-		const destDirIsWritable = await testDirectoryIsWritable(destDir)
 
-		if (!destDirIsWritable) {
-			throw new Error(`The directory ${destDir} is not writable. There may be a permissions issue.`)
+		await ensureDir(destDir)
+	}
+
+	try {
+		// fs.rename() replaces an existing destination file in a single step on
+		// POSIX and on Windows (where libuv uses MoveFileExW with
+		// MOVEFILE_REPLACE_EXISTING, the atomic replace flag available to Node).
+		await rename(source, dest)
+	} catch (e) {
+		const error = e as NodeJS.ErrnoException
+
+		if (error.code === 'EXDEV') {
+			// The source and destination are on different volumes, where an
+			// atomic rename is impossible; fall back to a copy and delete.
+			await copyFile(source, dest)
+			await tryFsyncFile(dest)
+			await remove(source)
+		} else {
+			throw error
 		}
 	}
 
-	return fsExtra.move(source, dest, { overwrite: true })
+	// The rename is now durable: flush the destination directory so the name
+	// change survives a crash (best effort, see the helper).
+	await tryFsyncDirectory(parsePath(dest).dir)
 }
 
 ///////////////////////////////////////////////////////////////////////////////////////////
-// Misc operations
+// Synchronization operations
 ///////////////////////////////////////////////////////////////////////////////////////////
-export async function existsAndIsWritable(targetPath: string) {
+export async function tryFsyncFile(filePath: string) {
+	let fileHandle: number | undefined
+
+	let succeeded = false
+
 	try {
-		await access(targetPath, gracefulFS.constants.W_OK);
+		fileHandle = await open(filePath, 'r+')
+
+		await fsync(fileHandle)
+
+		succeeded = true
 	} catch {
-		return false
+	} finally {
+		if (fileHandle !== undefined) {
+			try {
+				await close(fileHandle)
+			} catch {
+			}
+		}
 	}
 
-	return true
+	return succeeded
+}
+
+export async function tryFsyncDirectory(dirPath: string) {
+	let dirHandle: number | undefined
+
+	let succeeded = false
+
+	try {
+		dirHandle = await open(dirPath, 'r')
+
+		await fsync(dirHandle)
+
+		succeeded = true
+	} catch {
+	} finally {
+		if (dirHandle !== undefined) {
+			try {
+				await close(dirHandle)
+			} catch {
+			}
+		}
+	}
+
+	return succeeded
 }
 
+///////////////////////////////////////////////////////////////////////////////////////////
+// Misc operations
+///////////////////////////////////////////////////////////////////////////////////////////
 export async function chmodRecursive(rootPath: string, newMode: number) {
 	const rootPathStat = await stat(rootPath)
 
@@ -323,3 +387,78 @@ export async function computeFileSha256Hex(filePath: string) {
 
 	return result
 }
+
+export async function existsAndIsWritable(targetPath: string) {
+	try {
+		await access(targetPath, gracefulFS.constants.W_OK);
+	} catch {
+		return false
+	}
+
+	return true
+}
+
+// The OS temporary directory keeps temporary files away from the caller's
+// directories, and the OS itself eventually cleans it up. It is used when it is
+// on the same volume as the destination, since then the final rename is atomic.
+//
+// When it is on a different volume, the temporary file is written next to the
+// destination instead: an atomic rename across volumes is impossible and would
+// have to fall back to a copy and delete.
+async function getTemporaryDirectory(destDir: string) {
+	const osTempDir = getAppTempDir(appName)
+
+	await ensureDir(osTempDir)
+
+	if (await areOnSameDevice(osTempDir, destDir)) {
+		return osTempDir
+	} else {
+		return destDir
+	}
+}
+
+async function areOnSameDevice(firstPath: string, secondPath: string) {
+	const firstDevice = await getDeviceOfExistingPath(firstPath)
+	const secondDevice = await getDeviceOfExistingPath(secondPath)
+
+	return firstDevice !== undefined && firstDevice === secondDevice
+}
+
+// Resolves the device of the closest existing ancestor directory of a target
+// path, since the path itself may not exist yet. The device identifies the
+// volume a path lives on (on Windows it is the drive number).
+async function getDeviceOfExistingPath(targetPath: string) {
+	let currentPath = normalizePath(targetPath)
+
+	while (true) {
+		try {
+			const targetStats = await stat(currentPath)
+
+			return targetStats.dev
+		} catch (e) {
+			const error = e as NodeJS.ErrnoException
+
+			if (error.code !== 'ENOENT') {
+				// If the path cannot be inspected, treat the volumes as different;
+				// falling back to the destination directory always remains safe.
+				return undefined
+			}
+
+			const parentPath = getDirName(currentPath)
+
+			if (parentPath === currentPath) {
+				return undefined
+			}
+
+			currentPath = parentPath
+		}
+	}
+}
+
+///////////////////////////////////////////////////////////////////////////////////////////
+// Types
+///////////////////////////////////////////////////////////////////////////////////////////
+
+interface WriteFileOptions {
+	fsync?: boolean
+}

package/src/utilities/FileWriter.ts

@@ -1,4 +1,4 @@
-import { write, open, close } from "./FileSystem.js"
+import { write, open, close, fsync } from './FileSystem.js'
 
 export class FileWriter {
 	private fileHandle?: number
@@ -45,6 +45,16 @@ export class FileWriter {
 		this.fileHandle = await open(this.filePath, 'w')
 	}
 
+	async fsync() {
+		if (this.isDisposed) {
+			throw new Error(`FileWriter has been disposed`)
+		}
+
+		await this.openIfNeeded()
+
+		await fsync(this.fileHandle!)
+	}
+
 	async dispose() {
 		if (this.isDisposed) {
 			return

package/src/utilities/ObjectUtilities.ts

@@ -1,11 +1,12 @@
 export function extendDeep(base: any, extension: any): any {
 	const baseClone = deepClone(base)
 
-	if (isPlainObject(base) && extension === undefined) {
+	if (extension === undefined) {
 		return baseClone
 	}
 
 	const extensionClone = deepClone(extension)
+
 	if (!isPlainObject(base) || !isPlainObject(extension)) {
 		return extensionClone
 	}
@@ -21,15 +22,15 @@ export function extendDeep(base: any, extension: any): any {
 	return baseClone
 }
 
-export function shallowClone<T>(val: T) {
-	return clone(val, false)
+export function shallowClone<T>(val: T): T {
+	return clone(val, false, new Set())
 }
 
-export function deepClone<T>(val: T) {
-	return clone(val, true)
+export function deepClone<T>(val: T): T {
+	return clone(val, true, new Set())
 }
 
-function clone<T>(val: T, deep = true, seenObjects: any[] = []): T {
+function clone<T>(val: T, deep = true, seenObjects: Set<any>): T {
 	if (val === undefined || val === null || typeof val !== 'object') {
 		return val
 	}
@@ -39,11 +40,11 @@ function clone<T>(val: T, deep = true, seenObjects: any[] = []): T {
 
 	switch (prototypeIdentifier) {
 		case '[object Array]': {
-			if (seenObjects.includes(obj)) {
+			if (seenObjects.has(obj)) {
 				throw new Error('deepClone: encountered a cyclic object')
 			}
 
-			seenObjects.push(obj)
+			seenObjects.add(obj)
 
 			const clonedArray = new Array(obj.length)
 
@@ -55,7 +56,7 @@ function clone<T>(val: T, deep = true, seenObjects: any[] = []): T {
 				}
 			}
 
-			seenObjects.pop()
+			seenObjects.delete(obj)
 
 			return clonedArray as any
 		}
@@ -149,7 +150,10 @@ function clone<T>(val: T, deep = true, seenObjects: any[] = []): T {
 		}
 
 		case '[object RegExp]': {
-			return obj
+			const clonedRegExp = new RegExp(obj.source, obj.flags)
+			clonedRegExp.lastIndex = obj.lastIndex
+
+			return clonedRegExp as any
 		}
 
 		case '[object Function]': {
@@ -157,11 +161,11 @@ function clone<T>(val: T, deep = true, seenObjects: any[] = []): T {
 		}
 
 		case '[object Object]': {
-			if (seenObjects.includes(obj)) {
-				throw new Error('deepClone: encountered a cyclic object')
+			if (seenObjects.has(obj)) {
+				throw new Error('deepClone: encountered a cyclic object.')
 			}
 
-			seenObjects.push(obj)
+			seenObjects.add(obj)
 
 			const clonedObj: any = {}
 
@@ -177,13 +181,13 @@ function clone<T>(val: T, deep = true, seenObjects: any[] = []): T {
 				}
 			}
 
-			seenObjects.pop()
+			seenObjects.delete(obj)
 
 			return clonedObj
 		}
 
 		default: {
-			throw new Error(`Cloning of type ${prototypeIdentifier} is not supported`)
+			throw new Error(`Cloning of type ${prototypeIdentifier} is not supported.`)
 		}
 	}
 }

package/src/utilities/PackageManager.ts

@@ -1,7 +1,7 @@
 import { downloadAndExtractTarball, FileDownloaderCallbacks } from './FileDownloader.js'
-import { getAppDataDir, ensureDir, existsSync, remove } from './FileSystem.js'
+import { ensureDir, existsSync, remove } from './FileSystem.js'
 import { appName } from '../api/Common.js'
-import { getAppTempDir, joinPath } from './PathUtilities.js'
+import { getAppDataDir, getAppTempDir, joinPath } from './PathUtilities.js'
 import { getGlobalOption } from '../api/GlobalOptions.js'
 import { EasierHttpRequestConfig } from 'easier-http-request'
 

package/src/utilities/PathUtilities.ts

@@ -41,6 +41,25 @@ export function getAppTempDir(appName: string) {
 	return tempDir
 }
 
+export function getAppDataDir(appName: string) {
+	let dataDir: string
+
+	const platform = process.platform
+	const homeDir = os.homedir()
+
+	if (platform == 'win32') {
+		dataDir = joinPath(homeDir, 'AppData', 'Local', appName)
+	} else if (platform == 'darwin') {
+		dataDir = joinPath(homeDir, 'Library', 'Application Support', appName)
+	} else if (platform == 'linux') {
+		dataDir = joinPath(homeDir, '.local', 'share', appName)
+	} else {
+		throw new Error(`Unsupport platform ${platform}`)
+	}
+
+	return dataDir
+}
+
 export function joinPath(...paths: string[]) {
 	return NodePath.join(...paths)
 }

package/src/utilities/StringUtilities.ts

@@ -1,5 +1,3 @@
-import { TypedArray } from '../typings/TypedArray.js'
-
 export function includesAnyOf(str: string, substrings: string[]) {
 	return indexOfAnyOf(str, substrings) >= 0
 }

package/src/utilities/Timeline.ts

@@ -1,6 +1,6 @@
 import { addMissingPunctuationWordsToWordSequence, segmentWordSequence, WordSequence } from '@echogarden/text-segmentation'
 import { ParagraphBreakType, WhitespaceProcessing } from '../api/Common.js'
-import { applyWhitespaceProcessing, isWord, isWordOrEmojiOrSymbolWord, splitToParagraphs } from '../nlp/Segmentation.js'
+import { applyWhitespaceProcessing, includesWordCharacter, isWordOrEmojiOrSymbolWord, splitToParagraphs } from '../nlp/Segmentation.js'
 import { deepClone } from './ObjectUtilities.js'
 import { getUTF32Chars } from './StringUtilities.js'
 import { roundToDigits } from './Utilities.js'
@@ -213,7 +213,7 @@ function replaceSentenceEndersWithinWordsWithMaskingCharacter(transcript: string
 	for (const wordEntry of wordTimeline) {
 		const wordText = wordEntry.text
 
-		if (!isWord(wordText)) {
+		if (!includesWordCharacter(wordText)) {
 			continue
 		}
 

package/src/utilities/WikipediaReader.ts

@@ -1,10 +1,10 @@
 import { OperationCallbacks } from '../api/Common.js'
-import { splitToParagraphs, wordCharacterRegExp } from '../nlp/Segmentation.js'
+import { isWordOrEmojiOrSymbolWord, splitToParagraphs } from '../nlp/Segmentation.js'
 
 export async function fetchAndParseWikipediaArticle(articleName: string, language: string, callbacks: OperationCallbacks) {
-	const { default: wtf } = await import('wtf_wikipedia')
+	const { default: wtfWikipedia } = await import('wtf_wikipedia')
 
-	const document = await wtf.fetch(articleName, language)
+	const document = await wtfWikipedia.fetch(articleName, language)
 
 	if (!document) {
 		throw new Error('Error fetching Wikipedia article')
@@ -16,7 +16,7 @@ export async function fetchAndParseWikipediaArticle(articleName: string, languag
 	for (const section of sections) {
 		const sectionTitle = section.title()
 
-		if (wordCharacterRegExp.test(sectionTitle)) {
+		if (isWordOrEmojiOrSymbolWord(sectionTitle)) {
 			sectionsText.push(sectionTitle)
 		}
 
@@ -25,7 +25,7 @@ export async function fetchAndParseWikipediaArticle(articleName: string, languag
 		for (const paragraph of sectionParagraphs) {
 			const paragraphText = paragraph
 
-			if (wordCharacterRegExp.test(paragraphText)) {
+			if (isWordOrEmojiOrSymbolWord(paragraphText)) {
 				sectionsText.push(paragraphText)
 			}
 		}