@inlang/sdk 0.35.4 → 0.35.6

package/src/persistence/filelock/acquireFileLock.ts

@@ -21,16 +21,19 @@ export async function acquireFileLock(
 	try {
 		debug(lockOrigin + " tries to acquire a lockfile Retry Nr.: " + tryCount)
 		await fs.mkdir(lockDirPath)
-		const stats = await fs.stat(lockDirPath)
+		// NOTE: fs.stat does not need to be atomic since mkdir would crash atomically - if we are here its save to consider the lock held by this process
+		const stats = await fs.stat(lockDirPath) 
 		debug(lockOrigin + " acquired a lockfile Retry Nr.: " + tryCount)
+
 		return stats.mtimeMs
 	} catch (error: any) {
 		if (error.code !== "EEXIST") {
-			// we only expect the error that the file exists already (locked by other process)
+		// NOTE in case we have an EEXIST - this is an expected error: the folder existed - another process already acquired the lock. Rethrow all other errors
 			throw error
 		}
 	}
 
+	// land here if the lockDirPath already exists => lock is held by other process
 	let currentLockTime: number
 
 	try {

package/src/persistence/filelock/releaseLock.ts

@@ -11,8 +11,9 @@ export async function releaseLock(
 	debug(lockOrigin + " releasing the lock ")
 	try {
 		const stats = await fs.stat(lockDirPath)
+		// I believe this check associates the lock with the aquirer
 		if (stats.mtimeMs === lockTime) {
-			// this can be corrupt as welll since the last getStat and the current a modification could have occured :-/
+		// NOTE: since we have to use a timeout for stale detection (uTimes is not exposed via mermoryfs) the check for the locktime is not sufficient and can fail in rare cases when another process accuires a lock that was identifiert as tale between call to fs.state and rmDir
 			await fs.rmdir(lockDirPath)
 		}
 	} catch (statError: any) {

package/src/persistence/store.test.ts

@@ -0,0 +1,102 @@
+import { test, expect } from "vitest"
+import { createNodeishMemoryFs } from "../test-utilities/index.js"
+import type { MessageBundle } from "../v2/types.js"
+import { createMessageBundle, createMessage, injectJSONNewlines, addSlots } from "../v2/helper.js"
+import { openStore, readJSON, writeJSON } from "./store.js"
+
+const locales = ["en", "de"]
+
+const mockMessages: MessageBundle[] = [
+	createMessageBundle({
+		id: "first_message",
+		messages: [
+			createMessage({
+				locale: "en",
+				text: "If this fails I will be sad",
+			}),
+		],
+	}),
+	createMessageBundle({
+		id: "second_message",
+		messages: [
+			createMessage({ locale: "en", text: "Let's see if this works" }),
+			createMessage({ locale: "de", text: "Mal sehen ob das funktioniert" }),
+		],
+	}),
+]
+
+test("roundtrip readJSON/writeJSON", async () => {
+	const nodeishFs = createNodeishMemoryFs()
+	const projectPath = "/test/project.inlang"
+	const filePath = projectPath + "/messages.json"
+	const persistedMessages = injectJSONNewlines(
+		JSON.stringify(mockMessages.map((bundle) => addSlots(bundle, locales)))
+	)
+
+	await nodeishFs.mkdir(projectPath, { recursive: true })
+	await nodeishFs.writeFile(filePath, persistedMessages)
+
+	const firstMessageLoad = await readJSON({
+		filePath,
+		nodeishFs: nodeishFs,
+	})
+
+	expect(firstMessageLoad).toStrictEqual(mockMessages)
+
+	await writeJSON({
+		filePath,
+		nodeishFs,
+		messages: firstMessageLoad,
+		locales,
+	})
+
+	const afterRoundtrip = await nodeishFs.readFile(filePath, { encoding: "utf-8" })
+
+	expect(afterRoundtrip).toStrictEqual(persistedMessages)
+
+	const messagesAfterRoundtrip = await readJSON({
+		filePath,
+		nodeishFs,
+	})
+
+	expect(messagesAfterRoundtrip).toStrictEqual(firstMessageLoad)
+})
+
+test("openStore does minimal CRUD on messageBundles", async () => {
+	const nodeishFs = createNodeishMemoryFs()
+	const projectPath = "/test/project.inlang"
+	const filePath = projectPath + "/messages.json"
+	const persistedMessages = JSON.stringify(mockMessages)
+
+	await nodeishFs.mkdir(projectPath, { recursive: true })
+	await nodeishFs.writeFile(filePath, persistedMessages)
+
+	const store = await openStore({ projectPath, nodeishFs, locales })
+
+	const messages = await store.messageBundles.getAll()
+	expect(messages).toStrictEqual(mockMessages)
+
+	const firstMessageBundle = await store.messageBundles.get({ id: "first_message" })
+	expect(firstMessageBundle).toStrictEqual(mockMessages[0])
+
+	const modifedMessageBundle = structuredClone(firstMessageBundle) as MessageBundle
+	const newMessage = createMessage({ locale: "de", text: "Wenn dies schiefläuft, bin ich sauer" })
+	modifedMessageBundle.messages.push(newMessage)
+	await store.messageBundles.set({ data: modifedMessageBundle })
+
+	const setMessageBundle = await store.messageBundles.get({ id: "first_message" })
+	expect(setMessageBundle).toStrictEqual(modifedMessageBundle)
+
+	// no need to sleep here
+	// with batchedIO, the set should await until the write is done
+	const messagesAfterRoundtrip = await readJSON({
+		filePath,
+		nodeishFs,
+	})
+	const expected = [setMessageBundle, ...mockMessages.slice(1)]
+	expect(messagesAfterRoundtrip).toStrictEqual(expected)
+
+	await store.messageBundles.delete({ id: "first_message" })
+	const messagesAfterDelete = await store.messageBundles.getAll()
+	expect(messagesAfterDelete).toStrictEqual(mockMessages.slice(1))
+})

package/src/persistence/store.ts

@@ -0,0 +1,119 @@
+import type { MessageBundle } from "../v2/types.js"
+import { addSlots, removeSlots, injectJSONNewlines } from "../v2/helper.js"
+import { getDirname, type NodeishFilesystem } from "@lix-js/fs"
+import { acquireFileLock } from "./filelock/acquireFileLock.js"
+import { releaseLock } from "./filelock/releaseLock.js"
+import { batchedIO } from "./batchedIO.js"
+import type { StoreApi } from "./storeApi.js"
+
+import _debug from "debug"
+const debug = _debug("sdk:store")
+
+export async function openStore(args: {
+	projectPath: string
+	nodeishFs: NodeishFilesystem
+	locales: string[]
+}): Promise<StoreApi> {
+	const nodeishFs = args.nodeishFs
+	const filePath = args.projectPath + "/messages.json"
+	const lockDirPath = args.projectPath + "/messagelock"
+
+	// the index holds the in-memory state
+	// TODO: reload when file changes on disk
+	// https://github.com/opral/inlang-message-sdk/issues/80
+	let index = await load()
+
+	const batchedSave = batchedIO(acquireSaveLock, releaseSaveLock, save)
+
+	return {
+		messageBundles: {
+			reload: async () => {
+				index.clear()
+				index = await load()
+			},
+			get: async (args: { id: string }) => {
+				return index.get(args.id)
+			},
+			set: async (args: { data: MessageBundle }) => {
+				index.set(args.data.id, args.data)
+				await batchedSave(args.data.id)
+			},
+			delete: async (args: { id: string }) => {
+				index.delete(args.id)
+				await batchedSave(args.id)
+			},
+			getAll: async () => {
+				return [...index.values()]
+			},
+		},
+	}
+
+	// load and save messages from file system atomically
+	// using a lock file to prevent partial reads and writes
+	async function load() {
+		const lockTime = await acquireFileLock(nodeishFs, lockDirPath, "load")
+		const messages = await readJSON({ filePath, nodeishFs: nodeishFs })
+		const index = new Map<string, MessageBundle>(messages.map((message) => [message.id, message]))
+		await releaseLock(nodeishFs, lockDirPath, "load", lockTime)
+		return index
+	}
+	async function acquireSaveLock() {
+		return await acquireFileLock(nodeishFs, lockDirPath, "save")
+	}
+	async function releaseSaveLock(lock: number) {
+		return await releaseLock(nodeishFs, lockDirPath, "save", lock)
+	}
+	async function save() {
+		await writeJSON({
+			filePath,
+			nodeishFs: nodeishFs,
+			messages: [...index.values()],
+			locales: args.locales,
+		})
+	}
+}
+
+export async function readJSON(args: { filePath: string; nodeishFs: NodeishFilesystem }) {
+	let result: MessageBundle[] = []
+
+	debug("loadAll", args.filePath)
+	try {
+		const file = await args.nodeishFs.readFile(args.filePath, { encoding: "utf-8" })
+		result = JSON.parse(file)
+	} catch (error) {
+		if ((error as any)?.code !== "ENOENT") {
+			debug("loadMessages", error)
+			throw error
+		}
+	}
+	return result.map(removeSlots)
+}
+
+export async function writeJSON(args: {
+	filePath: string
+	nodeishFs: NodeishFilesystem
+	messages: MessageBundle[]
+	locales: string[]
+}) {
+	debug("saveall", args.filePath)
+	try {
+		await createDirectoryIfNotExits(getDirname(args.filePath), args.nodeishFs)
+		const output = injectJSONNewlines(
+			JSON.stringify(args.messages.map((bundle) => addSlots(bundle, args.locales)))
+		)
+		await args.nodeishFs.writeFile(args.filePath, output)
+	} catch (error) {
+		debug("saveMessages", error)
+		throw error
+	}
+}
+
+async function createDirectoryIfNotExits(path: string, nodeishFs: NodeishFilesystem) {
+	try {
+		await nodeishFs.mkdir(path, { recursive: true })
+	} catch (error: any) {
+		if (error.code !== "EEXIST") {
+			throw error
+		}
+	}
+}

package/src/persistence/storeApi.ts

@@ -0,0 +1,19 @@
+import type * as V2 from "../v2/types.js"
+
+/**
+ * WIP async V2 store interface
+ * E.g. `await project.store.messageBundles.get({ id: "..." })`
+ **/
+export type StoreApi = {
+	messageBundles: Store<V2.MessageBundle>
+}
+
+export interface Store<T> {
+	// TODO: remove reload when fs.watch can trigger auto-invalidation
+	reload: () => Promise<void>
+
+	get: (args: { id: string }) => Promise<T | undefined>
+	set: (args: { data: T }) => Promise<void>
+	delete: (args: { id: string }) => Promise<void>
+	getAll: () => Promise<T[]>
+}

package/src/reactivity/solid.test.ts

@@ -1,4 +1,5 @@
 import { describe, it, expect } from "vitest"
+import { sleep, delay } from "../test-utilities/sleep.js"
 import {
 	createRoot,
 	createSignal,
@@ -9,14 +10,6 @@ import {
 } from "./solid.js"
 import { ReactiveMap } from "./map.js"
 
-function sleep(ms: number) {
-	return new Promise((resolve) => setTimeout(resolve, ms))
-}
-
-function delay(v: unknown, ms: number) {
-	return new Promise((resolve) => setTimeout(() => resolve(v), ms))
-}
-
 describe("vitest", () => {
 	it("waits for the result of an async function", async () => {
 		const rval = await delay(42, 1)

package/src/resolve-modules/plugins/resolvePlugins.ts

@@ -1,10 +1,6 @@
 /* eslint-disable @typescript-eslint/no-non-null-assertion */
 import type { ResolvePluginsFunction } from "./types.js"
 import { Plugin } from "@inlang/plugin"
-import {
-	loadMessages as sdkLoadMessages,
-	saveMessages as sdkSaveMessages,
-} from "../../persistence/plugin.js"
 import {
 	PluginReturnedInvalidCustomApiError,
 	PluginLoadMessagesFunctionAlreadyDefinedError,
@@ -121,15 +117,10 @@ export const resolvePlugins: ResolvePluginsFunction = async (args) => {
 
 	// --- LOADMESSAGE / SAVEMESSAGE NOT DEFINED ---
 
-	if (experimentalPersistence) {
-		debug("Override load/save for experimental persistence")
-		// @ts-ignore - type mismatch error
-		result.data.loadMessages = sdkLoadMessages
-		// @ts-ignore - type mismatch error
-		result.data.saveMessages = sdkSaveMessages
-	} else if (
-		typeof result.data.loadMessages !== "function" ||
-		typeof result.data.saveMessages !== "function"
+	if (
+		!experimentalPersistence &&
+		(typeof result.data.loadMessages !== "function" ||
+			typeof result.data.saveMessages !== "function")
 	) {
 		result.errors.push(new PluginsDoNotProvideLoadOrSaveMessagesError())
 	}

package/src/test-utilities/sleep.ts

@@ -0,0 +1,11 @@
+export function sleep(ms: number) {
+	return new Promise((resolve) => setTimeout(resolve, ms))
+}
+
+export function delay(v: unknown, ms: number) {
+	return new Promise((resolve) => setTimeout(() => resolve(v), ms))
+}
+
+export function fail(ms: number) {
+	return new Promise((resolve, reject) => setTimeout(() => reject(new Error("fail")), ms))
+}

package/src/v2/{createMessageBundle.test.ts → helper.test.ts}

@@ -1,5 +1,5 @@
 import { describe, it, expect } from "vitest"
-import { createMessageBundle, createMessage } from "./createMessageBundle.js"
+import { createMessageBundle, createMessage } from "./helper.js"
 import { MessageBundle } from "./types.js"
 import { Value } from "@sinclair/typebox/value"
 

package/src/v2/helper.ts

@@ -0,0 +1,98 @@
+import {
+	LanguageTag,
+	MessageBundle,
+	MessageBundleWithSlots,
+	Message,
+	MessageSlot,
+	Text,
+} from "./types.js"
+
+/**
+ * create v2 MessageBundle
+ * @example createMessageBundle({
+ * 	 id: "greeting",
+ *   messages: [
+ * 		 createMessage({locale: "en", text: "Hello world!"})
+ * 		 createMessage({locale: "de", text: "Hallo Welt!"})
+ *   ]
+ * })
+ */
+export function createMessageBundle(args: {
+	id: string
+	messages: Message[]
+	alias?: MessageBundle["alias"]
+}): MessageBundle {
+	return {
+		id: args.id,
+		alias: args.alias ?? {},
+		messages: args.messages,
+	}
+}
+
+/**
+ * create v2 Messsage AST with text-only pattern
+ * @example createMessage({locale: "en", text: "Hello world"})
+ */
+export function createMessage(args: {
+	locale: LanguageTag
+	text: string
+	match?: Array<string>
+}): Message {
+	return {
+		locale: args.locale,
+		declarations: [],
+		selectors: [],
+		variants: [{ match: args.match ? args.match : [], pattern: [toTextElement(args.text ?? "")] }],
+	}
+}
+
+export function toTextElement(text: string): Text {
+	return {
+		type: "text",
+		value: text,
+	}
+}
+
+// ****************************
+// WIP experimental persistence
+// ****************************
+
+/**
+ * create MessageSlot for a locale (only used for persistence)
+ */
+export function createMessageSlot(locale: LanguageTag): MessageSlot {
+	return {
+		locale,
+		slot: true,
+	}
+}
+
+/**
+ * return structuredClone with message slots for all locales not yet present
+ */
+export function addSlots(messageBundle: MessageBundle, locales: string[]): MessageBundleWithSlots {
+	const bundle = structuredClone(messageBundle) as MessageBundleWithSlots
+	bundle.messages = locales.map((locale) => {
+		return bundle.messages.find((message) => message.locale === locale) ?? createMessageSlot(locale)
+	})
+	return bundle
+}
+
+/**
+ * remove empty message slots without first creating a structured clone
+ */
+export function removeSlots(messageBundle: MessageBundleWithSlots) {
+	messageBundle.messages = messageBundle.messages.filter((message) => !("slot" in message))
+	return messageBundle as MessageBundle
+}
+
+/**
+ * Add newlines between bundles and messages to avoid merge conflicts
+ */
+export function injectJSONNewlines(json: string): string {
+	return json
+		.replace(/\{"id":"/g, '\n\n\n\n{"id":"')
+		.replace(/"messages":\[\{"locale":"/g, '"messages":[\n\n\n\n{"locale":"')
+		.replace(/\}\]\}\]\},\{"locale":"/g, '}]}]},\n\n\n\n{"locale":"')
+		.replace(/"slot":true\},\{"locale":/g, '"slot":true},\n\n\n\n{"locale":')
+}

package/src/v2/index.ts

@@ -1 +1,3 @@
 export type * from "./types.js"
+export * from "./helper.js"
+export * from "./shim.js"

package/src/v2/mocks/index.ts

@@ -0,0 +1,2 @@
+export * from "./plural/bundle.js"
+export * from "./multipleMatcher/bundle.js"

package/src/v2/mocks/multipleMatcher/bundle.test.ts

@@ -0,0 +1,11 @@
+import { describe, it, expect } from "vitest"
+import { multipleMatcherBundle } from "./bundle.js"
+import { MessageBundle } from "../../types.js"
+import { Value } from "@sinclair/typebox/value"
+
+describe("mock plural messageBundle", () => {
+	it("is valid", () => {
+		const messageBundle: unknown = multipleMatcherBundle
+		expect(Value.Check(MessageBundle, messageBundle)).toBe(true)
+	})
+})

package/src/v2/mocks/multipleMatcher/bundle.ts

@@ -0,0 +1,196 @@
+import type { MessageBundle } from "../../types.js"
+
+export const multipleMatcherBundle: MessageBundle = {
+	id: "mock-bundle-human-id",
+	alias: {
+		default: "mock-bundle-alias",
+	},
+	messages: [
+		{
+			locale: "en",
+			declarations: [],
+			selectors: [
+				{
+					type: "expression",
+					arg: {
+						type: "variable",
+						name: "count",
+					},
+					annotation: { type: "function", name: "plural", options: [] },
+				},
+			],
+			variants: [
+				{
+					match: ["one"],
+					pattern: [
+						{
+							type: "text",
+							value: "Show ",
+						},
+						{
+							type: "expression",
+							arg: {
+								type: "variable",
+								name: "count",
+							},
+						},
+						{
+							type: "text",
+							value: " message.",
+						},
+					],
+				},
+				{
+					match: ["many"],
+					pattern: [
+						{
+							type: "text",
+							value: "Show ",
+						},
+						{
+							type: "expression",
+							arg: {
+								type: "variable",
+								name: "count",
+							},
+						},
+						{
+							type: "text",
+							value: " messages.",
+						},
+					],
+				},
+				{
+					match: ["*"],
+					pattern: [
+						{
+							type: "text",
+							value: "Show ",
+						},
+						{
+							type: "expression",
+							arg: {
+								type: "variable",
+								name: "count",
+							},
+						},
+					],
+				},
+			],
+		},
+		{
+			locale: "de",
+			declarations: [],
+			selectors: [
+				{
+					type: "expression",
+					arg: {
+						type: "variable",
+						name: "count",
+					},
+					annotation: { type: "function", name: "plural", options: [] },
+				},
+				{
+					type: "expression",
+					arg: {
+						type: "variable",
+						name: "formal",
+					},
+					annotation: { type: "function", name: "bool", options: [] },
+				},
+			],
+			variants: [
+				{
+					match: ["one", "formal"],
+					pattern: [
+						{
+							type: "text",
+							value: "Zeigen Sie bitte Ihre Nachricht.",
+						},
+					],
+				},
+				{
+					match: ["one", "informal"],
+					pattern: [
+						{
+							type: "text",
+							value: "Zeigen Deine Nachricht.",
+						},
+					],
+				},
+				{
+					match: ["many", "formal"],
+					pattern: [
+						{
+							type: "text",
+							value: "Zeigen Sie bitte Ihre ",
+						},
+						{
+							type: "expression",
+							arg: {
+								type: "variable",
+								name: "count",
+							},
+						},
+						{
+							type: "text",
+							value: " Nachrichten.",
+						},
+					],
+				},
+				{
+					match: ["many", "informal"],
+					pattern: [
+						{
+							type: "text",
+							value: "Zeigen Deine ",
+						},
+						{
+							type: "expression",
+							arg: {
+								type: "variable",
+								name: "count",
+							},
+						},
+						{
+							type: "text",
+							value: " Nachrichten.",
+						},
+					],
+				},
+				{
+					match: ["many", "*"],
+					pattern: [
+						{
+							type: "expression",
+							arg: {
+								type: "variable",
+								name: "count",
+							},
+						},
+						{
+							type: "text",
+							value: " Nachrichten zeigen.",
+						},
+					],
+				},
+				{
+					match: ["*"],
+					pattern: [
+						{
+							type: "text",
+							value: "Zeige ",
+						},
+						{
+							type: "expression",
+							arg: {
+								type: "variable",
+								name: "count",
+							},
+						},
+					],
+				},
+			],
+		},
+	],
+}

package/src/v2/mocks/plural/bundle.test.ts

@@ -1,18 +1,18 @@
 /* eslint-disable @typescript-eslint/no-non-null-assertion */
 
 import { describe, it, expect } from "vitest"
-import { bundle } from "./bundle.js"
+import { pluralBundle } from "./bundle.js"
 import { MessageBundle } from "../../types.js"
 import { Value } from "@sinclair/typebox/value"
 
 describe("mock plural messageBundle", () => {
 	it("is valid", () => {
-		const messageBundle: unknown = bundle
+		const messageBundle: unknown = pluralBundle
 		expect(Value.Check(MessageBundle, messageBundle)).toBe(true)
 
-		expect(bundle.messages.length).toBe(2)
-		expect(bundle.messages[0]!.declarations.length).toBe(1)
-		expect(bundle.messages[0]!.selectors.length).toBe(1)
-		expect(bundle.messages[0]!.variants.length).toBe(3)
+		expect(pluralBundle.messages.length).toBe(2)
+		expect(pluralBundle.messages[0]!.declarations.length).toBe(1)
+		expect(pluralBundle.messages[0]!.selectors.length).toBe(1)
+		expect(pluralBundle.messages[0]!.variants.length).toBe(3)
 	})
 })