@inlang/sdk 0.34.4 → 0.34.6

package/src/createMessageLintReportsQuery.ts

@@ -3,23 +3,19 @@ import type {
 	InstalledMessageLintRule,
 	MessageLintReportsQueryApi,
 	MessageQueryApi,
+	MessageQueryDelegate,
 } from "./api.js"
 import type { ProjectSettings } from "@inlang/project-settings"
 import type { resolveModules } from "./resolve-modules/index.js"
 import type { MessageLintReport, Message } from "./versionedInterfaces.js"
 import { lintSingleMessage } from "./lint/index.js"
-import { createRoot, createEffect } from "./reactivity/solid.js"
-
-import { throttle } from "throttle-debounce"
-import _debug from "debug"
-const debug = _debug("sdk:lintReports")
 
 function sleep(ms: number) {
 	return new Promise((resolve) => setTimeout(resolve, ms))
 }
 
 /**
- * Creates a reactive query API for messages.
+ * Creates a ~~reactive~~ query API for lint reports.
  */
 export function createMessageLintReportsQuery(
 	messagesQuery: MessageQueryApi,
@@ -42,72 +38,54 @@ export function createMessageLintReportsQuery(
 		}
 	}
 
-	const messages = messagesQuery.getAll() as Message[]
-
-	const trackedMessages: Map<string, () => void> = new Map()
-
-	debug(`createMessageLintReportsQuery ${rulesArray?.length} rules, ${messages.length} messages`)
-
-	// TODO: don't throttle when no debug
-	let lintMessageCount = 0
-	const throttledLogLintMessage = throttle(2000, (messageId) => {
-		debug(`lintSingleMessage: ${lintMessageCount} id: ${messageId}`)
-	})
-
-	createEffect(() => {
-		const currentMessageIds = new Set(messagesQuery.includedMessageIds())
-
-		const deletedTrackedMessages = [...trackedMessages].filter(
-			(tracked) => !currentMessageIds.has(tracked[0])
-		)
-
-		if (rulesArray) {
-			for (const messageId of currentMessageIds) {
-				if (!trackedMessages.has(messageId)) {
-					createRoot((dispose) => {
-						createEffect(() => {
-							const message = messagesQuery.get({ where: { id: messageId } })
-							if (!message) {
-								return
-							}
-							if (!trackedMessages?.has(messageId)) {
-								// initial effect execution - add dispose function
-								trackedMessages?.set(messageId, dispose)
-							}
+	const lintMessage = (message: Message, messages: Message[]) => {
+		if (!rulesArray) {
+			return
+		}
 
-							lintSingleMessage({
-								rules: rulesArray,
-								settings: settingsObject(),
-								messages: messages,
-								message: message,
-							}).then((report) => {
-								lintMessageCount++
-								throttledLogLintMessage(messageId)
-								if (report.errors.length === 0 && index.get(messageId) !== report.data) {
-									// console.log("lintSingleMessage", messageId, report.data.length)
-									index.set(messageId, report.data)
-								}
-							})
-						})
-					})
-				}
+		// TODO unhandled promise rejection (as before the refactor) but won't tackle this in this pr
+		lintSingleMessage({
+			rules: rulesArray,
+			settings: settingsObject(),
+			messages: messages,
+			message: message,
+		}).then((report) => {
+			if (report.errors.length === 0 && index.get(message.id) !== report.data) {
+				// console.log("lintSingleMessage", messageId, report.data.length)
+				index.set(message.id, report.data)
 			}
+		})
+	}
 
-			for (const deletedMessage of deletedTrackedMessages) {
-				const deletedMessageId = deletedMessage[0]
+	const messages = messagesQuery.getAll() as Message[]
+	// load report for all messages once
+	for (const message of messages) {
+		// NOTE: this potentually creates thousands of promisses we could create a promise that batches linting
+		lintMessage(message, messages)
+	}
 
-				// call dispose to cleanup the effect
-				const messageEffectDisposeFunction = trackedMessages.get(deletedMessageId)
-				if (messageEffectDisposeFunction) {
-					messageEffectDisposeFunction()
-					trackedMessages.delete(deletedMessageId)
-					// remove lint report result
-					index.delete(deletedMessageId)
-					debug(`delete lint message id: ${deletedMessageId}`)
-				}
+	const messageQueryChangeDelegate: MessageQueryDelegate = {
+		onCleanup: () => {
+			// NOTE: we could cancel all running lint rules - but results get overritten anyway
+			index.clear()
+		},
+		onLoaded: (messages: Message[]) => {
+			for (const message of messages) {
+				lintMessage(message, messages)
 			}
-		}
-	})
+		},
+		onMessageCreate: (messageId: string, message: Message) => {
+			lintMessage(message, messages)
+		},
+		onMessageUpdate: (messageId: string, message: Message) => {
+			lintMessage(message, messages)
+		},
+		onMessageDelete: (messageId: string) => {
+			index.delete(messageId)
+		},
+	}
+
+	messagesQuery.setDelegate(messageQueryChangeDelegate)
 
 	return {
 		getAll: async () => {

package/src/createMessagesQuery.ts

@@ -2,7 +2,7 @@ import type { Message } from "@inlang/message"
 import { ReactiveMap } from "./reactivity/map.js"
 import { createEffect } from "./reactivity/solid.js"
 import { createSubscribable } from "./loadProject.js"
-import type { InlangProject, MessageQueryApi } from "./api.js"
+import type { InlangProject, MessageQueryApi, MessageQueryDelegate } from "./api.js"
 import type { ResolvedPluginApi } from "./resolve-modules/plugins/types.js"
 import type { resolveModules } from "./resolve-modules/resolveModules.js"
 import { createNodeishFsWithWatcher } from "./createNodeishFsWithWatcher.js"
@@ -17,6 +17,10 @@ import { PluginLoadMessagesError, PluginSaveMessagesError } from "./errors.js"
 import { humanIdHash } from "./storage/human-id/human-readable-id.js"
 const debug = _debug("sdk:createMessagesQuery")
 
+function sleep(ms: number) {
+	return new Promise((resolve) => setTimeout(resolve, ms))
+}
+
 type MessageState = {
 	messageDirtyFlags: {
 		[messageId: string]: boolean
@@ -62,6 +66,12 @@ export function createMessagesQuery({
 	// filepath for the lock folder
 	const messageLockDirPath = projectPath + "/messagelock"
 
+	let delegate: MessageQueryDelegate | undefined = undefined
+
+	const setDelegate = (newDelegate: MessageQueryDelegate) => {
+		delegate = newDelegate
+	}
+
 	// Map default alias to message
 	// Assumes that aliases are only created and deleted, not updated
 	// TODO #2346 - handle updates to aliases
@@ -98,6 +108,7 @@ export function createMessagesQuery({
 		onCleanup(() => {
 			// stop listening on fs events
 			abortController.abort()
+			delegate?.onCleanup()
 		})
 
 		const fsWithWatcher = createNodeishFsWithWatcher({
@@ -111,6 +122,7 @@ export function createMessagesQuery({
 					messageLockDirPath,
 					messageStates,
 					index,
+					delegate,
 					_settings, // NOTE we bang here - we don't expect the settings to become null during the livetime of a project
 					resolvedPluginApi
 				)
@@ -133,6 +145,7 @@ export function createMessagesQuery({
 			messageLockDirPath,
 			messageStates,
 			index,
+			delegate,
 			_settings, // NOTE we bang here - we don't expect the settings to become null during the livetime of a project
 			resolvedPluginApi
 		)
@@ -142,6 +155,7 @@ export function createMessagesQuery({
 			})
 			.then(() => {
 				onInitialMessageLoadResult()
+				delegate?.onLoaded([...index.values()])
 			})
 	})
 
@@ -165,6 +179,7 @@ export function createMessagesQuery({
 			messageLockDirPath,
 			messageStates,
 			index,
+			delegate,
 			_settings, // NOTE we bang here - we don't expect the settings to become null during the livetime of a project
 			resolvedPluginApi
 		)
@@ -181,6 +196,7 @@ export function createMessagesQuery({
 	}
 
 	return {
+		setDelegate,
 		create: ({ data }): boolean => {
 			if (index.has(data.id)) return false
 			index.set(data.id, data)
@@ -189,6 +205,7 @@ export function createMessagesQuery({
 			}
 
 			messageStates.messageDirtyFlags[data.id] = true
+			delegate?.onMessageCreate(data.id, index.get(data.id))
 			scheduleSave()
 			return true
 		},
@@ -215,6 +232,7 @@ export function createMessagesQuery({
 			if (message === undefined) return false
 			index.set(where.id, { ...message, ...data })
 			messageStates.messageDirtyFlags[where.id] = true
+			delegate?.onMessageCreate(where.id, index.get(data.id))
 			scheduleSave()
 			return true
 		},
@@ -225,10 +243,13 @@ export function createMessagesQuery({
 				if ("default" in data.alias) {
 					defaultAliasIndex.set(data.alias.default, data)
 				}
+				messageStates.messageDirtyFlags[where.id] = true
+				delegate?.onMessageCreate(data.id, index.get(data.id))
 			} else {
 				index.set(where.id, { ...message, ...data })
+				messageStates.messageDirtyFlags[where.id] = true
+				delegate?.onMessageUpdate(data.id, index.get(data.id))
 			}
-			messageStates.messageDirtyFlags[where.id] = true
 			scheduleSave()
 			return true
 		},
@@ -240,6 +261,7 @@ export function createMessagesQuery({
 			}
 			index.delete(where.id)
 			messageStates.messageDirtyFlags[where.id] = true
+			delegate?.onMessageDelete(where.id)
 			scheduleSave()
 			return true
 		},
@@ -253,6 +275,8 @@ export function createMessagesQuery({
 // - saving a message in two different languages would lead to a write in de.json first
 // - This will leads to a load of the messages and since en.json has not been saved yet the english variant in the message would get overritten with the old state again
 
+const maxMessagesPerTick = 500
+
 /**
  * Messsage that loads messages from a plugin - this method synchronizes with the saveMessage funciton.
  * If a save is in progress loading will wait until saving is done. If another load kicks in during this load it will queue the
@@ -271,6 +295,7 @@ async function loadMessagesViaPlugin(
 	lockDirPath: string,
 	messageState: MessageState,
 	messages: Map<string, Message>,
+	delegate: MessageQueryDelegate | undefined,
 	settingsValue: ProjectSettings,
 	resolvedPluginApi: ResolvedPluginApi
 ) {
@@ -298,6 +323,8 @@ async function loadMessagesViaPlugin(
 			})
 		)
 
+		let loadedMessageCount = 0
+
 		for (const loadedMessage of loadedMessages) {
 			const loadedMessageClone = structuredClone(loadedMessage)
 
@@ -339,6 +366,8 @@ async function loadMessagesViaPlugin(
 				messages.set(loadedMessageClone.id, loadedMessageClone)
 				// NOTE could use hash instead of the whole object JSON to save memory...
 				messageState.messageLoadHash[loadedMessageClone.id] = importedEnecoded
+				delegate?.onMessageUpdate(loadedMessageClone.id, loadedMessageClone)
+				loadedMessageCount++
 			} else {
 				// message with the given alias does not exist so far
 				loadedMessageClone.alias = {} as any
@@ -365,6 +394,15 @@ async function loadMessagesViaPlugin(
 				// we don't have to check - done before hand if (messages.has(loadedMessageClone.id)) return false
 				messages.set(loadedMessageClone.id, loadedMessageClone)
 				messageState.messageLoadHash[loadedMessageClone.id] = importedEnecoded
+				delegate?.onMessageUpdate(loadedMessageClone.id, loadedMessageClone)
+				loadedMessageCount++
+			}
+			if (loadedMessageCount > maxMessagesPerTick) {
+				// move loading of the next messages to the next ticks to allow solid to cleanup resources
+				// solid needs some time to settle and clean up
+				// https://github.com/solidjs-community/solid-primitives/blob/9ca76a47ffa2172770e075a90695cf933da0ff48/packages/trigger/src/index.ts#L64
+				await sleep(0)
+				loadedMessageCount = 0
 			}
 		}
 		await releaseLock(fs as NodeishFilesystem, lockDirPath, "loadMessage", lockTime)
@@ -388,7 +426,15 @@ async function loadMessagesViaPlugin(
 		messageState.sheduledLoadMessagesViaPlugin = undefined
 
 		// recall load unawaited to allow stack to pop
-		loadMessagesViaPlugin(fs, lockDirPath, messageState, messages, settingsValue, resolvedPluginApi)
+		loadMessagesViaPlugin(
+			fs,
+			lockDirPath,
+			messageState,
+			messages,
+			delegate,
+			settingsValue,
+			resolvedPluginApi
+		)
 			.then(() => {
 				// resolve the scheduled load message promise
 				executingScheduledMessages.resolve()
@@ -405,6 +451,7 @@ async function saveMessagesViaPlugin(
 	lockDirPath: string,
 	messageState: MessageState,
 	messages: Map<string, Message>,
+	delegate: MessageQueryDelegate | undefined,
 	settingsValue: ProjectSettings,
 	resolvedPluginApi: ResolvedPluginApi
 ): Promise<void> {
@@ -491,6 +538,7 @@ async function saveMessagesViaPlugin(
 					lockDirPath,
 					messageState,
 					messages,
+					delegate,
 					settingsValue,
 					resolvedPluginApi
 				)
@@ -530,7 +578,15 @@ async function saveMessagesViaPlugin(
 		const executingSheduledSaveMessages = messageState.sheduledSaveMessages
 		messageState.sheduledSaveMessages = undefined
 
-		saveMessagesViaPlugin(fs, lockDirPath, messageState, messages, settingsValue, resolvedPluginApi)
+		saveMessagesViaPlugin(
+			fs,
+			lockDirPath,
+			messageState,
+			messages,
+			delegate,
+			settingsValue,
+			resolvedPluginApi
+		)
 			.then(() => {
 				executingSheduledSaveMessages.resolve()
 			})

package/src/loadProject.ts

@@ -33,7 +33,6 @@ import { identifyProject } from "./telemetry/groupIdentify.js"
 import _debug from "debug"
 const debug = _debug("sdk:loadProject")
 
-
 const settingsCompiler = TypeCompiler.Compile(ProjectSettings)
 
 /**

package/src/v2/index.ts

@@ -0,0 +1 @@
+export type * from "./types.js"

package/src/v2/types.ts

@@ -0,0 +1,142 @@
+import { Type, type Static } from "@sinclair/typebox"
+
+/**
+ * Follows the IETF BCP 47 language tag schema.
+ *
+ * @see https://www.ietf.org/rfc/bcp/bcp47.txt
+ * @see https://www.iana.org/assignments/language-subtag-registry/language-subtag-registry
+ */
+export type LanguageTag = Static<typeof LanguageTag>
+/**
+ * Follows the IETF BCP 47 language tag schema with modifications.
+ * @see REAMDE.md file for more information on the validation.
+ */
+
+export const pattern =
+	"^((?<grandfathered>(en-GB-oed|i-ami|i-bnn|i-default|i-enochian|i-hak|i-klingon|i-lux|i-mingo|i-navajo|i-pwn|i-tao|i-tay|i-tsu|sgn-BE-FR|sgn-BE-NL|sgn-CH-DE)|(art-lojban|cel-gaulish|no-bok|no-nyn|zh-guoyu|zh-hakka|zh-min|zh-min-nan|zh-xiang))|((?<language>([A-Za-z]{2,3}(-(?<extlang>[A-Za-z]{3}(-[A-Za-z]{3}){0,2}))?))(-(?<script>[A-Za-z]{4}))?(-(?<region>[A-Za-z]{2}|[0-9]{3}))?(-(?<variant>[A-Za-z0-9]{5,8}|[0-9][A-Za-z0-9]{3}))*))$"
+
+export const LanguageTag = Type.String({
+	pattern: pattern,
+	description: "The language tag must be a valid IETF BCP 47 language tag.",
+	examples: ["en", "de", "en-US", "zh-Hans", "es-419"],
+})
+
+export type Literal = Static<typeof Literal>
+export const Literal = Type.Object({
+	type: Type.Literal("literal"),
+	value: Type.String(),
+})
+
+/**
+ * A (text) element that is translatable and rendered to the UI.
+ */
+export type Text = Static<typeof Text>
+export const Text = Type.Object({
+	type: Type.Literal("text"),
+	value: Type.String(),
+})
+
+export type VariableReference = Static<typeof VariableReference>
+export const VariableReference = Type.Object({
+	type: Type.Literal("variable"),
+	name: Type.String(),
+})
+
+export type Option = Static<typeof Option>
+export const Option = Type.Object({
+	name: Type.String(),
+	value: Type.Union([Literal, VariableReference]),
+})
+
+export type FunctionAnnotation = Static<typeof FunctionAnnotation>
+export const FunctionAnnotation = Type.Object({
+	type: Type.Literal("function"),
+	name: Type.String(),
+	options: Type.Array(Option),
+})
+
+/**
+ * An expression is a reference to a variable or a function.
+ *
+ * Think of expressions as elements that are rendered to a
+ * text value during runtime.
+ */
+export type Expression = Static<typeof Expression>
+export const Expression = Type.Object({
+	type: Type.Literal("expression"),
+	arg: Type.Union([Literal, VariableReference]),
+	annotation: Type.Optional(FunctionAnnotation),
+})
+
+// export type FunctionReference = {
+// 	type: "function"
+// 	name: string
+// 	operand?: Text | VariableReference
+// 	options?: Option[]
+// }
+
+/**
+ * A pattern is a sequence of elements that comprise
+ * a message that is rendered to the UI.
+ */
+export type Pattern = Static<typeof Pattern>
+export const Pattern = Type.Array(Type.Union([Text, Expression]))
+
+/**
+ * A variant contains a pattern that is rendered to the UI.
+ */
+export type Variant = Static<typeof Variant>
+export const Variant = Type.Object({
+	/**
+	 * The number of keys in each variant match MUST equal the number of expressions in the selectors.
+	 *
+	 * Inspired by: https://github.com/unicode-org/message-format-wg/blob/main/spec/formatting.md#pattern-selection
+	 */
+	// a match can always only be string-based because a string is what is rendered to the UI
+	match: Type.Array(Type.String()),
+	pattern: Pattern,
+})
+
+export type InputDeclaration = Static<typeof InputDeclaration>
+export const InputDeclaration = Type.Object({
+	type: Type.Literal("input"),
+	name: Type.String(),
+
+	//TODO make this generic so that only Variable-Ref Expressions are allowed
+	value: Expression,
+})
+
+// local declarations are not supported.
+// Will only add when required. See discussion:
+// https://github.com/opral/monorepo/pull/2700#discussion_r1591070701
+//
+// export type LocalDeclaration = Static<typeof InputDeclaration>
+// export const LocalDeclaration = Type.Object({
+// 	type: Type.Literal("local"),
+// 	name: Type.String(),
+// 	value: Expression,
+// })
+//
+// export type Declaration = Static<typeof Declaration>
+// export const Declaration = Type.Union([LocalDeclaration, InputDeclaration])
+
+export type Declaration = Static<typeof Declaration>
+export const Declaration = Type.Union([InputDeclaration])
+
+export type Message = Static<typeof Message>
+export const Message = Type.Object({
+	locale: LanguageTag,
+	declarations: Type.Array(Declaration),
+	/**
+	 * The order in which the selectors are placed determines the precedence of patterns.
+	 */
+	selectors: Type.Array(Expression),
+	variants: Type.Array(Variant),
+})
+
+export type MessageBundle = Static<typeof MessageBundle>
+export const MessageBundle = Type.Object({
+	id: Type.String(),
+	alias: Type.Record(Type.String(), Type.String()),
+	messages: Type.Array(Message),
+})