@inlang/sdk 0.36.2 → 2.0.0-prerelease.0

package/src/lix-plugin/resolveConflictBySelecting.test.ts

@@ -0,0 +1,188 @@
+// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+// @ts-nocheck
+
+import { test, expect } from "vitest";
+import { newProject } from "../project/newProject.js";
+import { loadProjectInMemory } from "../project/loadProjectInMemory.js";
+import {
+	isInSimulatedCurrentBranch,
+	resolveConflictBySelecting,
+} from "@lix-js/sdk";
+import { contentFromDatabase } from "sqlite-wasm-kysely";
+
+test.skip("it should resolve a conflict with the selected change", async () => {
+	const project = await loadProjectInMemory({ blob: await newProject() });
+
+	const dbFile = await project.lix.db
+		.selectFrom("file")
+		.select("id")
+		.where("path", "=", "/db.sqlite")
+		.executeTakeFirstOrThrow();
+
+	const changes = await project.lix.db
+		.insertInto("change")
+		.values([
+			{
+				id: "e59b2fc6-612d-4087-9790-03b2ab724274",
+				author: "Anonymous",
+				file_id: dbFile.id,
+				plugin_key: "inlang-lix-plugin-v1",
+				operation: "create",
+				type: "bundle",
+				value: {
+					id: "even_hour_mule_drum",
+				},
+				created_at: "2024-08-28 20:58:04",
+			},
+			{
+				id: "c0c9e3f3-0eba-4233-813f-e1901401a653",
+				author: "Anonymous",
+				file_id: dbFile.id,
+				plugin_key: "inlang-lix-plugin-v1",
+				operation: "create",
+				type: "message",
+				value: {
+					id: "c2684c3d-3e14-47e4-96b7-8d33579bb7e9",
+					bundleId: "even_hour_mule_drum",
+					locale: "en",
+					selectors: [],
+				},
+				created_at: "2024-08-28 20:58:04",
+			},
+			{
+				id: "b92a634f-1e25-410f-af5a-e8702cc92ef0",
+				author: "Anonymous",
+				file_id: dbFile.id,
+				plugin_key: "inlang-lix-plugin-v1",
+				operation: "create",
+				type: "variant",
+				value: {
+					id: "6a860f96-0cf3-477c-80ad-7893d8fde852",
+					messageId: "c2684c3d-3e14-47e4-96b7-8d33579bb7e9",
+					matches: [],
+					pattern: [
+						{
+							type: "text",
+							value: "My Awesome Todo App",
+						},
+					],
+				},
+				created_at: "2024-08-28 20:58:04",
+			},
+			{
+				id: "samuels-change",
+				author: "Samuel",
+				parent_id: "b92a634f-1e25-410f-af5a-e8702cc92ef0",
+				file_id: dbFile.id,
+				plugin_key: "inlang-lix-plugin-v1",
+				operation: "update",
+				type: "variant",
+				value: {
+					id: "6a860f96-0cf3-477c-80ad-7893d8fde852",
+					messageId: "c2684c3d-3e14-47e4-96b7-8d33579bb7e9",
+					matches: [],
+					pattern: [
+						{
+							type: "text",
+							value: "My awesome lix app",
+						},
+					],
+				},
+				meta: {
+					id: "6a860f96-0cf3-477c-80ad-7893d8fde852",
+				},
+				created_at: "2024-08-28 21:29:05",
+			},
+			{
+				id: "peters-change",
+				author: "Peter",
+				parent_id: "b92a634f-1e25-410f-af5a-e8702cc92ef0",
+				file_id: dbFile.id,
+				plugin_key: "inlang-lix-plugin-v1",
+				operation: "update",
+				type: "variant",
+				value: {
+					id: "6a860f96-0cf3-477c-80ad-7893d8fde852",
+					messageId: "c2684c3d-3e14-47e4-96b7-8d33579bb7e9",
+					matches: [],
+					pattern: [
+						{
+							type: "text",
+							value: "You are building an app",
+						},
+					],
+				},
+				meta: {
+					id: "6a860f96-0cf3-477c-80ad-7893d8fde852",
+				},
+				created_at: "2024-08-28 21:29:35",
+			},
+		])
+		.returningAll()
+		.execute();
+
+	// ---------------
+	// Todo inlang sdk should auto write file to lix and differ must be
+	// fault tolerant. without this code, the differ captures an old
+	// change again
+
+	for (const change of changes) {
+		await project.db
+			.insertInto(change.type as any)
+			.values(change.value as any)
+			.onConflict((c) => c.doNothing())
+			.execute();
+	}
+	await project.lix.db
+		.updateTable("file")
+		.set({ data: contentFromDatabase(project._sqlite) })
+		.where("path", "=", "/db.sqlite")
+		.execute();
+	// ---------------
+
+	const conflicts = await project.lix.db
+		.insertInto("conflict")
+		.values([
+			{
+				change_id: "samuels-change",
+				conflicting_change_id: "peters-change",
+				reason: "",
+			},
+		])
+		.returningAll()
+		.execute();
+
+	const changesInCurrentBranchBefore = await project.lix.db
+		.selectFrom("change")
+		.selectAll()
+		.where(isInSimulatedCurrentBranch)
+		.execute();
+
+	expect(changesInCurrentBranchBefore.map((c) => c.id)).toEqual([
+		changes[0]?.id,
+		changes[1]?.id,
+		changes[2]?.id,
+		"samuels-change",
+	]);
+
+	await resolveConflictBySelecting({
+		lix: project.lix,
+		conflict: conflicts[0]!,
+		selectChangeId: "peters-change",
+	});
+
+	await project.lix.settled();
+
+	const changesInCurrentBranch = await project.lix.db
+		.selectFrom("change")
+		.selectAll()
+		.where(isInSimulatedCurrentBranch)
+		.execute();
+
+	expect(changesInCurrentBranch.map((c) => c.id)).toEqual([
+		changes[0]?.id,
+		changes[1]?.id,
+		changes[2]?.id,
+		"peters-change",
+	]);
+});

package/src/migrations/v2/createMessageV1.ts

@@ -0,0 +1,37 @@
+import type {
+	MessageV1,
+	PatternV1,
+} from "../../json-schema/old-v1-message/schemaV1.js";
+
+/**
+ * @deprecated don't use the messageV1 type anymore.
+ *
+ * This function exists to ease migrations of v1 plugins to v2 plugins.
+ * The tests can call `createMessageV1` to create a message object that satisfies the v1 schema.
+ *
+ * @example
+ *   - createMessage("id", { "en": "Hello" })
+ *   + createMessageV1("id", { "en": "Hello" })
+ */
+export const createMessageV1 = (
+	id: string,
+	patterns: Record<string, PatternV1 | string>
+) =>
+	({
+		id,
+		alias: {},
+		selectors: [],
+		variants: Object.entries(patterns).map(([languageTag, patterns]) => ({
+			languageTag,
+			match: [],
+			pattern:
+				typeof patterns === "string"
+					? [
+							{
+								type: "Text",
+								value: patterns,
+							},
+						]
+					: patterns,
+		})),
+	}) satisfies MessageV1;

package/src/migrations/v2/withLanguageTagToLocaleMigration.test.ts

@@ -0,0 +1,51 @@
+import { expect, test } from "vitest";
+import type { ProjectSettings } from "../../json-schema/settings.js";
+import { withLanguageTagToLocaleMigration } from "./withLanguageTagToLocaleMigration.js";
+
+test("it should set sourceLanguageTag and languageTags if non-existent to make v1 plugins work", async () => {
+	const settings: ProjectSettings = {
+		baseLocale: "en",
+		locales: ["en", "de"],
+	};
+
+	const withMigration = withLanguageTagToLocaleMigration(settings);
+
+	expect(withMigration.baseLocale).toBe("en");
+	expect(withMigration.sourceLanguageTag).toBe("en");
+	expect(withMigration.locales).toEqual(["en", "de"]);
+	expect(withMigration.languageTags).toEqual(["en", "de"]);
+});
+
+test("it should set baseLocale and locales if non-existent to ensure first load of an old project works", async () => {
+	const settings: ProjectSettings = {
+		sourceLanguageTag: "en",
+		languageTags: ["en", "de"],
+	} as ProjectSettings;
+
+	const withMigration = withLanguageTagToLocaleMigration(settings);
+
+	expect(withMigration.baseLocale).toBe("en");
+	expect(withMigration.sourceLanguageTag).toBe("en");
+	expect(withMigration.languageTags).toEqual(["en", "de"]);
+	expect(withMigration.locales).toEqual(["en", "de"]);
+});
+
+/**
+ * Users are expected to treat baseLocale and locales as the source of truth.
+ * The alternative is complicated merging logic which is not worth the effort.
+ */
+test("it should treat baseLocale and locales as source of truth", async () => {
+	const settings: ProjectSettings = {
+		baseLocale: "de",
+		locales: ["de", "en", "fr", "es"],
+		sourceLanguageTag: "en",
+		languageTags: ["en", "nl"],
+	};
+
+	const withMigration = withLanguageTagToLocaleMigration(settings);
+
+	expect(withMigration.baseLocale).toBe("de");
+	expect(withMigration.sourceLanguageTag).toBe("de");
+	expect(withMigration.languageTags).toEqual(["de", "en", "fr", "es"]);
+	expect(withMigration.locales).toEqual(["de", "en", "fr", "es"]);
+});

package/src/migrations/v2/withLanguageTagToLocaleMigration.ts

@@ -0,0 +1,31 @@
+import type { ProjectSettings } from "../../json-schema/settings.js";
+
+/**
+ * Change introduced in v2.
+ *
+ * The sourceLanguageTag and languageTags are now deprecated
+ * in favor of baseLocale and locales.
+ */
+export function withLanguageTagToLocaleMigration(
+	settings: ProjectSettings
+): ProjectSettings {
+	if (settings.sourceLanguageTag === undefined) {
+		settings.sourceLanguageTag = settings.baseLocale;
+	}
+	if (settings.languageTags === undefined) {
+		settings.languageTags = settings.locales;
+	}
+	if (settings.baseLocale === undefined) {
+		settings.baseLocale = settings.sourceLanguageTag;
+	}
+	if (settings.locales === undefined) {
+		settings.locales = settings.languageTags;
+	}
+	if (settings.sourceLanguageTag !== settings.baseLocale) {
+		settings.sourceLanguageTag = settings.baseLocale;
+	}
+	if (settings.languageTags !== settings.locales) {
+		settings.languageTags = settings.locales;
+	}
+	return settings;
+}

package/src/plugin/cache.test.ts

@@ -0,0 +1,83 @@
+import { test, expect, vi } from "vitest";
+import { withCache } from "./cache.js";
+import { newLixFile, openLixInMemory } from "@lix-js/sdk";
+import { sql } from "kysely";
+
+test("it should be network-first", async () => {
+	const mockLoader = vi
+		.fn()
+		.mockResolvedValueOnce("module content 1")
+		.mockResolvedValueOnce("module content 2");
+
+	const mockModulePath = "https://mock.com/module.js";
+
+	const lix = await openLixInMemory({ blob: await newLixFile() });
+
+	const result1 = await withCache(mockLoader, lix)(mockModulePath);
+
+	expect(mockLoader).toHaveBeenCalledTimes(1);
+	expect(result1).toBe("module content 1");
+
+	const cachedPlugins = await lix.db
+		.selectFrom("file")
+		.selectAll()
+		// @ts-expect-error - kysely doesn't know about GLOB
+		.where(sql`path GLOB '/cache/plugins/*'`)
+		.execute();
+
+	expect(cachedPlugins.length).toBe(1);
+
+	const parsed = new TextDecoder().decode(cachedPlugins[0]!.data);
+
+	expect(parsed).toBe("module content 1");
+
+	const result2 = await withCache(mockLoader, lix)(mockModulePath);
+
+	expect(mockLoader).toHaveBeenCalledTimes(2);
+	expect(result2).toBe("module content 2");
+
+	const cachedPlugins2 = await lix.db
+		.selectFrom("file")
+		.selectAll()
+		// @ts-expect-error - kysely doesn't know about GLOB
+		.where(sql`path GLOB '/cache/plugins/*'`)
+		.execute();
+
+	expect(cachedPlugins2.length).toBe(1);
+
+	const parsed2 = new TextDecoder().decode(cachedPlugins2[0]!.data);
+
+	expect(parsed2).toBe("module content 2");
+});
+
+test("it should throw the error from the loader if the cache does not exist", async () => {
+	const mockLoader = vi.fn().mockRejectedValueOnce(new Error("Network error"));
+
+	const mockModulePath = "https://mock.com/module.js";
+
+	const lix = await openLixInMemory({ blob: await newLixFile() });
+
+	expect(
+		async () => await withCache(mockLoader, lix)(mockModulePath)
+	).rejects.toThrowError("Network error");
+});
+
+test("it should fallback to the cache if the loader fails", async () => {
+	const mockLoader = vi.fn().mockRejectedValueOnce(new Error("Network error"));
+
+	const mockModulePath = "https://mock.com/module.js";
+	const mockModuleCachePath = "/cache/plugins/31i1etp0l413h";
+
+	const lix = await openLixInMemory({ blob: await newLixFile() });
+
+	await lix.db
+		.insertInto("file")
+		.values({
+			path: mockModuleCachePath,
+			data: new TextEncoder().encode("cached module content"),
+		})
+		.execute();
+
+	const result = await withCache(mockLoader, lix)(mockModulePath);
+	expect(result).toBe("cached module content");
+});

package/src/plugin/cache.ts

@@ -0,0 +1,79 @@
+import type { Lix } from "@lix-js/sdk";
+
+function escape(url: string) {
+	const bytes = new TextEncoder().encode(url);
+
+	// 64-bit FNV1a hash to make the file-names shorter
+	// https://en.wikipedia.org/wiki/FNV-1a
+	const hash = bytes.reduce(
+		(hash, byte) =>
+			BigInt.asUintN(64, (hash ^ BigInt(byte)) * 1_099_511_628_211n),
+		14_695_981_039_346_656_037n
+	);
+
+	return hash.toString(36);
+}
+
+async function readModuleFromCache(
+	moduleURI: string,
+	lix: Lix
+): Promise<string | undefined> {
+	const moduleHash = escape(moduleURI);
+	const filePath = `/cache/plugins/${moduleHash}`;
+
+	const file = await lix.db
+		.selectFrom("file")
+		.where("path", "=", filePath)
+		.selectAll()
+		.executeTakeFirst();
+
+	if (file) {
+		return new TextDecoder().decode(file.data);
+	}
+	return undefined;
+}
+
+async function writeModuleToCache(
+	moduleURI: string,
+	moduleContent: string,
+	lix: Lix
+): Promise<void> {
+	const moduleHash = escape(moduleURI);
+	const filePath = `/cache/plugins/${moduleHash}`;
+
+	await lix.db
+		.insertInto("file")
+		.values({
+			path: filePath,
+			data: new TextEncoder().encode(moduleContent),
+		})
+		// update the cache
+		.onConflict((oc) =>
+			oc.doUpdateSet({ data: new TextEncoder().encode(moduleContent) })
+		)
+		.execute();
+}
+
+/**
+ * Implements a "Network-First" caching strategy.
+ */
+export function withCache(
+	moduleLoader: (uri: string) => Promise<string>,
+	lix: Lix
+): (uri: string) => Promise<string> {
+	return async (uri: string) => {
+		try {
+			const moduleAsText = await moduleLoader(uri);
+			await writeModuleToCache(uri, moduleAsText, lix);
+			return moduleAsText;
+		} catch (e) {
+			// network fetch failed, try to read from cache
+			const cacheResult = await readModuleFromCache(uri, lix);
+			if (cacheResult) {
+				return cacheResult;
+			} else {
+				throw e;
+			}
+		}
+	};
+}

package/src/plugin/errors.ts

@@ -0,0 +1,67 @@
+import type { ValueError } from "@sinclair/typebox/errors";
+
+export class PluginError extends Error {
+	public readonly plugin: string;
+
+	constructor(message: string, options: { plugin: string; cause?: Error }) {
+		super(message);
+		this.name = "PluginError";
+		this.plugin = options.plugin;
+		this.cause = options.cause;
+	}
+}
+
+/**
+ * Error when a plugin cannot be imported.
+ */
+export class PluginImportError extends PluginError {
+	constructor(options: { plugin: string; cause: Error }) {
+		super(
+			`Couldn't import the plugin "${options.plugin}":\n\n${options.cause}`,
+			options
+		);
+		this.name = "PluginImportError";
+	}
+}
+
+export class PluginSettingsAreInvalidError extends PluginError {
+	constructor(options: { plugin: string; errors: ValueError[] }) {
+		super(
+			`The settings of "${options.plugin}" are invalid:\n\n${options.errors
+				.map(
+					(error) =>
+						`Path "${error.path}" with value "${JSON.stringify(
+							error.value,
+							undefined,
+							2
+						)}": "${error.message}"`
+				)
+				.join("\n")}`,
+			options
+		);
+		this.name = "PluginSettingsAreInvalidError";
+	}
+}
+
+/**
+ * Error when a plugin does not implement a required function
+ */
+export class PluginDoesNotImplementFunctionError extends PluginError {
+	constructor(options: { plugin: string; function: string }) {
+		super(
+			`The plugin "${options.plugin}" does not implement the "${options.function}" function.`,
+			options
+		);
+		this.name = "PluginDoesNotImplementFunction";
+	}
+}
+
+/**
+ * Error when a plugin was expected to exist, but doesn't
+ */
+export class PluginMissingError extends PluginError {
+	constructor(options: { plugin: string }) {
+		super(`The plugin "${options.plugin}" does not exist`, options);
+		this.name = "PluginMissingError";
+	}
+}

package/src/plugin/importPlugins.test.ts

@@ -0,0 +1,107 @@
+import { test, expect, vi } from "vitest";
+import { importPlugins } from "./importPlugins.js";
+import { PluginImportError } from "./errors.js";
+import { newLixFile, openLixInMemory } from "@lix-js/sdk";
+
+test("it should preprocess a plugin", async () => {
+	global.fetch = vi.fn().mockResolvedValue({
+		ok: true,
+		text: vi.fn().mockResolvedValue("export default { key: 'mock' }"),
+	});
+
+	const lix = await openLixInMemory({ blob: await newLixFile() });
+	const result = await importPlugins({
+		lix,
+		settings: {
+			baseLocale: "en",
+			locales: ["en"],
+			modules: ["https://mock.com/module.js"],
+		},
+		preprocessPluginBeforeImport: async (moduleText) => {
+			return moduleText.replace("mock", "preprocessed");
+		},
+	});
+
+	expect(global.fetch).toHaveBeenCalledTimes(1);
+	expect(result.plugins.length).toBe(1);
+	expect(result.errors.length).toBe(0);
+	expect(result.plugins[0]?.key).toBe("preprocessed");
+});
+
+test("if a fetch fails, a plugin import error is expected", async () => {
+	global.fetch = vi.fn().mockResolvedValue({
+		ok: false,
+		statusText: "HTTP 404",
+	});
+	const lix = await openLixInMemory({ blob: await newLixFile() });
+
+	const result = await importPlugins({
+		lix,
+		settings: {
+			baseLocale: "en",
+			locales: ["en"],
+			modules: ["https://mock.com/module.js"],
+		},
+	});
+
+	expect(global.fetch).toHaveBeenCalledTimes(1);
+	expect(result.plugins.length).toBe(0);
+	expect(result.errors.length).toBe(1);
+	expect(result.errors[0]).toBeInstanceOf(PluginImportError);
+});
+
+test("it should filter message lint rules for legacy reasons", async () => {
+	global.fetch = vi.fn().mockResolvedValue({
+		ok: true,
+		text: vi
+			.fn()
+			.mockResolvedValue("export default { id: 'messageLintRule.something' }"),
+	});
+
+	const lix = await openLixInMemory({ blob: await newLixFile() });
+
+	const result = await importPlugins({
+		lix,
+		settings: {
+			baseLocale: "en",
+			locales: ["en"],
+			modules: ["https://mock.com/module.js"],
+		},
+	});
+
+	expect(result.plugins.length).toBe(0);
+	expect(result.errors.length).toBe(0);
+});
+
+// more tests are found in cache.test.ts
+test("cache should work", async () => {
+	global.fetch = vi.fn().mockResolvedValue({
+		ok: false,
+	});
+
+	const mockModulePath = "https://mock.com/module.js";
+	const mockModuleCachePath = "/cache/plugins/31i1etp0l413h";
+
+	const lix = await openLixInMemory({ blob: await newLixFile() });
+
+	await lix.db
+		.insertInto("file")
+		.values({
+			path: mockModuleCachePath,
+			data: new TextEncoder().encode("export default { key: 'mock' }"),
+		})
+		.execute();
+
+	const result = await importPlugins({
+		lix,
+		settings: {
+			baseLocale: "en",
+			locales: ["en"],
+			modules: [mockModulePath],
+		},
+	});
+
+	expect(result.errors).lengthOf(0);
+	expect(result.plugins).lengthOf(1);
+	expect(result.plugins[0]?.key).toBe("mock");
+});

package/src/plugin/importPlugins.ts

@@ -0,0 +1,60 @@
+import type { Lix } from "@lix-js/sdk";
+import type { ProjectSettings } from "../json-schema/settings.js";
+import { PluginError, PluginImportError } from "./errors.js";
+import type { InlangPlugin } from "./schema.js";
+import { withCache } from "./cache.js";
+
+/**
+ * Function that preprocesses the plugin before importing it.
+ *
+ * - used by sherlock to convert ESM to CJS
+ */
+export type PreprocessPluginBeforeImportFunction = (
+	moduleText: string
+) => Promise<string> | string;
+
+export async function importPlugins(args: {
+	lix: Lix;
+	settings: ProjectSettings;
+	preprocessPluginBeforeImport?: PreprocessPluginBeforeImportFunction;
+}): Promise<{
+	plugins: InlangPlugin[];
+	errors: PluginError[];
+}> {
+	const plugins: InlangPlugin[] = [];
+	const errors: PluginError[] = [];
+	for (const uri of args.settings.modules ?? []) {
+		try {
+			let moduleAsText = await withCache(fetchPlugin, args.lix)(uri);
+			if (args.preprocessPluginBeforeImport) {
+				moduleAsText = await args.preprocessPluginBeforeImport(moduleAsText);
+			}
+			const moduleWithMimeType =
+				"data:application/javascript," + encodeURIComponent(moduleAsText);
+			const { default: module } = await import(
+				/* @vite-ignore */ moduleWithMimeType
+			);
+			// old legacy message lint rules are not supported
+			// and ingored for backwards compatibility
+			if (module.id?.includes("messageLintRule")) {
+				continue;
+			}
+			plugins.push(module);
+		} catch (e) {
+			errors.push(new PluginImportError({ plugin: uri, cause: e as Error }));
+		}
+	}
+	return { plugins, errors };
+}
+
+async function fetchPlugin(uri: string): Promise<string> {
+	try {
+		const response = await fetch(uri);
+		return await response.text();
+	} catch (error) {
+		throw new PluginImportError({
+			plugin: uri,
+			cause: error as Error,
+		});
+	}
+}