@inlang/sdk 0.36.3 → 2.0.0-prerelease.0

package/src/import-export/exportFiles.ts

@@ -0,0 +1,36 @@
+import type { Kysely } from "kysely";
+import {
+	PluginDoesNotImplementFunctionError,
+	PluginMissingError,
+} from "../plugin/errors.js";
+import type { ProjectSettings } from "../json-schema/settings.js";
+import type { InlangDatabaseSchema } from "../database/schema.js";
+import type { InlangPlugin } from "../plugin/schema.js";
+
+export async function exportFiles(opts: {
+	readonly pluginKey: string;
+	readonly settings: ProjectSettings;
+	readonly plugins: readonly InlangPlugin[];
+	readonly db: Kysely<InlangDatabaseSchema>;
+}) {
+	const plugin = opts.plugins.find((p) => p.key === opts.pluginKey);
+	if (!plugin) throw new PluginMissingError({ plugin: opts.pluginKey });
+	if (!plugin.exportFiles) {
+		throw new PluginDoesNotImplementFunctionError({
+			plugin: opts.pluginKey,
+			function: "exportFiles",
+		});
+	}
+
+	const bundles = await opts.db.selectFrom("bundle").selectAll().execute();
+	const messages = await opts.db.selectFrom("message").selectAll().execute();
+	const variants = await opts.db.selectFrom("variant").selectAll().execute();
+
+	const files = await plugin.exportFiles({
+		settings: structuredClone(opts.settings),
+		bundles,
+		messages,
+		variants,
+	});
+	return files;
+}

package/src/import-export/importFiles.test.ts

@@ -0,0 +1,211 @@
+import { test, expect } from "vitest";
+import { importFiles } from "./importFiles.js";
+import { loadProjectInMemory } from "../project/loadProjectInMemory.js";
+import { newProject } from "../project/newProject.js";
+import type { InlangPlugin } from "../plugin/schema.js";
+
+test("it should insert a message as is if the id is provided", async () => {
+	const project = await loadProjectInMemory({ blob: await newProject() });
+
+	const mockPlugin: InlangPlugin = {
+		key: "mock",
+		importFiles: async () => ({
+			bundles: [{ id: "mock-bundle" }],
+			messages: [{ id: "alfa23", bundleId: "mock-bundle", locale: "en" }],
+			variants: [],
+		}),
+	};
+
+	await importFiles({
+		db: project.db,
+		files: [{ content: new Uint8Array(), locale: "mock" }],
+		pluginKey: "mock",
+		plugins: [mockPlugin],
+		settings: {} as any,
+	});
+
+	const messages = await project.db.selectFrom("message").selectAll().execute();
+
+	expect(messages.length).toBe(1);
+	expect(messages[0]?.id).toBe("alfa23");
+});
+
+test("it should match an existing message if the id is not provided", async () => {
+	const project = await loadProjectInMemory({ blob: await newProject() });
+
+	await project.db.insertInto("bundle").values({ id: "mock-bundle" }).execute();
+	await project.db
+		.insertInto("message")
+		.values({
+			id: "alfa23",
+			bundleId: "mock-bundle",
+			locale: "en",
+			selectors: [],
+		})
+		.execute();
+
+	const mockPlugin: InlangPlugin = {
+		key: "mock",
+		importFiles: async () => ({
+			bundles: [],
+			messages: [
+				{
+					bundleId: "mock-bundle",
+					locale: "en",
+					selectors: [{ type: "variable-reference", name: "platform" }],
+				},
+			],
+			variants: [],
+		}),
+	};
+
+	await importFiles({
+		db: project.db,
+		files: [{ content: new Uint8Array(), locale: "mock" }],
+		pluginKey: "mock",
+		plugins: [mockPlugin],
+		settings: {} as any,
+	});
+
+	const messages = await project.db.selectFrom("message").selectAll().execute();
+
+	expect(messages.length).toBe(1);
+	expect(messages[0]?.id).toBe("alfa23");
+	expect(messages[0]?.selectors).toStrictEqual([
+		{ type: "variable-reference", name: "platform" },
+	]);
+});
+
+test("it should create a bundle for a message if the bundle does not exist to avoid foreign key conflicts and enable partial imports", async () => {
+	const project = await loadProjectInMemory({ blob: await newProject() });
+
+	const mockPlugin: InlangPlugin = {
+		key: "mock",
+		importFiles: async () => ({
+			bundles: [],
+			messages: [{ bundleId: "non-existent-bundle", locale: "en" }],
+			variants: [],
+		}),
+	};
+
+	await importFiles({
+		db: project.db,
+		files: [{ content: new Uint8Array(), locale: "mock" }],
+		pluginKey: "mock",
+		plugins: [mockPlugin],
+		settings: {} as any,
+	});
+
+	const bundles = await project.db.selectFrom("bundle").selectAll().execute();
+
+	expect(bundles.length).toBe(1);
+	expect(bundles[0]?.id).toBe("non-existent-bundle");
+});
+
+test("it should insert a variant as is if the id is provided", async () => {
+	const project = await loadProjectInMemory({ blob: await newProject() });
+
+	await project.db.insertInto("bundle").values({ id: "mock-bundle" }).execute();
+	await project.db
+		.insertInto("message")
+		.values({
+			id: "mock-message",
+			bundleId: "mock-bundle",
+			locale: "en",
+		})
+		.execute();
+
+	const mockPlugin: InlangPlugin = {
+		key: "mock",
+		importFiles: async () => ({
+			bundles: [],
+			messages: [],
+			variants: [{ id: "variant-id-23", messageId: "mock-message" }],
+		}),
+	};
+
+	await importFiles({
+		db: project.db,
+		files: [{ content: new Uint8Array(), locale: "mock" }],
+		pluginKey: "mock",
+		plugins: [mockPlugin],
+		settings: {} as any,
+	});
+
+	const variants = await project.db.selectFrom("variant").selectAll().execute();
+
+	expect(variants.length).toBe(1);
+	expect(variants[0]?.id).toBe("variant-id-23");
+});
+
+test("it should match an existing variant if the id is not provided", async () => {
+	const project = await loadProjectInMemory({ blob: await newProject() });
+
+	await project.db.insertInto("bundle").values({ id: "mock-bundle" }).execute();
+	await project.db
+		.insertInto("message")
+		.values({
+			id: "mock-message",
+			bundleId: "mock-bundle",
+			locale: "en",
+		})
+		.execute();
+
+	const mockPlugin: InlangPlugin = {
+		key: "mock",
+		importFiles: async () => ({
+			bundles: [],
+			messages: [],
+			variants: [{ messageBundleId: "mock-bundle", messageLocale: "en" }],
+		}),
+	};
+
+	await importFiles({
+		db: project.db,
+		files: [{ content: new Uint8Array(), locale: "mock" }],
+		pluginKey: "mock",
+		plugins: [mockPlugin],
+		settings: {} as any,
+	});
+
+	const variants = await project.db.selectFrom("variant").selectAll().execute();
+
+	expect(variants.length).toBe(1);
+	expect(variants[0]?.messageId).toBe("mock-message");
+	expect(variants[0]?.id).toBeDefined();
+});
+
+test("it should create a message for a variant if the message does not exist to avoid foreign key conflicts and enable partial imports", async () => {
+	const project = await loadProjectInMemory({ blob: await newProject() });
+
+	await project.db.insertInto("bundle").values({ id: "mock-bundle" }).execute();
+
+	const mockPlugin: InlangPlugin = {
+		key: "mock",
+		importFiles: async () => ({
+			bundles: [],
+			messages: [],
+			variants: [{ messageBundleId: "mock-bundle", messageLocale: "en" }],
+		}),
+	};
+
+	await importFiles({
+		db: project.db,
+		files: [{ content: new Uint8Array(), locale: "mock" }],
+		pluginKey: "mock",
+		plugins: [mockPlugin],
+		settings: {} as any,
+	});
+
+	const bundles = await project.db.selectFrom("bundle").selectAll().execute();
+	const messages = await project.db.selectFrom("message").selectAll().execute();
+	const variants = await project.db.selectFrom("variant").selectAll().execute();
+
+	expect(bundles.length).toBe(1);
+	expect(messages.length).toBe(1);
+	expect(variants.length).toBe(1);
+
+	expect(messages[0]?.bundleId).toBe("mock-bundle");
+	expect(messages[0]?.locale).toBe("en");
+	expect(variants[0]?.messageId).toBe(messages[0]?.id);
+});

package/src/import-export/importFiles.ts

@@ -0,0 +1,142 @@
+import type { Kysely } from "kysely";
+import {
+	PluginDoesNotImplementFunctionError,
+	PluginMissingError,
+} from "../plugin/errors.js";
+import type { ProjectSettings } from "../json-schema/settings.js";
+import type { InlangDatabaseSchema, NewVariant } from "../database/schema.js";
+import type { InlangPlugin, VariantImport } from "../plugin/schema.js";
+import type { ImportFile } from "../project/api.js";
+
+export async function importFiles(args: {
+	files: ImportFile[];
+	readonly pluginKey: string;
+	readonly settings: ProjectSettings;
+	readonly plugins: readonly InlangPlugin[];
+	readonly db: Kysely<InlangDatabaseSchema>;
+}) {
+	const plugin = args.plugins.find((p) => p.key === args.pluginKey);
+
+	if (!plugin) throw new PluginMissingError({ plugin: args.pluginKey });
+
+	if (!plugin.importFiles) {
+		throw new PluginDoesNotImplementFunctionError({
+			plugin: args.pluginKey,
+			function: "importFiles",
+		});
+	}
+
+	const imported = await plugin.importFiles({
+		files: args.files,
+		settings: structuredClone(args.settings),
+	});
+
+	await args.db.transaction().execute(async (trx) => {
+		// upsert every bundle
+		for (const bundle of imported.bundles) {
+			await trx
+				.insertInto("bundle")
+				.values(bundle)
+				.onConflict((oc) => oc.column("id").doUpdateSet(bundle))
+				.execute();
+		}
+		// upsert every message
+		for (const message of imported.messages) {
+			// match the message by bundle id and locale if
+			// no id is provided by the importer
+			if (message.id === undefined) {
+				const exisingMessage = await trx
+					.selectFrom("message")
+					.where("bundleId", "=", message.bundleId)
+					.where("locale", "=", message.locale)
+					.select("id")
+					.executeTakeFirst();
+				message.id = exisingMessage?.id;
+			}
+			try {
+				await trx
+					.insertInto("message")
+					.values(message)
+					.onConflict((oc) => oc.column("id").doUpdateSet(message))
+					.execute();
+			} catch (e) {
+				// 787 = SQLITE_CONSTRAINT_FOREIGNKEY
+				// handle foreign key violation
+				// e.g. a message references a bundle that doesn't exist
+				// by creating the bundle
+				if ((e as any)?.resultCode === 787) {
+					await trx
+						.insertInto("bundle")
+						.values({ id: message.bundleId })
+						.execute();
+					await trx.insertInto("message").values(message).execute();
+				} else {
+					throw e;
+				}
+			}
+		}
+		// upsert every variant
+		for (const variant of imported.variants) {
+			// match the variant by message id and matches if
+			// no id is provided by the importer
+			if (variant.id === undefined) {
+				let existingMessage = await trx
+					.selectFrom("message")
+					.where("bundleId", "=", variant.messageBundleId)
+					.where("locale", "=", variant.messageLocale)
+					.select("id")
+					.executeTakeFirst();
+
+				// if the message does not exist, create it
+				if (existingMessage === undefined) {
+					const existingBundle = await trx
+						.selectFrom("bundle")
+						.where("id", "=", variant.messageBundleId)
+						.select("id")
+						.executeTakeFirst();
+					// if the bundle does not exist, create it
+					if (existingBundle === undefined) {
+						await trx
+							.insertInto("bundle")
+							.values({ id: variant.messageBundleId })
+							.execute();
+					}
+					// insert the message
+					existingMessage = await trx
+						.insertInto("message")
+						.values({
+							bundleId: variant.messageBundleId,
+							locale: variant.messageLocale,
+						})
+						.returningAll()
+						.executeTakeFirstOrThrow();
+				}
+
+				const existingVariants = await trx
+					.selectFrom("variant")
+					.where("messageId", "=", existingMessage.id)
+					.selectAll()
+					.execute();
+
+				const existingVariant = existingVariants.find(
+					(v) => JSON.stringify(v.matches) === JSON.stringify(variant.matches)
+				);
+
+				// need to reset typescript's type narrowing
+				(variant as VariantImport).id = existingVariant?.id;
+				(variant as VariantImport).messageId = existingMessage.id;
+			}
+			const toBeInsertedVariant: NewVariant = {
+				...variant,
+				// @ts-expect-error - bundle id is provided by VariantImport but not needed when inserting
+				messageBundleId: undefined,
+				messageLocale: undefined,
+			};
+			await trx
+				.insertInto("variant")
+				.values(toBeInsertedVariant)
+				.onConflict((oc) => oc.column("id").doUpdateSet(toBeInsertedVariant))
+				.execute();
+		}
+	});
+}

package/src/import-export/roundtrip.test.ts

@@ -0,0 +1,313 @@
+import { test, expect } from "vitest";
+import type {
+	BundleImport,
+	InlangPlugin,
+	MessageImport,
+	VariantImport,
+} from "../plugin/schema.js";
+import type { Message } from "../database/schema.js";
+import type { Text } from "../json-schema/pattern.js";
+import { loadProjectInMemory } from "../project/loadProjectInMemory.js";
+import { newProject } from "../project/newProject.js";
+import { importFiles } from "./importFiles.js";
+import { exportFiles } from "./exportFiles.js";
+
+test("the file should be identical after a roundtrip if no modifications occured", async () => {
+	const project = await loadProjectInMemory({
+		blob: await newProject(),
+	});
+
+	const enResource = JSON.stringify({
+		hello_world: "value1",
+	});
+
+	await importFiles({
+		files: [{ content: new TextEncoder().encode(enResource), locale: "en" }],
+		pluginKey: "mock",
+		plugins: [mockPluginSimple],
+		settings: await project.settings.get(),
+		db: project.db,
+	});
+
+	const importedBundles = await project.db
+		.selectFrom("bundle")
+		.selectAll()
+		.execute();
+	const importedMessages = await project.db
+		.selectFrom("message")
+		.selectAll()
+		.execute();
+	const importedVariants = await project.db
+		.selectFrom("variant")
+		.selectAll()
+		.execute();
+
+	expect(importedBundles.length).toBe(1);
+	expect(importedMessages.length).toBe(1);
+	expect(importedVariants.length).toBe(1);
+	expect(importedBundles[0]?.id).toBe("hello_world");
+	expect(importedMessages[0]?.bundleId).toBe("hello_world");
+	expect(importedVariants[0]?.messageId).toBe(importedMessages[0]?.id);
+
+	const exportedFiles = await exportFiles({
+		pluginKey: "mock",
+		plugins: [mockPluginSimple],
+		settings: await project.settings.get(),
+		db: project.db,
+	});
+
+	expect(exportedFiles.length).toBe(1);
+	expect(exportedFiles[0]?.name).toBe("en.json");
+	expect(new TextDecoder().decode(exportedFiles[0]?.content)).toBe(enResource);
+});
+
+test("a variant with an existing match should update the existing variant and not create a new one to enable roundtrips", async () => {
+	const project = await loadProjectInMemory({
+		blob: await newProject(),
+	});
+
+	const existing = {
+		bundles: [
+			{
+				id: "mock-bundle-id",
+				declarations: [],
+			},
+		],
+		messages: [
+			{
+				id: "mock-message-id",
+				locale: "en",
+				selectors: [],
+				bundleId: "mock-bundle-id",
+			},
+		],
+		variants: [
+			{
+				id: "mock-variant-id",
+				messageId: "mock-message-id",
+				matches: [
+					{
+						type: "literal-match",
+						key: "color",
+						value: "blue",
+					},
+				],
+				pattern: [
+					{
+						type: "text",
+						value: "You have blue eyes.",
+					},
+				],
+			},
+		],
+	};
+
+	const updated = structuredClone(existing);
+	// @ts-expect-error - we know this is a text pattern
+	updated.variants[0].pattern[0].value = "You have beautiful blue eyes.";
+
+	const enResource = JSON.stringify(updated);
+
+	await importFiles({
+		files: [{ content: new TextEncoder().encode(enResource), locale: "en" }],
+		pluginKey: "mock",
+		plugins: [mockPluginAst],
+		settings: await project.settings.get(),
+		db: project.db,
+	});
+
+	const bundles = await project.db.selectFrom("bundle").selectAll().execute();
+	const messages = await project.db.selectFrom("message").selectAll().execute();
+	const variants = await project.db.selectFrom("variant").selectAll().execute();
+
+	expect(bundles.length).toBe(1);
+	expect(messages.length).toBe(1);
+	expect(variants.length).toBe(1);
+	// @ts-expect-error - we know this is a text pattern
+	expect(variants[0]?.pattern[0].value).toBe("You have beautiful blue eyes.");
+});
+
+test("if a message for the bundle id and locale already exists, update it. don't create a new message", async () => {
+	const project = await loadProjectInMemory({
+		blob: await newProject(),
+	});
+
+	const existing: any = {
+		bundles: [{ id: "mock-bundle-id", declarations: [] }],
+		messages: [
+			{
+				id: "mock-message-id",
+				locale: "en",
+				selectors: [],
+				bundleId: "mock-bundle-id",
+			},
+		],
+		variants: [],
+	};
+
+	const updated = structuredClone(existing);
+	updated.messages[0]!.selectors = [
+		{
+			type: "variable-reference",
+			name: "variable2",
+		},
+	] satisfies Message["selectors"];
+
+	const enResource = new TextEncoder().encode(JSON.stringify(updated));
+
+	await importFiles({
+		files: [{ content: enResource, locale: "en" }],
+		pluginKey: "mock",
+		plugins: [mockPluginAst],
+		settings: await project.settings.get(),
+		db: project.db,
+	});
+
+	const bundles = await project.db.selectFrom("bundle").selectAll().execute();
+	const messages = await project.db.selectFrom("message").selectAll().execute();
+
+	expect(bundles.length).toBe(1);
+	expect(messages.length).toBe(1);
+
+	expect(messages[0]?.selectors).toStrictEqual([
+		{
+			type: "variable-reference",
+			name: "variable2",
+		},
+	]);
+});
+
+test("keys should be ordered alphabetically for .json to minimize git diffs", async () => {
+	const project = await loadProjectInMemory({
+		blob: await newProject(),
+	});
+
+	const enResource = JSON.stringify({
+		a: "value1",
+		b: "value2",
+		d: "value4",
+	});
+
+	await importFiles({
+		files: [{ content: new TextEncoder().encode(enResource), locale: "en" }],
+		pluginKey: "mock",
+		plugins: [mockPluginSimple],
+		settings: await project.settings.get(),
+		db: project.db,
+	});
+
+	await project.db.insertInto("bundle").values({ id: "c" }).execute();
+	await project.db
+		.insertInto("message")
+		.values({ id: "c-en", bundleId: "c", locale: "en" })
+		.execute();
+	await project.db
+		.insertInto("variant")
+		.values({ messageId: "c-en", pattern: [{ type: "text", value: "value3" }] })
+		.execute();
+
+	const exportedFiles = await exportFiles({
+		pluginKey: "mock",
+		plugins: [mockPluginSimple],
+		settings: await project.settings.get(),
+		db: project.db,
+	});
+
+	expect(
+		JSON.parse(new TextDecoder().decode(exportedFiles[0]?.content))
+	).toStrictEqual({
+		a: "value1",
+		b: "value2",
+		c: "value3",
+		d: "value4",
+	});
+});
+
+const mockPluginAst: InlangPlugin = {
+	key: "mock",
+	exportFiles: async ({ bundles, messages, variants }) => {
+		return [
+			{
+				locale: "every",
+				name: "x.json",
+				content: new TextEncoder().encode(
+					JSON.stringify({ bundles, messages, variants })
+				),
+			},
+		];
+	},
+	importFiles: async ({ files }) => {
+		let bundles: any[] = [];
+		let messages: any[] = [];
+		let variants: any[] = [];
+		for (const file of files) {
+			const parsed = JSON.parse(new TextDecoder().decode(file.content));
+			bundles = [...bundles, ...parsed.bundles];
+			messages = [...messages, ...parsed.messages];
+			variants = [...variants, ...parsed.variants];
+		}
+		return { bundles, messages, variants };
+	},
+};
+
+// a simple mock plugin to test roundtrips
+//
+// purposefully not imported a real plugin
+// to keep the interdepenedencies low and
+// increase maintainability
+const mockPluginSimple: InlangPlugin = {
+	key: "mock",
+	exportFiles: async ({ messages, variants }) => {
+		const jsons: any = {};
+		for (const message of messages) {
+			const key = message.bundleId;
+			const value = (
+				variants.find((v) => v.messageId === message.id)?.pattern[0] as Text
+			).value;
+			if (!jsons[message.locale]) {
+				jsons[message.locale] = {};
+			}
+			jsons[message.locale][key] = value;
+		}
+		return Object.entries(jsons).map(([locale, json]) => ({
+			locale,
+			name: locale + ".json",
+			content: new TextEncoder().encode(JSON.stringify(json)),
+		}));
+	},
+	importFiles: async ({ files }) => {
+		const bundles: BundleImport[] = [];
+		const messages: MessageImport[] = [];
+		const variants: VariantImport[] = [];
+		for (const file of files) {
+			const parsed = JSON.parse(new TextDecoder().decode(file.content));
+			for (const key in parsed) {
+				bundles.push({
+					id: key,
+					declarations: [],
+				});
+				messages.push({
+					bundleId: key,
+					locale: file.locale,
+					selectors: [],
+				});
+				variants.push({
+					messageBundleId: key,
+					messageLocale: file.locale,
+					matches: [],
+					pattern: [
+						{
+							type: "text",
+							value: parsed[key],
+						},
+					],
+				});
+			}
+		}
+		return {
+			bundles,
+			messages,
+			variants,
+		};
+	},
+};