@inlang/sdk 0.36.4 → 2.0.0-beta.2

package/src/project/loadProjectInMemory.test.ts

@@ -0,0 +1,29 @@
+import { expect, test } from "vitest";
+import { newProject } from "./newProject.js";
+import { loadProjectInMemory } from "./loadProjectInMemory.js";
+
+test("roundtrip should succeed", async () => {
+	const file1 = await newProject();
+	const project1 = await loadProjectInMemory({ blob: file1 });
+	const numBundles1 = (
+		await project1.db.selectFrom("bundle").select("id").execute()
+	).length;
+	expect(numBundles1).toBe(0);
+
+	// modify project
+	const insertedBundle = await project1.db
+		.insertInto("bundle")
+		.values({
+			id: "mock245",
+		})
+		.returning("id")
+		.executeTakeFirstOrThrow();
+
+	const file1AfterUpdates = await project1.toBlob();
+	await project1.close();
+
+	const project2 = await loadProjectInMemory({ blob: file1AfterUpdates });
+	const bundles = await project2.db.selectFrom("bundle").select("id").execute();
+	expect(bundles.length).toBe(1);
+	expect(bundles[0]?.id).toBe(insertedBundle.id);
+});

package/src/project/loadProjectInMemory.ts

@@ -0,0 +1,36 @@
+import { openLixInMemory } from "@lix-js/sdk";
+import { createInMemoryDatabase, importDatabase } from "sqlite-wasm-kysely";
+import { loadProject } from "./loadProject.js";
+
+/**
+ * Load a project from a blob in memory.
+ */
+export async function loadProjectInMemory(
+	args: {
+		blob: Blob;
+	} & Omit<Parameters<typeof loadProject>[0], "sqlite" | "lix">
+) {
+	const lix = await openLixInMemory({
+		blob: args.blob,
+		account: args.account,
+		providePlugins: [
+			// inlangLixPluginV1
+		],
+	});
+
+	const dbFile = await lix.db
+		.selectFrom("file")
+		.select("data")
+		.where("path", "=", "/db.sqlite")
+		.executeTakeFirstOrThrow();
+
+	const sqlite = await createInMemoryDatabase({});
+	importDatabase({ db: sqlite, content: new Uint8Array(dbFile.data) });
+
+	return await loadProject({
+		// pass common arguments to loadProject
+		...args,
+		sqlite,
+		lix,
+	});
+}

package/src/project/maybeCaptureTelemetry.test.ts

@@ -0,0 +1,79 @@
+import { expect, test, vi } from "vitest";
+import { loadProjectInMemory } from "./loadProjectInMemory.js";
+import { newProject } from "./newProject.js";
+import { maybeCaptureLoadedProject } from "./maybeCaptureTelemetry.js";
+import { capture } from "../services/telemetry/capture.js";
+
+test("it should capture as expected", async () => {
+	vi.mock("../services/telemetry/capture.js", async () => {
+		return {
+			capture: vi.fn(() => Promise.resolve()),
+		};
+	});
+
+	vi.mock("../services/env-variables/index", async () => {
+		return {
+			ENV_VARIABLES: {
+				PUBLIC_POSTHOG_TOKEN: "mock-defined",
+				SDK_VERSION: "1.0.0-mock",
+			},
+		};
+	});
+
+	const project = await loadProjectInMemory({
+		blob: await newProject(),
+	});
+
+	const account = await project.lix.db
+		.selectFrom("active_account")
+		.select("id")
+		.executeTakeFirstOrThrow();
+
+	const bundle = await project.db
+		.insertInto("bundle")
+		.defaultValues()
+		.returningAll()
+		.executeTakeFirstOrThrow();
+
+	const message = await project.db
+		.insertInto("message")
+		.values({ bundleId: bundle.id, locale: "en" })
+		.returningAll()
+		.executeTakeFirstOrThrow();
+
+	await project.db
+		.insertInto("variant")
+		.values({ messageId: message.id })
+		.returningAll()
+		.executeTakeFirst();
+
+	const settings = await project.settings.get();
+
+	const id = await project.id.get();
+
+	const plugins = await project.plugins.get();
+
+	await maybeCaptureLoadedProject({
+		id,
+		settings,
+		plugins,
+		lix: project.lix,
+		appId: "test",
+		db: project.db,
+	});
+
+	expect(capture).toHaveBeenCalledWith("SDK loaded project", {
+		projectId: await project.id.get(),
+		settings: await project.settings.get(),
+		accountId: account.id,
+		properties: {
+			appId: "test",
+			settings: await project.settings.get(),
+			pluginKeys: [],
+			sdkVersion: "1.0.0-mock",
+			numBundles: 1,
+			numMessages: 1,
+			numVariants: 1,
+		},
+	});
+});

package/src/project/maybeCaptureTelemetry.ts

@@ -0,0 +1,69 @@
+import { Kysely } from "kysely";
+import { capture } from "../services/telemetry/capture.js";
+import type { InlangDatabaseSchema } from "../database/schema.js";
+import { ENV_VARIABLES } from "../services/env-variables/index.js";
+import type { ProjectSettings } from "../json-schema/settings.js";
+import { captureError } from "../services/error-reporting/index.js";
+import type { Lix } from "@lix-js/sdk";
+
+export async function maybeCaptureLoadedProject(args: {
+	id: string;
+	lix: Lix;
+	settings: ProjectSettings;
+	plugins: Readonly<Array<{ key: string }>>;
+	appId?: string;
+	db: Kysely<InlangDatabaseSchema>;
+}) {
+	if (args.settings.telemetry === "off") {
+		return;
+	}
+
+	try {
+		const activeAccount = await args.lix.db
+			.selectFrom("active_account")
+			.select("id")
+			.executeTakeFirstOrThrow();
+
+		const bundles = await args.db
+			.selectFrom("bundle")
+			.select((s) => s.fn.count("id").as("count"))
+			.executeTakeFirst();
+		const messages = await args.db
+			.selectFrom("message")
+			.select((s) => s.fn.count("id").as("count"))
+			.executeTakeFirst();
+		const variants = await args.db
+			.selectFrom("variant")
+			.select((s) => s.fn.count("id").as("count"))
+			.executeTakeFirst();
+
+		await capture("SDK loaded project", {
+			projectId: args.id,
+			settings: args.settings,
+			accountId: activeAccount.id,
+			properties: {
+				// Insight: Which app is used by the SDK
+				appId: args.appId,
+				// Insight: How many languages are used, etc.
+				settings: args.settings,
+				// Insight on the used plugins (which one's to prioritize)
+				pluginKeys: args.plugins.map((plugin) => plugin.key),
+				// Insight: Which version of the SDK is used (can be used to deprecate old versions)
+				sdkVersion: ENV_VARIABLES.SDK_VERSION,
+				// Insight: Scale of projects (what project size to optimize for)
+				numBundles: bundles?.count,
+				numMessages: messages?.count,
+				numVariants: variants?.count,
+			},
+		});
+	} catch (e) {
+		if (
+			e instanceof Error &&
+			e.message.includes("driver has already been destroyed")
+		) {
+			// The project has been closed, nothing to capture
+			return;
+		}
+		captureError(e);
+	}
+}

package/src/project/newProject.test.ts

@@ -0,0 +1,36 @@
+import { expect, test } from "vitest";
+import { defaultProjectSettings, newProject } from "./newProject.js";
+import { loadProjectInMemory } from "./loadProjectInMemory.js";
+import { validate } from "uuid";
+
+test("it should be possible to provide settings for testing or other purposes", async () => {
+	const project = await loadProjectInMemory({
+		blob: await newProject({
+			settings: { baseLocale: "fr", locales: ["fr"], modules: [] },
+		}),
+	});
+	const settings = await project.settings.get();
+	expect(settings.baseLocale).toBe("fr");
+	expect(settings.locales).toEqual(["fr"]);
+	expect(settings.modules).toEqual([]);
+});
+
+test("it should be possible to create a project with default settings", async () => {
+	const project = await loadProjectInMemory({
+		blob: await newProject(),
+	});
+	const settings = await project.settings.get();
+	// pruning old settings that might exist for legacy reasons
+	delete settings.languageTags;
+	delete settings.sourceLanguageTag;
+	expect(settings).toStrictEqual(defaultProjectSettings);
+});
+
+test("it should have a uuid as project id", async () => {
+	const project = await loadProjectInMemory({
+		blob: await newProject(),
+	});
+	const projectId = await project.id.get();
+	expect(projectId).toBeDefined();
+	expect(validate(projectId)).toBe(true);
+});

package/src/project/newProject.ts

@@ -0,0 +1,78 @@
+import { newLixFile, openLixInMemory, toBlob } from "@lix-js/sdk";
+import { v4 } from "uuid";
+import type { ProjectSettings } from "../json-schema/settings.js";
+import {
+	contentFromDatabase,
+	createInMemoryDatabase,
+} from "sqlite-wasm-kysely";
+import { initDb } from "../database/initDb.js";
+import { captureError } from "../services/error-reporting/index.js";
+
+/**
+ * Creates a new inlang project.
+ *
+ * The app is responsible for saving the project "whereever"
+ * e.g. the user's computer, cloud storage, or OPFS in the browser.
+ */
+export async function newProject(args?: {
+	settings?: ProjectSettings;
+}): Promise<Blob> {
+	const sqlite = await createInMemoryDatabase({
+		readOnly: false,
+	});
+	initDb({ sqlite });
+
+	try {
+		const inlangDbContent = contentFromDatabase(sqlite);
+
+		const lix = await openLixInMemory({ blob: await newLixFile() });
+
+		// write files to lix
+		await lix.db
+			.insertInto("file")
+			.values([
+				{
+					path: "/db.sqlite",
+					data: inlangDbContent,
+				},
+				{
+					path: "/settings.json",
+					data: new TextEncoder().encode(
+						JSON.stringify(args?.settings ?? defaultProjectSettings)
+					),
+				},
+				{
+					path: "/project_id",
+					data: new TextEncoder().encode(v4()),
+				},
+			])
+			.execute();
+		const blob = toBlob({ lix });
+		lix.sqlite.close();
+		return blob;
+	} catch (e) {
+		const error = new Error(`Failed to create new inlang project: ${e}`, {
+			cause: e,
+		});
+		captureError(error);
+		throw error;
+	} finally {
+		sqlite.close();
+	}
+}
+
+export const defaultProjectSettings = {
+	$schema: "https://inlang.com/schema/project-settings",
+	baseLocale: "en",
+	locales: ["en"],
+	modules: [
+		// for instant gratification, we're adding common rules
+		// "https://cdn.jsdelivr.net/npm/@inlang/message-lint-rule-empty-pattern@latest/dist/index.js",
+		// "https://cdn.jsdelivr.net/npm/@inlang/message-lint-rule-missing-translation@latest/dist/index.js",
+		// "https://cdn.jsdelivr.net/npm/@inlang/message-lint-rule-without-source@latest/dist/index.js",
+		// default to the message format plugin because it supports all features
+		// "https://cdn.jsdelivr.net/npm/@inlang/plugin-message-format@latest/dist/index.js",
+		// the m function matcher should be installed by default in case Sherlock (VS Code extension) is adopted
+		// "https://cdn.jsdelivr.net/npm/@inlang/plugin-m-function-matcher@latest/dist/index.js",
+	],
+} satisfies ProjectSettings;

package/src/project/saveProjectToDirectory.test.ts

@@ -0,0 +1,310 @@
+import { test, expect, vi } from "vitest";
+import { saveProjectToDirectory } from "./saveProjectToDirectory.js";
+import { Volume } from "memfs";
+import { loadProjectInMemory } from "./loadProjectInMemory.js";
+import { newProject } from "./newProject.js";
+import type { InlangPlugin } from "../plugin/schema.js";
+import type { Bundle, NewMessage, Variant } from "../database/schema.js";
+import { loadProjectFromDirectory } from "./loadProjectFromDirectory.js";
+import { selectBundleNested } from "../query-utilities/selectBundleNested.js";
+import type { ProjectSettings } from "../json-schema/settings.js";
+import type { MessageV1 } from "../json-schema/old-v1-message/schemaV1.js";
+
+test("it should throw if the path doesn't end with .inlang", async () => {
+	await expect(() =>
+		saveProjectToDirectory({
+			fs: {} as any,
+			project: {} as any,
+			path: "/foo/bar",
+		})
+	).rejects.toThrowError("The path must end with .inlang");
+});
+
+test("it should overwrite all files to the directory except the db.sqlite file", async () => {
+	const mockFs = Volume.fromJSON({
+		"/foo/bar.inlang/settings.json": JSON.stringify({
+			baseLocale: "en",
+			locales: ["en"],
+		}),
+	}).promises as any;
+
+	const project = await loadProjectInMemory({
+		blob: await newProject({
+			settings: {
+				baseLocale: "en",
+				locales: ["en", "fr", "mock"],
+			},
+		}),
+	});
+
+	await saveProjectToDirectory({
+		fs: mockFs,
+		project,
+		path: "/foo/bar.inlang",
+	});
+
+	const files = await mockFs.readdir("/foo/bar.inlang");
+	const updatedSettingsFile = await mockFs.readFile(
+		"/foo/bar.inlang/settings.json",
+		"utf-8"
+	);
+	const updatedSettings = JSON.parse(updatedSettingsFile);
+
+	// only testing known files at the time of the test.
+	// this test should be updated for files that should NOT
+	// be contained in the directory in the future
+	expect(files).toContain("settings.json");
+	expect(files).not.toContain("db.sqlite");
+	expect(updatedSettings.baseLocale).toBe("en");
+	expect(updatedSettings.locales).toEqual(["en", "fr", "mock"]);
+});
+
+test("a roundtrip should work", async () => {
+	const bundles: Bundle[] = [{ id: "mock-bundle", declarations: [] }];
+	const messages: NewMessage[] = [{ bundleId: "mock-bundle", locale: "en" }];
+	const variants: Variant[] = [];
+
+	const volume = Volume.fromJSON({
+		"/mock-file.json": JSON.stringify({ bundles, messages, variants }),
+	});
+
+	const mockPlugin: InlangPlugin = {
+		key: "mock-plugin",
+		toBeImportedFiles: async () => {
+			return [{ path: "/mock-file.json", locale: "mock" }];
+		},
+		importFiles: async ({ files }) => {
+			const { bundles, messages, variants } = JSON.parse(
+				new TextDecoder().decode(files[0]?.content)
+			);
+			return { bundles, messages, variants };
+		},
+		exportFiles: async ({ bundles }) => {
+			return [
+				{
+					content: new TextEncoder().encode(JSON.stringify(bundles)),
+					name: "mock-file.json",
+					locale: "mock",
+				},
+			];
+		},
+	};
+
+	const exportFilesSpy = vi.spyOn(mockPlugin, "exportFiles");
+	const importFilesSpy = vi.spyOn(mockPlugin, "importFiles");
+
+	const project = await loadProjectInMemory({
+		blob: await newProject(),
+		providePlugins: [mockPlugin],
+	});
+
+	await project.db.insertInto("bundle").values(bundles).execute();
+	await project.db.insertInto("message").values(messages).execute();
+
+	await saveProjectToDirectory({
+		fs: volume.promises as any,
+		project,
+		path: "/foo/bar.inlang",
+	});
+
+	// const fileTree = volume.toJSON();
+
+	expect(exportFilesSpy).toHaveBeenCalled();
+	expect(importFilesSpy).not.toHaveBeenCalled();
+
+	// TODO deactivated since mockBundleNested no longer contains the id of the messages
+	// expect(fileTree).toEqual(
+	// 	expect.objectContaining({
+	// 		"/foo/mock-file.json": JSON.stringify([mockBundleNested]),
+	// 	})
+	// );
+
+	// testing roundtrip
+
+	const project2 = await loadProjectFromDirectory({
+		fs: volume as any,
+		path: "/foo/bar.inlang",
+		providePlugins: [mockPlugin],
+	});
+
+	expect(mockPlugin.importFiles).toHaveBeenCalled();
+
+	const bundlesAfter = await project2.db
+		.selectFrom("bundle")
+		.selectAll()
+		.execute();
+	const messagesAfter = await project2.db
+		.selectFrom("message")
+		.selectAll()
+		.execute();
+	const variantsAfter = await project2.db
+		.selectFrom("variant")
+		.selectAll()
+		.execute();
+
+	expect(bundlesAfter).lengthOf(1);
+	expect(messagesAfter).lengthOf(1);
+	expect(variantsAfter).lengthOf(0);
+
+	expect(bundlesAfter[0]).toStrictEqual(expect.objectContaining(bundles[0]));
+	expect(messagesAfter[0]).toStrictEqual(
+		expect.objectContaining(messagesAfter[0])
+	);
+});
+
+test.todo(
+	"a roundtrip with legacy load and save messages should work",
+	async () => {
+		const mockMessageV1: MessageV1 = {
+			id: "mock-legacy-message",
+			alias: {},
+			selectors: [],
+			variants: [
+				{
+					languageTag: "en",
+					match: [],
+					pattern: [{ type: "Text", value: "Hello from legacy message" }],
+				},
+			],
+		};
+
+		const volume = Volume.fromJSON({
+			"/foo/bar.inlang/settings.json": JSON.stringify({
+				baseLocale: "en",
+				locales: ["en"],
+			} satisfies ProjectSettings),
+			"/foo/i18n/en.json": JSON.stringify([mockMessageV1]),
+		});
+
+		const mockPlugin: InlangPlugin = {
+			id: "mock-legacy-plugin",
+			key: "mock-legacy-plugin",
+			loadMessages: async ({ nodeishFs }) => {
+				// expecting `loadMessages` to transform the relative path
+				// to an absolute path `./i18n/en.json` -> `/foo/i18n/en.json`
+				const file = await nodeishFs.readFile("./i18n/en.json", {
+					encoding: "utf-8",
+				});
+				return JSON.parse(file as string);
+			},
+			saveMessages: async ({ messages, nodeishFs }) => {
+				await nodeishFs.writeFile(
+					"./i18n/en.json",
+					new TextEncoder().encode(JSON.stringify(messages))
+						.buffer as ArrayBuffer
+				);
+			},
+		};
+
+		const loadMessagesSpy = vi.spyOn(mockPlugin, "loadMessages");
+		const saveMessagesSpy = vi.spyOn(mockPlugin, "saveMessages");
+
+		const project = await loadProjectFromDirectory({
+			fs: volume as any,
+			path: "/foo/bar.inlang",
+			providePlugins: [mockPlugin],
+		});
+
+		expect(loadMessagesSpy).toHaveBeenCalled();
+		expect(saveMessagesSpy).not.toHaveBeenCalled();
+
+		const bundles1 = await selectBundleNested(project.db).execute();
+
+		expect(bundles1[0]?.messages).lengthOf(1);
+		expect(bundles1[0]?.messages[0]?.variants).toEqual([
+			expect.objectContaining({
+				pattern: [
+					{
+						type: "text",
+						value: "Hello from legacy message",
+					},
+				],
+			}),
+		]);
+
+		// await project.db
+		// 	.updateTable("variant")
+		// 	.set({
+		// 		pattern: [{ type: "text", value: "Updated message" }],
+		// 	})
+		// 	.where("id", "=", bundles1[0]?.messages[0]?.variants[0]?.id as string)
+		// 	.execute();
+
+		// testing the saveMessages function by removing the en.json file
+		await volume.promises.rm("/foo/i18n/en.json");
+
+		await saveProjectToDirectory({
+			fs: volume.promises as any,
+			project,
+			path: "/foo/bar.inlang",
+		});
+
+		expect(saveMessagesSpy).toHaveBeenCalled();
+
+		const fileTree = volume.toJSON();
+		const parsed = JSON.parse(fileTree["/foo/i18n/en.json"] as string);
+
+		expect(parsed).toEqual(expect.objectContaining([mockMessageV1]));
+
+		// testing roundtrip
+
+		const project2 = await loadProjectFromDirectory({
+			fs: volume as any,
+			path: "/foo/bar.inlang",
+			providePlugins: [mockPlugin],
+		});
+
+		const bundles2 = await selectBundleNested(project2.db).execute();
+
+		// TODO deactivated since the ids must not be equal for separate imports - matching happens on language and matcher now
+		expect(bundles1).toStrictEqual(bundles2);
+	}
+);
+
+test("it should preserve the formatting of existing json resource files", async () => {
+	const mockJson =
+		JSON.stringify(
+			{ key: "value" },
+			undefined,
+			// tab spacing
+			"\t"
+		) +
+		// ends with new line
+		"\n";
+
+	const mockPlugin: InlangPlugin = {
+		key: "mock",
+		exportFiles: async () => {
+			return [
+				{
+					name: "en.json",
+					// no beautified json
+					content: new TextEncoder().encode(JSON.stringify({ key: "value" })),
+					locale: "en",
+				},
+			];
+		},
+	};
+
+	const volume = Volume.fromJSON({
+		"/foo/project.inlang/settings.json": JSON.stringify({
+			baseLocale: "en",
+			locales: ["en"],
+		} satisfies ProjectSettings),
+		"/foo/en.json": mockJson,
+	});
+
+	const project = await loadProjectInMemory({
+		blob: await newProject(),
+		providePlugins: [mockPlugin],
+	});
+
+	await saveProjectToDirectory({
+		path: "/foo/project.inlang",
+		fs: volume.promises as any,
+		project,
+	});
+
+	const fileAfterSave = await volume.promises.readFile("/foo/en.json", "utf-8");
+	expect(fileAfterSave).toBe(mockJson);
+});