@inlang/sdk 0.36.2 → 2.0.0-prerelease.0

package/src/project/saveProjectToDirectory.ts

@@ -0,0 +1,100 @@
+/* eslint-disable no-restricted-imports */
+import type fs from "node:fs/promises";
+import type { InlangProject } from "./api.js";
+import path from "node:path";
+import { toMessageV1 } from "../json-schema/old-v1-message/toMessageV1.js";
+import {
+	absolutePathFromProject,
+	withAbsolutePaths,
+} from "./loadProjectFromDirectory.js";
+import { detectJsonFormatting } from "../utilities/detectJsonFormatting.js";
+import { selectBundleNested } from "../query-utilities/selectBundleNested.js";
+
+export async function saveProjectToDirectory(args: {
+	fs: typeof fs;
+	project: InlangProject;
+	path: string;
+}): Promise<void> {
+	if (args.path.endsWith(".inlang") === false) {
+		throw new Error("The path must end with .inlang");
+	}
+	const files = await args.project.lix.db
+		.selectFrom("file")
+		.selectAll()
+		.execute();
+
+	// write all files to the directory
+	for (const file of files) {
+		if (file.path.endsWith("db.sqlite")) {
+			continue;
+		}
+		const p = path.join(args.path, file.path);
+		await args.fs.mkdir(path.dirname(p), { recursive: true });
+		await args.fs.writeFile(p, new Uint8Array(file.data));
+	}
+
+	// run exporters
+	const plugins = await args.project.plugins.get();
+	const settings = await args.project.settings.get();
+
+	for (const plugin of plugins) {
+		// old legacy remove with v3
+		if (plugin.saveMessages) {
+			// in-efficient re-qeuery but it's a legacy function that will be removed.
+			// the effort of adjusting the code to not re-query is not worth it.
+			const bundlesNested = await selectBundleNested(args.project.db).execute();
+			await plugin.saveMessages({
+				messages: bundlesNested.map((b) => toMessageV1(b)),
+				// @ts-expect-error - legacy
+				nodeishFs: withAbsolutePaths(args.fs, args.path),
+				settings,
+			});
+		}
+
+		if (plugin.exportFiles) {
+			const bundles = await args.project.db
+				.selectFrom("bundle")
+				.selectAll()
+				.execute();
+			const messages = await args.project.db
+				.selectFrom("message")
+				.selectAll()
+				.execute();
+			const variants = await args.project.db
+				.selectFrom("variant")
+				.selectAll()
+				.execute();
+			const files = await plugin.exportFiles({
+				bundles,
+				messages,
+				variants,
+				settings,
+			});
+			for (const file of files) {
+				const p = absolutePathFromProject(args.path, file.name);
+				const dirname = path.dirname(p);
+				if ((await args.fs.stat(dirname)).isDirectory() === false) {
+					await args.fs.mkdir(dirname, { recursive: true });
+				}
+				if (p.endsWith(".json")) {
+					try {
+						const existing = await args.fs.readFile(p, "utf-8");
+						const stringify = detectJsonFormatting(existing);
+						await args.fs.writeFile(
+							p,
+							new TextEncoder().encode(
+								stringify(JSON.parse(new TextDecoder().decode(file.content)))
+							)
+						);
+					} catch {
+						// write the file to disk (json doesn't exist yet)
+						// yeah ugly duplication of write file but it works.
+						await args.fs.writeFile(p, new Uint8Array(file.content));
+					}
+				} else {
+					await args.fs.writeFile(p, new Uint8Array(file.content));
+				}
+			}
+		}
+	}
+}

package/src/project/state/README.md

@@ -0,0 +1,22 @@
+Reactivity is a bitch. Hard to reason about. Hard to debug. Hard to maintain.
+
+The `/state` has been modelled under the following principles:
+
+1. The state is derived from the files in lix.
+
+   The files in lix are the source of truth. That's great for
+   reproducibility (you can send the file(s) around, the state
+   is always reproducible), crash resistance, and ensures that
+   the interplay with lix (tracking changes) works.
+
+2. Every setter writes to a lix file, not the in-memory state.
+
+   Avoids running out of sync with the project file and ensures
+   that lix is tracking changes correctly.
+
+3. Reactivity is contained in `/state` and does not leak outside.
+
+   Maintaining reactivity is a high effort. Especially if reactivity
+   is all over the codebase. We learned this the hard way
+   (like full refactor hard way). Thus, `/state` reactive side effects
+   are modeled in `/state` alone.

package/src/project/state/id$.test.ts

@@ -0,0 +1,40 @@
+import { expect, test } from "vitest";
+import { newLixFile, openLixInMemory } from "@lix-js/sdk";
+import { firstValueFrom } from "rxjs";
+import { createId$ } from "./id$.js";
+
+test("if the settings file has been updated in lix, the observable should emit the new id", async () => {
+	const lix = await openLixInMemory({ blob: await newLixFile() });
+
+	await lix.db
+		.insertInto("file")
+		.values({
+			path: "/project_id",
+			data: new TextEncoder().encode("foo-bar"),
+		})
+		.execute();
+
+	const id$ = createId$({ lix });
+
+	// todo the emitted events are always too high
+	// that could be due to polling. not important rn.
+	// let emittedEvents = 0;
+	// settings$.subscribe(() => {
+	// 	emittedEvents++;
+	// });
+
+	const id = await firstValueFrom(id$);
+
+	expect(id).toBe("foo-bar");
+
+	await lix.db
+		.updateTable("file")
+		.where("file.path", "=", "/project_id")
+		.set({
+			data: new TextEncoder().encode("foo-bar-box"),
+		})
+		.execute();
+
+	const id2 = await firstValueFrom(id$);
+	expect(id2).toBe("foo-bar-box");
+});

package/src/project/state/id$.ts

@@ -0,0 +1,21 @@
+import { type Lix } from "@lix-js/sdk";
+import { filter, map, share, type Observable } from "rxjs";
+import { pollQuery } from "../../query-utilities/pollQuery.js";
+
+export function createId$(args: { lix: Lix }): Observable<string> {
+	const settings$ = pollQuery(() =>
+		args.lix.db
+			.selectFrom("file")
+			.where("path", "=", "/project_id")
+			.selectAll()
+			.executeTakeFirst()
+	).pipe(
+		// undefined can happen if a project has no id yet
+		// the project id will be automatically generated
+		// shortly after
+		filter((file) => file !== undefined),
+		map((file) => new TextDecoder().decode(file.data)),
+		share()
+	);
+	return settings$;
+}

package/src/project/state/setSettings.test.ts

@@ -0,0 +1,73 @@
+import { expect, test } from "vitest";
+import { newLixFile, openLixInMemory } from "@lix-js/sdk";
+import { ProjectSettings } from "../../json-schema/settings.js";
+import { createProjectState } from "./state.js";
+import { setSettings } from "./setSettings.js";
+
+test("setSettings an update to baseLocale and locales should update sourceLanguageTag and languageTags", async () => {
+	const lix = await openLixInMemory({ blob: await newLixFile() });
+
+	const settings: ProjectSettings = {
+		baseLocale: "en",
+		locales: ["en", "fr"],
+	};
+
+	await lix.db
+		.insertInto("file")
+		.values({
+			path: "/settings.json",
+			data: new TextEncoder().encode(JSON.stringify(settings)),
+		})
+		.execute();
+
+	await setSettings({ newSettings: settings, lix });
+
+	const settingsFile = await lix.db
+		.selectFrom("file")
+		.select("data")
+		.where("path", "=", "/settings.json")
+		.executeTakeFirstOrThrow();
+
+	const updatedSettings = JSON.parse(
+		new TextDecoder().decode(settingsFile.data)
+	) as ProjectSettings;
+
+	expect(updatedSettings.baseLocale).toBe("en");
+	expect(updatedSettings.sourceLanguageTag).toBe("en");
+	expect(updatedSettings.locales).toEqual(["en", "fr"]);
+	expect(updatedSettings.languageTags).toEqual(["en", "fr"]);
+});
+
+test("the settings should be persisted to lix", async () => {
+	const lix = await openLixInMemory({ blob: await newLixFile() });
+
+	await lix.db
+		.insertInto("file")
+		.values({
+			path: "/settings.json",
+			data: new TextEncoder().encode("{}"),
+		})
+		.execute();
+
+	const settings: ProjectSettings = {
+		baseLocale: "en",
+		locales: ["en", "fr"],
+	};
+
+	const state = createProjectState({ settings, lix });
+
+	await state.settings.set(settings);
+
+	const settingsFile = await lix.db
+		.selectFrom("file")
+		.select("data")
+		.where("path", "=", "/settings.json")
+		.executeTakeFirstOrThrow();
+
+	const updatedSettingsInLix = JSON.parse(
+		new TextDecoder().decode(settingsFile.data)
+	) as ProjectSettings;
+
+	expect(updatedSettingsInLix.baseLocale).toEqual("en");
+	expect(updatedSettingsInLix.locales).toEqual(["en", "fr"]);
+});

package/src/project/state/setSettings.ts

@@ -0,0 +1,19 @@
+import type { Lix } from "@lix-js/sdk";
+import type { ProjectSettings } from "../../json-schema/settings.js";
+
+export async function setSettings(args: {
+	newSettings: ProjectSettings;
+	lix: Lix;
+}) {
+	const cloned = structuredClone(args.newSettings);
+	cloned.languageTags = cloned.locales;
+	cloned.sourceLanguageTag = cloned.baseLocale;
+
+	await args.lix.db
+		.updateTable("file")
+		.where("path", "=", "/settings.json")
+		.set({
+			data: new TextEncoder().encode(JSON.stringify(cloned, undefined, 2)),
+		})
+		.execute();
+}

package/src/project/state/settings$.test.ts

@@ -0,0 +1,63 @@
+import { expect, test } from "vitest";
+import { createSettings$ } from "./settings$.js";
+import { newLixFile, openLixInMemory } from "@lix-js/sdk";
+import type { ProjectSettings } from "../../json-schema/settings.js";
+import { firstValueFrom } from "rxjs";
+import { withLanguageTagToLocaleMigration } from "../../migrations/v2/withLanguageTagToLocaleMigration.js";
+
+test("if the settings file has been updated in lix, the observable should emit the new settings", async () => {
+	const lix = await openLixInMemory({ blob: await newLixFile() });
+
+	const mockSettings: ProjectSettings = {
+		baseLocale: "en",
+		locales: ["en", "de"],
+		modules: [],
+	};
+
+	await lix.db
+		.insertInto("file")
+		.values({
+			path: "/settings.json",
+			data: new TextEncoder().encode(JSON.stringify(mockSettings)),
+		})
+		.execute();
+
+	const settings$ = createSettings$({ lix });
+
+	// todo the emitted events are always too high
+	// that could be due to polling. not important rn.
+	// let emittedEvents = 0;
+	// settings$.subscribe(() => {
+	// 	emittedEvents++;
+	// });
+
+	const settings = await firstValueFrom(settings$);
+
+	expect(settings).toStrictEqual(
+		withLanguageTagToLocaleMigration(mockSettings)
+	);
+
+	settings.baseLocale = "mock-locale";
+
+	// mutating a value from the observable should not affect the observable
+	const settings2 = await firstValueFrom(settings$);
+	expect(settings2.baseLocale).toBe("en");
+
+	await lix.db
+		.updateTable("file")
+		.where("file.path", "=", "/settings.json")
+		.set({
+			data: new TextEncoder().encode(
+				JSON.stringify({
+					...settings,
+					baseLocale: "mock-locale",
+					"plugin.mock": "mock",
+				})
+			),
+		})
+		.execute();
+
+	const settings3 = await firstValueFrom(settings$);
+	expect(settings3.baseLocale).toBe("mock-locale");
+	expect(settings3["plugin.mock"]).toBe("mock");
+});

package/src/project/state/settings$.ts

@@ -0,0 +1,26 @@
+import { type Lix } from "@lix-js/sdk";
+import { map, share, type Observable } from "rxjs";
+import type { ProjectSettings } from "../../json-schema/settings.js";
+import { withLanguageTagToLocaleMigration } from "../../migrations/v2/withLanguageTagToLocaleMigration.js";
+import { pollQuery } from "../../query-utilities/pollQuery.js";
+
+export function createSettings$(args: {
+	lix: Lix;
+}): Observable<ProjectSettings> {
+	const settings$ = pollQuery(() =>
+		args.lix.db
+			.selectFrom("file")
+			.where("path", "=", "/settings.json")
+			.selectAll()
+			.executeTakeFirstOrThrow()
+	).pipe(
+		map((settingsFile) => {
+			const decoded = new TextDecoder().decode(settingsFile.data);
+			const parsed = JSON.parse(decoded);
+			const withMigration = withLanguageTagToLocaleMigration(parsed);
+			return withMigration;
+		}),
+		share()
+	);
+	return settings$;
+}

package/src/project/state/state.test.ts

@@ -0,0 +1,128 @@
+import { test, expect, vi } from "vitest";
+import { createProjectState } from "./state.js";
+import { newLixFile, openLixInMemory } from "@lix-js/sdk";
+import type { ProjectSettings } from "../../json-schema/settings.js";
+import { loadProjectInMemory } from "../loadProjectInMemory.js";
+import { newProject } from "../newProject.js";
+import type { InlangPlugin } from "../../plugin/schema.js";
+
+test("plugins should be re-imported if the settings have been updated", async () => {
+	const mockImportPlugins = vi.hoisted(() =>
+		vi.fn().mockImplementation(async (args) => {
+			const plugins = args.settings.modules.map((uri: any) => {
+				return { key: uri };
+			});
+			return { plugins, errors: [] };
+		})
+	);
+
+	vi.mock(import("../../plugin/importPlugins.js"), async (importOriginal) => {
+		const mod = await importOriginal();
+		return {
+			...mod,
+			importPlugins: mockImportPlugins,
+		};
+	});
+
+	const lix = await openLixInMemory({ blob: await newLixFile() });
+
+	await lix.db
+		.insertInto("file")
+		.values({
+			path: "/settings.json",
+			data: new TextEncoder().encode(
+				JSON.stringify({
+					baseLocale: "en",
+					locales: ["en"],
+					modules: [],
+				} satisfies ProjectSettings)
+			),
+		})
+		.execute();
+
+	const state = createProjectState({
+		lix,
+		settings: {
+			baseLocale: "en",
+			locales: ["en"],
+			modules: [],
+		},
+	});
+
+	const plugins1 = await state.plugins.get();
+
+	expect(plugins1).toEqual([]);
+	// expect(mockImportPlugins).toHaveBeenCalledTimes(1);
+
+	const settings1 = {
+		baseLocale: "en",
+		locales: ["en"],
+		modules: ["@inlang/plugin-react"],
+	};
+
+	await state.settings.set(settings1);
+
+	const plugins2 = await state.plugins.get();
+	expect(plugins2).toEqual([{ key: "@inlang/plugin-react" }]);
+
+	await state.settings.set({
+		baseLocale: "en",
+		locales: ["en"],
+		modules: ["@inlang/plugin-react"],
+	});
+
+	const plugins3 = await state.plugins.get();
+	expect(plugins3).toEqual([{ key: "@inlang/plugin-react" }]);
+	// expect(mockImportPlugins).toHaveBeenCalledTimes(2);
+
+	state.settings.set({
+		baseLocale: "en",
+		locales: ["en"],
+		modules: [],
+	});
+
+	const plugins4 = await state.plugins.get();
+
+	expect(plugins4).toEqual([]);
+	// expect(mockImportPlugins).toHaveBeenCalledTimes(3);
+});
+
+test("subscribing to plugins should work", async () => {
+	// vite hoists the import plugin from the previous test :/
+	const project = await loadProjectInMemory({ blob: await newProject() });
+
+	const state = createProjectState({
+		lix: project.lix,
+		settings: {
+			baseLocale: "en",
+			locales: ["en"],
+			modules: [],
+		},
+	});
+
+	expect(await state.plugins.get()).toEqual([]);
+
+	let pluginsFromSub: readonly InlangPlugin[] = [];
+	const sub = state.plugins.subscribe((value) => {
+		pluginsFromSub = value;
+	});
+
+	await project.lix.db
+		.updateTable("file")
+		.where("path", "=", "/settings.json")
+		.set({
+			data: new TextEncoder().encode(
+				JSON.stringify({
+					baseLocale: "en",
+					locales: ["en"],
+					modules: ["broken-module.js"],
+				})
+			),
+		})
+		.execute();
+
+	const plugins = await state.plugins.get();
+	expect(plugins.length).toBe(1);
+	expect(pluginsFromSub).toStrictEqual(plugins);
+	sub.unsubscribe();
+});

package/src/project/state/state.ts

@@ -0,0 +1,94 @@
+import { firstValueFrom, map, merge, share, switchMap } from "rxjs";
+import type { InlangPlugin } from "../../plugin/schema.js";
+import type { ProjectSettings } from "../../json-schema/settings.js";
+import {
+	importPlugins,
+	type PreprocessPluginBeforeImportFunction,
+} from "../../plugin/importPlugins.js";
+import { setSettings } from "./setSettings.js";
+import type { Lix } from "@lix-js/sdk";
+import { createSettings$ } from "./settings$.js";
+import { createId$ } from "./id$.js";
+
+/**
+ * State of a project.
+ */
+export type ProjectState = Awaited<ReturnType<typeof createProjectState>>;
+
+/**
+ * The state of a project derived from the settings file in lix.
+ *
+ * Everything that's not in the database is stored here. Settings,
+ * loaded plugins etc. The principal is "everything is derived
+ * from the settings file in lix":
+ *
+ *   - the source of truth is the file.
+ *      - crash resistance: if the state is lost, it can be re-imported.
+ *      - portable: copy the file, open on another machine = same state.
+ *
+ *   - simple(r) unit testing (provide the settings file observable, done)
+ */
+export function createProjectState(args: {
+	lix: Lix;
+	providePlugins?: InlangPlugin[];
+	preprocessPluginBeforeImport?: PreprocessPluginBeforeImportFunction;
+	settings: ProjectSettings;
+}) {
+	const settings$ = createSettings$(args);
+
+	const _importPluginsResult$ = settings$.pipe(
+		// distinctUntilChanges is bypassed for whatever reason
+		// not important for now if the plugins re-import even if the modules are identical.
+		// distinctUntilChanged((prev, curr) => {
+		// 	console.log("distinct called");
+		// 	return true;
+		// }),
+		switchMap((settings) =>
+			// re-import the plugins when the settings change
+			importPlugins({
+				settings,
+				lix: args.lix,
+				preprocessPluginBeforeImport: args.preprocessPluginBeforeImport,
+			})
+		),
+		share()
+	);
+
+	const plugins$ = _importPluginsResult$.pipe(
+		map((result) => [...(args.providePlugins ?? []), ...result.plugins]),
+		share()
+	);
+
+	const errors$ = merge(
+		_importPluginsResult$.pipe(map((result) => result.errors))
+	).pipe(share());
+
+	const id$ = createId$({ lix: args.lix });
+
+	return {
+		id: {
+			get: () => firstValueFrom(id$),
+			// bind to keep the context of this
+			subscribe: id$.subscribe.bind(id$),
+		},
+		settings: {
+			get: () => firstValueFrom(settings$),
+			set: (settings: ProjectSettings) =>
+				setSettings({
+					newSettings: settings,
+					lix: args.lix,
+				}),
+			subscribe: settings$.subscribe,
+		},
+		errors: {
+			get: () => firstValueFrom(errors$),
+			// bind to keep the context of this
+			subscribe: errors$.subscribe.bind(errors$),
+		},
+		plugins: {
+			get: () => firstValueFrom(plugins$),
+			// bind to keep the context of this
+			subscribe: plugins$.subscribe.bind(plugins$),
+		},
+	};
+}

package/src/query-utilities/index.ts

@@ -0,0 +1,5 @@
+export { selectBundleNested } from "./selectBundleNested.js";
+export { insertBundleNested } from "./insertBundleNested.js";
+export { updateBundleNested } from "./updateBundleNested.js";
+export { upsertBundleNested } from "./upsertBundleNested.js";
+export { pollQuery } from "./pollQuery.js";

package/src/query-utilities/insertBundleNested.ts

@@ -0,0 +1,46 @@
+import type { Kysely } from "kysely";
+import type {
+	InlangDatabaseSchema,
+	NewBundleNested,
+} from "../database/schema.js";
+
+export const insertBundleNested = async (
+	db: Kysely<InlangDatabaseSchema>,
+	bundle: NewBundleNested
+): Promise<void> => {
+	await db.transaction().execute(async (trx) => {
+		const insertedBundle = await trx
+			.insertInto("bundle")
+			.values({
+				id: bundle.id,
+				declarations: bundle.declarations,
+			})
+			.returning("id")
+			.executeTakeFirstOrThrow();
+
+		for (const message of bundle.messages) {
+			const insertedMessage = await trx
+				.insertInto("message")
+				.values({
+					id: message.id,
+					bundleId: insertedBundle.id,
+					locale: message.locale,
+					selectors: message.selectors,
+				})
+				.returning("id")
+				.executeTakeFirstOrThrow();
+
+			for (const variant of message.variants) {
+				await trx
+					.insertInto("variant")
+					.values({
+						id: variant.id,
+						messageId: insertedMessage.id,
+						matches: variant.matches,
+						pattern: variant.pattern,
+					})
+					.execute();
+			}
+		}
+	});
+};