@inlang/sdk 0.36.3 → 2.0.0-prerelease.0

package/src/plugin/meta/ideExtension.ts

@@ -0,0 +1,56 @@
+// Basic structure for a position in the document
+interface Position {
+	line: number;
+	character: number;
+}
+
+// Basic structure for a message reference match
+interface MessageReferenceMatch {
+	bundleId: string;
+	position: {
+		start: Position;
+		end: Position;
+	};
+}
+
+// Main IDE extension configuration schema
+interface IdeExtensionConfigSchema {
+	/**
+	 * Defines matchers for message references inside the code.
+	 *
+	 * @param args - Represents the data to conduct the search on.
+	 * @returns A promise with matched message references.
+	 */
+	messageReferenceMatchers: Array<
+		(args: { documentText: string }) => Promise<MessageReferenceMatch[]>
+	>;
+
+	/**
+	 * Defines options to extract messages.
+	 */
+	extractMessageOptions: Array<{
+		/**
+		 * Function which is called when the user finishes the message extraction command.
+		 *
+		 * @param args - Contains messageId and selection.
+		 * @returns The code which is inserted into the document.
+		 */
+		callback: (args: { bundleId: string; selection: string }) => {
+			bundleId: string;
+			messageReplacement: string;
+		};
+	}>;
+
+	/**
+	 * An array of Visual Studio Code DocumentSelectors.
+	 *
+	 * The document selectors specify for which files/programming languages
+	 * (typescript, svelte, etc.) the extension should be activated.
+	 *
+	 * See https://code.visualstudio.com/api/references/document-selector
+	 */
+	documentSelectors?: Array<{ language?: string }>;
+}
+
+// Export the type for external use
+export type IdeExtensionConfig = IdeExtensionConfigSchema;

package/src/plugin/schema.ts

@@ -0,0 +1,165 @@
+import type { TObject } from "@sinclair/typebox";
+import type { MessageV1 } from "../json-schema/old-v1-message/schemaV1.js";
+import type { ProjectSettings } from "../json-schema/settings.js";
+import type {
+	Bundle,
+	Message,
+	NewBundle,
+	NewMessage,
+	NewVariant,
+	Variant,
+} from "../database/schema.js";
+import type { ExportFile, ImportFile } from "../project/api.js";
+
+export type InlangPlugin<
+	ExternalSettings extends Record<string, any> | unknown = unknown,
+> = {
+	/**
+	 * @deprecated Use `key` instead.
+	 */
+	id?: string;
+	/**
+	 * The key of the plugin.
+	 */
+	key: string;
+	settingsSchema?: TObject;
+	/**
+	 * @deprecated Use `importFiles` instead.
+	 */
+	loadMessages?: (args: {
+		settings: ProjectSettings;
+		nodeishFs: NodeFsPromisesSubsetLegacy;
+	}) => Promise<MessageV1[]> | MessageV1[];
+	/**
+	 * @deprecated Use `exportFiles` instead.
+	 */
+	saveMessages?: (args: {
+		messages: MessageV1[];
+		settings: ProjectSettings;
+		nodeishFs: NodeFsPromisesSubsetLegacy;
+	}) => Promise<void> | void;
+	/**
+	 * Files that should be imported by the inlang SDK.
+	 *
+	 * - `metadata` is optional and can be used to store additional information
+	 *   that is accessible in `importFiles` via `toBeImportedMetadata`. See
+	 *   https://github.com/opral/inlang-sdk/issues/218 for more info.
+	 *
+	 */
+	toBeImportedFiles?: (args: {
+		settings: ProjectSettings & ExternalSettings;
+	}) => MaybePromise<
+		Array<{ path: string; locale: string; metadata?: Record<string, any> }>
+	>;
+	importFiles?: (args: {
+		files: ImportFile[];
+		settings: ProjectSettings & ExternalSettings; // we expose the settings in case the importFunction needs to access the plugin config
+	}) => MaybePromise<{
+		bundles: BundleImport[];
+		messages: MessageImport[];
+		variants: VariantImport[];
+	}>;
+	exportFiles?: (args: {
+		bundles: Bundle[];
+		messages: Message[];
+		variants: Variant[];
+		settings: ProjectSettings & ExternalSettings;
+	}) => MaybePromise<Array<ExportFile>>;
+	/**
+	 * @deprecated Use the `meta` field instead.
+	 */
+	addCustomApi?: (args: {
+		settings: ProjectSettings & ExternalSettings;
+	}) => Record<string, unknown>;
+	/**
+	 * Define app-specific APIs under a `meta` field.
+	 *
+	 * @example
+	 * meta: {
+	 *   "app.inlang.ide-extension": {
+	 *     documentPaths: ["*.json"]
+	 *   }
+	 * }
+	 */
+	meta?: Record<string, Record<string, unknown>>;
+};
+
+/**
+ * Exposing only a subset to ease mapping of fs functions.
+ *
+ * https://github.com/opral/inlang-sdk/issues/136
+ */
+export type NodeFsPromisesSubsetLegacy = {
+	readFile:
+		| ((path: string) => Promise<ArrayBuffer>)
+		| ((path: string, options?: { encoding: "utf-8" }) => Promise<string>);
+	readdir: (path: string) => Promise<string[]>;
+	writeFile: (path: string, data: ArrayBuffer | string) => Promise<void>;
+	mkdir: (path: string) => Promise<void>;
+};
+
+/**
+ * A to be imported bundle.
+ */
+export type BundleImport = NewBundle;
+
+/**
+ * A to be imported message.
+ *
+ * The `id` property is omitted because it is generated by the SDK.
+ */
+export type MessageImport = Omit<NewMessage, "id"> & {
+	/**
+	 * If the id is not provided, the SDK will generate one.
+	 */
+	id?: string;
+};
+
+/**
+ * A to be imported variant.
+ *
+ * - The `id` and `messageId` properties are omitted because they are generated by the SDK.
+ * - The `bundleId` and `locale` properties are added to the import variant to match the variant
+ *   with a message.
+ */
+export type VariantImport =
+	| (NewVariant & {
+			/**
+			 * If the id is not provided, the SDK will generate one.
+			 */
+			id: string;
+			/**
+			 * If the messageId is not provided, the SDK will match the variant
+			 * with a message based on the `messageBundleId` and `messageLocale` properties.
+			 */
+			messageId: string;
+			/**
+			 * Required to match the variant with a message in case the `id` and `messageId` are undefined.
+			 */
+			messageBundleId?: undefined;
+			/**
+			 * Required to match the variant with a message in case the `id` and `messageId` are undefined.
+			 */
+			messageLocale?: undefined;
+	  })
+	| (Omit<NewVariant, "id" | "messageId"> & {
+			/**
+			 * If the id is not provided, the SDK will generate one.
+			 */
+			id?: undefined;
+			/**
+			 * If the messageId is not provided, the SDK will match the variant
+			 * with a message based on the `messageBundleId` and `messageLocale` properties.
+			 */
+			messageId?: undefined;
+			/**
+			 * Required to match the variant with a message in case the `id` and `messageId` are undefined.
+			 */
+			messageBundleId: string;
+			/**
+			 * Required to match the variant with a message in case the `id` and `messageId` are undefined.
+			 */
+			messageLocale: string;
+	  });
+
+type MaybePromise<T> = T | Promise<T>;

package/src/project/api.ts

@@ -0,0 +1,81 @@
+import type { Kysely } from "kysely";
+import type { InlangDatabaseSchema } from "../database/schema.js";
+import type { InlangPlugin } from "../plugin/schema.js";
+import type { ProjectSettings } from "../json-schema/settings.js";
+import type { Lix } from "@lix-js/sdk";
+import type { SqliteDatabase } from "sqlite-wasm-kysely";
+
+export type InlangProject = {
+	db: Kysely<InlangDatabaseSchema>;
+	/**
+	 * @deprecated Don't use this. Only an internal hack to unblock
+	 * fink v2.
+	 *
+	 * TODO remove this
+	 */
+	_sqlite: SqliteDatabase;
+	id: {
+		get: () => Promise<string>;
+		subscribe: Subscription<string>;
+	};
+	plugins: {
+		get: () => Promise<readonly InlangPlugin[]>;
+		subscribe: Subscription<readonly InlangPlugin[]>;
+	};
+	errors: {
+		get: () => Promise<readonly Error[]>;
+		subscribe: Subscription<readonly Error[]>;
+	};
+	settings: {
+		get: () => Promise<ProjectSettings>;
+		set: (settings: ProjectSettings) => Promise<void>;
+		subscribe: Subscription<ProjectSettings>;
+	};
+	lix: Lix;
+	importFiles: (args: {
+		pluginKey: InlangPlugin["key"];
+		files: ImportFile[];
+	}) => Promise<void>;
+	exportFiles: (args: {
+		pluginKey: InlangPlugin["key"];
+	}) => Promise<ExportFile[]>;
+	close: () => Promise<void>;
+	toBlob: () => Promise<Blob>;
+};
+
+export type ImportFile = {
+	/** The locale of the resource file */
+	locale: string;
+	/** The binary content of the resource */
+	content: Uint8Array;
+	/**
+	 * The metadata of the file to be imported.
+	 *
+	 * Used to store additional information that is accessible in `importFiles` via `toBeImportedFilesMetadata`.
+	 * https://github.com/opral/inlang-sdk/issues/218
+	 */
+	toBeImportedFilesMetadata?: Record<string, any>;
+};
+
+export type ExportFile = {
+	/** The locale of the resource file */
+	locale: string;
+	/**
+	 * The name of the file.
+	 *
+	 * @example
+	 *   "en.json"
+	 *   "common-de.json"
+	 *
+	 */
+	name: string;
+	/** The binary content of the resource */
+	content: Uint8Array;
+};
+
+/**
+ * Minimal RxJS compatible (generic) subscription type.
+ */
+export type Subscription<T> = (callback: (value: T) => void) => {
+	unsubscribe: () => void;
+};

package/src/project/initHandleSaveToLixOnChange.ts

@@ -0,0 +1,88 @@
+import { Kysely, sql } from "kysely";
+import { contentFromDatabase, type SqliteDatabase } from "sqlite-wasm-kysely";
+import type { Lix } from "@lix-js/sdk";
+import type { InlangDatabaseSchema } from "../database/schema.js";
+
+/**
+ * Saves updates of the database (file) to lix.
+ */
+export async function initHandleSaveToLixOnChange(args: {
+	sqlite: SqliteDatabase;
+	db: Kysely<InlangDatabaseSchema>;
+	lix: Lix;
+	pendingPromises: Promise<unknown>[];
+}) {
+	args.sqlite.createFunction({
+		name: "save_db_file_to_lix",
+		arity: 0,
+		// @ts-expect-error - dynamic function
+		xFunc: () => {
+			// TODO handle in cancellable queue. only the last entry
+			// in the queue needs to be pushed to the file table
+			// because all previous entries are overwritten anyways
+			args.pendingPromises.push(
+				(async () => {
+					// We have to await the database operations to be finished and the database to eb in a consistent state, otherwise contentFromDatabase will crash! 100 ms is too short for current test cases, but we need a proper solution for this
+					await new Promise((resolve) => setTimeout(resolve, 400));
+					const data = contentFromDatabase(args.sqlite);
+					await args.lix.db
+						.updateTable("file")
+						.set("data", data)
+						.where("path", "=", "/db.sqlite")
+						.execute();
+				})()
+			);
+			return;
+		},
+	});
+
+	// better way to write to lix on updates?
+	await sql`
+	CREATE TEMP TRIGGER bundle_insert AFTER INSERT ON bundle
+	BEGIN
+	  SELECT save_db_file_to_lix();
+	END;
+
+	CREATE TEMP TRIGGER bundle_update AFTER UPDATE ON bundle
+	BEGIN
+	  SELECT save_db_file_to_lix();
+	END;
+
+	CREATE TEMP TRIGGER bundle_delete AFTER DELETE ON bundle
+	BEGIN
+	  SELECT save_db_file_to_lix();
+	END;
+
+		CREATE TEMP TRIGGER message_insert AFTER INSERT ON message
+	BEGIN
+	  SELECT save_db_file_to_lix();
+	END;
+
+	CREATE TEMP TRIGGER message_update AFTER UPDATE ON message
+	BEGIN
+	  SELECT save_db_file_to_lix();
+	END;
+
+	CREATE TEMP TRIGGER message_delete AFTER DELETE ON message
+	BEGIN
+	  SELECT save_db_file_to_lix();
+	END;
+
+	CREATE TEMP TRIGGER variant_insert AFTER INSERT ON variant
+	BEGIN
+	  SELECT save_db_file_to_lix();
+	END;
+
+	CREATE TEMP TRIGGER variant_update AFTER UPDATE ON variant
+	BEGIN
+	  SELECT save_db_file_to_lix();
+	END;
+
+	CREATE TEMP TRIGGER variant_delete AFTER DELETE ON variant
+	BEGIN
+	  SELECT save_db_file_to_lix();
+	END;
+
+
+	`.execute(args.db);
+}

package/src/project/loadProject.test.ts

@@ -0,0 +1,148 @@
+import { expect, test } from "vitest";
+import { newProject } from "./newProject.js";
+import { loadProjectInMemory } from "./loadProjectInMemory.js";
+import { validate } from "uuid";
+
+test("it should persist changes of bundles, messages, and variants to lix ", async () => {
+	const file1 = await newProject();
+	const project1 = await loadProjectInMemory({ blob: file1 });
+	const bundle = await project1.db
+		.insertInto("bundle")
+		.defaultValues()
+		.returning("id")
+		.executeTakeFirstOrThrow();
+
+	const message = await project1.db
+		.insertInto("message")
+		.values({
+			bundleId: bundle.id,
+			locale: "en",
+		})
+		.returning("id")
+		.executeTakeFirstOrThrow();
+
+	await project1.db
+		.insertInto("variant")
+		.values({
+			messageId: message.id,
+		})
+		.execute();
+
+	const file1AfterUpdates = await project1.toBlob();
+	await project1.close();
+
+	const project2 = await loadProjectInMemory({ blob: file1AfterUpdates });
+	const bundles = await project2.db.selectFrom("bundle").select("id").execute();
+	const messages = await project2.db
+		.selectFrom("message")
+		.select("id")
+		.execute();
+	const variants = await project2.db
+		.selectFrom("variant")
+		.select("id")
+		.execute();
+	expect(bundles.length).toBe(1);
+	expect(messages.length).toBe(1);
+	expect(variants.length).toBe(1);
+});
+
+test("get and set settings", async () => {
+	const project = await loadProjectInMemory({ blob: await newProject() });
+	const settings = await project.settings.get();
+
+	expect(settings["plugin.key"]).toBeUndefined();
+
+	const copied = structuredClone(settings);
+
+	copied["plugin.key"] = { test: "value" };
+
+	await project.settings.set(copied);
+
+	const updatedSettings = await project.settings.get();
+	expect(updatedSettings["plugin.key"]).toEqual({ test: "value" });
+});
+
+test("it should set sourceLanguageTag and languageTags if non-existent to make v1 plugins work", async () => {
+	const project = await loadProjectInMemory({
+		blob: await newProject({
+			settings: {
+				baseLocale: "en",
+				locales: ["en", "de"],
+			},
+		}),
+	});
+	const settings = await project.settings.get();
+	expect(settings.baseLocale).toBe("en");
+	expect(settings.sourceLanguageTag).toBe("en");
+	expect(settings.languageTags).toEqual(["en", "de"]);
+	expect(settings.locales).toEqual(["en", "de"]);
+});
+
+test("providing plugins should work", async () => {
+	const project = await loadProjectInMemory({
+		blob: await newProject({
+			settings: {
+				baseLocale: "en",
+				locales: ["en"],
+				modules: [],
+			},
+		}),
+		providePlugins: [{ key: "my-provided-plugin" }],
+	});
+
+	const plugins = await project.plugins.get();
+	const errors = await project.errors.get();
+
+	expect(plugins.length).toBe(1);
+	expect(plugins[0]?.key).toBe("my-provided-plugin");
+	expect(errors.length).toBe(0);
+});
+
+test("if a project has no id, it should be generated", async () => {
+	const project = await loadProjectInMemory({ blob: await newProject() });
+
+	await project.lix.db
+		.deleteFrom("file")
+		.where("path", "=", "/project_id")
+		.execute();
+
+	const blob = await project.toBlob();
+
+	const project2 = await loadProjectInMemory({ blob });
+
+	const id = await project2.id.get();
+
+	expect(id).toBeDefined();
+	expect(validate(id)).toBe(true);
+});
+
+test("subscribing to errors should work", async () => {
+	const project = await loadProjectInMemory({ blob: await newProject() });
+
+	expect(await project.errors.get()).toEqual([]);
+
+	let errorsFromSub: readonly Error[] = [];
+	project.errors.subscribe((value) => {
+		errorsFromSub = value;
+	});
+
+	await project.lix.db
+		.updateTable("file")
+		.where("path", "=", "/settings.json")
+		.set({
+			data: new TextEncoder().encode(
+				JSON.stringify({
+					baseLocale: "en",
+					locales: ["en"],
+					modules: ["invalid-module.js"],
+				})
+			),
+		})
+		.execute();
+
+	const errors = await project.errors.get();
+
+	expect(errors.length).toBe(1);
+	expect(errorsFromSub.length).toBe(1);
+	expect(errorsFromSub).toStrictEqual(errors);
+});

package/src/project/loadProject.ts

@@ -0,0 +1,153 @@
+import { toBlob, type Lix } from "@lix-js/sdk";
+import type { InlangPlugin } from "../plugin/schema.js";
+import type { ProjectSettings } from "../json-schema/settings.js";
+import { type SqliteDatabase } from "sqlite-wasm-kysely";
+import { initDb } from "../database/initDb.js";
+import { initHandleSaveToLixOnChange } from "./initHandleSaveToLixOnChange.js";
+import { type PreprocessPluginBeforeImportFunction } from "../plugin/importPlugins.js";
+import type { InlangProject } from "./api.js";
+import { createProjectState } from "./state/state.js";
+import { withLanguageTagToLocaleMigration } from "../migrations/v2/withLanguageTagToLocaleMigration.js";
+import { v4 } from "uuid";
+import { initErrorReporting } from "../services/error-reporting/index.js";
+import { maybeCaptureLoadedProject } from "./maybeCaptureTelemetry.js";
+import { importFiles } from "../import-export/importFiles.js";
+import { exportFiles } from "../import-export/exportFiles.js";
+
+/**
+ * Common load project logic.
+ */
+export async function loadProject(args: {
+	sqlite: SqliteDatabase;
+	lix: Lix;
+	/**
+	 * Provide plugins to the project.
+	 *
+	 * This is useful for testing or providing plugins that are
+	 * app specific. Keep in mind that provided plugins
+	 * are not shared with other instances.
+	 */
+	providePlugins?: InlangPlugin[];
+	/**
+	 * Function that preprocesses the plugin before importing it.
+	 *
+	 * The callback can be used to process plugins as needed in the
+	 * environment of the app. For example, Sherlock uses this to convert
+	 * ESM, which all inlang plugins are written in, to CJS which Sherlock
+	 * runs in.
+	 *
+	 * @example
+	 *   const project = await loadProject({ preprocessPluginBeforeImport: (moduleText) => convertEsmToCjs(moduleText) })
+	 *
+	 */
+	preprocessPluginBeforeImport?: PreprocessPluginBeforeImportFunction;
+	/**
+	 * The id of the app that is using the SDK.
+	 *
+	 * The is used for telemetry purposes. To derive insights like
+	 * which app is using the SDK, how many projects are loaded, etc.
+	 *
+	 * The app id can be removed at any time in the future
+	 */
+	appId?: string;
+}): Promise<InlangProject> {
+	const db = initDb({ sqlite: args.sqlite });
+
+	await maybeMigrateFirstProjectId({ lix: args.lix });
+
+	const settingsFile = await args.lix.db
+		.selectFrom("file")
+		.select("data")
+		.where("path", "=", "/settings.json")
+		.executeTakeFirstOrThrow();
+
+	const settings = withLanguageTagToLocaleMigration(
+		JSON.parse(new TextDecoder().decode(settingsFile.data)) as ProjectSettings
+	);
+
+	const state = createProjectState({
+		...args,
+		settings,
+	});
+
+	// TODO implement garbage collection/a proper queue.
+	//      for the protoype and tests, it seems good enough
+	//      without garbage collection of old promises.
+	const pendingSaveToLixPromises: Promise<unknown>[] = [];
+
+	await initHandleSaveToLixOnChange({
+		sqlite: args.sqlite,
+		db,
+		lix: args.lix,
+		pendingPromises: pendingSaveToLixPromises,
+	});
+
+	// not awaiting to not block the load time of a project
+	maybeCaptureLoadedProject({
+		db,
+		state,
+		appId: args.appId,
+	});
+
+	initErrorReporting({ projectId: await state.id.get() });
+
+	return {
+		db,
+		id: state.id,
+		settings: state.settings,
+		plugins: state.plugins,
+		errors: state.errors,
+		importFiles: async ({ files, pluginKey }) => {
+			return await importFiles({
+				files,
+				pluginKey,
+				settings: await state.settings.get(),
+				plugins: await state.plugins.get(),
+				db,
+			});
+		},
+		exportFiles: async ({ pluginKey }) => {
+			return (
+				await exportFiles({
+					pluginKey,
+					db,
+					settings: await state.settings.get(),
+					plugins: await state.plugins.get(),
+				})
+			).map((output) => ({ ...output, pluginKey }));
+		},
+		close: async () => {
+			await db.destroy();
+			await args.lix.db.destroy();
+		},
+		_sqlite: args.sqlite,
+		toBlob: async () => {
+			await Promise.all(pendingSaveToLixPromises);
+			return await toBlob({ lix: args.lix });
+		},
+		lix: args.lix,
+	};
+}
+
+/**
+ * Old leftover migration from v1. Probably not needed anymore.
+ *
+ * Kept it in just in case.
+ */
+async function maybeMigrateFirstProjectId(args: { lix: Lix }): Promise<void> {
+	const firstProjectIdFile = await args.lix.db
+		.selectFrom("file")
+		.select("data")
+		.where("path", "=", "/project_id")
+		.executeTakeFirst();
+
+	if (!firstProjectIdFile) {
+		await args.lix.db
+			.insertInto("file")
+			.values({
+				path: "/project_id",
+				data: new TextEncoder().encode(v4()),
+			})
+			.execute();
+	}
+}