@inlang/sdk 0.36.4 → 2.0.0-beta.1

package/src/json-schema/settings.ts

@@ -0,0 +1,138 @@
+import {
+	type Static,
+	Type,
+	type TTemplateLiteral,
+	type TLiteral,
+} from "@sinclair/typebox";
+
+const SDKSettings = Type.Object({
+	// TODO SDK-v2 SETTINGS do we need to generate a settings v2 schema?
+	$schema: Type.Optional(
+		Type.Literal("https://inlang.com/schema/project-settings")
+	),
+	baseLocale: Type.String({
+		title: "Base locale",
+		description:
+			"The base locale of the project. We recommend BCP-47 language tags.",
+	}),
+	locales: Type.Array(Type.String(), {
+		uniqueItems: true,
+		title: "Project Locales",
+		description:
+			"Set the locales that are available in your project. All locales needs to be a valid BCP-47 language tag. Needs to include the base locale tag.",
+	}),
+	// exits for backwards compatibility
+	// remove in SDK-v3
+	sourceLanguageTag: Type.Optional(
+		Type.String({
+			description: "Use baseLocale instead.",
+			deprecated: true,
+		})
+	),
+	// exits for backwards compatibility
+	// remove in SDK-v3
+	languageTags: Type.Optional(
+		Type.Array(Type.String(), {
+			uniqueItems: true,
+			deprecated: true,
+			description: "Use locales instead.",
+		})
+	),
+	/**
+	 * The modules to load.
+	 *
+	 * @example
+	 *  modules: [
+	 * 	  "https://cdn.jsdelivr.net/npm/@inlang/plugin-i18next@3/dist/index.js",
+	 * 	  "https://cdn.jsdelivr.net/npm/@inlang/plugin-csv@1/dist/index.js",
+	 *  ]
+	 */
+	modules: Type.Optional(
+		Type.Array(
+			Type.Intersect([
+				Type.String({
+					description: "The module must be a valid URI according to RFC 3986.",
+					pattern:
+						"(?:[A-Za-z][A-Za-z0-9+.-]*:/{2})?(?:(?:[A-Za-z0-9-._~]|%[A-Fa-f0-9]{2})+(?::([A-Za-z0-9-._~]?|[%][A-Fa-f0-9]{2})+)?@)?(?:[A-Za-z0-9](?:[A-Za-z0-9-]*[A-Za-z0-9])?\\.){1,126}[A-Za-z0-9](?:[A-Za-z0-9-]*[A-Za-z0-9])?(?::[0-9]+)?(?:/(?:[A-Za-z0-9-._~]|%[A-Fa-f0-9]{2})*)*(?:\\?(?:[A-Za-z0-9-._~]+(?:=(?:[A-Za-z0-9-._~+]|%[A-Fa-f0-9]{2})+)?)(?:&|;[A-Za-z0-9-._~]+(?:=(?:[A-Za-z0-9-._~+]|%[A-Fa-f0-9]{2})+)?)*)?",
+				}),
+				Type.String({
+					description: "The module must end with `.js`.",
+					pattern: ".*\\.js$",
+				}),
+				Type.String({
+					description: "The module can only contain a major version number.",
+					pattern: "^(?!.*@\\d\\.)[^]*$",
+				}),
+			]),
+			{
+				uniqueItems: true,
+				description:
+					"The modules to load. Must be a valid URI but can be relative.",
+				examples: [
+					"https://cdn.jsdelivr.net/npm/@inlang/plugin-i18next@3/dist/index.js",
+					"https://cdn.jsdelivr.net/npm/@inlang/plugin-csv@1/dist/index.js",
+					"./local-testing-plugin.js",
+				],
+			}
+		)
+	),
+	telemetry: Type.Optional(
+		Type.Union(
+			[
+				Type.Literal("off", {
+					description: "No telemetry events ",
+				}),
+			],
+			{ description: "If not set, defaults to all" }
+		)
+	),
+	experimental: Type.Optional(
+		Type.Record(Type.String(), Type.Literal(true), {
+			title: "Experimental settings",
+			description:
+				"Experimental settings that are used for product development.",
+		})
+	),
+	/**
+	 * plugin.*: JSONObject
+	 *
+	 * The plugin settings are validated when importing plugins
+	 */
+});
+
+/**
+ * Settings defined via apps, plugins, lint rules, etc.
+ *
+ * Using external settings to only allow `plugin.*` keys
+ * and don't block the SDK from adding new settings.
+ */
+const ExternalSettings = Type.Record(
+	Type.String({
+		pattern: `^((plugin)\\.([a-z][a-zA-Z0-9]*(?:[A-Z][a-z0-9]*)*)|\\$schema|${
+			// pattern must include the settings properties
+			Object.keys(SDKSettings.properties)
+				.map((key) => key.replaceAll(".", "\\."))
+				.join("|")
+		})$`,
+		description: "The key must be conform to `plugin.*`.",
+		examples: ["plugin.csv-importer", "plugin.i18next"],
+	}) as unknown as TTemplateLiteral<[TLiteral<`${"plugin"}.${string}`>]>,
+	// Using JSON (array and object) as a workaround to make the
+	// intersection between `InternalSettings`, which contains an array,
+	// and `ExternalSettings` which are objects possible
+	Type.Record(Type.String(), Type.Any()),
+	{ description: "Settings defined by apps, plugins, etc." }
+);
+
+export type ProjectSettings = Omit<
+	Static<typeof ProjectSettings>,
+	"languageTags" | "sourceLanguageTag"
+> & {
+	/** @deprecated Use `baseLocale` */
+	sourceLanguageTag?: string;
+	/** @deprecated Use `locales` */
+	languageTags?: string[];
+	/** @deprecated This will soon be replaced by `Lix Validation Rules` */
+	messageLintRuleLevels?: Record<string, "error" | "warning">;
+};
+export const ProjectSettings = Type.Intersect([SDKSettings, ExternalSettings]);

package/src/lix-plugin/applyChanges.test.ts

@@ -0,0 +1,150 @@
+// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+// @ts-nocheck
+
+import { test, expect } from "vitest";
+import { loadProjectInMemory } from "../project/loadProjectInMemory.js";
+import { newProject } from "../project/newProject.js";
+import type { Change, NewChange } from "@lix-js/sdk";
+import { applyChanges } from "./applyChanges.js";
+import { loadDatabaseInMemory } from "sqlite-wasm-kysely";
+import { initDb } from "../database/initDb.js";
+import type { Bundle } from "../database/schema.js";
+
+test.skip("it should be able to delete", async () => {
+	const project = await loadProjectInMemory({
+		blob: await newProject(),
+	});
+
+	const changes: NewChange[] = [
+		{
+			id: "1",
+			operation: "create",
+			file_id: "mock",
+			plugin_key: "mock",
+			type: "bundle",
+			meta: { id: "mock" },
+			value: {
+				id: "mock",
+				declarations: [],
+			} satisfies Bundle,
+		},
+		{
+			id: "2",
+			parent_id: "1",
+			operation: "update",
+			file_id: "mock",
+			plugin_key: "mock",
+			type: "bundle",
+			meta: {
+				id: "mock",
+			},
+			value: {
+				id: "mock",
+				declarations: [],
+			} satisfies Bundle,
+		},
+		{
+			id: "3",
+			parent_id: "2",
+			operation: "delete",
+			file_id: "mock",
+			plugin_key: "mock",
+			meta: {
+				id: "mock",
+			},
+			type: "bundle",
+			value: undefined,
+		},
+	];
+
+	await project.db
+		.insertInto("bundle")
+		.values({
+			id: "mock",
+		})
+		.execute();
+
+	const dbFile = await project.lix.db
+		.selectFrom("file")
+		.selectAll()
+		.where("path", "=", "/db.sqlite")
+		.executeTakeFirstOrThrow();
+
+	const dbFileAfter = await applyChanges({
+		lix: project.lix,
+		file: dbFile,
+		changes: changes as Change[],
+	});
+
+	const db = initDb({
+		sqlite: await loadDatabaseInMemory(dbFileAfter.fileData),
+	});
+
+	const bundles = await db.selectFrom("bundle").selectAll().execute();
+
+	expect(bundles).toHaveLength(0);
+});
+
+test.skip("it should be able to upsert (insert & update)", async () => {
+	const project = await loadProjectInMemory({
+		blob: await newProject(),
+	});
+
+	const changes: NewChange[] = [
+		{
+			id: "1",
+			parent_id: undefined,
+			operation: "create",
+			file_id: "mock",
+			plugin_key: "mock",
+			type: "bundle",
+			meta: { id: "mock" },
+			value: {
+				id: "mock",
+				declarations: [],
+			} satisfies Bundle,
+		},
+		{
+			id: "2",
+			parent_id: "1",
+			operation: "update",
+			file_id: "mock",
+			plugin_key: "mock",
+			type: "bundle",
+			meta: {
+				id: "mock",
+			},
+			value: {
+				id: "mock",
+				declarations: [],
+			} satisfies Bundle,
+		},
+	];
+
+	await project.db
+		.insertInto("bundle")
+		.values({
+			id: "mock",
+		})
+		.execute();
+
+	const dbFile = await project.lix.db
+		.selectFrom("file")
+		.selectAll()
+		.where("path", "=", "/db.sqlite")
+		.executeTakeFirstOrThrow();
+
+	const dbFileAfter = await applyChanges({
+		lix: project.lix,
+		file: dbFile,
+		changes: changes as Change[],
+	});
+
+	const db = initDb({
+		sqlite: await loadDatabaseInMemory(dbFileAfter.fileData),
+	});
+
+	const bundles = await db.selectFrom("bundle").selectAll().execute();
+
+	expect(bundles).toHaveLength(1);
+});

package/src/lix-plugin/applyChanges.ts

@@ -0,0 +1,171 @@
+// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+// @ts-nocheck
+
+import {
+	getLeafChange,
+	type Change,
+	type LixPlugin,
+	type LixReadonly,
+} from "@lix-js/sdk";
+import { contentFromDatabase, loadDatabaseInMemory } from "sqlite-wasm-kysely";
+import { initDb } from "../database/initDb.js";
+import type { Kysely } from "kysely";
+import type { InlangDatabaseSchema } from "../database/schema.js";
+
+export const applyChanges: NonNullable<LixPlugin["applyChanges"]> = async ({
+	lix,
+	file,
+	changes,
+}) => {
+	if (file.path?.endsWith("db.sqlite") === false) {
+		throw new Error(
+			"Unimplemented. Only the db.sqlite file can be handled for now."
+		);
+	}
+
+	// todo make transactional
+
+	const sqlite = await loadDatabaseInMemory(file.data);
+	const db = initDb({ sqlite });
+
+	// the award for the most inefficient deduplication goes to...
+	const leafChanges = [
+		...new Set(
+			await Promise.all(
+				changes.map(async (change) => {
+					const leafChange = await getLeafChange({ change, lix });
+					// enable string comparison to avoid duplicates
+					return JSON.stringify(leafChange);
+				})
+			)
+		),
+	].map((v) => JSON.parse(v));
+
+	// changes need to be applied in order of foreign keys to avoid constraint violations
+	// 1. bundles
+	// 2. messages
+	// 3. variants
+	const applyOrder: Record<string, number> = {
+		bundle: 1,
+		message: 2,
+		variant: 3,
+	};
+
+	// future optimization potential here but sorting in one go
+	const orderedLeafChanges = [...leafChanges].sort((a, b) => {
+		const orderA = applyOrder[a.type];
+		const orderB = applyOrder[b.type];
+
+		if (orderA === undefined || orderB === undefined) {
+			throw new Error(
+				`Received an unknown entity type: ${a.type} && ${
+					b.type
+				}. Expected one of: ${Object.keys(applyOrder)}`
+			);
+		}
+
+		return orderA - orderB;
+	});
+	for (const leafChange of orderedLeafChanges) {
+		// deletion
+		if (leafChange.value === undefined) {
+			await db
+				.deleteFrom(leafChange.type as "bundle" | "message" | "variant")
+				.where("id", "=", leafChange.meta?.id)
+				.execute();
+			continue;
+		}
+
+		// upsert the value
+		const value = leafChange.value as any;
+
+		try {
+			await db
+				.insertInto(leafChange.type as "bundle" | "message" | "variant")
+				.values(value)
+				.onConflict((c) => c.column("id").doUpdateSet(value))
+				.execute();
+		} catch (e) {
+			// 787 = SQLITE_CONSTRAINT_FOREIGNKEY
+			if (e instanceof Error && (e as any)?.resultCode === 787) {
+				await handleForeignKeyViolation({ change: leafChange, lix, db });
+			} else {
+				throw e;
+			}
+		}
+	}
+	return { fileData: contentFromDatabase(sqlite) };
+};
+
+/**
+ * Handles foreign key violations e.g. a change
+ * doesn't exist in the target database but is referenced
+ * by an entity.
+ */
+async function handleForeignKeyViolation(args: {
+	change: Change;
+	lix: LixReadonly;
+	db: Kysely<InlangDatabaseSchema>;
+}) {
+	const lastKnown = async (
+		type: "bundle" | "message" | "variant",
+		id: string
+	) =>
+		await args.lix.db
+			.selectFrom("change")
+			.selectAll()
+			// heuristic that getting the last bundle value is fine
+			// and using created_at is fine too. if the change is undesired
+			// , a user can revert it with lix change control
+			.orderBy("created_at desc")
+			.where("type", "=", type)
+			.where((eb) => eb.ref("value", "->>").key("id"), "=", id)
+			.where("operation", "in", ["create", "update"])
+			// TODO shouldn't throw. The API needs to be able to
+			// report issues back to the app without throwing and potentially failing
+			// to apply 1000 changes because 1 change is invalid
+			// same requirement as in inlang, see https://github.com/opral/inlang-sdk/issues/213
+			.executeTakeFirstOrThrow();
+
+	if (args.change.type === "message") {
+		const lastKnownBundle = await lastKnown(
+			"bundle",
+			args.change.value?.bundleId
+		);
+		await args.db
+			.insertInto("bundle")
+			.values(lastKnownBundle.value as any)
+			.execute();
+	} else if (args.change.type === "variant") {
+		const lastKnownMessage = await lastKnown(
+			"message",
+			args.change.value?.messageId
+		);
+		// getting the bundle too out of precaution
+		const lastKnownBundle = await lastKnown(
+			"bundle",
+			lastKnownMessage.value?.bundleId
+		);
+		await args.db
+			.insertInto("bundle")
+			.values(lastKnownBundle.value as any)
+			// the bundle exists, so we can ignore the conflict
+			.onConflict((c) => c.doNothing())
+			.execute();
+		await args.db
+			.insertInto("message")
+			.values(lastKnownMessage.value as any)
+			.execute();
+		await args.db
+			.insertInto(args.change.type as "bundle" | "message" | "variant")
+			.values(args.change.value as any)
+			.onConflict((c) => c.column("id").doUpdateSet(args.change.value as any))
+			.execute();
+	}
+	// re-execute applying the change
+	await args.db
+		.insertInto(args.change.type as "bundle" | "message" | "variant")
+		.values(args.change.value as any)
+		.onConflict((c) => c.column("id").doUpdateSet(args.change.value as any))
+		.execute();
+}