@inlang/sdk 0.36.3 → 2.0.0-beta.1

package/src/project/saveProjectToDirectory.ts

@@ -0,0 +1,100 @@
+/* eslint-disable no-restricted-imports */
+import type fs from "node:fs/promises";
+import type { InlangProject } from "./api.js";
+import path from "node:path";
+import { toMessageV1 } from "../json-schema/old-v1-message/toMessageV1.js";
+import {
+	absolutePathFromProject,
+	withAbsolutePaths,
+} from "./loadProjectFromDirectory.js";
+import { detectJsonFormatting } from "../utilities/detectJsonFormatting.js";
+import { selectBundleNested } from "../query-utilities/selectBundleNested.js";
+
+export async function saveProjectToDirectory(args: {
+	fs: typeof fs;
+	project: InlangProject;
+	path: string;
+}): Promise<void> {
+	if (args.path.endsWith(".inlang") === false) {
+		throw new Error("The path must end with .inlang");
+	}
+	const files = await args.project.lix.db
+		.selectFrom("file")
+		.selectAll()
+		.execute();
+
+	// write all files to the directory
+	for (const file of files) {
+		if (file.path.endsWith("db.sqlite")) {
+			continue;
+		}
+		const p = path.join(args.path, file.path);
+		await args.fs.mkdir(path.dirname(p), { recursive: true });
+		await args.fs.writeFile(p, new Uint8Array(file.data));
+	}
+
+	// run exporters
+	const plugins = await args.project.plugins.get();
+	const settings = await args.project.settings.get();
+
+	for (const plugin of plugins) {
+		// old legacy remove with v3
+		if (plugin.saveMessages) {
+			// in-efficient re-qeuery but it's a legacy function that will be removed.
+			// the effort of adjusting the code to not re-query is not worth it.
+			const bundlesNested = await selectBundleNested(args.project.db).execute();
+			await plugin.saveMessages({
+				messages: bundlesNested.map((b) => toMessageV1(b)),
+				// @ts-expect-error - legacy
+				nodeishFs: withAbsolutePaths(args.fs, args.path),
+				settings,
+			});
+		}
+
+		if (plugin.exportFiles) {
+			const bundles = await args.project.db
+				.selectFrom("bundle")
+				.selectAll()
+				.execute();
+			const messages = await args.project.db
+				.selectFrom("message")
+				.selectAll()
+				.execute();
+			const variants = await args.project.db
+				.selectFrom("variant")
+				.selectAll()
+				.execute();
+			const files = await plugin.exportFiles({
+				bundles,
+				messages,
+				variants,
+				settings,
+			});
+			for (const file of files) {
+				const p = absolutePathFromProject(args.path, file.name);
+				const dirname = path.dirname(p);
+				if ((await args.fs.stat(dirname)).isDirectory() === false) {
+					await args.fs.mkdir(dirname, { recursive: true });
+				}
+				if (p.endsWith(".json")) {
+					try {
+						const existing = await args.fs.readFile(p, "utf-8");
+						const stringify = detectJsonFormatting(existing);
+						await args.fs.writeFile(
+							p,
+							new TextEncoder().encode(
+								stringify(JSON.parse(new TextDecoder().decode(file.content)))
+							)
+						);
+					} catch {
+						// write the file to disk (json doesn't exist yet)
+						// yeah ugly duplication of write file but it works.
+						await args.fs.writeFile(p, new Uint8Array(file.content));
+					}
+				} else {
+					await args.fs.writeFile(p, new Uint8Array(file.content));
+				}
+			}
+		}
+	}
+}

package/src/query-utilities/index.ts

@@ -0,0 +1,4 @@
+export { selectBundleNested } from "./selectBundleNested.js";
+export { insertBundleNested } from "./insertBundleNested.js";
+export { updateBundleNested } from "./updateBundleNested.js";
+export { upsertBundleNested } from "./upsertBundleNested.js";

package/src/query-utilities/insertBundleNested.ts

@@ -0,0 +1,46 @@
+import type { Kysely } from "kysely";
+import type {
+	InlangDatabaseSchema,
+	NewBundleNested,
+} from "../database/schema.js";
+
+export const insertBundleNested = async (
+	db: Kysely<InlangDatabaseSchema>,
+	bundle: NewBundleNested
+): Promise<void> => {
+	await db.transaction().execute(async (trx) => {
+		const insertedBundle = await trx
+			.insertInto("bundle")
+			.values({
+				id: bundle.id,
+				declarations: bundle.declarations,
+			})
+			.returning("id")
+			.executeTakeFirstOrThrow();
+
+		for (const message of bundle.messages) {
+			const insertedMessage = await trx
+				.insertInto("message")
+				.values({
+					id: message.id,
+					bundleId: insertedBundle.id,
+					locale: message.locale,
+					selectors: message.selectors,
+				})
+				.returning("id")
+				.executeTakeFirstOrThrow();
+
+			for (const variant of message.variants) {
+				await trx
+					.insertInto("variant")
+					.values({
+						id: variant.id,
+						messageId: insertedMessage.id,
+						matches: variant.matches,
+						pattern: variant.pattern,
+					})
+					.execute();
+			}
+		}
+	});
+};

package/src/query-utilities/selectBundleNested.ts

@@ -0,0 +1,46 @@
+import type { Kysely } from "kysely";
+import { jsonArrayFrom } from "kysely/helpers/sqlite";
+import type { InlangDatabaseSchema } from "../database/schema.js";
+
+/**
+ * Select bundles with nested messages and variants.
+ *
+ * `{ bundle, messages: [{ message, variants: [{ variant }] }] }`
+ *
+ * @example
+ *   // getting one bundle where id is 123
+ *   await selectBundleNested(db)
+ *     .where("bundle.id", "=", "123")
+ *     .executeTakeFirst()
+ *
+ *   // getting all bundles
+ *   await selectBundleNested(db)
+ *     .execute()
+ */
+export const selectBundleNested = (db: Kysely<InlangDatabaseSchema>) => {
+	return db.selectFrom("bundle").select((eb) => [
+		// select all columns from bundle
+		"id",
+		"declarations",
+		// select all columns from messages as "messages"
+		jsonArrayFrom(
+			eb
+				.selectFrom("message")
+				.select((eb) => [
+					// select all columns from message
+					"id",
+					"bundleId",
+					"locale",
+					"selectors",
+					// select all columns from variants as "variants"
+					jsonArrayFrom(
+						eb
+							.selectFrom("variant")
+							.select(["id", "messageId", "matches", "pattern"])
+							.whereRef("variant.messageId", "=", "message.id")
+					).as("variants"),
+				])
+				.whereRef("message.bundleId", "=", "bundle.id")
+		).as("messages"),
+	]);
+};

package/src/query-utilities/updateBundleNested.ts

@@ -0,0 +1,35 @@
+import type { Kysely } from "kysely";
+import type {
+	BundleNestedUpdate,
+	InlangDatabaseSchema,
+} from "../database/schema.js";
+
+export const updateBundleNested = async (
+	db: Kysely<InlangDatabaseSchema>,
+	bundle: BundleNestedUpdate & {
+		id: string;
+		messages: { id: string; variants: { id: string }[] }[];
+	}
+): Promise<void> => {
+	await db
+		.updateTable("bundle")
+		.set(bundle)
+		.where("id", "=", bundle.id)
+		.execute();
+
+	for (const message of bundle.messages) {
+		await db
+			.updateTable("message")
+			.set(message)
+			.where("id", "=", message.id)
+			.execute();
+
+		for (const variant of message.variants) {
+			await db
+				.updateTable("variant")
+				.set(variant)
+				.where("id", "=", variant.id)
+				.execute();
+		}
+	}
+};

package/src/query-utilities/upsertBundleNested.ts

@@ -0,0 +1,61 @@
+import type { Kysely } from "kysely";
+import type {
+	InlangDatabaseSchema,
+	NewBundleNested,
+} from "../database/schema.js";
+
+export const upsertBundleNested = async (
+	db: Kysely<InlangDatabaseSchema>,
+	bundle: NewBundleNested
+): Promise<void> => {
+	await db.transaction().execute(async (trx) => {
+		const insertedBundle = await trx
+			.insertInto("bundle")
+			.values({
+				id: bundle.id,
+				declarations: bundle.declarations,
+			})
+			.returning("id")
+			.onConflict((oc) =>
+				oc.column("id").doUpdateSet({
+					...bundle,
+					// @ts-expect-error - undefined
+					messages: undefined,
+				})
+			)
+			.executeTakeFirstOrThrow();
+
+		for (const message of bundle.messages) {
+			const insertedMessage = await trx
+				.insertInto("message")
+				.values({
+					id: message.id,
+					bundleId: insertedBundle.id,
+					locale: message.locale,
+					selectors: message.selectors,
+				})
+				.onConflict((oc) =>
+					oc.column("id").doUpdateSet({
+						...message,
+						// @ts-expect-error - undefined
+						variants: undefined,
+					})
+				)
+				.returning("id")
+				.executeTakeFirstOrThrow();
+
+			for (const variant of message.variants) {
+				await trx
+					.insertInto("variant")
+					.values({
+						id: variant.id,
+						messageId: insertedMessage.id,
+						matches: variant.matches,
+						pattern: variant.pattern,
+					})
+					.onConflict((oc) => oc.column("id").doUpdateSet(variant))
+					.execute();
+			}
+		}
+	});
+};

package/src/services/env-variables/createIndexFile.js

@@ -0,0 +1,40 @@
+/* eslint-disable no-restricted-imports */
+/* eslint-disable no-undef */
+/**
+ * This script writes public environment variables
+ * to an importable env file.
+ *
+ * - The SDK must bundle this file with the rest of the SDK
+ * - This scripts avoids the need for a bundler
+ * - Must be ran before building the SDK
+ */
+
+import fs from "node:fs/promises";
+import url from "node:url";
+import path from "node:path";
+
+const dirname = path.dirname(url.fileURLToPath(import.meta.url));
+
+const packageJson = JSON.parse(
+	await fs.readFile(path.resolve(dirname, "../../../package.json"), "utf-8")
+);
+
+await fs.writeFile(
+	dirname + "/index.ts",
+	`
+export const ENV_VARIABLES = {
+  PUBLIC_POSTHOG_TOKEN: ${ifDefined(process.env.PUBLIC_POSTHOG_TOKEN)},
+	PUBLIC_INLANG_SDK_SENTRY_DSN: ${ifDefined(
+		process.env.PUBLIC_INLANG_SDK_SENTRY_DSN
+	)},
+	SDK_VERSION: ${ifDefined(packageJson.version)},
+}
+`
+);
+
+// eslint-disable-next-line no-console
+// console.log("✅ Created env variable index file.");
+
+function ifDefined(value) {
+	return value ? `"${value}"` : undefined;
+}

package/src/{env-variables → services/env-variables}/index.d.ts

@@ -5,9 +5,12 @@
 
 /**
  * Env variables that are available at runtime.
- *
- * - assume that each env variable might be undefined (to ease development/contributions)
  */
-export declare const ENV_VARIABLES: Partial<{
-	PUBLIC_POSTHOG_TOKEN: string
-}>
+export declare const ENV_VARIABLES: {
+	PUBLIC_POSTHOG_TOKEN?: string;
+	PUBLIC_INLANG_SDK_SENTRY_DSN?: string;
+	/**
+	 * As defined in the package.json
+	 */
+	SDK_VERSION: string;
+};

package/src/services/error-reporting/index.ts

@@ -0,0 +1,28 @@
+import { init, setUser, captureException } from "@sentry/browser";
+import { ENV_VARIABLES } from "../env-variables/index.js";
+
+export function initErrorReporting(args: { projectId: string }) {
+	if (ENV_VARIABLES.PUBLIC_INLANG_SDK_SENTRY_DSN) {
+		// in order to receive alerts like "if more than 100 "users" are effected, notify the team"
+		// the user id is set to the project id
+		setUser({ id: args.projectId });
+		init({
+			dsn: ENV_VARIABLES.PUBLIC_INLANG_SDK_SENTRY_DSN,
+			release: ENV_VARIABLES.SDK_VERSION,
+		});
+	}
+}
+
+/**
+ * Capture an error.
+ *
+ * @example
+ *   try {
+ *     throw new Error("Something went wrong");
+ *   } catch (error) {
+ *     captureError(error);
+ *   }
+ */
+export function captureError(error: Error | any) {
+	return captureException(error);
+}

package/src/services/telemetry/capture.test.ts

@@ -0,0 +1,48 @@
+import { expect, test, vi } from "vitest";
+import { capture } from "./capture.js";
+
+test("it should not capture if telemetry is off", async () => {
+	// @ts-expect-error - global.fetch is not defined
+	global.fetch = vi.fn(() => Promise.resolve());
+
+	vi.mock("../env-variables/index.js", async () => {
+		return {
+			ENV_VARIABLES: {
+				PUBLIC_POSTHOG_TOKEN: "mock-defined",
+			},
+		};
+	});
+
+	await capture("SDK loaded project", {
+		projectId: "test",
+		settings: {
+			telemetry: "off",
+		},
+		properties: {},
+	});
+
+	expect(global.fetch).not.toHaveBeenCalled();
+});
+
+test("it should not capture if telemetry is NOT off", async () => {
+	// @ts-expect-error - global.fetch is not defined
+	global.fetch = vi.fn(() => Promise.resolve());
+
+	vi.mock("../env-variables/index.js", async () => {
+		return {
+			ENV_VARIABLES: {
+				PUBLIC_POSTHOG_TOKEN: "mock-defined",
+			},
+		};
+	});
+
+	await capture("SDK loaded project", {
+		projectId: "test",
+		settings: {
+			telemetry: undefined,
+		},
+		properties: {},
+	});
+
+	expect(global.fetch).toHaveBeenCalled();
+});

package/src/services/telemetry/capture.ts

@@ -0,0 +1,98 @@
+// import { ENV_VARIABLES } from "../env-variables/index.js";
+
+import type { ProjectSettings } from "../../json-schema/settings.js";
+import { ENV_VARIABLES } from "../env-variables/index.js";
+import { captureError } from "../error-reporting/index.js";
+
+/**
+ * List of telemetry events for typesafety.
+ *
+ * - prefix with `SDK` to avoid collisions with other apps
+ * - use past tense to indicate that the event is completed
+ */
+type TelemetryEvent = "SDK loaded project";
+
+/**
+ * Capture an event.
+ *
+ * - manually calling the PostHog API because the SDKs were not platform angostic (and generally bloated)
+ */
+export const capture = async (
+	event: TelemetryEvent,
+	args: {
+		projectId: string;
+		/**
+		 * Please use snake_case for property names.
+		 */
+		properties: Record<string, any>;
+		settings: Pick<ProjectSettings, "telemetry">;
+	}
+) => {
+	if (args.settings.telemetry === "off") {
+		return;
+	} else if (ENV_VARIABLES.PUBLIC_POSTHOG_TOKEN === undefined) {
+		return;
+	}
+	try {
+		await fetch("https://eu.posthog.com/capture/", {
+			method: "POST",
+			body: JSON.stringify({
+				api_key: ENV_VARIABLES.PUBLIC_POSTHOG_TOKEN,
+				event,
+				// id is "unknown" because no user information is available
+				distinct_id: "unknown",
+				properties: {
+					$groups: { project: args.projectId },
+					...args.properties,
+				},
+			}),
+		});
+		await identifyProject({
+			projectId: args.projectId,
+			// using the id for now as a name but can be changed in the future
+			// we need at least one property to make a project visible in the dashboar
+			properties: { name: args.projectId },
+		});
+	} catch (e) {
+		captureError(e);
+	}
+};
+
+/**
+ * Identifying a project is needed.
+ *
+ * Otherwise, the project will not be visible in the PostHog dashboard.
+ */
+const identifyProject = async (args: {
+	projectId: string;
+	/**
+	 * Please use snake_case for property names.
+	 */
+	properties: Record<string, any>;
+}) => {
+	// do not send events if the token is not set
+	// (assuming this eases testing)
+	if (ENV_VARIABLES.PUBLIC_POSTHOG_TOKEN === undefined) {
+		return;
+	}
+	try {
+		await fetch("https://eu.posthog.com/capture/", {
+			method: "POST",
+			body: JSON.stringify({
+				api_key: ENV_VARIABLES.PUBLIC_POSTHOG_TOKEN,
+				event: "$groupidentify",
+				// id is "unknown" because no user information is available
+				distinct_id: "unknown",
+				properties: {
+					$group_type: "project",
+					$group_key: args.projectId,
+					$group_set: {
+						...args.properties,
+					},
+				},
+			}),
+		});
+	} catch (e) {
+		captureError(e);
+	}
+};

package/src/utilities/detectJsonFormatting.test.ts

@@ -0,0 +1,38 @@
+import { expect, it } from "vitest";
+import { detectJsonFormatting } from "./detectJsonFormatting.js";
+
+it("should detect spacing", () => {
+	// test all possible spacings
+	for (const value of [1, 2, 3, 4, 6, 8, "\t"]) {
+		const spacing = value === "\t" ? "\t" : " ".repeat(value as number);
+		const objectWithSpacing = `{\n${spacing}"test": "test"\n}`;
+
+		const serialize = detectJsonFormatting(objectWithSpacing);
+		expect(serialize(JSON.parse(objectWithSpacing))).toBe(objectWithSpacing);
+	}
+});
+
+it("should detect spacing if the json is an array", () => {
+	// testing with one element only because dynamic generation of
+	// arrays with different spacings is too complex
+	const objectWithSpacing = `[\n\t"test",\n\t"test"\n]`;
+
+	const serialize = detectJsonFormatting(objectWithSpacing);
+	expect(serialize(JSON.parse(objectWithSpacing))).toBe(objectWithSpacing);
+});
+
+it("should detect new lines correctly", () => {
+	const withNewLine = `{"test":"test"}\n`;
+	const withoutNewLine = `{"test":"test"}`;
+	const withNewLineAndSpacing = `{\n\t"test": "test"\n}`;
+
+	const serialize1 = detectJsonFormatting(withNewLine);
+	const serialize2 = detectJsonFormatting(withoutNewLine);
+	const serialize3 = detectJsonFormatting(withNewLineAndSpacing);
+
+	expect(serialize1(JSON.parse(withNewLine))).toBe(withNewLine);
+	expect(serialize2(JSON.parse(withoutNewLine))).toBe(withoutNewLine);
+	expect(serialize3(JSON.parse(withNewLineAndSpacing))).toBe(
+		withNewLineAndSpacing
+	);
+});

package/src/utilities/detectJsonFormatting.ts

@@ -0,0 +1,109 @@
+/**
+ * Detects the formatting of a JSON file and returns a function
+ * that can be used to stringify JSON with the same formatting.
+ *
+ * @example
+ *   const file = await fs.readFile("./messages.json", { encoding: "utf-8" })
+ *   const stringify = detectJsonFormatting(file)
+ *   const newFile = stringify(json)
+ */
+export const detectJsonFormatting = (
+	file: string
+): ((
+	value: Parameters<typeof JSON.stringify>[0],
+	replacer?: Parameters<typeof JSON.stringify>[1]
+	// space is provided by the function
+) => string) => {
+	const endsWithNewLine = file.endsWith("\n");
+	const spacing = guessJsonIndent(file);
+
+	return (value, replacer) =>
+		JSON.stringify(value, replacer, spacing as any) +
+		(endsWithNewLine ? "\n" : "");
+};
+
+/**
+ * vendored from https://github.com/ehmicky/guess-json-indent
+ */
+// Guess the indentation of a JSON string
+const guessJsonIndent = (jsonString: string) => {
+	const firstIndex = skipWhitespaces(jsonString, 0);
+
+	if (
+		firstIndex === undefined ||
+		!isJsonObjectOrArray(jsonString[firstIndex])
+	) {
+		return;
+	}
+
+	const secondIndex = skipWhitespaces(jsonString, firstIndex + 1);
+
+	if (secondIndex === undefined) {
+		return;
+	}
+
+	return getIndent(jsonString, firstIndex, secondIndex);
+};
+
+// Whitespaces are ignored before|between|after tokens in JSON.
+// Uses imperative logic for performance.
+const skipWhitespaces = (jsonString: string | any[], startIndex: number) => {
+	for (let index = startIndex; index < jsonString.length; index += 1) {
+		const character = jsonString[index];
+
+		if (!isJsonWhitespace(character)) {
+			return index;
+		}
+	}
+	return;
+};
+
+// JSON defines only those are valid whitespaces
+const isJsonWhitespace = (character: string) =>
+	character === " " ||
+	character === "\t" ||
+	character === "\n" ||
+	character === "\r";
+
+// If the top-level value is another type than an object or an array, there is
+// no possible indentation
+const isJsonObjectOrArray = (character: string | undefined) =>
+	character === "{" || character === "[";
+
+// Uses imperative logic for performance
+// @ts-expect-error - not all code paths return a value
+const getIndent = (
+	jsonString: string | any[],
+	firstIndex: number,
+	secondIndex: number
+) => {
+	let indent;
+
+	for (let index = secondIndex - 1; index > firstIndex; index -= 1) {
+		const character = jsonString[index];
+
+		if (character === "\r") {
+			return;
+		}
+
+		if (character === "\n") {
+			return normalizeIndent(indent);
+		}
+
+		if (indent === undefined) {
+			indent = character;
+		} else if (indent[0] === character) {
+			indent += character;
+		} else {
+			return;
+		}
+	}
+};
+
+const normalizeIndent = (indent: string | any[] | undefined) => {
+	if (indent === undefined) {
+		return 0;
+	}
+
+	return indent[0] === " " ? indent.length : indent;
+};