@inlang/sdk 2.0.0-prerelease.0 → 2.0.0

package/src/project/loadProject.ts

@@ -1,12 +1,14 @@
-import { toBlob, type Lix } from "@lix-js/sdk";
+import { toBlob, type Account, type Lix } from "@lix-js/sdk";
 import type { InlangPlugin } from "../plugin/schema.js";
 import type { ProjectSettings } from "../json-schema/settings.js";
 import { type SqliteDatabase } from "sqlite-wasm-kysely";
 import { initDb } from "../database/initDb.js";
 import { initHandleSaveToLixOnChange } from "./initHandleSaveToLixOnChange.js";
-import { type PreprocessPluginBeforeImportFunction } from "../plugin/importPlugins.js";
+import {
+	importPlugins,
+	type PreprocessPluginBeforeImportFunction,
+} from "../plugin/importPlugins.js";
 import type { InlangProject } from "./api.js";
-import { createProjectState } from "./state/state.js";
 import { withLanguageTagToLocaleMigration } from "../migrations/v2/withLanguageTagToLocaleMigration.js";
 import { v4 } from "uuid";
 import { initErrorReporting } from "../services/error-reporting/index.js";
@@ -20,6 +22,16 @@ import { exportFiles } from "../import-export/exportFiles.js";
 export async function loadProject(args: {
 	sqlite: SqliteDatabase;
 	lix: Lix;
+	/**
+	 * The account that loaded the project.
+	 *
+	 * Defaults to an anonymous/new account if undefined.
+	 *
+	 * @example
+	 *   const account = localStorage.getItem("account")
+	 *   const project = await loadProject({ account })
+	 */
+	account?: Account;
 	/**
 	 * Provide plugins to the project.
 	 *
@@ -65,11 +77,27 @@ export async function loadProject(args: {
 		JSON.parse(new TextDecoder().decode(settingsFile.data)) as ProjectSettings
 	);
 
-	const state = createProjectState({
-		...args,
+	const importedPlugins = await importPlugins({
 		settings,
+		lix: args.lix,
+		preprocessPluginBeforeImport: args.preprocessPluginBeforeImport,
 	});
 
+	const plugins = [...(args.providePlugins ?? []), ...importedPlugins.plugins];
+
+	const idFile = await args.lix.db
+		.selectFrom("file")
+		.where("path", "=", "/project_id")
+		.select("data")
+		.executeTakeFirstOrThrow();
+
+	const id = new TextDecoder().decode(idFile.data);
+
+	// const state = createProjectState({
+	// 	...args,
+	// 	settings,
+	// });
+
 	// TODO implement garbage collection/a proper queue.
 	//      for the protoype and tests, it seems good enough
 	//      without garbage collection of old promises.
@@ -85,34 +113,99 @@ export async function loadProject(args: {
 	// not awaiting to not block the load time of a project
 	maybeCaptureLoadedProject({
 		db,
-		state,
+		id,
+		settings,
+		plugins,
+		lix: args.lix,
 		appId: args.appId,
 	});
 
-	initErrorReporting({ projectId: await state.id.get() });
+	initErrorReporting({ projectId: id });
 
 	return {
 		db,
-		id: state.id,
-		settings: state.settings,
-		plugins: state.plugins,
-		errors: state.errors,
+		id: {
+			get: async () => {
+				const file = await args.lix.db
+					.selectFrom("file")
+					.where("path", "=", "/project_id")
+					.select("file.data")
+					.executeTakeFirstOrThrow();
+				return new TextDecoder().decode(file.data);
+			},
+		},
+		settings: {
+			get: async () => {
+				const file = await args.lix.db
+					.selectFrom("file")
+					.where("path", "=", "/settings.json")
+					.select("file.data")
+					.executeTakeFirstOrThrow();
+				return withLanguageTagToLocaleMigration(
+					JSON.parse(new TextDecoder().decode(file.data))
+				);
+			},
+			set: async (newSettings) => {
+				const cloned = JSON.parse(JSON.stringify(newSettings));
+				cloned.languageTags = cloned.locales;
+				cloned.sourceLanguageTag = cloned.baseLocale;
+
+				await args.lix.db
+					.updateTable("file")
+					.where("path", "=", "/settings.json")
+					.set({
+						data: new TextEncoder().encode(
+							JSON.stringify(cloned, undefined, 2)
+						),
+					})
+					.execute();
+			},
+		},
+		plugins: {
+			get: async () => plugins,
+		},
+		errors: {
+			get: async () => [...importedPlugins.errors],
+		},
+		// errors: state.errors,
 		importFiles: async ({ files, pluginKey }) => {
+			const settingsFile = await args.lix.db
+				.selectFrom("file")
+				.where("path", "=", "/settings.json")
+				.select("file.data")
+				.executeTakeFirstOrThrow();
+
+			const settings = JSON.parse(
+				new TextDecoder().decode(settingsFile.data)
+			) as ProjectSettings;
+
 			return await importFiles({
 				files,
 				pluginKey,
-				settings: await state.settings.get(),
-				plugins: await state.plugins.get(),
+				settings,
+				// TODO don't use global state, might be stale
+				plugins,
 				db,
 			});
 		},
 		exportFiles: async ({ pluginKey }) => {
+			const settingsFile = await args.lix.db
+				.selectFrom("file")
+				.where("path", "=", "/settings.json")
+				.select("file.data")
+				.executeTakeFirstOrThrow();
+
+			const settings = JSON.parse(
+				new TextDecoder().decode(settingsFile.data)
+			) as ProjectSettings;
+
 			return (
 				await exportFiles({
 					pluginKey,
 					db,
-					settings: await state.settings.get(),
-					plugins: await state.plugins.get(),
+					settings,
+					// TODO don't use global state, might be stale
+					plugins,
 				})
 			).map((output) => ({ ...output, pluginKey }));
 		},

package/src/project/loadProjectFromDirectory.test.ts

@@ -1,5 +1,5 @@
 /* eslint-disable @typescript-eslint/no-unused-vars */
-/* eslint-disable @typescript-eslint/no-non-null-assertion */
+
 import { beforeEach, describe, expect, test, vi } from "vitest";
 import { ProjectSettings } from "../json-schema/settings.js";
 import { fs, vol, Volume } from "memfs";
@@ -7,7 +7,6 @@ import {
 	loadProjectFromDirectory,
 	ResourceFileImportError,
 	WarningDeprecatedLintRule,
-	WarningLocalPluginImport,
 } from "./loadProjectFromDirectory.js";
 import { selectBundleNested } from "../query-utilities/selectBundleNested.js";
 import { Text } from "../json-schema/pattern.js";
@@ -698,13 +697,8 @@ test("it should provide plugins from disk for backwards compatibility but warn t
 	expect(plugins.length).toBe(1);
 	expect(plugins[0]?.key).toBe("mock-plugin");
 
-	// old mock lint rule import is number two import
-	// it's hard to model the import of a lint rule
-	// best if they are removed
-	expect(errors.length).toBe(3);
-	expect(errors[0]).toBeInstanceOf(WarningLocalPluginImport);
-	expect(errors[1]).toBeInstanceOf(WarningLocalPluginImport);
-	expect(errors[2]).toBeInstanceOf(WarningDeprecatedLintRule);
+	expect(errors.length).toBe(1);
+	expect(errors[0]).toBeInstanceOf(WarningDeprecatedLintRule);
 
 	// it should not remove the module from the settings
 	// else roundtrips would not work
@@ -817,3 +811,44 @@ test("plugin calls that use fs should be intercepted to use an absolute path", a
 	// expect(saveMessagesSpy).toHaveBeenCalled();
 	// expect(toBeImportedFilesSpy).toHaveBeenCalled();
 });
+
+test("it can import plugins via http", async () => {
+	const mockRepo = {
+		"/project.inlang/settings.json": JSON.stringify({
+			baseLocale: "en",
+			locales: ["en"],
+			modules: ["https://example.com/plugin.js"],
+		} satisfies ProjectSettings),
+	};
+
+	const fs = Volume.fromJSON(mockRepo);
+
+	const mockPluginModule = `export default {
+			key: "mock-plugin"	
+		}`;
+
+	global.fetch = vi.fn(() => Promise.resolve(new Response(mockPluginModule)));
+
+	const project = await loadProjectFromDirectory({
+		fs: fs as any,
+		path: "/project.inlang",
+	});
+
+	const plugins = await project.plugins.get();
+
+	expect(global.fetch).toHaveBeenCalledWith("https://example.com/plugin.js");
+	expect(plugins.length).toBe(1);
+
+	const pluginCache = await project.lix.db
+		.selectFrom("file")
+		.selectAll()
+		.where("path", "like", "/cache/plugins/%")
+		.execute();
+
+	expect(
+		pluginCache.some(
+			(file) => new TextDecoder().decode(file.data) === mockPluginModule
+		),
+		"expecting the plugin to be cached"
+	).toBe(true);
+});

package/src/project/loadProjectFromDirectory.ts

@@ -13,7 +13,6 @@ import { fromMessageV1 } from "../json-schema/old-v1-message/fromMessageV1.js";
 import type { ProjectSettings } from "../json-schema/settings.js";
 import type { PreprocessPluginBeforeImportFunction } from "../plugin/importPlugins.js";
 import { PluginImportError } from "../plugin/errors.js";
-import type { InlangProject } from "./api.js";
 import { upsertBundleNestedMatchByProperties } from "../import-export/upsertBundleNestedMatchByProperties.js";
 
 /**
@@ -28,8 +27,14 @@ export async function loadProjectFromDirectory(
 		"blob"
 	>
 ) {
+	const settingsPath = nodePath.join(args.path, "settings.json");
+	const settings = JSON.parse(
+		await args.fs.promises.readFile(settingsPath, "utf8")
+	) as ProjectSettings;
+
 	const localImport = await importLocalPlugins({
 		fs: args.fs,
+		settings,
 		path: args.path,
 	});
 
@@ -45,7 +50,9 @@ export async function loadProjectFromDirectory(
 	const project = await loadProjectInMemory({
 		...args,
 		providePlugins: providePluginsWithLocalPlugins,
-		blob: await newProject(),
+		blob: await newProject({
+			settings,
+		}),
 	});
 
 	await syncLixFsFiles({
@@ -154,24 +161,19 @@ export async function loadProjectFromDirectory(
 		...project,
 		errors: {
 			get: async () => {
-				const errors = await project.errors.get();
-				return [
-					...withLocallyImportedPluginWarning(errors),
-					...localImport.errors,
-					...importedResourceFileErrors,
-				];
-			},
-			subscribe: (
-				callback: Parameters<InlangProject["errors"]["subscribe"]>[0]
-			) => {
-				return project.errors.subscribe((value) => {
-					callback([
-						...withLocallyImportedPluginWarning(value),
-						...localImport.errors,
-						...importedResourceFileErrors,
-					]);
-				});
+				return [...localImport.errors, ...importedResourceFileErrors];
 			},
+			// subscribe: (
+			// 	callback: Parameters<InlangProject["errors"]["subscribe"]>[0]
+			// ) => {
+			// 	return project.errors.subscribe((value) => {
+			// 		callback([
+			// 			...withLocallyImportedPluginWarning(value),
+			// 			...localImport.errors,
+			// 			...importedResourceFileErrors,
+			// 		]);
+			// 	});
+			// },
 		},
 	};
 }
@@ -473,9 +475,13 @@ async function syncLixFsFiles(args: {
 			if (!statesToSync.fsFileStates[path]) {
 				if (lixState.state == "unknown") {
 					// ADD TO FS (6)
+					// create directory if not exists
+					args.fs.mkdirSync(nodePath.dirname(nodePath.join(args.path, path)), {
+						recursive: true,
+					});
+					// write file
 					args.fs.writeFileSync(
-						// TODO check platform dependent folder separator
-						args.path + path,
+						nodePath.join(args.path, path),
 						Buffer.from(lixState.content)
 					);
 					statesToSync.fsFileStates[path] = {
@@ -632,16 +638,16 @@ function categorizePlugins(plugins: readonly InlangPlugin[]): {
  */
 async function importLocalPlugins(args: {
 	fs: typeof fs;
+	settings: ProjectSettings;
 	path: string;
 	preprocessPluginBeforeImport?: PreprocessPluginBeforeImportFunction;
 }) {
 	const errors: Error[] = [];
 	const locallyImportedPlugins = [];
-	const settingsPath = nodePath.join(args.path, "settings.json");
-	const settings = JSON.parse(
-		await args.fs.promises.readFile(settingsPath, "utf8")
-	) as ProjectSettings;
-	for (const module of settings.modules ?? []) {
+	for (const module of args.settings.modules ?? []) {
+		if (module.startsWith("http")) {
+			continue;
+		}
 		const modulePath = absolutePathFromProject(args.path, module);
 		try {
 			let moduleAsText = await args.fs.promises.readFile(modulePath, "utf8");
@@ -669,27 +675,6 @@ async function importLocalPlugins(args: {
 	};
 }
 
-function withLocallyImportedPluginWarning(errors: readonly Error[]) {
-	return errors.map((error) => {
-		if (
-			error instanceof PluginImportError &&
-			error.plugin.startsWith("http") === false
-		) {
-			return new WarningLocalPluginImport(error.plugin);
-		}
-		return error;
-	});
-}
-
-export class WarningLocalPluginImport extends Error {
-	constructor(module: string) {
-		super(
-			`Plugin ${module} is imported from a local path. This will work fine in dev tools like Sherlock or Paraglide JS but is not portable. Web apps like Fink or Parrot won't be able to import this plugin. It is recommended to use an http url to import plugins. The plugins are cached locally and will be available offline.`
-		);
-		this.name = "WarningLocalImport";
-	}
-}
-
 export class WarningDeprecatedLintRule extends Error {
 	constructor(module: string) {
 		super(

package/src/project/loadProjectInMemory.ts

@@ -12,6 +12,7 @@ export async function loadProjectInMemory(
 ) {
 	const lix = await openLixInMemory({
 		blob: args.blob,
+		account: args.account,
 		providePlugins: [
 			// inlangLixPluginV1
 		],

package/src/project/maybeCaptureTelemetry.test.ts

@@ -2,7 +2,6 @@ import { expect, test, vi } from "vitest";
 import { loadProjectInMemory } from "./loadProjectInMemory.js";
 import { newProject } from "./newProject.js";
 import { maybeCaptureLoadedProject } from "./maybeCaptureTelemetry.js";
-import type { ProjectState } from "./state/state.js";
 import { capture } from "../services/telemetry/capture.js";
 
 test("it should capture as expected", async () => {
@@ -25,6 +24,11 @@ test("it should capture as expected", async () => {
 		blob: await newProject(),
 	});
 
+	const account = await project.lix.db
+		.selectFrom("active_account")
+		.select("id")
+		.executeTakeFirstOrThrow();
+
 	const bundle = await project.db
 		.insertInto("bundle")
 		.defaultValues()
@@ -43,8 +47,17 @@ test("it should capture as expected", async () => {
 		.returningAll()
 		.executeTakeFirst();
 
+	const settings = await project.settings.get();
+
+	const id = await project.id.get();
+
+	const plugins = await project.plugins.get();
+
 	await maybeCaptureLoadedProject({
-		state: project as unknown as ProjectState,
+		id,
+		settings,
+		plugins,
+		lix: project.lix,
 		appId: "test",
 		db: project.db,
 	});
@@ -52,6 +65,7 @@ test("it should capture as expected", async () => {
 	expect(capture).toHaveBeenCalledWith("SDK loaded project", {
 		projectId: await project.id.get(),
 		settings: await project.settings.get(),
+		accountId: account.id,
 		properties: {
 			appId: "test",
 			settings: await project.settings.get(),

package/src/project/maybeCaptureTelemetry.ts

@@ -1,51 +1,69 @@
 import { Kysely } from "kysely";
 import { capture } from "../services/telemetry/capture.js";
-import { type ProjectState } from "./state/state.js";
 import type { InlangDatabaseSchema } from "../database/schema.js";
 import { ENV_VARIABLES } from "../services/env-variables/index.js";
+import type { ProjectSettings } from "../json-schema/settings.js";
+import { captureError } from "../services/error-reporting/index.js";
+import type { Lix } from "@lix-js/sdk";
 
 export async function maybeCaptureLoadedProject(args: {
-	state: ProjectState;
+	id: string;
+	lix: Lix;
+	settings: ProjectSettings;
+	plugins: Readonly<Array<{ key: string }>>;
 	appId?: string;
 	db: Kysely<InlangDatabaseSchema>;
 }) {
-	const id = await args.state.id.get();
-	const settings = await args.state.settings.get();
-	const plugins = await args.state.plugins.get();
-
-	if (settings.telemetry === "off") {
+	if (args.settings.telemetry === "off") {
 		return;
 	}
 
-	const bundles = await args.db
-		.selectFrom("bundle")
-		.select((s) => s.fn.count("id").as("count"))
-		.executeTakeFirst();
-	const messages = await args.db
-		.selectFrom("message")
-		.select((s) => s.fn.count("id").as("count"))
-		.executeTakeFirst();
-	const variants = await args.db
-		.selectFrom("variant")
-		.select((s) => s.fn.count("id").as("count"))
-		.executeTakeFirst();
+	try {
+		const activeAccount = await args.lix.db
+			.selectFrom("active_account")
+			.select("id")
+			.executeTakeFirstOrThrow();
+
+		const bundles = await args.db
+			.selectFrom("bundle")
+			.select((s) => s.fn.count("id").as("count"))
+			.executeTakeFirst();
+		const messages = await args.db
+			.selectFrom("message")
+			.select((s) => s.fn.count("id").as("count"))
+			.executeTakeFirst();
+		const variants = await args.db
+			.selectFrom("variant")
+			.select((s) => s.fn.count("id").as("count"))
+			.executeTakeFirst();
 
-	await capture("SDK loaded project", {
-		projectId: id,
-		settings,
-		properties: {
-			// Insight: Which app is used by the SDK
-			appId: args.appId,
-			// Insight: How many languages are used, etc.
-			settings,
-			// Insight on the used plugins (which one's to prioritize)
-			pluginKeys: plugins.map((plugin) => plugin.key),
-			// Insight: Which version of the SDK is used (can be used to deprecate old versions)
-			sdkVersion: ENV_VARIABLES.SDK_VERSION,
-			// Insight: Scale of projects (what project size to optimize for)
-			numBundles: bundles?.count,
-			numMessages: messages?.count,
-			numVariants: variants?.count,
-		},
-	});
+		await capture("SDK loaded project", {
+			projectId: args.id,
+			settings: args.settings,
+			accountId: activeAccount.id,
+			properties: {
+				// Insight: Which app is used by the SDK
+				appId: args.appId,
+				// Insight: How many languages are used, etc.
+				settings: args.settings,
+				// Insight on the used plugins (which one's to prioritize)
+				pluginKeys: args.plugins.map((plugin) => plugin.key),
+				// Insight: Which version of the SDK is used (can be used to deprecate old versions)
+				sdkVersion: ENV_VARIABLES.SDK_VERSION,
+				// Insight: Scale of projects (what project size to optimize for)
+				numBundles: bundles?.count,
+				numMessages: messages?.count,
+				numVariants: variants?.count,
+			},
+		});
+	} catch (e) {
+		if (
+			e instanceof Error &&
+			e.message.includes("driver has already been destroyed")
+		) {
+			// The project has been closed, nothing to capture
+			return;
+		}
+		captureError(e);
+	}
 }

package/src/project/newProject.ts

@@ -20,7 +20,7 @@ export async function newProject(args?: {
 	const sqlite = await createInMemoryDatabase({
 		readOnly: false,
 	});
-	const db = initDb({ sqlite });
+	initDb({ sqlite });
 
 	try {
 		const inlangDbContent = contentFromDatabase(sqlite);
@@ -47,7 +47,9 @@ export async function newProject(args?: {
 				},
 			])
 			.execute();
-		return toBlob({ lix });
+		const blob = toBlob({ lix });
+		lix.sqlite.close();
+		return blob;
 	} catch (e) {
 		const error = new Error(`Failed to create new inlang project: ${e}`, {
 			cause: e,
@@ -56,7 +58,6 @@ export async function newProject(args?: {
 		throw error;
 	} finally {
 		sqlite.close();
-		await db.destroy();
 	}
 }
 

package/src/project/saveProjectToDirectory.test.ts

@@ -11,7 +11,7 @@ import type { ProjectSettings } from "../json-schema/settings.js";
 import type { MessageV1 } from "../json-schema/old-v1-message/schemaV1.js";
 
 test("it should throw if the path doesn't end with .inlang", async () => {
-	expect(() =>
+	await expect(() =>
 		saveProjectToDirectory({
 			fs: {} as any,
 			project: {} as any,

package/src/query-utilities/index.ts

@@ -2,4 +2,3 @@ export { selectBundleNested } from "./selectBundleNested.js";
 export { insertBundleNested } from "./insertBundleNested.js";
 export { updateBundleNested } from "./updateBundleNested.js";
 export { upsertBundleNested } from "./upsertBundleNested.js";
-export { pollQuery } from "./pollQuery.js";

package/src/services/telemetry/capture.test.ts

@@ -15,6 +15,7 @@ test("it should not capture if telemetry is off", async () => {
 
 	await capture("SDK loaded project", {
 		projectId: "test",
+		accountId: "test",
 		settings: {
 			telemetry: "off",
 		},
@@ -38,6 +39,7 @@ test("it should not capture if telemetry is NOT off", async () => {
 
 	await capture("SDK loaded project", {
 		projectId: "test",
+		accountId: "test",
 		settings: {
 			telemetry: undefined,
 		},

package/src/services/telemetry/capture.ts

@@ -21,6 +21,7 @@ export const capture = async (
 	event: TelemetryEvent,
 	args: {
 		projectId: string;
+		accountId: string;
 		/**
 		 * Please use snake_case for property names.
 		 */
@@ -39,8 +40,7 @@ export const capture = async (
 			body: JSON.stringify({
 				api_key: ENV_VARIABLES.PUBLIC_POSTHOG_TOKEN,
 				event,
-				// id is "unknown" because no user information is available
-				distinct_id: "unknown",
+				distinct_id: args.accountId,
 				properties: {
 					$groups: { project: args.projectId },
 					...args.properties,
@@ -49,6 +49,7 @@ export const capture = async (
 		});
 		await identifyProject({
 			projectId: args.projectId,
+			accountId: args.accountId,
 			// using the id for now as a name but can be changed in the future
 			// we need at least one property to make a project visible in the dashboar
 			properties: { name: args.projectId },
@@ -65,6 +66,7 @@ export const capture = async (
  */
 const identifyProject = async (args: {
 	projectId: string;
+	accountId: string;
 	/**
 	 * Please use snake_case for property names.
 	 */
@@ -81,8 +83,7 @@ const identifyProject = async (args: {
 			body: JSON.stringify({
 				api_key: ENV_VARIABLES.PUBLIC_POSTHOG_TOKEN,
 				event: "$groupidentify",
-				// id is "unknown" because no user information is available
-				distinct_id: "unknown",
+				distinct_id: args.accountId,
 				properties: {
 					$group_type: "project",
 					$group_key: args.projectId,

package/dist/project/state/id$.d.ts

@@ -1,6 +0,0 @@
-import { type Lix } from "@lix-js/sdk";
-import { type Observable } from "rxjs";
-export declare function createId$(args: {
-    lix: Lix;
-}): Observable<string>;
-//# sourceMappingURL=id$.d.ts.map

package/dist/project/state/id$.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"id$.d.ts","sourceRoot":"/","sources":["project/state/id$.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,GAAG,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EAAsB,KAAK,UAAU,EAAE,MAAM,MAAM,CAAC;AAG3D,wBAAgB,SAAS,CAAC,IAAI,EAAE;IAAE,GAAG,EAAE,GAAG,CAAA;CAAE,GAAG,UAAU,CAAC,MAAM,CAAC,CAgBhE"}