@inlang/sdk 2.0.0-prerelease.0 → 2.0.1

package/src/project/loadProject.ts

@@ -1,12 +1,14 @@
-import { toBlob, type Lix } from "@lix-js/sdk";
+import { toBlob, type Account, type Lix } from "@lix-js/sdk";
 import type { InlangPlugin } from "../plugin/schema.js";
 import type { ProjectSettings } from "../json-schema/settings.js";
 import { type SqliteDatabase } from "sqlite-wasm-kysely";
 import { initDb } from "../database/initDb.js";
 import { initHandleSaveToLixOnChange } from "./initHandleSaveToLixOnChange.js";
-import { type PreprocessPluginBeforeImportFunction } from "../plugin/importPlugins.js";
+import {
+	importPlugins,
+	type PreprocessPluginBeforeImportFunction,
+} from "../plugin/importPlugins.js";
 import type { InlangProject } from "./api.js";
-import { createProjectState } from "./state/state.js";
 import { withLanguageTagToLocaleMigration } from "../migrations/v2/withLanguageTagToLocaleMigration.js";
 import { v4 } from "uuid";
 import { initErrorReporting } from "../services/error-reporting/index.js";
@@ -20,6 +22,16 @@ import { exportFiles } from "../import-export/exportFiles.js";
 export async function loadProject(args: {
 	sqlite: SqliteDatabase;
 	lix: Lix;
+	/**
+	 * The account that loaded the project.
+	 *
+	 * Defaults to an anonymous/new account if undefined.
+	 *
+	 * @example
+	 *   const account = localStorage.getItem("account")
+	 *   const project = await loadProject({ account })
+	 */
+	account?: Account;
 	/**
 	 * Provide plugins to the project.
 	 *
@@ -65,11 +77,27 @@ export async function loadProject(args: {
 		JSON.parse(new TextDecoder().decode(settingsFile.data)) as ProjectSettings
 	);
 
-	const state = createProjectState({
-		...args,
+	const importedPlugins = await importPlugins({
 		settings,
+		lix: args.lix,
+		preprocessPluginBeforeImport: args.preprocessPluginBeforeImport,
 	});
 
+	const plugins = [...(args.providePlugins ?? []), ...importedPlugins.plugins];
+
+	const idFile = await args.lix.db
+		.selectFrom("file")
+		.where("path", "=", "/project_id")
+		.select("data")
+		.executeTakeFirstOrThrow();
+
+	const id = new TextDecoder().decode(idFile.data);
+
+	// const state = createProjectState({
+	// 	...args,
+	// 	settings,
+	// });
+
 	// TODO implement garbage collection/a proper queue.
 	//      for the protoype and tests, it seems good enough
 	//      without garbage collection of old promises.
@@ -85,34 +113,99 @@ export async function loadProject(args: {
 	// not awaiting to not block the load time of a project
 	maybeCaptureLoadedProject({
 		db,
-		state,
+		id,
+		settings,
+		plugins,
+		lix: args.lix,
 		appId: args.appId,
 	});
 
-	initErrorReporting({ projectId: await state.id.get() });
+	initErrorReporting({ projectId: id });
 
 	return {
 		db,
-		id: state.id,
-		settings: state.settings,
-		plugins: state.plugins,
-		errors: state.errors,
+		id: {
+			get: async () => {
+				const file = await args.lix.db
+					.selectFrom("file")
+					.where("path", "=", "/project_id")
+					.select("file.data")
+					.executeTakeFirstOrThrow();
+				return new TextDecoder().decode(file.data);
+			},
+		},
+		settings: {
+			get: async () => {
+				const file = await args.lix.db
+					.selectFrom("file")
+					.where("path", "=", "/settings.json")
+					.select("file.data")
+					.executeTakeFirstOrThrow();
+				return withLanguageTagToLocaleMigration(
+					JSON.parse(new TextDecoder().decode(file.data))
+				);
+			},
+			set: async (newSettings) => {
+				const cloned = JSON.parse(JSON.stringify(newSettings));
+				cloned.languageTags = cloned.locales;
+				cloned.sourceLanguageTag = cloned.baseLocale;
+
+				await args.lix.db
+					.updateTable("file")
+					.where("path", "=", "/settings.json")
+					.set({
+						data: new TextEncoder().encode(
+							JSON.stringify(cloned, undefined, 2)
+						),
+					})
+					.execute();
+			},
+		},
+		plugins: {
+			get: async () => plugins,
+		},
+		errors: {
+			get: async () => [...importedPlugins.errors],
+		},
+		// errors: state.errors,
 		importFiles: async ({ files, pluginKey }) => {
+			const settingsFile = await args.lix.db
+				.selectFrom("file")
+				.where("path", "=", "/settings.json")
+				.select("file.data")
+				.executeTakeFirstOrThrow();
+
+			const settings = JSON.parse(
+				new TextDecoder().decode(settingsFile.data)
+			) as ProjectSettings;
+
 			return await importFiles({
 				files,
 				pluginKey,
-				settings: await state.settings.get(),
-				plugins: await state.plugins.get(),
+				settings,
+				// TODO don't use global state, might be stale
+				plugins,
 				db,
 			});
 		},
 		exportFiles: async ({ pluginKey }) => {
+			const settingsFile = await args.lix.db
+				.selectFrom("file")
+				.where("path", "=", "/settings.json")
+				.select("file.data")
+				.executeTakeFirstOrThrow();
+
+			const settings = JSON.parse(
+				new TextDecoder().decode(settingsFile.data)
+			) as ProjectSettings;
+
 			return (
 				await exportFiles({
 					pluginKey,
 					db,
-					settings: await state.settings.get(),
-					plugins: await state.plugins.get(),
+					settings,
+					// TODO don't use global state, might be stale
+					plugins,
 				})
 			).map((output) => ({ ...output, pluginKey }));
 		},

package/src/project/loadProjectFromDirectory.test.ts

@@ -1,5 +1,5 @@
 /* eslint-disable @typescript-eslint/no-unused-vars */
-/* eslint-disable @typescript-eslint/no-non-null-assertion */
+
 import { beforeEach, describe, expect, test, vi } from "vitest";
 import { ProjectSettings } from "../json-schema/settings.js";
 import { fs, vol, Volume } from "memfs";
@@ -7,7 +7,6 @@ import {
 	loadProjectFromDirectory,
 	ResourceFileImportError,
 	WarningDeprecatedLintRule,
-	WarningLocalPluginImport,
 } from "./loadProjectFromDirectory.js";
 import { selectBundleNested } from "../query-utilities/selectBundleNested.js";
 import { Text } from "../json-schema/pattern.js";
@@ -659,7 +658,6 @@ test("it should pass toBeImportedMetadata on import", async () => {
 		expect.objectContaining({
 			files: [
 				expect.objectContaining({
-					name: "en.json",
 					toBeImportedFilesMetadata: {
 						foo: "bar",
 					},
@@ -698,13 +696,8 @@ test("it should provide plugins from disk for backwards compatibility but warn t
 	expect(plugins.length).toBe(1);
 	expect(plugins[0]?.key).toBe("mock-plugin");
 
-	// old mock lint rule import is number two import
-	// it's hard to model the import of a lint rule
-	// best if they are removed
-	expect(errors.length).toBe(3);
-	expect(errors[0]).toBeInstanceOf(WarningLocalPluginImport);
-	expect(errors[1]).toBeInstanceOf(WarningLocalPluginImport);
-	expect(errors[2]).toBeInstanceOf(WarningDeprecatedLintRule);
+	expect(errors.length).toBe(1);
+	expect(errors[0]).toBeInstanceOf(WarningDeprecatedLintRule);
 
 	// it should not remove the module from the settings
 	// else roundtrips would not work
@@ -817,3 +810,113 @@ test("plugin calls that use fs should be intercepted to use an absolute path", a
 	// expect(saveMessagesSpy).toHaveBeenCalled();
 	// expect(toBeImportedFilesSpy).toHaveBeenCalled();
 });
+
+test("it can import plugins via http", async () => {
+	const mockRepo = {
+		"/project.inlang/settings.json": JSON.stringify({
+			baseLocale: "en",
+			locales: ["en"],
+			modules: ["https://example.com/plugin.js"],
+		} satisfies ProjectSettings),
+	};
+
+	const fs = Volume.fromJSON(mockRepo);
+
+	const mockPluginModule = `export default {
+			key: "mock-plugin"	
+		}`;
+
+	global.fetch = vi.fn(() => Promise.resolve(new Response(mockPluginModule)));
+
+	const project = await loadProjectFromDirectory({
+		fs: fs as any,
+		path: "/project.inlang",
+	});
+
+	const plugins = await project.plugins.get();
+
+	expect(global.fetch).toHaveBeenCalledWith("https://example.com/plugin.js");
+	expect(plugins.length).toBe(1);
+
+	const pluginCache = await project.lix.db
+		.selectFrom("file")
+		.selectAll()
+		.where("path", "like", "/cache/plugins/%")
+		.execute();
+
+	expect(
+		pluginCache.some(
+			(file) => new TextDecoder().decode(file.data) === mockPluginModule
+		),
+		"expecting the plugin to be cached"
+	).toBe(true);
+});
+
+test("plugins that provide both loadMessages and importFiles should be allowed and the importFiles should be called", async () => {
+	const mockRepo = {
+		"/project.inlang/settings.json": JSON.stringify({
+			baseLocale: "en",
+			locales: ["en"],
+			modules: [],
+		} satisfies ProjectSettings),
+	};
+
+	const fs = Volume.fromJSON(mockRepo);
+
+	const mockPlugin: InlangPlugin = {
+		key: "mock-plugin",
+		loadMessages: vi.fn(async () => []),
+		importFiles: vi.fn(async () => {
+			return { bundles: [], messages: [], variants: [] };
+		}),
+	};
+
+	const project = await loadProjectFromDirectory({
+		fs: fs as any,
+		path: "/project.inlang",
+		providePlugins: [mockPlugin],
+	});
+
+	expect(mockPlugin.importFiles).toHaveBeenCalled();
+	expect(mockPlugin.loadMessages).not.toHaveBeenCalled();
+});
+
+test("providing multiple plugins that have legacy loadMessages and saveMessages function should be possible if they have import/export functions", async () => {
+	const mockRepo = {
+		"/project.inlang/settings.json": JSON.stringify({
+			baseLocale: "en",
+			locales: ["en"],
+			modules: [],
+		} satisfies ProjectSettings),
+	};
+
+	const fs = Volume.fromJSON(mockRepo);
+
+	const mockPlugin1: InlangPlugin = {
+		key: "mock-plugin1",
+		loadMessages: vi.fn(async () => []),
+		importFiles: vi.fn(async () => {
+			return { bundles: [], messages: [], variants: [] };
+		}),
+	};
+
+	const mockPlugin2: InlangPlugin = {
+		key: "mock-plugin2",
+		loadMessages: vi.fn(async () => []),
+		importFiles: vi.fn(async () => {
+			return { bundles: [], messages: [], variants: [] };
+		}),
+	};
+
+	const project = await loadProjectFromDirectory({
+		fs: fs as any,
+		path: "/project.inlang",
+		providePlugins: [mockPlugin1, mockPlugin2],
+	});
+
+	expect(mockPlugin1.importFiles).toHaveBeenCalled();
+	expect(mockPlugin1.loadMessages).not.toHaveBeenCalled();
+
+	expect(mockPlugin2.importFiles).toHaveBeenCalled();
+	expect(mockPlugin2.loadMessages).not.toHaveBeenCalled();
+});

package/src/project/loadProjectFromDirectory.ts

@@ -1,9 +1,7 @@
 import { newProject } from "./newProject.js";
 import { loadProjectInMemory } from "./loadProjectInMemory.js";
 import { type Lix } from "@lix-js/sdk";
-
 import fs from "node:fs";
-
 import nodePath from "node:path";
 import type {
 	InlangPlugin,
@@ -13,8 +11,8 @@ import { fromMessageV1 } from "../json-schema/old-v1-message/fromMessageV1.js";
 import type { ProjectSettings } from "../json-schema/settings.js";
 import type { PreprocessPluginBeforeImportFunction } from "../plugin/importPlugins.js";
 import { PluginImportError } from "../plugin/errors.js";
-import type { InlangProject } from "./api.js";
 import { upsertBundleNestedMatchByProperties } from "../import-export/upsertBundleNestedMatchByProperties.js";
+import type { ImportFile } from "./api.js";
 
 /**
  * Loads a project from a directory.
@@ -28,8 +26,14 @@ export async function loadProjectFromDirectory(
 		"blob"
 	>
 ) {
+	const settingsPath = nodePath.join(args.path, "settings.json");
+	const settings = JSON.parse(
+		await args.fs.promises.readFile(settingsPath, "utf8")
+	) as ProjectSettings;
+
 	const localImport = await importLocalPlugins({
 		fs: args.fs,
+		settings,
 		path: args.path,
 	});
 
@@ -45,7 +49,9 @@ export async function loadProjectFromDirectory(
 	const project = await loadProjectInMemory({
 		...args,
 		providePlugins: providePluginsWithLocalPlugins,
-		blob: await newProject(),
+		blob: await newProject({
+			settings,
+		}),
 	});
 
 	await syncLixFsFiles({
@@ -55,69 +61,39 @@ export async function loadProjectFromDirectory(
 		syncInterval: args.syncInterval,
 	});
 
-	const {
-		loadMessagesPlugins,
-		saveMessagesPlugins,
-		importPlugins,
-		exportPlugins,
-	} = categorizePlugins(await project.plugins.get());
+	const allPlugins = await project.plugins.get();
+	const { loadSavePlugins, importExportPlugins } =
+		categorizePlugins(allPlugins);
 
 	// TODO i guess we should move this validation logic into sdk2/src/project/loadProject.ts
 	// Two scenarios could arise:
 	// 1. set settings is called from an app - it should detect and reject the setting of settings -> app need to be able to validate before calling set
 	// 2. the settings file loaded from disc here is corrupted -> user has to fix the file on disc
-	if (loadMessagesPlugins.length > 1 || saveMessagesPlugins.length > 1) {
+	if (loadSavePlugins.length > 1) {
 		throw new Error(
 			"Max one loadMessages (found: " +
-				loadMessagesPlugins.length +
+				loadSavePlugins.length +
 				") and one saveMessages plugins (found: " +
-				saveMessagesPlugins.length +
+				loadSavePlugins.length +
 				") are allowed "
 		);
 	}
 	const importedResourceFileErrors: Error[] = [];
 
-	if (
-		(loadMessagesPlugins.length > 0 || saveMessagesPlugins.length > 0) &&
-		(exportPlugins.length > 0 || importPlugins.length > 0)
-	) {
-		throw new Error(
-			"Plugins for loadMessages (found: " +
-				loadMessagesPlugins.length +
-				") and saveMessages plugins (found: " +
-				saveMessagesPlugins.length +
-				") must not coexist with import (found: " +
-				importPlugins.length +
-				") or export (found: " +
-				exportPlugins.length +
-				") "
-		);
-	} else if (loadMessagesPlugins.length > 1 || saveMessagesPlugins.length > 1) {
-		throw new Error(
-			"Max one loadMessages (found: " +
-				loadMessagesPlugins.length +
-				") and one saveMessages plugins (found: " +
-				saveMessagesPlugins.length +
-				") are allowed "
-		);
-	} else if (importPlugins[0]) {
-		const importer = importPlugins[0];
-		const files = [];
-
-		if (importer.toBeImportedFiles) {
-			const toBeImportedFiles = await importer.toBeImportedFiles({
+	// import files from local fs
+	for (const plugin of importExportPlugins) {
+		const files: ImportFile[] = [];
+		if (plugin.toBeImportedFiles) {
+			const toBeImportedFiles = await plugin.toBeImportedFiles({
 				settings: await project.settings.get(),
 			});
 			for (const toBeImported of toBeImportedFiles) {
 				const absolute = absolutePathFromProject(args.path, toBeImported.path);
 				try {
 					const data = await args.fs.promises.readFile(absolute);
-					const name = nodePath.basename(toBeImported.path);
 					files.push({
-						name,
 						locale: toBeImported.locale,
 						content: data,
-						pluginKey: importer.key,
 						toBeImportedFilesMetadata: toBeImported.metadata,
 					});
 				} catch (e) {
@@ -136,17 +112,18 @@ export async function loadProjectFromDirectory(
 		}
 
 		await project.importFiles({
-			pluginKey: importer.key,
-			files: files as any,
+			pluginKey: plugin.key,
+			files,
 		});
-	} else if (loadMessagesPlugins[0] !== undefined) {
-		// TODO create resource files from loadMessageFn call - to poll?
+	}
+
+	for (const plugin of loadSavePlugins) {
 		await loadLegacyMessages({
 			project,
+			pluginKey: plugin.key ?? plugin.id,
+			loadMessagesFn: plugin.loadMessages!,
 			projectPath: args.path,
 			fs: args.fs,
-			pluginKey: loadMessagesPlugins[0].key ?? loadMessagesPlugins[0].id,
-			loadMessagesFn: loadMessagesPlugins[0].loadMessages,
 		});
 	}
 
@@ -154,24 +131,19 @@ export async function loadProjectFromDirectory(
 		...project,
 		errors: {
 			get: async () => {
-				const errors = await project.errors.get();
-				return [
-					...withLocallyImportedPluginWarning(errors),
-					...localImport.errors,
-					...importedResourceFileErrors,
-				];
-			},
-			subscribe: (
-				callback: Parameters<InlangProject["errors"]["subscribe"]>[0]
-			) => {
-				return project.errors.subscribe((value) => {
-					callback([
-						...withLocallyImportedPluginWarning(value),
-						...localImport.errors,
-						...importedResourceFileErrors,
-					]);
-				});
+				return [...localImport.errors, ...importedResourceFileErrors];
 			},
+			// subscribe: (
+			// 	callback: Parameters<InlangProject["errors"]["subscribe"]>[0]
+			// ) => {
+			// 	return project.errors.subscribe((value) => {
+			// 		callback([
+			// 			...withLocallyImportedPluginWarning(value),
+			// 			...localImport.errors,
+			// 			...importedResourceFileErrors,
+			// 		]);
+			// 	});
+			// },
 		},
 	};
 }
@@ -473,9 +445,13 @@ async function syncLixFsFiles(args: {
 			if (!statesToSync.fsFileStates[path]) {
 				if (lixState.state == "unknown") {
 					// ADD TO FS (6)
+					// create directory if not exists
+					args.fs.mkdirSync(nodePath.dirname(nodePath.join(args.path, path)), {
+						recursive: true,
+					});
+					// write file
 					args.fs.writeFileSync(
-						// TODO check platform dependent folder separator
-						args.path + path,
+						nodePath.join(args.path, path),
 						Buffer.from(lixState.content)
 					);
 					statesToSync.fsFileStates[path] = {
@@ -563,9 +539,14 @@ async function upsertFileInLix(
 	path: string,
 	data: ArrayBuffer
 ) {
-	// file is in known state with lix - means we have only changes on the fs - easy
-	// NOTE we use file_internal for now see: https://linear.app/opral/issue/LIXDK-102/re-visit-simplifying-the-change-queue-implementation#comment-65eb3485
-	// This means we don't see changes for the file we update via this method!
+	// force posix path when upserting into lix
+	// https://github.com/opral/inlang-sdk/issues/229
+	let posixPath = path.split(nodePath.win32.sep).join(nodePath.posix.sep);
+
+	if (posixPath.startsWith("/") === false) {
+		posixPath = "/" + posixPath;
+	}
+
 	await args.lix.db
 		.insertInto("file") // change queue
 		.values({
@@ -577,52 +558,28 @@ async function upsertFileInLix(
 		)
 		.execute();
 }
+/**
+ * Filters legacy load and save messages plugins.
+ *
+ * Legacy plugins are plugins that implement loadMessages and saveMessages but not importFiles and exportFiles.
+ */
+function categorizePlugins(plugins: readonly InlangPlugin[]) {
+	const loadSavePlugins: InlangPlugin[] = [];
+	const importExportPlugins: InlangPlugin[] = [];
 
-// TODO i guess we should move this validation logic into sdk2/src/project/loadProject.ts
-function categorizePlugins(plugins: readonly InlangPlugin[]): {
-	loadMessagesPlugins: (InlangPlugin &
-		Required<Pick<InlangPlugin, "loadMessages">>)[];
-	saveMessagesPlugins: (InlangPlugin &
-		Required<Pick<InlangPlugin, "saveMessages">>)[];
-	importPlugins: (InlangPlugin &
-		Required<Pick<InlangPlugin, "importFiles" | "toBeImportedFiles">>)[];
-	exportPlugins: (InlangPlugin & Required<Pick<InlangPlugin, "exportFiles">>)[];
-} {
-	const loadMessagesPlugins = plugins.filter(
-		(
-			plugin
-		): plugin is InlangPlugin & Required<Pick<InlangPlugin, "loadMessages">> =>
-			plugin.loadMessages !== undefined
-	);
-
-	const saveMessagesPlugins = plugins.filter(
-		(
-			plugin
-		): plugin is InlangPlugin & Required<Pick<InlangPlugin, "saveMessages">> =>
-			plugin.saveMessages !== undefined
-	);
-
-	const importPlugins = plugins.filter(
-		(
-			plugin
-		): plugin is InlangPlugin &
-			Required<Pick<InlangPlugin, "importFiles" | "toBeImportedFiles">> =>
-			plugin.importFiles !== undefined && plugin.toBeImportedFiles !== undefined
-	);
-
-	const exportPlugins = plugins.filter(
-		(
-			plugin
-		): plugin is InlangPlugin & Required<Pick<InlangPlugin, "exportFiles">> =>
-			plugin.exportFiles !== undefined
-	);
+	for (const plugin of plugins) {
+		if (
+			plugin.loadMessages &&
+			plugin.saveMessages &&
+			!(plugin.importFiles && plugin.exportFiles)
+		) {
+			loadSavePlugins.push(plugin);
+		} else if (plugin.importFiles || plugin.exportFiles) {
+			importExportPlugins.push(plugin);
+		}
+	}
 
-	return {
-		loadMessagesPlugins,
-		saveMessagesPlugins,
-		importPlugins,
-		exportPlugins,
-	};
+	return { loadSavePlugins, importExportPlugins };
 }
 
 /**
@@ -632,16 +589,16 @@ function categorizePlugins(plugins: readonly InlangPlugin[]): {
  */
 async function importLocalPlugins(args: {
 	fs: typeof fs;
+	settings: ProjectSettings;
 	path: string;
 	preprocessPluginBeforeImport?: PreprocessPluginBeforeImportFunction;
 }) {
 	const errors: Error[] = [];
 	const locallyImportedPlugins = [];
-	const settingsPath = nodePath.join(args.path, "settings.json");
-	const settings = JSON.parse(
-		await args.fs.promises.readFile(settingsPath, "utf8")
-	) as ProjectSettings;
-	for (const module of settings.modules ?? []) {
+	for (const module of args.settings.modules ?? []) {
+		if (module.startsWith("http")) {
+			continue;
+		}
 		const modulePath = absolutePathFromProject(args.path, module);
 		try {
 			let moduleAsText = await args.fs.promises.readFile(modulePath, "utf8");
@@ -669,27 +626,6 @@ async function importLocalPlugins(args: {
 	};
 }
 
-function withLocallyImportedPluginWarning(errors: readonly Error[]) {
-	return errors.map((error) => {
-		if (
-			error instanceof PluginImportError &&
-			error.plugin.startsWith("http") === false
-		) {
-			return new WarningLocalPluginImport(error.plugin);
-		}
-		return error;
-	});
-}
-
-export class WarningLocalPluginImport extends Error {
-	constructor(module: string) {
-		super(
-			`Plugin ${module} is imported from a local path. This will work fine in dev tools like Sherlock or Paraglide JS but is not portable. Web apps like Fink or Parrot won't be able to import this plugin. It is recommended to use an http url to import plugins. The plugins are cached locally and will be available offline.`
-		);
-		this.name = "WarningLocalImport";
-	}
-}
-
 export class WarningDeprecatedLintRule extends Error {
 	constructor(module: string) {
 		super(

package/src/project/loadProjectInMemory.ts

@@ -12,6 +12,7 @@ export async function loadProjectInMemory(
 ) {
 	const lix = await openLixInMemory({
 		blob: args.blob,
+		account: args.account,
 		providePlugins: [
 			// inlangLixPluginV1
 		],