@inlang/sdk 0.36.4 → 2.0.0-beta.1

package/src/database/jsonbPlugin.ts

@@ -0,0 +1,215 @@
+import {
+	OperationNodeTransformer,
+	sql,
+	ValueListNode,
+	ValueNode,
+	ValuesNode,
+	ParseJSONResultsPlugin,
+	type KyselyPlugin,
+	type PluginTransformQueryArgs,
+	type PluginTransformResultArgs,
+	type QueryResult,
+	type RootOperationNode,
+	type UnknownRow,
+	OnConflictNode,
+} from "kysely";
+import type { SqliteDatabase } from "sqlite-wasm-kysely";
+
+export class JsonbPlugin implements KyselyPlugin {
+	#serializeJsonTransformer = new SerializeJsonbTransformer();
+	#parseJsonPlugin = new ParseJSONResultsPlugin();
+	#database: SqliteDatabase;
+
+	constructor(args: { database: SqliteDatabase }) {
+		this.#database = args.database;
+	}
+
+	/**
+	 * For an outgoing query like insert or update, the JSON
+	 * values are transformed into `jsonb` function calls when
+	 * executed against the database.
+	 */
+	transformQuery(args: PluginTransformQueryArgs): RootOperationNode {
+		if (
+			args.node.kind === "InsertQueryNode" ||
+			args.node.kind === "UpdateQueryNode"
+		) {
+			const result = this.#serializeJsonTransformer.transformNode(args.node);
+			return result;
+		}
+		return args.node;
+	}
+
+	/**
+	 * For incoming query results, the JSON binaries are parsed
+	 * into JSON objects.
+	 */
+	async transformResult(
+		args: PluginTransformResultArgs
+	): Promise<QueryResult<UnknownRow>> {
+		for (const row of args.result.rows) {
+			for (const key in row) {
+				if (
+					row[key] instanceof ArrayBuffer ||
+					// uint8array, etc
+					ArrayBuffer.isView(row[key])
+				) {
+					try {
+						const res = this.#database.exec(`SELECT json(?)`, {
+							returnValue: "resultRows",
+							bind: [row[key] as any],
+						});
+
+						row[key] = JSON.parse(res[0] as any);
+					} catch {
+						// it's not a json binary
+					}
+				}
+			}
+		}
+		// in case it's a regular (text) json, run it through kyseley's json parser
+		return this.#parseJsonPlugin.transformResult(args);
+	}
+}
+
+class SerializeJsonbTransformer extends OperationNodeTransformer {
+	protected override transformOnConflict(node: OnConflictNode): OnConflictNode {
+		return super.transformOnConflict({
+			...node,
+			updates: node.updates?.map((updateItem) => {
+				if (updateItem.kind !== "ColumnUpdateNode") {
+					return updateItem;
+				}
+				return {
+					kind: "ColumnUpdateNode",
+					column: updateItem.column,
+					// @ts-expect-error - we know that the value is a ValueNode
+					value: this.transformValue(updateItem.value),
+				};
+			}),
+		});
+	}
+
+	protected override transformValue(node: ValueNode): ValueNode {
+		const { value } = node;
+		const serializedValue = maybeSerializeJson(value);
+		if (value === serializedValue) {
+			return node;
+		}
+		// @ts-expect-error - we know that the node is a ValueNode
+		return sql`jsonb(${serializedValue})`.toOperationNode();
+	}
+	/**
+	 * Transforms the value list node by replacing all JSON objects with `jsonb` function calls.
+	 */
+	protected override transformValueList(node: ValueListNode): ValueListNode {
+		return super.transformValueList({
+			...node,
+			values: node.values.map((listNodeItem) => {
+				if (listNodeItem.kind !== "ValueNode") {
+					return listNodeItem;
+				}
+				// @ts-expect-error - we know that the node is a ValueNode
+				const { value } = listNodeItem;
+				const serializedValue = maybeSerializeJson(value);
+
+				if (value === serializedValue) {
+					return listNodeItem;
+				}
+				return sql`jsonb(${serializedValue})`.toOperationNode();
+			}),
+		});
+	}
+
+	/**
+	 * Why this function is needed or why this works remains a mystery.
+	 */
+	override transformValues(node: ValuesNode): ValuesNode {
+		return super.transformValues({
+			...node,
+			values: node.values.map((valueItemNode) => {
+				if (valueItemNode.kind !== "PrimitiveValueListNode") {
+					return valueItemNode;
+				}
+
+				// change valueItem to ValueListNode
+				return {
+					kind: "ValueListNode",
+					values: valueItemNode.values.map(
+						(value) =>
+							({
+								kind: "ValueNode",
+								value,
+							}) as ValueNode
+					),
+				} as ValueListNode;
+			}),
+		});
+	}
+}
+
+function maybeSerializeJson(value: any): any {
+	if (
+		// binary data
+		value instanceof ArrayBuffer ||
+		// uint8array, etc
+		ArrayBuffer.isView(value) ||
+		value === null ||
+		value === undefined
+	) {
+		return value;
+	} else if (typeof value === "object" || Array.isArray(value)) {
+		return JSON.stringify(value);
+	}
+	return value;
+}
+
+// The code here didn't work https://github.com/opral/inlang-sdk/issues/132#issuecomment-2339321910
+// but would be the "right" solution to avoid heuristics which column might or might not be a json column
+// // modifies the query in place for readability and performance
+// function mapQuery(
+// 	node: InsertQueryNode,
+// 	jsonColumns: TableSchema
+// ): InsertQueryNode {
+// 	// if the query is not an insert query, we don't need to do anything
+// 	if (node.into === undefined) {
+// 		return node;
+// 	}
+// 	// if the table is not in the schema that has json columns, we don't need to do anything
+// 	const columnsWithJson = jsonColumns[node.into.table.identifier.name];
+// 	if (columnsWithJson === undefined) {
+// 		return node;
+// 	}
+// 	// find the indexes of the values that need to be transformed
+// 	// SQL query: INSERT INTO table (col1, col2) VALUES (val1, val2)
+// 	const indexesThatNeedToBeTransformed: [number, string][] = [];
+// 	for (const [i, col] of node.columns?.entries() ?? []) {
+// 		const jsonType = columnsWithJson[col.column.name];
+// 		if (jsonType !== undefined) {
+// 			indexesThatNeedToBeTransformed.push([i, jsonType]);
+// 		}
+// 	}
+// 	const values = structuredClone(node.values);
+// 	for (const [i, jsonType] of indexesThatNeedToBeTransformed) {
+// 		if (
+// 			// top level values node that should contain a list of values
+// 			node.values?.kind !== "ValuesNode" &&
+// 			// the node we are interested in must be a value node
+// 			// @ts-expect-error - we know that the node is a ValuesNode with values
+// 			(node.values as ValuesNode).values?.[i].kind !== "ValueNode"
+// 		) {
+// 			throw new Error("Unexpected node structure");
+// 		}
+// 		const serializedJson = JSON.stringify(node.values.values[0].values[i]);
+// 		// @ts-expect-error - we know that the node is a ValuesNode with values
+// 		values.values[0].values[i] =
+// 			jsonType === "jsonb"
+// 				? sql`jsonb(${serializedJson})`.toOperationNode()
+// 				: sql`json(${serializedJson})`.toOperationNode();
+// 	}
+
+// 	return {
+// 		...node,
+// 		values,
+// 	};
+// }

package/src/database/schema.ts

@@ -0,0 +1,127 @@
+import type { Generated, Insertable, Selectable, Updateable } from "kysely";
+import type { SqliteDatabase } from "sqlite-wasm-kysely";
+import {
+	Declaration,
+	Pattern,
+	VariableReference,
+} from "../json-schema/pattern.js";
+
+export function applySchema(args: { sqlite: SqliteDatabase }) {
+	const foreignKeyActivated: any = args.sqlite.exec("PRAGMA foreign_keys", {
+		returnValue: "resultRows",
+	});
+	if (
+		// first row that is returned
+		// first column of the first row
+		// is equal to 0, then foreign keys are disabled
+		foreignKeyActivated[0][0] === 0
+	) {
+		args.sqlite.exec("PRAGMA foreign_keys = ON", {
+			returnValue: "resultRows",
+		});
+	}
+
+	args.sqlite.exec(`
+CREATE TABLE IF NOT EXISTS bundle (
+  id TEXT PRIMARY KEY DEFAULT (human_id()),
+	declarations BLOB NOT NULL DEFAULT (jsonb('[]'))
+) strict;
+
+CREATE TABLE IF NOT EXISTS message (
+  id TEXT PRIMARY KEY DEFAULT (uuid_v7()), 
+  bundle_id TEXT NOT NULL,
+  locale TEXT NOT NULL,
+  selectors BLOB NOT NULL DEFAULT (jsonb('[]')),
+  FOREIGN KEY (bundle_id) REFERENCES bundle(id) ON DELETE CASCADE
+) strict;
+
+
+CREATE TABLE IF NOT EXISTS variant (
+  id TEXT PRIMARY KEY DEFAULT (uuid_v7()), 
+  message_id TEXT NOT NULL,
+  matches BLOB NOT NULL DEFAULT (jsonb('[]')),
+  pattern BLOB NOT NULL DEFAULT (jsonb('[]')),
+  FOREIGN KEY (message_id) REFERENCES message(id) ON DELETE CASCADE
+) strict;
+  
+CREATE INDEX IF NOT EXISTS idx_message_bundle_id ON message (bundle_id);
+CREATE INDEX IF NOT EXISTS idx_variant_message_id ON variant (message_id);
+		`);
+}
+
+export type InlangDatabaseSchema = {
+	bundle: BundleTable;
+	message: MessageTable;
+	variant: VariantTable;
+};
+
+type BundleTable = {
+	id: Generated<string>;
+	declarations: Generated<Array<Declaration>>;
+};
+
+type MessageTable = {
+	id: Generated<string>;
+	bundleId: string;
+	locale: string;
+	selectors: Generated<Array<VariableReference>>;
+};
+
+type VariantTable = {
+	id: Generated<string>;
+	messageId: string;
+	matches: Generated<Array<Match>>;
+	pattern: Generated<Pattern>;
+};
+
+/**
+ * A match is a variable reference that is either a literal or a catch-all.
+ *
+ * https://github.com/opral/inlang-sdk/issues/205
+ *
+ * @example
+ *   match = { type: "match", name: "gender", value: { type: "literal", value: "male"  }}
+ */
+export type Match = LiteralMatch | CatchAllMatch;
+
+export type LiteralMatch = {
+	type: "literal-match";
+	key: VariableReference["name"];
+	value: string;
+};
+export type CatchAllMatch = {
+	type: "catchall-match";
+	key: VariableReference["name"];
+};
+
+export type Bundle = Selectable<BundleTable>;
+export type NewBundle = Insertable<BundleTable>;
+export type BundleUpdate = Updateable<BundleTable>;
+
+export type Message = Selectable<MessageTable>;
+export type NewMessage = Insertable<MessageTable>;
+export type MessageUpdate = Updateable<MessageTable>;
+
+export type Variant = Selectable<VariantTable>;
+export type NewVariant = Insertable<VariantTable>;
+export type VariantUpdate = Updateable<VariantTable>;
+
+export type MessageNested = Message & {
+	variants: Variant[];
+};
+export type NewMessageNested = NewMessage & {
+	variants: NewVariant[];
+};
+export type MessageNestedUpdate = Updateable<MessageTable> & {
+	variants: VariantUpdate[];
+};
+
+export type BundleNested = Bundle & {
+	messages: MessageNested[];
+};
+export type NewBundleNested = NewBundle & {
+	messages: NewMessageNested[];
+};
+export type BundleNestedUpdate = BundleUpdate & {
+	messages: MessageNestedUpdate[];
+};

package/src/helper.ts

@@ -0,0 +1,113 @@
+import { v4 as uuid } from "uuid";
+import type { ProjectSettings } from "./json-schema/settings.js";
+import { humanId } from "./human-id/human-id.js";
+import type {
+	Match,
+	NewBundleNested,
+	NewMessageNested,
+	Variant,
+} from "./database/schema.js";
+import type { Text } from "./json-schema/pattern.js";
+
+/**
+ * create v2 Bundle with a random human ID
+ *
+ * @deprecated
+ *
+ * use the database directly
+ *
+ * - less code because the database has default values
+ * - `createMessage` is misleading because it does not treat expressions in the text
+ *
+ * @example createBundle({
+ *   messages: [
+ * 		 createMessage({locale: "en", text: "Hello world!"})
+ * 		 createMessage({locale: "de", text: "Hallo Welt!"})
+ *   ]
+ * })
+ */
+export function createBundle(args: {
+	id?: string;
+	messages: NewMessageNested[];
+}): NewBundleNested {
+	return {
+		id: args.id ?? humanId(),
+		messages: args.messages,
+	};
+}
+
+/**
+ *
+ * @deprecated
+ * use the database directly
+ *
+ * - text will always be a string, no matter
+ *   if an expression is provided like hello "{username}"
+ * - the database has default values
+ *
+ * ```
+ * await project.db.insertInto("message").values({
+ * 		bundleId: "bundleId",
+ *    pattern: []
+ * 		...
+ * })
+ * ```
+ *
+ * create v2 Messsage AST with a randomId, and text-only pattern
+ * @example createMessage({locale: "en", text: "Hello world"})
+ */
+export function createMessage(args: {
+	bundleId: string;
+	locale: ProjectSettings["locales"][number];
+	text: string;
+	matches?: Match[];
+}): NewMessageNested {
+	const messageId = uuid();
+	return {
+		bundleId: args.bundleId,
+		id: messageId,
+		locale: args.locale,
+		selectors: [],
+		variants: [
+			createVariant({
+				messageId: messageId,
+				text: args.text,
+				matches: args.matches,
+			}),
+		],
+	};
+}
+
+/**
+ *
+ * @deprecated
+ *
+ * use the database directly
+ *
+ * - less code because the database has default values
+ * - `text` is misleading because it does not treat expressions in the text
+ *
+ * create v2 Variant AST with text-only pattern
+ * @example createVariant({match: ["*"], text: "Hello world"})
+ */
+export function createVariant(args: {
+	messageId: string;
+	id?: string;
+	text?: string;
+	matches?: Match[];
+	pattern?: Variant["pattern"];
+}): Variant {
+	return {
+		messageId: args.messageId,
+		id: args.id ? args.id : uuid(),
+		matches: args.matches ? args.matches : [],
+		pattern: args.pattern ? args.pattern : [toTextElement(args.text ?? "")],
+	};
+}
+
+export function toTextElement(text: string): Text {
+	return {
+		type: "text",
+		value: text,
+	};
+}

package/src/human-id/human-id.ts

@@ -0,0 +1,14 @@
+import { adjectives, animals, verbs } from "./words.js";
+
+export function humanId() {
+	return `${adjectives[Math.floor(Math.random() * 256)]}_${
+		adjectives[Math.floor(Math.random() * 256)]
+	}_${animals[Math.floor(Math.random() * 256)]}_${
+		verbs[Math.floor(Math.random() * 256)]
+	}`;
+}
+
+export function isHumanId(id: string): boolean {
+	// naive implementation (good enough for now)
+	return id.split("_").length === 4;
+}

package/src/{storage/human-id → human-id}/words.test.ts

@@ -1,27 +1,27 @@
-import { describe, it, expect } from "vitest"
-import { adjectives, animals, adverbs, verbs } from "./words.js"
+import { describe, it, expect } from "vitest";
+import { adjectives, animals, adverbs, verbs } from "./words.js";
 
-const wordlists = [adjectives, animals, adverbs, verbs]
-const allwords = [...adjectives, ...animals, ...adverbs, ...verbs]
+const wordlists = [adjectives, animals, adverbs, verbs];
+const allwords = [...adjectives, ...animals, ...adverbs, ...verbs];
 
 describe("wordlists", () => {
 	it("should have 256 words", () => {
 		for (const wordlist of wordlists) {
-			expect(wordlist.length).toBe(256)
+			expect(wordlist.length).toBe(256);
 		}
-	})
+	});
 	it("words should be unique across lists", () => {
-		const unique = new Set(allwords)
-		expect(unique.size).toBe(256 * 4)
-	})
+		const unique = new Set(allwords);
+		expect(unique.size).toBe(256 * 4);
+	});
 	it("words should have < 10 characters", () => {
 		for (const word of allwords) {
-			expect(word.length).toBeLessThan(10)
+			expect(word.length).toBeLessThan(10);
 		}
-	})
+	});
 	it("words should match /^[a-z]+$/", () => {
 		for (const word of allwords) {
-			expect(word.match(/^[a-z]+$/) !== null).toBe(true)
+			expect(word.match(/^[a-z]+$/) !== null).toBe(true);
 		}
-	})
-})
+	});
+});

package/src/{storage/human-id → human-id}/words.ts

@@ -255,7 +255,7 @@ export const animals = [
 	"wren",
 	"yak",
 	"zebra",
-]
+];
 
 export const adjectives = [
 	"acidic",
@@ -514,7 +514,7 @@ export const adjectives = [
 	"zany",
 	"zesty",
 	"zippy",
-]
+];
 
 export const adverbs = [
 	"ablaze",
@@ -773,7 +773,7 @@ export const adverbs = [
 	"within",
 	"wrongly",
 	"yonder",
-]
+];
 
 export const verbs = [
 	"absorb",
@@ -1032,4 +1032,4 @@ export const verbs = [
 	"zap",
 	"zip",
 	"zoom",
-]
+];

package/src/import-export/exportFiles.ts

@@ -0,0 +1,36 @@
+import type { Kysely } from "kysely";
+import {
+	PluginDoesNotImplementFunctionError,
+	PluginMissingError,
+} from "../plugin/errors.js";
+import type { ProjectSettings } from "../json-schema/settings.js";
+import type { InlangDatabaseSchema } from "../database/schema.js";
+import type { InlangPlugin } from "../plugin/schema.js";
+
+export async function exportFiles(opts: {
+	readonly pluginKey: string;
+	readonly settings: ProjectSettings;
+	readonly plugins: readonly InlangPlugin[];
+	readonly db: Kysely<InlangDatabaseSchema>;
+}) {
+	const plugin = opts.plugins.find((p) => p.key === opts.pluginKey);
+	if (!plugin) throw new PluginMissingError({ plugin: opts.pluginKey });
+	if (!plugin.exportFiles) {
+		throw new PluginDoesNotImplementFunctionError({
+			plugin: opts.pluginKey,
+			function: "exportFiles",
+		});
+	}
+
+	const bundles = await opts.db.selectFrom("bundle").selectAll().execute();
+	const messages = await opts.db.selectFrom("message").selectAll().execute();
+	const variants = await opts.db.selectFrom("variant").selectAll().execute();
+
+	const files = await plugin.exportFiles({
+		settings: structuredClone(opts.settings),
+		bundles,
+		messages,
+		variants,
+	});
+	return files;
+}