@lobb-js/core 0.13.0

package/src/api/schema/SchemaService.ts

@@ -0,0 +1,71 @@
+import type { CollectionConfig } from "../../types/index.ts";
+import { Lobb } from "../../Lobb.ts";
+import type { DatabaseDriver } from "../../types/index.ts";
+
+export class SchemaService {
+  private dbDriver: DatabaseDriver;
+
+  constructor(dbDriver: DatabaseDriver) {
+    this.dbDriver = dbDriver;
+  }
+
+  public static async init() {
+    const driver = Lobb.instance.databaseService.getDriver();
+    const databaseSyncManager = new SchemaService(driver);
+    return databaseSyncManager;
+  }
+
+  public async getDiff() {
+    return {
+      configSchema: Lobb.instance.configManager.getDbSchema(),
+      dbSchema: await this.dbDriver.getDbSchema(),
+    };
+  }
+
+  public async createCollection(
+    collectionName: string,
+    collection: CollectionConfig,
+    effectDb: boolean = true,
+  ) {
+    if (effectDb) {
+      await this.dbDriver.createCollection(
+        collectionName,
+        collection,
+      );
+    }
+    Lobb.instance.configManager.addCollection(
+      collectionName,
+      collection,
+    );
+  }
+
+  public async updateCollection(
+    collectionName: string,
+    collection: CollectionConfig,
+    effectDb: boolean = true,
+  ) {
+    if (effectDb) {
+      await Lobb.instance.databaseSyncManager.syncDatabase(
+        collectionName,
+      );
+    }
+    Lobb.instance.configManager.updateCollection(
+      collectionName,
+      collection,
+    );
+  }
+
+  public async deleteCollection(
+    collectionName: string,
+    effectDb: boolean = true,
+  ) {
+    if (effectDb) {
+      await this.dbDriver.dropCollection(
+        collectionName,
+      );
+    }
+    Lobb.instance.configManager.removeCollection(
+      collectionName,
+    );
+  }
+}

package/src/api/schema/schemaRoutes.ts

@@ -0,0 +1,13 @@
+import { type Context, Hono } from "hono";
+import { Lobb } from "@lobb-js/core";
+
+export function getSchemaRoutes() {
+  const route = new Hono();
+
+  route.get("diff", async (c: Context) => {
+    const difference = await Lobb.instance.schemaService.getDiff();
+    return c.json(difference);
+  });
+
+  return route;
+}

package/src/config/ConfigManager.ts

@@ -0,0 +1,252 @@
+import type { Config } from "../types/index.ts";
+import {
+  type CollectionConfig,
+  CollectionConfigSchema,
+  type CollectionsConfig,
+} from "../types/index.ts";
+import type { CollectionField } from "../types/index.ts";
+import type { Extension } from "../types/index.ts";
+import type { RelationsConfig } from "../types/index.ts";
+
+import _ from "lodash";
+import { coreCollections } from "../coreCollections/index.ts";
+import { LobbError } from "../LobbError.ts";
+import { validateConfig } from "./validations.ts";
+
+/**
+ * Config class is responsible for loading the user configuration and checking if
+ * it's valid
+ */
+export class ConfigManager {
+  public config!: Config;
+
+  constructor(config: Config) {
+    this.config = config;
+  }
+
+  /**
+   * Loads the data and checks if its valid
+   */
+  static async init(config: Config) {
+    const configManager = new ConfigManager(config);
+    configManager.loadCoreCollections();
+    validateConfig(configManager.config, configManager);
+    return configManager;
+  }
+
+  private loadCoreCollections() {
+    this.config.collections = {
+      ...this.config.collections,
+      ...coreCollections(),
+    };
+  }
+
+  public addRelationsFromIntegerRefrences() {
+    for (
+      const [collectionName, collectionValue] of Object.entries(
+        this.config.collections,
+      )
+    ) {
+      for (
+        const [fieldName, fieldValue] of Object.entries(
+          this.config.collections[collectionName].fields,
+        )
+      ) {
+        if (fieldValue.type === "integer" && fieldValue.references) {
+          if (!this.config.collections[fieldValue.references.collection]) {
+            throw new LobbError({
+              code: "INTERNAL_SERVER_ERROR",
+              message: `Field (${collectionName}.${fieldName}) references collection (${fieldValue.references.collection}) which does not exist`,
+            });
+          }
+          if (!this.config.relations) {
+            this.config.relations = [];
+          }
+          this.config.relations.push({
+            from: {
+              collection: collectionName,
+              field: fieldName,
+            },
+            to: {
+              collection: fieldValue.references.collection,
+              field: fieldValue.references.field,
+            },
+          });
+        }
+      }
+    }
+  }
+
+  public getDbSchema(): CollectionsConfig {
+    const collections = _.cloneDeep(this.config.collections);
+
+    // Iterate over the collections and map each field to only include the 'type' property
+    for (const collectionName in collections) {
+      // Keep only the 'indexes' and 'fields' properties in a collection
+      collections[collectionName] = {
+        indexes: collections[collectionName].indexes,
+        fields: collections[collectionName].fields,
+      };
+      const fields = collections[collectionName].fields;
+      for (const fieldName in fields) {
+        // including only the properties that are db related
+        const field: any = {};
+        field["type"] = fields[fieldName].type;
+        // TODO: this is a property related to only varchar fields. have a better way to
+        // include local field properties that are the related to the db instead of
+        // hardcoding them here
+        if (fields[fieldName].type === "string" && fields[fieldName].length) {
+          field["length"] = fields[fieldName].length;
+        }
+
+        fields[fieldName] = field;
+      }
+    }
+
+    return collections;
+  }
+
+  public getCollections(): CollectionsConfig {
+    const collection = this.config.collections;
+
+    return collection;
+  }
+
+  public getCollection(collectionName: string): CollectionConfig {
+    const collection = this.config.collections[collectionName];
+
+    if (!collection) {
+      throw new LobbError({
+        code: "INTERNAL_SERVER_ERROR",
+        message: `The (${collectionName}) collection doesnt exist`,
+      });
+    }
+
+    return collection;
+  }
+
+  public getFieldNames(collectionName: string) {
+    return Object.keys(
+      this.getCollection(collectionName).fields,
+    );
+  }
+
+  public fieldExists(
+    fieldName: string,
+    collectionName: string,
+  ): boolean {
+    return Boolean(
+      this.getCollection(collectionName).fields[fieldName],
+    );
+  }
+
+  public collectionExists(collectionName: string): boolean {
+    return Boolean(this.config.collections[collectionName]);
+  }
+
+  public getField(
+    fieldName: string,
+    collectionName: string,
+  ): CollectionField {
+    const field = this.getCollection(collectionName).fields[fieldName];
+    if (!field) {
+      throw new LobbError({
+        code: "BAD_REQUEST",
+        message: `The (${fieldName}) Field Doesnt Exist`,
+      });
+    }
+    return field;
+  }
+
+  public getCollectionsNames() {
+    if (!this.config) {
+      throw new LobbError({
+        code: "INTERNAL_SERVER_ERROR",
+        message: "the data object is not found",
+      });
+    }
+    return Object.keys(this.config.collections);
+  }
+
+  public getExtensions() {
+    return this.config.extensions;
+  }
+
+  public getExtension(extensionName: string): Extension | null {
+    const extensions = this.config.extensions;
+
+    if (extensions) {
+      for (let index = 0; index < extensions.length; index++) {
+        const extension = extensions[index];
+        if (extensionName === extension.name) {
+          return extension;
+        }
+      }
+    }
+
+    return null;
+  }
+
+  public getExtensionNames() {
+    const extensions = this.config.extensions;
+    if (extensions) {
+      return extensions.map((ext) => ext.name);
+    }
+    return [];
+  }
+
+  public getVirtualCollections(): CollectionsConfig {
+    const virtualCollections: CollectionsConfig = {};
+    const collections = this.getCollections();
+    for (const [collectionName, collection] of Object.entries(collections)) {
+      const collectionFieldsNames = Object.keys(collection.fields);
+      if (collectionFieldsNames.length <= 1) {
+        virtualCollections[collectionName] = collection;
+      }
+    }
+    return virtualCollections;
+  }
+
+  public addCollection(collectionName: string, collectionValue: unknown) {
+    const collectionExists = this.collectionExists(collectionName);
+    if (collectionExists) {
+      throw new LobbError({
+        code: "BAD_REQUEST",
+        message: `The (${collectionName}) collection already exists`,
+      });
+    }
+    const parsedCollectionValue = CollectionConfigSchema.parse(collectionValue);
+    this.config.collections[collectionName] = parsedCollectionValue;
+  }
+
+  public updateCollection(collectionName: string, collectionValue: unknown) {
+    const collectionExists = this.collectionExists(collectionName);
+    if (!collectionExists) {
+      throw new LobbError({
+        code: "BAD_REQUEST",
+        message: `The (${collectionName}) collection doesnt exist`,
+      });
+    }
+    const parsedCollectionValue = CollectionConfigSchema.parse(collectionValue);
+    this.config.collections[collectionName] = parsedCollectionValue;
+  }
+
+  public removeCollection(collectionName: string) {
+    delete this.config.collections[collectionName];
+  }
+
+  public getChildRelations(collectionName: string): RelationsConfig {
+    if (this.config.relations) {
+      const childrenRelations = this.config.relations.filter((relation) =>
+        relation.to.collection === collectionName
+      );
+      return childrenRelations;
+    }
+
+    return [];
+  }
+
+  public isCollectionSingleton(collectionName: string): boolean {
+    return Boolean(this.config.collections[collectionName]?.singleton);
+  }
+}

package/src/config/validations.ts

@@ -0,0 +1,49 @@
+import type { Config } from "../types/index.ts";
+import type { ConfigManager } from "./ConfigManager.ts";
+
+export function validateConfig(config: Config, configManager: ConfigManager) {
+  validateRelations(config, configManager);
+  validateDBNamesAreSnakeCase(config);
+}
+
+function validateRelations(config: Config, configManager: ConfigManager) {
+  if (config.relations) {
+    for (let index = 0; index < config.relations.length; index++) {
+      const relation = config.relations[index];
+      configManager.getCollection(relation.from.collection);
+      configManager.getField(relation.from.field, relation.from.collection);
+      if (!Array.isArray(relation.to)) {
+        configManager.getCollection(relation.to.collection);
+        configManager.getField(relation.to.field, relation.to.collection);
+      }
+    }
+  }
+}
+
+function validateDBNamesAreSnakeCase(
+  config: Config,
+) {
+  function isSnakeCase(str: string): boolean {
+    return /^[a-z]+(_[a-z]+)*$/.test(str);
+  }
+  for (
+    const [collectionName, collectionValue] of Object.entries(
+      config.collections,
+    )
+  ) {
+    if (!isSnakeCase(collectionName)) {
+      throw new Error(
+        `The name of the (${collectionName}) collection should be snake case`,
+      );
+    }
+    for (
+      const [fieldName, _] of Object.entries(collectionValue.fields)
+    ) {
+      if (!isSnakeCase(fieldName)) {
+        throw new Error(
+          `The name of the (${collectionName}.${fieldName}) field name should be snake case`,
+        );
+      }
+    }
+  }
+}

package/src/coreCollections/collectionsCollection.ts

@@ -0,0 +1,56 @@
+import {
+  type CollectionConfig,
+  CollectionConfigSchema,
+} from "../types/index.ts";
+
+import { convertSchemaToTypeString } from "../utils/utils.ts";
+
+export function getCollectionsCollection(): CollectionConfig {
+  return {
+    "indexes": {
+      "core_collections_unique_index": {
+        "unique": true,
+        "fields": {
+          "name": {
+            "order": "asc",
+          },
+        },
+      },
+    },
+    "fields": {
+      "id": {
+        "type": "integer",
+      },
+      "name": {
+        "type": "string",
+        "length": 255,
+        "validators": {
+          "required": true,
+        },
+      },
+      "value": {
+        "type": "text",
+        "validators": {
+          "required": true,
+        },
+        "pre_processors": {
+          "default":
+            'async function collection(): Promise<Collection> {\n\treturn {\n\t\tindexes: {},\n\t\tfields: {\n\t\t\tid: {\n\t\t\t\ttype: "integer",\n\t\t\t},\n\t\t},\n\t};\n}',
+        },
+        "ui": {
+          "input": {
+            "type": "code",
+            "args": {
+              "type": "typescript",
+              "height": 200,
+              "types": convertSchemaToTypeString(
+                "Collection",
+                CollectionConfigSchema,
+              ),
+            },
+          },
+        },
+      },
+    },
+  };
+}

package/src/coreCollections/index.ts

@@ -0,0 +1,14 @@
+import type { CollectionConfig } from "../types/index.ts";
+import { migrationsCollection } from "./migrationsCollection.ts";
+import { workflowsCollection } from "./workflowsCollection.ts";
+import { getCollectionsCollection } from "./collectionsCollection.ts";
+import { getQueryCollection } from "./queryCollection.ts";
+
+export function coreCollections(): Record<string, CollectionConfig> {
+  const collectionsSchemas: Record<string, CollectionConfig> = {};
+  collectionsSchemas["core_workflows"] = workflowsCollection;
+  collectionsSchemas["core_migrations"] = migrationsCollection;
+  collectionsSchemas["core_collections"] = getCollectionsCollection();
+  collectionsSchemas["core_query"] = getQueryCollection();
+  return collectionsSchemas;
+}

package/src/coreCollections/migrationsCollection.ts

@@ -0,0 +1,36 @@
+import type { CollectionConfig } from "../types/index.ts";
+
+export const migrationsCollection: CollectionConfig = {
+  "indexes": {
+    "core_migrations_unique_index": {
+      "unique": true,
+      "fields": {
+        "owner": {
+          "order": "asc",
+        },
+        "migration_name": {
+          "order": "asc",
+        },
+      },
+    },
+  },
+  "fields": {
+    "id": {
+      "type": "integer",
+    },
+    "owner": {
+      "type": "string",
+      "length": 255,
+      "validators": {
+        "required": true,
+      },
+    },
+    "migration_name": {
+      "type": "string",
+      "length": 255,
+      "validators": {
+        "required": true,
+      },
+    },
+  },
+};

package/src/coreCollections/queryCollection.ts

@@ -0,0 +1,26 @@
+import type { CollectionConfig } from "../types/index.ts";
+
+export function getQueryCollection(): CollectionConfig {
+  return {
+    "indexes": {},
+    "fields": {
+      "id": {
+        "type": "integer",
+      },
+      "query": {
+        "type": "text",
+        "validators": {
+          "required": true,
+        },
+        "ui": {
+          "input": {
+            "type": "code",
+            "args": {
+              "type": "sql",
+            },
+          },
+        },
+      },
+    },
+  };
+}

package/src/coreCollections/workflowsCollection.ts

@@ -0,0 +1,73 @@
+import type { CollectionConfig } from "../types/index.ts";
+
+export const workflowsCollection: CollectionConfig = {
+  "indexes": {},
+  "fields": {
+    "id": {
+      "type": "integer",
+    },
+    "name": {
+      "type": "string",
+      "length": 255,
+      "validators": {
+        "required": true,
+      },
+    },
+    "event_name": {
+      "type": "string",
+      "length": 255,
+    },
+    "directory": {
+      "type": "string",
+      "length": 255,
+    },
+    "input_schema": {
+      "type": "text",
+      "pre_processors": {
+        "default": "function schema(z) {\n\treturn z.object({});\n}",
+      },
+      "ui": {
+        "input": {
+          "type": "code",
+          "args": {
+            "type": "typescript",
+            "height": 200,
+          },
+        },
+      },
+    },
+    "output_schema": {
+      "type": "text",
+      "pre_processors": {
+        "default": "function schema(z) {\n\treturn z.object({});\n}",
+      },
+      "ui": {
+        "input": {
+          "type": "code",
+          "args": {
+            "type": "typescript",
+            "height": 200,
+          },
+        },
+      },
+    },
+    "handler": {
+      "type": "text",
+      "validators": {
+        "required": true,
+      },
+      "pre_processors": {
+        "default":
+          "async function workflow(input: Input, ctx: Context): Promise<Output> {\n\treturn input;\n}",
+      },
+      "ui": {
+        "input": {
+          "type": "code",
+          "args": {
+            "type": "typescript",
+          },
+        },
+      },
+    },
+  },
+};

package/src/coreDbSetup/index.ts

@@ -0,0 +1,72 @@
+import type { DatabaseDriver } from "../types/index.ts";
+import { Lobb } from "../Lobb.ts";
+
+// DB setup tasks run on every startup, before migrations and schema sync.
+// They are idempotent — safe to run multiple times — and handle DB setup
+// that must be in place before anything else can proceed.
+type DbSetupTask = (driver: DatabaseDriver) => Promise<void>;
+
+const coreDbSetupTasks: DbSetupTask[] = [
+
+  // Creates the core_migrations table if it doesn't exist, and ensures its
+  // schema is up to date. We use raw SQL with IF NOT EXISTS / ADD COLUMN IF
+  // NOT EXISTS so this is safe to run against both fresh and old databases.
+  // We cannot use driver.createCollection here because it calls CREATE TABLE
+  // IF NOT EXISTS (which silently skips when the table already exists) and
+  // then unconditionally tries to create indexes — crashing on old databases
+  // that are missing the migration_name column.
+  // TODO: REMOVE THIS AFTER SOME TIME BECAUSE ITS NOT NEEDED
+  async (driver) => {
+    await driver.query(`
+      CREATE TABLE IF NOT EXISTS core_migrations (
+        "id"             SERIAL PRIMARY KEY,
+        "owner"          VARCHAR(255) NOT NULL,
+        "migration_name" VARCHAR(255) NOT NULL
+      )
+    `);
+
+    // Older versions of Lobb created this table without migration_name.
+    await driver.query(
+      `ALTER TABLE core_migrations ADD COLUMN IF NOT EXISTS "migration_name" VARCHAR(255) NOT NULL DEFAULT ''`,
+    );
+
+    await driver.query(`
+      CREATE UNIQUE INDEX IF NOT EXISTS core_migrations_unique_index
+      ON core_migrations ("owner" ASC, "migration_name" ASC)
+    `);
+  },
+
+  // Converts any timezone-naive timestamp/time columns to their timezone-aware
+  // counterparts. Lobb only works with TIMESTAMPTZ / TIMETZ — older lobb version generated databases
+  // that have these columns in the wrong type, which would crash schema reading.
+  // TODO: REMOVE THIS AFTER SOME TIME BECAUSE ITS NOT NEEDED
+  async (driver) => {
+    const columns = await driver.query<{
+      table_name: string;
+      column_name: string;
+      data_type: string;
+    }>(
+      `SELECT table_name, column_name, data_type
+       FROM information_schema.columns
+       WHERE table_schema = 'public'
+         AND data_type IN ('timestamp without time zone', 'time without time zone')`,
+    );
+
+    for (const col of columns) {
+      const newType = col.data_type === "timestamp without time zone"
+        ? "TIMESTAMPTZ"
+        : "TIMETZ";
+      await driver.query(
+        `ALTER TABLE "${col.table_name}" ALTER COLUMN "${col.column_name}" TYPE ${newType}`,
+      );
+    }
+  },
+
+];
+
+export async function runCoreDbSetup() {
+  const driver = Lobb.instance.databaseService.getDriver();
+  for (const task of coreDbSetupTasks) {
+    await task(driver);
+  }
+}

package/src/coreMigrations/index.ts

@@ -0,0 +1,3 @@
+import type { Migrations } from "../types/index.ts";
+
+export const coreMigrations: Migrations = {};