@raclettejs/core 0.0.1

package/services/backend/src/core/pluginSystem/pluginLoader.ts

@@ -0,0 +1,780 @@
+import type { PluginOptions } from "@/types"
+import { PluginType } from "@/types"
+import type { PluginMetadata } from "@raclettejs/core"
+import type { FastifyInstance } from "fastify"
+import fs from "fs"
+import path from "path"
+import { createPluginFastifyInstance } from "./pluginFastifyInstance"
+import configService from "@c/config/configService"
+import { slugify } from "@shared/helper/pluginHelper"
+import { fileURLToPath } from "url"
+import { glob } from "glob"
+
+const __filename = fileURLToPath(import.meta.url)
+const __dirname = path.dirname(path.join(__filename, ".."))
+
+const MAX_SCAN_DEPTH = 3
+
+// Error handling flags
+const PLUGIN_ERROR_FLAGS = {
+  CORE: 1,
+  NPM: 2,
+  APP: 4,
+} as const
+
+const DEFAULT_ERROR_FLAGS = PLUGIN_ERROR_FLAGS.CORE | PLUGIN_ERROR_FLAGS.APP // 5
+
+// Interface to represent plugin information regardless of source
+interface PluginInfo {
+  author: string
+  pluginName: string
+  modulePath: string
+  pluginPath: string
+  type: PluginType
+  metadata: PluginMetadata
+}
+
+// Logging helper functions
+const createLogPrefix = (type: PluginType) => `[${type.toUpperCase()} PLUGIN]`
+
+const logPluginDiscovery = (
+  fastify: FastifyInstance,
+  type: PluginType,
+  message: string,
+) => {
+  fastify.log.info(`${createLogPrefix(type)} ${message}`)
+}
+
+const logPluginSuccess = (
+  fastify: FastifyInstance,
+  type: PluginType,
+  author: string,
+  name: string,
+  action: string,
+) => {
+  fastify.log.info(`${createLogPrefix(type)} ${author}/${name}: ${action}`)
+}
+
+const logPluginSkipped = (
+  fastify: FastifyInstance,
+  type: PluginType,
+  identifier: string,
+  reason: string,
+) => {
+  fastify.log.debug(
+    `${createLogPrefix(type)} ${identifier}: SKIPPED - ${reason}`,
+  )
+}
+
+const logPluginError = (
+  fastify: FastifyInstance,
+  type: PluginType,
+  identifier: string,
+  error: string,
+) => {
+  fastify.log.debug(`${createLogPrefix(type)} ${identifier}: ERROR - ${error}`)
+}
+
+/**
+ * Get app plugin paths for current environment
+ */
+const getAppPluginPaths = async (): Promise<string[]> => {
+  const basePath =
+    process.env.NODE_ENV === "production" ? "/app/dist/src" : "/app/src"
+
+  // Use glob to find all raclette.plugin files and get their directories
+  const pattern = path.join(basePath, "**/raclette.plugin.{ts,js}")
+  const pluginConfigs = await glob(pattern, { ignore: ["**/node_modules/**"] })
+
+  // Get unique directories from found configs
+  const pluginDirs = [
+    ...new Set(pluginConfigs.map((config) => path.dirname(config))),
+  ]
+
+  // Filter to only include allowed paths
+  const allowedBasePaths = [
+    path.join(basePath, "appPlugins"),
+    ...configService
+      .getAdditonalPluginPaths()
+      .map((p) =>
+        process.env.NODE_ENV === "production" && p.startsWith("/app/src/")
+          ? p.replace("/app/src/", "/app/dist/src/")
+          : p,
+      ),
+  ]
+
+  return pluginDirs.filter((dir) =>
+    allowedBasePaths.some((allowedPath) => dir.startsWith(allowedPath)),
+  )
+}
+
+/**
+ * Register plugins in three phases: core, npm, then app plugins
+ */
+export const registerPlugins = async (
+  fastify: FastifyInstance,
+  errorFlags: number = DEFAULT_ERROR_FLAGS,
+) => {
+  // Constants
+  const CORE_PLUGIN_PATHS = [path.join(__dirname, "..", "corePlugins")]
+  const APP_PLUGIN_PATHS = await getAppPluginPaths()
+
+  const allPlugins: PluginInfo[] = []
+
+  fastify.log.info("=== PLUGIN LOADING STARTED ===")
+
+  // Phase 1: Core plugins (always required, crash on error (default, PLUGIN_ERROR_FLAGS))
+  await collectCorePlugins(fastify, CORE_PLUGIN_PATHS, allPlugins, errorFlags)
+
+  // Phase 2: NPM plugins (standalone, continue on error unless flag set)
+  await collectNpmPlugins(fastify, allPlugins, errorFlags)
+
+  // Phase 3: App/User plugins (might depend on core/npm, crash on error (default, PLUGIN_ERROR_FLAGS))
+  await collectAppPlugins(fastify, APP_PLUGIN_PATHS, allPlugins, errorFlags)
+
+  // Register all collected plugins
+  await registerCollectedPlugins(fastify, allPlugins, errorFlags)
+
+  fastify.log.info("=== PLUGIN LOADING COMPLETED ===")
+}
+
+/**
+ * Collect core plugins from hardcoded paths
+ */
+const collectCorePlugins = async (
+  fastify: FastifyInstance,
+  paths: string[],
+  plugins: PluginInfo[],
+  errorFlags: number,
+) => {
+  logPluginDiscovery(fastify, PluginType.CORE, "Discovery started")
+  logPluginDiscovery(
+    fastify,
+    PluginType.CORE,
+    `Scanning paths: ${paths.join(", ")}`,
+  )
+
+  const initialCount = plugins.length
+
+  for (const pluginPath of paths) {
+    await collectLocalPluginsFromPath(
+      fastify,
+      plugins,
+      pluginPath,
+      PluginType.CORE,
+      errorFlags,
+    )
+  }
+
+  const foundCount = plugins.length - initialCount
+  logPluginDiscovery(
+    fastify,
+    PluginType.CORE,
+    `Discovery completed - found ${foundCount} plugin(s)`,
+  )
+}
+
+/**
+ * Collect app/user plugins from hardcoded paths
+ */
+const collectAppPlugins = async (
+  fastify: FastifyInstance,
+  paths: string[],
+  plugins: PluginInfo[],
+  errorFlags: number,
+) => {
+  logPluginDiscovery(fastify, PluginType.APP, "Discovery started")
+  logPluginDiscovery(
+    fastify,
+    PluginType.APP,
+    `Scanning paths: ${paths.join(", ")}`,
+  )
+
+  const initialCount = plugins.length
+
+  for (const pluginPath of paths) {
+    await collectLocalPluginsFromPath(
+      fastify,
+      plugins,
+      pluginPath,
+      PluginType.APP,
+      errorFlags,
+    )
+  }
+
+  const foundCount = plugins.length - initialCount
+  logPluginDiscovery(
+    fastify,
+    PluginType.APP,
+    `Discovery completed - found ${foundCount} plugin(s)`,
+  )
+}
+
+/**
+ * Collect local plugins from a specific path
+ */
+const collectLocalPluginsFromPath = async (
+  fastify: FastifyInstance,
+  plugins: PluginInfo[],
+  basePath: string,
+  pluginType: PluginType,
+  errorFlags: number,
+) => {
+  if (!fs.existsSync(basePath)) {
+    logPluginSkipped(fastify, pluginType, basePath, "path does not exist")
+    return
+  }
+
+  const pluginConfigs = await findPluginConfigs(basePath, 0, fastify)
+
+  fastify.log.debug(
+    `${createLogPrefix(pluginType)} Found ${pluginConfigs.length} plugin config(s) in ${basePath}`,
+  )
+
+  for (const configPath of pluginConfigs) {
+    const pluginPath = path.dirname(configPath)
+    const pluginIdentifier = path.basename(pluginPath)
+
+    try {
+      const metadata = await loadPluginMetadata(configPath)
+
+      if (!validatePluginMetadata(metadata)) {
+        logPluginSkipped(
+          fastify,
+          pluginType,
+          pluginIdentifier,
+          "invalid metadata - missing required fields (name, author)",
+        )
+        handlePluginError(
+          fastify,
+          `Invalid plugin metadata in ${configPath}: Missing required fields`,
+          pluginType,
+          errorFlags,
+        )
+        continue
+      }
+
+      // Get backend path from metadata or use default
+      const backendRelativePath = metadata.backendDir || "backend"
+      const backendPath = path.join(pluginPath, backendRelativePath)
+
+      if (!fs.existsSync(backendPath)) {
+        logPluginSkipped(
+          fastify,
+          pluginType,
+          pluginIdentifier,
+          `backend directory not found: ${backendRelativePath}`,
+        )
+        if (metadata.backendDir) {
+          // we only cancel when the dir was explicitly set
+          handlePluginError(
+            fastify,
+            `Server directory not found for plugin: ${configPath}`,
+            pluginType,
+            errorFlags,
+          )
+        }
+        continue
+      }
+
+      if (!fs.statSync(backendPath).isDirectory()) {
+        logPluginSkipped(
+          fastify,
+          pluginType,
+          pluginIdentifier,
+          `backend path is not a directory: ${backendRelativePath}`,
+        )
+        if (metadata.backendDir) {
+          handlePluginError(
+            fastify,
+            `Server directory not found for plugin: ${configPath}`,
+            pluginType,
+            errorFlags,
+          )
+        }
+        continue
+      }
+
+      // Find the main module file
+      const modulePath = getFilePathWithExtension(backendPath, "index")
+      if (!modulePath) {
+        logPluginSkipped(
+          fastify,
+          pluginType,
+          pluginIdentifier,
+          `no index file found in backend directory: ${backendRelativePath}`,
+        )
+        handlePluginError(
+          fastify,
+          `No plugin index file found in ${backendPath}`,
+          pluginType,
+          errorFlags,
+        )
+        continue
+      }
+
+      const author = slugify(metadata.author)
+      const pluginName = slugify(metadata.name)
+
+      plugins.push({
+        author,
+        pluginName,
+        modulePath,
+        pluginPath,
+        type: pluginType,
+        metadata,
+      })
+
+      logPluginSuccess(fastify, pluginType, author, pluginName, "discovered")
+    } catch (error: any) {
+      logPluginError(
+        fastify,
+        pluginType,
+        pluginIdentifier,
+        `processing failed: ${error.message}`,
+      )
+      handlePluginError(
+        fastify,
+        `Error processing ${pluginType} plugin ${configPath}: ${error.message}`,
+        pluginType,
+        errorFlags,
+      )
+    }
+  }
+}
+
+/**
+ * Collect npm-installable plugins
+ */
+const collectNpmPlugins = async (
+  fastify: FastifyInstance,
+  plugins: PluginInfo[],
+  errorFlags: number,
+) => {
+  const installedPlugins = configService.getPlugins()
+
+  logPluginDiscovery(fastify, PluginType.NPM, "Discovery started")
+
+  if (!installedPlugins.length) {
+    logPluginDiscovery(
+      fastify,
+      PluginType.NPM,
+      "Discovery completed - no plugins configured",
+    )
+    return
+  }
+
+  logPluginDiscovery(
+    fastify,
+    PluginType.NPM,
+    `Scanning ${installedPlugins.length} configured plugin(s): ${installedPlugins.join(", ")}`,
+  )
+
+  const initialCount = plugins.length
+
+  for (const pluginPackage of installedPlugins) {
+    try {
+      // Extract the plugin name without scope
+      const pluginNameWithoutScope = pluginPackage.replace(/^@[^/]+\//, "")
+
+      // Try to find the plugin in several locations (keep fallback logic)
+      const possibleLocations = [
+        // Regular node_modules location
+        path.resolve(process.cwd(), "node_modules", pluginPackage),
+        // Development convention (mounted directory)
+        path.resolve("/app", pluginNameWithoutScope),
+      ]
+
+      let pluginPath: string | null = null
+      let pluginMetadata: PluginMetadata | null = null
+
+      fastify.log.debug(
+        `${createLogPrefix(PluginType.NPM)} ${pluginPackage}: checking possible locations: ${possibleLocations.join(", ")}`,
+      )
+
+      // Find the plugin path and load its metadata
+      for (const location of possibleLocations) {
+        try {
+          fastify.log.debug(
+            `${createLogPrefix(PluginType.NPM)} ${pluginPackage}: checking location: ${location}`,
+          )
+
+          // Check if this path exists and is a directory
+          if (!fs.existsSync(location)) {
+            fastify.log.debug(
+              `${createLogPrefix(PluginType.NPM)} ${pluginPackage}: location does not exist: ${location}`,
+            )
+            continue
+          }
+
+          if (!fs.statSync(location).isDirectory()) {
+            fastify.log.debug(
+              `${createLogPrefix(PluginType.NPM)} ${pluginPackage}: location is not a directory: ${location}`,
+            )
+            continue
+          }
+
+          const pluginConfigPath = getFilePathWithExtension(
+            location,
+            "raclette.plugin",
+          )
+
+          if (!pluginConfigPath) {
+            fastify.log.debug(
+              `${createLogPrefix(PluginType.NPM)} ${pluginPackage}: no plugin config found in: ${location}`,
+            )
+            continue
+          }
+
+          pluginMetadata = await loadPluginMetadata(pluginConfigPath)
+
+          if (validatePluginMetadata(pluginMetadata)) {
+            pluginPath = location
+            fastify.log.debug(
+              `${createLogPrefix(PluginType.NPM)} ${pluginPackage}: found valid plugin at: ${location}`,
+            )
+            break
+          } else {
+            fastify.log.debug(
+              `${createLogPrefix(PluginType.NPM)} ${pluginPackage}: invalid metadata at: ${location}`,
+            )
+          }
+        } catch (error: any) {
+          fastify.log.debug(
+            `${createLogPrefix(PluginType.NPM)} ${pluginPackage}: error checking location ${location}: ${error.message}`,
+          )
+        }
+      }
+
+      if (!pluginPath || !pluginMetadata) {
+        logPluginSkipped(
+          fastify,
+          PluginType.NPM,
+          pluginPackage,
+          "not found in any checked location",
+        )
+        handlePluginError(
+          fastify,
+          `Could not find valid npm plugin ${pluginPackage}`,
+          PluginType.NPM,
+          errorFlags,
+        )
+        continue
+      }
+
+      // Get backend path from metadata or use default
+      const backendRelativePath = pluginMetadata.backendDir || "src/backend"
+      const backendPath = path.join(pluginPath, backendRelativePath)
+
+      if (!fs.existsSync(backendPath)) {
+        logPluginSkipped(
+          fastify,
+          PluginType.NPM,
+          pluginPackage,
+          `backend directory not found: ${backendRelativePath}`,
+        )
+        handlePluginError(
+          fastify,
+          `Server directory not found for npm plugin: ${pluginPackage}`,
+          PluginType.NPM,
+          errorFlags,
+        )
+        continue
+      }
+
+      if (!fs.statSync(backendPath).isDirectory()) {
+        logPluginSkipped(
+          fastify,
+          PluginType.NPM,
+          pluginPackage,
+          `backend path is not a directory: ${backendRelativePath}`,
+        )
+        handlePluginError(
+          fastify,
+          `Server directory not found for npm plugin: ${pluginPackage}`,
+          PluginType.NPM,
+          errorFlags,
+        )
+        continue
+      }
+
+      // Find the main module file
+      const modulePath = getFilePathWithExtension(backendPath, "index")
+      if (!modulePath) {
+        logPluginSkipped(
+          fastify,
+          PluginType.NPM,
+          pluginPackage,
+          `no index file found in backend directory: ${backendRelativePath}`,
+        )
+        handlePluginError(
+          fastify,
+          `No plugin module found in ${backendPath}`,
+          PluginType.NPM,
+          errorFlags,
+        )
+        continue
+      }
+
+      const author = slugify(pluginMetadata.author)
+      const pluginName = slugify(pluginMetadata.name)
+
+      plugins.push({
+        author,
+        pluginName,
+        modulePath,
+        pluginPath,
+        type: PluginType.NPM,
+        metadata: pluginMetadata,
+      })
+
+      logPluginSuccess(
+        fastify,
+        PluginType.NPM,
+        author,
+        pluginName,
+        "discovered",
+      )
+    } catch (error: any) {
+      logPluginError(
+        fastify,
+        PluginType.NPM,
+        pluginPackage,
+        `processing failed: ${error.message}`,
+      )
+      handlePluginError(
+        fastify,
+        `Error processing npm plugin ${pluginPackage}: ${error.message}`,
+        PluginType.NPM,
+        errorFlags,
+      )
+    }
+  }
+
+  const foundCount = plugins.length - initialCount
+  logPluginDiscovery(
+    fastify,
+    PluginType.NPM,
+    `Discovery completed - found ${foundCount} plugin(s)`,
+  )
+}
+
+/**
+ * Recursively find raclette.plugin.js files
+ */
+const findPluginConfigs = async (
+  dir: string,
+  currentDepth: number,
+  fastify: FastifyInstance,
+): Promise<string[]> => {
+  const configs: string[] = []
+
+  if (currentDepth >= MAX_SCAN_DEPTH) {
+    return configs
+  }
+
+  try {
+    const items = fs.readdirSync(dir)
+
+    // First check if current directory has a plugin config
+    const pluginConfigPath = getFilePathWithExtension(dir, "raclette.plugin")
+
+    if (pluginConfigPath && fs.existsSync(pluginConfigPath)) {
+      configs.push(pluginConfigPath)
+      return configs // If we found a config here, don't scan subdirectories
+    }
+
+    // If no config found, scan subdirectories
+    for (const item of items) {
+      const itemPath = path.join(dir, item)
+
+      if (fs.statSync(itemPath).isDirectory()) {
+        const subConfigs = await findPluginConfigs(
+          itemPath,
+          currentDepth + 1,
+          fastify,
+        )
+        configs.push(...subConfigs)
+      }
+    }
+  } catch (error) {
+    // Ignore permission errors and continue
+  }
+
+  return configs
+}
+
+/**
+ * Load and validate plugin metadata from raclette.plugin.ts/js
+ */
+const loadPluginMetadata = async (
+  configPath: string,
+): Promise<PluginMetadata> => {
+  const importedConfig = await import(`file://${configPath}`)
+  return importedConfig.default || importedConfig
+}
+
+/**
+ * Validate that plugin metadata has required fields
+ */
+const validatePluginMetadata = (metadata: any): metadata is PluginMetadata => {
+  return (
+    metadata &&
+    typeof metadata.name === "string" &&
+    typeof metadata.author === "string"
+  )
+}
+
+/**
+ * Handle plugin errors based on error flags
+ */
+const handlePluginError = (
+  fastify: FastifyInstance,
+  error: string,
+  pluginType: PluginType,
+  errorFlags: number,
+) => {
+  const shouldCrash =
+    (errorFlags &
+      PLUGIN_ERROR_FLAGS[
+        pluginType.toUpperCase() as keyof typeof PLUGIN_ERROR_FLAGS
+      ]) !==
+    0
+
+  if (shouldCrash) {
+    fastify.log.error(error)
+    throw new Error(error)
+  } else {
+    fastify.log.warn(error)
+  }
+}
+
+/**
+ * Register all collected plugins using a common registration process
+ */
+const registerCollectedPlugins = async (
+  fastify: FastifyInstance,
+  plugins: PluginInfo[],
+  errorFlags: number,
+) => {
+  if (plugins.length === 0) {
+    fastify.log.info("=== PLUGIN REGISTRATION === No plugins to register")
+    return
+  }
+
+  fastify.log.info(
+    `=== PLUGIN REGISTRATION === Starting registration of ${plugins.length} plugin(s)`,
+  )
+
+  // Group plugins by type for better logging
+  const pluginsByType = plugins.reduce(
+    (acc, plugin) => {
+      if (!acc[plugin.type]) acc[plugin.type] = []
+      acc[plugin.type].push(plugin)
+      return acc
+    },
+    {} as Record<PluginType, PluginInfo[]>,
+  )
+
+  // Log summary by type
+  for (const [type, typePlugins] of Object.entries(pluginsByType)) {
+    const pluginNames = typePlugins.map((p) => `${p.author}/${p.pluginName}`)
+    fastify.log.info(
+      `${createLogPrefix(type as PluginType)} Registering ${typePlugins.length} plugin(s): ${pluginNames.join(", ")}`,
+    )
+  }
+
+  let successCount = 0
+  let failureCount = 0
+
+  for (const plugin of plugins) {
+    const { author, pluginName, modulePath } = plugin
+    const key = `${author}__${pluginName}`
+
+    try {
+      // Construct the prefixed collection name and route prefix
+      const collectionPrefix = `${author}__${pluginName}-`
+      const routePrefix = `/${author}__${pluginName}`
+
+      const pluginOptions: PluginOptions = {
+        key,
+        author,
+        pluginName,
+        type: plugin.type,
+        collectionPrefix,
+        routePrefix,
+        absolutePath: plugin.pluginPath,
+      }
+
+      // Dynamically import the plugin module
+      const pluginModule = await import(modulePath)
+      const pluginFunction = pluginModule.default
+
+      if (!pluginModule || !pluginFunction) {
+        logPluginError(
+          fastify,
+          plugin.type,
+          key,
+          "no default export function found",
+        )
+        handlePluginError(
+          fastify,
+          `No plugin bootstrapping function found in ${modulePath}`,
+          plugin.type,
+          errorFlags,
+        )
+        failureCount++
+        continue
+      }
+
+      // Create a restricted fastify instance for the plugins to use
+      const customFastify = createPluginFastifyInstance(fastify, pluginOptions)
+
+      // Register the plugin, passing the restricted instance and options
+      await fastify.register(() => {
+        return pluginFunction(customFastify, pluginOptions)
+      })
+
+      logPluginSuccess(
+        fastify,
+        plugin.type,
+        author,
+        pluginName,
+        "successfully registered",
+      )
+      successCount++
+    } catch (error: any) {
+      logPluginError(
+        fastify,
+        plugin.type,
+        key,
+        `registration failed: ${error.message}`,
+      )
+      handlePluginError(
+        fastify,
+        `[PluginRegistration] ${key}: Error during registration: ${error.message}`,
+        plugin.type,
+        errorFlags,
+      )
+      failureCount++
+    }
+  }
+
+  fastify.log.info(
+    `=== PLUGIN REGISTRATION === Completed - ${successCount} successful, ${failureCount} failed`,
+  )
+}
+
+/**
+ * Helper function to find the plugin module file
+ */
+const getFilePathWithExtension = (
+  backendPath: string,
+  fileName: string,
+): string | undefined => {
+  const extensions = ["ts", "js"]
+
+  return extensions
+    .map((ext) => path.join(backendPath, `${fileName}.${ext}`))
+    .find((fullPath) => fs.existsSync(fullPath))
+}