@divizend/scratch-core 1.0.0 → 1.0.2

package/http-server/ExpressHttpServer.ts

@@ -0,0 +1,1175 @@
+/**
+ * ExpressHttpServer - Express-based HTTP server implementation
+ *
+ * Elegant and concise implementation using Express for dynamic endpoint handling
+ */
+
+import express, {
+  Express,
+  Request,
+  Response as ExpressResponse,
+  RequestHandler,
+} from "express";
+import { readFile, writeFile, mkdir, readdir } from "node:fs/promises";
+import { resolve, isAbsolute, join, basename, dirname } from "node:path";
+import { randomUUID } from "node:crypto";
+import { platform } from "node:process";
+import { Server } from "node:http";
+import { stat } from "node:fs/promises";
+import { getProjectRoot } from "../core/ProjectRoot";
+import { HttpServer } from "./HttpServer";
+import {
+  Universe,
+  ScratchContext,
+  ScratchEndpointDefinition,
+  ScratchBlock,
+  JsonSchema,
+  JsonSchemaValidator,
+  UniverseModule,
+  envOrDefault,
+} from "../index";
+
+// Minimal structured logger
+const log = (record: Record<string, unknown>) => {
+  if (!record.ts) record.ts = new Date().toISOString();
+  console.log(JSON.stringify(record));
+};
+
+// Type alias for schema property values
+type SchemaProperty = NonNullable<ScratchBlock["schema"]>[string];
+
+export class ExpressHttpServer implements HttpServer {
+  private app: Express;
+  private server: Server | null = null;
+  private universe: Universe;
+  private endpoints: Map<string, ScratchEndpointDefinition> = new Map();
+  private staticRoot: string | null = null;
+  private routeHandlers: Map<string, RequestHandler> = new Map();
+
+  constructor(universe: Universe) {
+    this.universe = universe;
+    this.app = express();
+    this.setupMiddleware();
+  }
+
+  async initializeDefaultEndpoints(): Promise<void> {
+    // Dynamically load all endpoints from default-endpoints directory
+    const projectRoot = await getProjectRoot();
+    const defaultEndpointsDir = join(
+      projectRoot,
+      "package",
+      "http-server",
+      "default-endpoints"
+    );
+    const files = await readdir(defaultEndpointsDir, { withFileTypes: true });
+
+    for (const file of files) {
+      if (
+        file.isFile() &&
+        file.name.endsWith(".ts") &&
+        file.name !== "index.ts"
+      ) {
+        const modulePath = join(defaultEndpointsDir, file.name);
+
+        try {
+          const module = await import(modulePath);
+          // Find the exported endpoint definition
+          const endpoint = Object.values(module).find(
+            (value): value is ScratchEndpointDefinition =>
+              value !== null &&
+              typeof value === "object" &&
+              "block" in value &&
+              "handler" in value
+          );
+
+          if (endpoint) {
+            const blockDef = await endpoint.block({});
+            if (blockDef.opcode) {
+              this.endpoints.set(blockDef.opcode, endpoint);
+              await this.registerRoute(blockDef.opcode, endpoint);
+            }
+          }
+        } catch (error) {
+          log({
+            level: "error",
+            event: "default_endpoint_load_failed",
+            file: file.name,
+            error: error instanceof Error ? error.message : String(error),
+          });
+        }
+      }
+    }
+
+    // Middleware is already set up in constructor - don't call it again
+
+    // Set up catch-all route handler for all endpoints
+    this.setupCatchAllRoute();
+
+    // Set up 404 handler - only triggers if catch-all route doesn't match
+    this.setup404Handler();
+  }
+
+  private setupCatchAllRoute(): void {
+    // Single catch-all route that handles all endpoint matching
+    // Express 5 (path-to-regexp v6) no longer accepts bare "*" patterns,
+    // so we use a regex to catch everything.
+    this.app.all(
+      /.*/,
+      async (
+        req: Request,
+        res: ExpressResponse,
+        next: express.NextFunction
+      ) => {
+        // Skip if response already sent
+        if (res.headersSent) return next();
+
+        // Handle root path specially
+        if (req.path === "/") {
+          if (this.routeHandlers.has("root")) {
+            const rootHandler = this.routeHandlers.get("root");
+            if (rootHandler) {
+              await rootHandler(req, res, next);
+              return;
+            }
+          }
+          // No root endpoint - delegate to loadEndpointsUI
+          if (this.routeHandlers.has("loadEndpointsUI")) {
+            const uiHandler = this.routeHandlers.get("loadEndpointsUI");
+            if (uiHandler) {
+              await uiHandler(req, res, next);
+              return;
+            }
+          }
+          return next(); // Fall through to 404
+        }
+
+        // Extract opcode from path (remove leading /)
+        const opcode = req.path.slice(1);
+        const endpoint = this.endpoints.get(opcode);
+
+        if (!endpoint) {
+          return next(); // No endpoint found, fall through to 404
+        }
+
+        // Get block definition to check method
+        const blockDef = await endpoint.block({});
+        const expectedMethod =
+          blockDef.blockType === "reporter" ? "GET" : "POST";
+
+        if (req.method !== expectedMethod) {
+          return next(); // Method mismatch, fall through to 404
+        }
+
+        // Get handler and call it
+        const handler = this.routeHandlers.get(opcode);
+        if (!handler) {
+          return next(); // No handler, fall through to 404
+        }
+
+        log({
+          level: "info",
+          event: "endpoint_matched",
+          opcode,
+          method: req.method,
+          path: req.path,
+        });
+
+        await handler(req, res, next);
+      }
+    );
+  }
+
+  private setup404Handler(): void {
+    // 404 handler - only triggers if catch-all route calls next()
+    this.app.use(
+      (req: Request, res: ExpressResponse, next: express.NextFunction) => {
+        if (res.headersSent) return next();
+        log({
+          level: "warn",
+          event: "route_not_found",
+          method: req.method,
+          path: req.path,
+          registeredRoutes: Array.from(this.routeHandlers.keys()),
+        });
+        res.status(404).json({ error: "Not found", path: req.path });
+      }
+    );
+  }
+
+  /**
+   * Register the catch-all 404 handler
+   * This is called during initialization and never changes
+   */
+  register404Handler(): void {
+    // No-op - 404 handler is set up in initializeDefaultEndpoints
+  }
+
+  private setupMiddleware(): void {
+    // Request logging - FIRST so we can track all requests
+    this.app.use(
+      (req: Request, res: ExpressResponse, next: express.NextFunction) => {
+        const reqId = randomUUID();
+        log({
+          level: "info",
+          event: "request",
+          req_id: reqId,
+          method: req.method,
+          path: req.path,
+          query: Object.keys(req.query).length > 0 ? req.query : undefined,
+        });
+        (req as any).reqId = reqId;
+        next();
+      }
+    );
+
+    // CORS
+    this.app.use(
+      (req: Request, res: ExpressResponse, next: express.NextFunction) => {
+        res.header("Access-Control-Allow-Origin", "*");
+        res.header(
+          "Access-Control-Allow-Methods",
+          "GET, POST, PUT, DELETE, OPTIONS"
+        );
+        res.header(
+          "Access-Control-Allow-Headers",
+          "Content-Type, Authorization"
+        );
+        if (req.method === "OPTIONS") return res.sendStatus(200);
+        next();
+      }
+    );
+
+    // Body parsing
+    this.app.use(express.json());
+    this.app.use(express.urlencoded({ extended: true }));
+  }
+
+  private async registerRoute(
+    opcode: string,
+    endpoint: ScratchEndpointDefinition
+  ): Promise<void> {
+    try {
+      const handler = await this.createHandlerForEndpoint(endpoint);
+
+      // Verify handler is a function
+      if (typeof handler !== "function") {
+        throw new Error(
+          `Handler for ${opcode} is not a function: ${typeof handler}`
+        );
+      }
+
+      // Just store the handler - the catch-all route will match it
+      this.routeHandlers.set(opcode, handler);
+
+      log({
+        level: "info",
+        event: "endpoint_registered",
+        opcode,
+      });
+    } catch (error) {
+      log({
+        level: "error",
+        event: "endpoint_registration_failed",
+        opcode,
+        error: error instanceof Error ? error.message : String(error),
+        stack: error instanceof Error ? error.stack : undefined,
+      });
+      throw error;
+    }
+  }
+
+  private async wrapHandlerWithAuthAndValidation(
+    endpoint: ScratchEndpointDefinition
+  ): Promise<
+    (
+      context: ScratchContext,
+      query?: Record<string, string>,
+      requestBody?: any,
+      authHeader?: string
+    ) => Promise<any>
+  > {
+    const universe = this.universe;
+    const noAuth = endpoint.noAuth || false;
+    const requiredModules = endpoint.requiredModules || [];
+
+    // Get the block definition to extract schema
+    const blockDef = await endpoint.block({});
+    const schema = blockDef.schema;
+    const isLoadEndpointsUI = blockDef.opcode === "loadEndpointsUI";
+
+    // Create the wrapped handler
+    return async (
+      context: ScratchContext,
+      query: Record<string, string> = {},
+      requestBody: any = undefined,
+      authHeader: string | undefined = undefined
+    ) => {
+      // Auth check - skip for loadEndpointsUI GET requests (it handles auth via query param)
+      if (!noAuth && !isLoadEndpointsUI) {
+        if (!universe.auth.isConfigured()) {
+          throw new Error("JWT authentication not configured");
+        }
+
+        if (!authHeader || !authHeader.startsWith("Bearer ")) {
+          throw new Error("Missing or invalid authorization header");
+        }
+
+        const token = authHeader.substring(7);
+        try {
+          const payload = await universe.auth.validateJwtToken(token);
+          if (!payload) {
+            throw new Error("Invalid or expired token");
+          }
+        } catch {
+          throw new Error("Invalid or expired token");
+        }
+      }
+
+      // Extract user email from auth header if present
+      let userEmail: string | undefined;
+      try {
+        if (authHeader?.startsWith("Bearer ")) {
+          const payload = await universe.auth.validateJwtToken(
+            authHeader.substring(7)
+          );
+          if (payload) userEmail = (payload as any)?.email;
+        }
+      } catch {}
+
+      // Update context with user email, authHeader, and ensure universe is set
+      const enrichedContext: ScratchContext = {
+        ...context,
+        userEmail,
+        universe: universe,
+        authHeader: authHeader,
+      };
+
+      // Module validation
+      if (requiredModules.length > 0) {
+        const missingModules = requiredModules.filter(
+          (module) => !universe.hasModule(module)
+        );
+        if (missingModules.length > 0) {
+          throw new Error(
+            `Required modules not available: ${missingModules.join(", ")}`
+          );
+        }
+      }
+
+      // Schema validation
+      if (schema) {
+        const validator =
+          universe?.jsonSchemaValidator || new JsonSchemaValidator();
+        const fullSchema = this.constructJsonSchema(schema);
+        const isGet = blockDef.blockType === "reporter";
+
+        // For GET requests, query params go directly into inputs
+        // For POST requests, request body goes into inputs
+        let data: any = isGet
+          ? Object.fromEntries(
+              Object.keys(schema).map((key) => [key, query[key] || undefined])
+            )
+          : requestBody || {};
+
+        // Handle JSON type properties
+        if (schema) {
+          for (const [key, propSchema] of Object.entries(schema)) {
+            const typedPropSchema = propSchema as SchemaProperty;
+            if (
+              typedPropSchema.type === "json" &&
+              data[key] !== undefined &&
+              data[key] !== null &&
+              data[key] !== ""
+            ) {
+              try {
+                // If data[key] is already an object, use it directly
+                let parsed = data[key];
+                if (typeof data[key] === "string") {
+                  parsed = JSON.parse(data[key]);
+                }
+                if (typedPropSchema.schema) {
+                  const wrappedSchema: JsonSchema = {
+                    type: "object",
+                    properties: { value: typedPropSchema.schema },
+                    required: ["value"],
+                  };
+                  const result = validator.validate(wrappedSchema, {
+                    value: parsed,
+                  });
+                  if (!result.valid) {
+                    throw new Error(
+                      `Validation failed for ${key}: ${JSON.stringify(
+                        result.errors
+                      )}`
+                    );
+                  }
+                  data[key] = result.data?.value ?? parsed;
+                } else {
+                  data[key] = parsed;
+                }
+              } catch (parseError) {
+                throw new Error(
+                  `Invalid JSON for ${key}: ${
+                    parseError instanceof Error
+                      ? parseError.message
+                      : "Unknown error"
+                  }`
+                );
+              }
+            }
+          }
+        }
+
+        // Validate the data
+        const dataForValidation: any = { ...data };
+        if (schema) {
+          for (const [key, propSchema] of Object.entries(schema)) {
+            const typedPropSchema = propSchema as SchemaProperty;
+            if (
+              typedPropSchema.type === "json" &&
+              dataForValidation[key] !== undefined
+            )
+              delete dataForValidation[key];
+          }
+        }
+
+        const result = validator.validate(fullSchema, dataForValidation);
+        if (!result.valid) {
+          throw new Error(
+            `Validation failed: ${JSON.stringify(result.errors)}`
+          );
+        }
+
+        const finalData = { ...result.data };
+        if (schema) {
+          for (const [key, propSchema] of Object.entries(schema)) {
+            const typedPropSchema = propSchema as SchemaProperty;
+            if (typedPropSchema.type === "json" && data[key] !== undefined)
+              finalData[key] = data[key];
+          }
+        }
+
+        enrichedContext.inputs = finalData;
+      } else {
+        // For endpoints without schema, set empty inputs
+        enrichedContext.inputs = {};
+      }
+
+      // Call the original handler
+      return await endpoint.handler(enrichedContext);
+    };
+  }
+
+  private constructJsonSchema(schema?: ScratchBlock["schema"]): JsonSchema {
+    if (!schema)
+      return {
+        type: "object",
+        properties: {},
+        required: [],
+        additionalProperties: false,
+      };
+    const properties: any = {};
+    const required: string[] = [];
+    for (const [key, propSchema] of Object.entries(schema)) {
+      const typedPropSchema = propSchema as SchemaProperty;
+      if (typedPropSchema.type === "json") {
+        if (!typedPropSchema.schema)
+          throw new Error(
+            `Property ${key} has type "json" but no schema provided`
+          );
+        properties[key] = {
+          type: "string",
+          description: typedPropSchema.description,
+          _jsonSchema: typedPropSchema.schema,
+        };
+      } else {
+        // Copy schema but exclude non-JSON-Schema fields
+        const {
+          default: _,
+          description: __,
+          ...jsonSchemaProps
+        } = typedPropSchema;
+        properties[key] = jsonSchemaProps;
+        // Only add to required if there's no default or default is a placeholder
+        if (
+          !typedPropSchema.default ||
+          typedPropSchema.default === `[${key}]`
+        ) {
+          required.push(key);
+        }
+      }
+    }
+    return {
+      type: "object",
+      properties,
+      required,
+      additionalProperties: false,
+    };
+  }
+
+  private async createHandlerForEndpoint(
+    endpoint: ScratchEndpointDefinition
+  ): Promise<RequestHandler> {
+    const blockDef = await endpoint.block({});
+    const opcode = blockDef.opcode || "unknown";
+
+    return async (
+      req: Request,
+      res: ExpressResponse,
+      next: express.NextFunction
+    ) => {
+      // Log when handler is invoked - this should ALWAYS be called if route matches
+      log({
+        level: "info",
+        event: "handler_invoked",
+        opcode,
+        path: req.path,
+        method: req.method,
+        url: req.url,
+      });
+
+      try {
+        const query = this.filterVercelParams(
+          req.query as Record<string, string>
+        );
+        const authHeader = req.headers.authorization || "";
+        const requestBody = req.body;
+
+        const scratchContext: ScratchContext = {
+          universe: this.universe,
+          authHeader: authHeader as string,
+          requestHost: req.headers.host || "",
+        };
+
+        // Get blockDef to check if this is loadEndpointsUI
+        const blockDef = await endpoint.block({});
+        const wrappedHandler = await this.wrapHandlerWithAuthAndValidation(
+          endpoint
+        );
+
+        const result = await wrappedHandler(
+          scratchContext,
+          query,
+          requestBody,
+          authHeader as string
+        );
+
+        // Handle different result types
+        if (result instanceof globalThis.Response) {
+          // Copy Fetch Response to Express response
+          const headers = Object.fromEntries(result.headers.entries());
+          res.status(result.status);
+          Object.entries(headers).forEach(([key, value]) => {
+            res.header(key, value);
+          });
+          const text = await result.text();
+          res.send(text);
+          return;
+        }
+
+        if (typeof result === "string") {
+          res.contentType("text/html");
+          res.send(result);
+          return;
+        }
+
+        if (result === null || result === undefined) {
+          res.json({ success: true });
+          return;
+        }
+
+        res.json(result);
+      } catch (error: any) {
+        const errorMessage =
+          error instanceof Error ? error.message : "Unknown error";
+        const statusCode =
+          errorMessage.includes("authentication") ||
+          errorMessage.includes("authorization") ||
+          errorMessage.includes("token")
+            ? 401
+            : errorMessage.includes("Validation failed") ||
+              errorMessage.includes("Invalid")
+            ? 400
+            : errorMessage.includes("modules not available")
+            ? 503
+            : 500;
+
+        log({
+          level: "error",
+          event: "handler_error",
+          req_id: (req as any).reqId,
+          opcode: (req as any).opcode || "unknown",
+          error: errorMessage,
+          statusCode,
+        });
+
+        res.status(statusCode).json({ error: errorMessage });
+      }
+    };
+  }
+
+  private filterVercelParams(
+    params: Record<string, string>
+  ): Record<string, string> {
+    const filtered: Record<string, string> = {};
+    for (const [key, value] of Object.entries(params)) {
+      if (!key.startsWith("...")) {
+        filtered[key] = value;
+      }
+    }
+    return filtered;
+  }
+
+  registerStaticFiles(rootPath: string): void {
+    this.staticRoot = rootPath;
+    // Register static files only for /public/* paths to avoid conflicts with API routes
+    this.app.use("/public", express.static(rootPath));
+  }
+
+  async start(port: number): Promise<void> {
+    return new Promise((resolve) => {
+      this.server = this.app.listen(port, () => {
+        console.log(`🚀 Server running on http://localhost:${port}`);
+
+        // Note: Router inspection removed - Express may lazy-load the router,
+        // and it's not critical for server operation. Routes are handled by
+        // the catch-all route and endpoint maps, which are the source of truth.
+
+        resolve();
+      });
+    });
+  }
+
+  async stop(): Promise<void> {
+    return new Promise((resolve) => {
+      if (!this.server) {
+        resolve();
+        return;
+      }
+      this.server.close(() => {
+        this.server = null;
+        resolve();
+      });
+    });
+  }
+
+  getExpressApp(): Express {
+    return this.app;
+  }
+
+  async registerEndpoints(
+    endpoints: ScratchEndpointDefinition[]
+  ): Promise<void> {
+    for (const endpoint of endpoints) {
+      const blockDef = await endpoint.block({});
+      if (!blockDef.opcode || blockDef.opcode === "") {
+        throw new Error("Endpoint opcode cannot be empty");
+      }
+      this.endpoints.set(blockDef.opcode, endpoint);
+      await this.registerRoute(blockDef.opcode, endpoint);
+    }
+  }
+
+  getAllEndpoints(): ScratchEndpointDefinition[] {
+    return Array.from(this.endpoints.values());
+  }
+
+  async getRegisteredEndpoints(): Promise<
+    Array<{
+      method: string;
+      endpoint: string;
+      blockType: string;
+      auth: string;
+      text: string;
+    }>
+  > {
+    const endpoints = this.getAllEndpoints();
+    const endpointInfos = await Promise.all(
+      endpoints.map(async (endpoint) => {
+        const blockDef = await endpoint.block({});
+        const opcode = blockDef.opcode!;
+        const method = blockDef.blockType === "reporter" ? "GET" : "POST";
+        const auth = endpoint.noAuth ? " (no auth)" : "";
+        return {
+          method,
+          endpoint: `/${opcode}`,
+          blockType: blockDef.blockType,
+          auth,
+          text: blockDef.text,
+        };
+      })
+    );
+    return endpointInfos.sort((a, b) => a.text.localeCompare(b.text));
+  }
+
+  async getEndpointHandlers(): Promise<
+    Record<string, (context: any) => Promise<any>>
+  > {
+    const handlers: Record<string, (context: any) => Promise<any>> = {};
+    for (const [opcode, endpoint] of this.endpoints) {
+      const wrappedHandler = await this.wrapHandlerWithAuthAndValidation(
+        endpoint
+      );
+      handlers[opcode] = wrappedHandler;
+    }
+    return handlers;
+  }
+
+  async getHandler(
+    opcode: string
+  ): Promise<
+    | ((
+        context: ScratchContext,
+        query?: Record<string, string>,
+        requestBody?: any,
+        authHeader?: string
+      ) => Promise<any>)
+    | undefined
+  > {
+    const endpoint = this.endpoints.get(opcode);
+    if (!endpoint) return undefined;
+
+    return await this.wrapHandlerWithAuthAndValidation(endpoint);
+  }
+
+  async registerEndpoint(source: string): Promise<{
+    success: boolean;
+    opcode?: string;
+    message?: string;
+    error?: string;
+  }> {
+    try {
+      const tempFile = `/tmp/endpoint_${Date.now()}_${randomUUID()}.ts`;
+      await Bun.write(tempFile, source);
+
+      try {
+        const module = await import(tempFile);
+        const endpoint = Object.values(module).find(
+          (value): value is ScratchEndpointDefinition =>
+            value !== null &&
+            typeof value === "object" &&
+            "block" in value &&
+            "handler" in value
+        );
+
+        if (!endpoint) {
+          throw new Error("No endpoint definition found in source code");
+        }
+
+        const blockDef = await endpoint.block({});
+        if (!blockDef.opcode || blockDef.opcode === "") {
+          throw new Error("Endpoint opcode cannot be empty");
+        }
+
+        this.endpoints.set(blockDef.opcode, endpoint);
+        await this.registerRoute(blockDef.opcode, endpoint);
+
+        return {
+          success: true,
+          opcode: blockDef.opcode,
+          message: `Endpoint "${blockDef.opcode}" registered successfully`,
+        };
+      } finally {
+        try {
+          await Bun.file(tempFile).unlink();
+        } catch {}
+      }
+    } catch (error: any) {
+      return {
+        success: false,
+        error: error.message || String(error),
+      };
+    }
+  }
+
+  async removeEndpoint(opcode: string): Promise<{
+    success: boolean;
+    message?: string;
+    error?: string;
+  }> {
+    try {
+      if (!opcode || opcode === "") {
+        throw new Error("Opcode cannot be empty");
+      }
+
+      const existed = this.endpoints.has(opcode);
+      this.endpoints.delete(opcode);
+      this.routeHandlers.delete(opcode);
+
+      // Note: Express doesn't have a direct way to remove routes, but since we track them
+      // in routeHandlers, new requests will use the updated endpoint map
+
+      return {
+        success: true,
+        message: existed
+          ? `Endpoint "${opcode}" removed successfully`
+          : `Endpoint "${opcode}" was not registered`,
+      };
+    } catch (error: any) {
+      return {
+        success: false,
+        error: error.message || String(error),
+      };
+    }
+  }
+
+  // Endpoint loading methods
+  async loadEndpointsFromUrl(
+    url: string
+  ): Promise<{ loaded: number; failed: number; errors: string[] }> {
+    log({
+      level: "info",
+      event: "loading_endpoints",
+      url,
+    });
+
+    try {
+      const parsedUrl = new URL(url);
+
+      if (parsedUrl.protocol === "file:") {
+        let localPath = parsedUrl.pathname;
+        if (platform === "win32" && localPath.match(/^\/[A-Za-z]:/)) {
+          localPath = localPath.substring(1);
+        }
+        localPath = decodeURIComponent(localPath);
+
+        if (!isAbsolute(localPath)) {
+          throw new Error(
+            `Expected absolute path from file:// URL, got: ${localPath}`
+          );
+        }
+
+        return await this.loadEndpointsFromFilesystem(localPath);
+      } else if (
+        parsedUrl.protocol === "https:" &&
+        parsedUrl.hostname === "github.com"
+      ) {
+        return await this.loadEndpointsFromGitHub(url);
+      } else {
+        throw new Error(
+          `Unsupported URL protocol: ${parsedUrl.protocol}. Supported: file://, https://github.com`
+        );
+      }
+    } catch (error) {
+      if (error instanceof TypeError) {
+        throw new Error(
+          `Invalid URL format: ${url}. Must be a valid file:// or https://github.com URL`
+        );
+      } else {
+        throw error;
+      }
+    }
+  }
+
+  private async loadEndpointsFromFilesystem(
+    directoryPath: string
+  ): Promise<{ loaded: number; failed: number; errors: string[] }> {
+    if (!isAbsolute(directoryPath)) {
+      throw new Error(`Expected absolute path, got: ${directoryPath}`);
+    }
+
+    log({
+      level: "info",
+      event: "loading_endpoints_from_filesystem",
+      directoryPath,
+    });
+
+    const filePaths = await this.iterateEndpointFiles(directoryPath);
+    log({
+      level: "info",
+      event: "endpoint_files_found",
+      count: filePaths.length,
+      files: filePaths.map((f) => f.split("/").pop()),
+    });
+
+    const loadedEndpoints: ScratchEndpointDefinition[] = [];
+    const errors: string[] = [];
+    let loadedCount = 0;
+    let failedCount = 0;
+
+    for (const filePath of filePaths) {
+      try {
+        const source = await readFile(filePath, "utf-8");
+        const endpoint = await this.parseEndpointFromSource(source, filePath);
+        const blockDef = await endpoint.block({});
+        if (!blockDef.opcode || blockDef.opcode === "") {
+          throw new Error(`Endpoint from ${filePath} has empty opcode`);
+        }
+        this.endpoints.set(blockDef.opcode, endpoint);
+        loadedEndpoints.push(endpoint);
+        loadedCount++;
+        await this.registerRoute(blockDef.opcode, endpoint);
+        log({
+          level: "info",
+          event: "endpoint_loaded",
+          opcode: blockDef.opcode,
+          file: filePath.split("/").pop(),
+        });
+      } catch (error) {
+        failedCount++;
+        const errorMessage =
+          error instanceof Error ? error.message : String(error);
+        const fileName = filePath.split("/").pop() || filePath;
+        errors.push(`${fileName}: ${errorMessage}`);
+        log({
+          level: "error",
+          event: "endpoint_load_failed",
+          file: fileName,
+          error: errorMessage,
+        });
+      }
+    }
+
+    await this.registerEndpoints(loadedEndpoints);
+
+    log({
+      level: "info",
+      event: "endpoints_loaded",
+      total: this.endpoints.size,
+      loaded: loadedCount,
+      failed: failedCount,
+    });
+
+    return { loaded: loadedCount, failed: failedCount, errors };
+  }
+
+  private async loadEndpointsFromGitHub(
+    githubUrl: string
+  ): Promise<{ loaded: number; failed: number; errors: string[] }> {
+    const errors: string[] = [];
+    let loadedCount = 0;
+    let failedCount = 0;
+
+    const urlMatch = githubUrl.match(
+      /https:\/\/github\.com\/([^\/]+)\/([^\/]+)\/tree\/([^\/]+)\/(.+)$/
+    );
+    if (!urlMatch) {
+      const errorMsg = `Invalid GitHub URL format. Expected: https://github.com/owner/repo/tree/branch/path, got: ${githubUrl}`;
+      errors.push(errorMsg);
+      return { loaded: 0, failed: 1, errors };
+    }
+
+    const [, owner, repo, branch, path] = urlMatch;
+    const apiUrl = `https://api.github.com/repos/${owner}/${repo}/contents/${path}?ref=${branch}`;
+    log({ level: "info", event: "fetching_github_file_list", apiUrl });
+
+    let files: Array<{ name: string; type: string; download_url?: string }> =
+      [];
+    try {
+      const response = await fetch(apiUrl, {
+        headers: {
+          Accept: "application/vnd.github.v3+json",
+          "User-Agent": "scratch-server",
+        },
+      });
+
+      if (!response.ok) {
+        const errorMsg = `Failed to fetch GitHub file list: ${response.status} ${response.statusText}`;
+        errors.push(errorMsg);
+        log({
+          level: "error",
+          event: "github_file_list_fetch_failed",
+          error: errorMsg,
+        });
+        return { loaded: 0, failed: 1, errors };
+      }
+
+      files = (await response.json()) as Array<{
+        name: string;
+        type: string;
+        download_url?: string;
+      }>;
+    } catch (error) {
+      const errorMsg = error instanceof Error ? error.message : String(error);
+      errors.push(`Failed to fetch GitHub file list: ${errorMsg}`);
+      log({
+        level: "error",
+        event: "github_file_list_fetch_error",
+        error: errorMsg,
+      });
+      return { loaded: 0, failed: 1, errors };
+    }
+
+    const tsFiles = files.filter(
+      (f) => f.type === "file" && f.name.endsWith(".ts")
+    );
+
+    log({
+      level: "info",
+      event: "github_files_found",
+      count: tsFiles.length,
+      files: tsFiles.map((f) => f.name),
+    });
+
+    const loadedEndpoints: ScratchEndpointDefinition[] = [];
+
+    for (const file of tsFiles) {
+      try {
+        const rawUrl = `https://raw.githubusercontent.com/${owner}/${repo}/${branch}/${path}/${file.name}`;
+        log({
+          level: "info",
+          event: "fetching_github_file",
+          file: file.name,
+          url: rawUrl,
+        });
+
+        const fileResponse = await fetch(rawUrl, {
+          headers: { "User-Agent": "scratch-server" },
+        });
+
+        if (!fileResponse.ok) {
+          throw new Error(
+            `Failed to fetch file ${file.name}: ${fileResponse.status} ${fileResponse.statusText}`
+          );
+        }
+
+        const source = await fileResponse.text();
+
+        const firstLines = source.split("\n").slice(0, 5).join("\n");
+        log({
+          level: "info",
+          event: "github_file_content_preview",
+          file: file.name,
+          firstLines: firstLines,
+          totalLines: source.split("\n").length,
+        });
+
+        const endpoint = await this.parseEndpointFromSource(source, file.name);
+        const blockDef = await endpoint.block({});
+        if (!blockDef.opcode || blockDef.opcode === "") {
+          throw new Error(`Endpoint from ${file.name} has empty opcode`);
+        }
+        this.endpoints.set(blockDef.opcode, endpoint);
+        loadedEndpoints.push(endpoint);
+        loadedCount++;
+        await this.registerRoute(blockDef.opcode, endpoint);
+        log({
+          level: "info",
+          event: "endpoint_loaded",
+          opcode: blockDef.opcode,
+          file: file.name,
+        });
+      } catch (error) {
+        failedCount++;
+        const errorMessage =
+          error instanceof Error ? error.message : String(error);
+        errors.push(`${file.name}: ${errorMessage}`);
+        log({
+          level: "error",
+          event: "endpoint_load_failed",
+          file: file.name,
+          error: errorMessage,
+        });
+      }
+    }
+
+    await this.registerEndpoints(loadedEndpoints);
+
+    log({
+      level: "info",
+      event: "github_endpoints_loaded",
+      total: this.endpoints.size,
+      loaded: loadedCount,
+      failed: failedCount,
+    });
+
+    return { loaded: loadedCount, failed: failedCount, errors };
+  }
+
+  private async iterateEndpointFiles(directoryPath: string): Promise<string[]> {
+    const files: string[] = [];
+    try {
+      const entries = await readdir(directoryPath, { withFileTypes: true });
+      for (const entry of entries) {
+        const fullPath = join(directoryPath, entry.name);
+        if (entry.isDirectory()) {
+          files.push(...(await this.iterateEndpointFiles(fullPath)));
+        } else if (entry.isFile() && entry.name.endsWith(".ts")) {
+          files.push(fullPath);
+        }
+      }
+    } catch (error) {
+      log({
+        level: "error",
+        event: "directory_read_failed",
+        directoryPath,
+        error: error instanceof Error ? error.message : String(error),
+      });
+    }
+    return files;
+  }
+
+  private async parseEndpointFromSource(
+    source: string,
+    filePath: string
+  ): Promise<ScratchEndpointDefinition> {
+    const absolutePath = resolve(filePath);
+    let finalPath = absolutePath;
+    let fileExists = false;
+    try {
+      await stat(absolutePath);
+      fileExists = true;
+    } catch {
+      // File doesn't exist - write GitHub source to temporary file in project directory
+      // This ensures npm packages can be resolved correctly
+      const projectRoot = await getProjectRoot();
+      const tempDir = join(
+        projectRoot,
+        ".scratch-endpoints-temp",
+        randomUUID()
+      );
+      await mkdir(tempDir, { recursive: true });
+      finalPath = join(tempDir, basename(filePath));
+      await writeFile(finalPath, source, "utf-8");
+    }
+
+    // Import the file using native import - works identically for local and GitHub files
+    const module = await import(finalPath);
+
+    // Extract the endpoint definition from the module
+    const endpointName = basename(filePath, ".ts");
+
+    // Try to find the endpoint in the module
+    let endpoint = module[endpointName] || module.default;
+
+    // If not found by name, search for any object with block and handler
+    if (!endpoint && module && typeof module === "object") {
+      endpoint =
+        Object.values(module).find(
+          (value): value is ScratchEndpointDefinition =>
+            value !== null &&
+            typeof value === "object" &&
+            "block" in value &&
+            "handler" in value
+        ) || null;
+    }
+
+    // If still not found, check if module itself is the endpoint
+    if (
+      !endpoint &&
+      module &&
+      typeof module === "object" &&
+      "block" in module &&
+      "handler" in module
+    ) {
+      endpoint = module as ScratchEndpointDefinition;
+    }
+
+    if (!endpoint) {
+      throw new Error(`No endpoint definition found in ${filePath}`);
+    }
+
+    if (
+      !endpoint.block ||
+      typeof endpoint.block !== "function" ||
+      !endpoint.handler ||
+      typeof endpoint.handler !== "function"
+    ) {
+      throw new Error(
+        `Invalid endpoint definition in ${filePath}: missing block or handler`
+      );
+    }
+
+    return endpoint as ScratchEndpointDefinition;
+  }
+}