@teamkeel/functions-runtime 0.412.0-next.2 → 0.412.0

package/src/casing.test.js

@@ -0,0 +1,56 @@
+import { test, expect } from "vitest";
+import { camelCaseObject, snakeCaseObject } from "./casing";
+
+const cases = {
+  FROM_SNAKE: {
+    input: {
+      id: "123",
+      slack_id: "xxx_2929",
+      api_key: "1234",
+      test_11: "1234",
+      test_1_test: "1234",
+      test12: "1234",
+      abc123_test: "1234",
+    },
+    expected: {
+      id: "123",
+      slackId: "xxx_2929",
+      apiKey: "1234",
+      test11: "1234",
+      test1Test: "1234",
+      test12: "1234",
+      abc123Test: "1234",
+    },
+  },
+  FROM_CAMEL: {
+    input: {
+      id: "123",
+      slackId: "xxx_2929",
+      apiKey: "1234",
+      test1: "test",
+      testA1: "test",
+      test1Test: "test",
+      test20: "test",
+      testURL: "test",
+    },
+    expected: {
+      id: "123",
+      slack_id: "xxx_2929",
+      api_key: "1234",
+      test_1: "test",
+      test_a_1: "test",
+      test_1_test: "test",
+      test_20: "test",
+      test_url: "test",
+    },
+  },
+};
+
+Object.entries(cases).map(([key, { input, expected }]) => {
+  test(key, () => {
+    const result =
+      key === "FROM_SNAKE" ? camelCaseObject(input) : snakeCaseObject(input);
+
+    expect(result).toEqual(expected);
+  });
+});

package/src/consts.js

@@ -0,0 +1,14 @@
+const PROTO_ACTION_TYPES = {
+  UNKNOWN: "ACTION_TYPE_UNKNOWN",
+  CREATE: "ACTION_TYPE_CREATE",
+  GET: "ACTION_TYPE_GET",
+  LIST: "ACTION_TYPE_LIST",
+  UPDATE: "ACTION_TYPE_UPDATE",
+  DELETE: "ACTION_TYPE_DELETE",
+  READ: "ACTION_TYPE_READ",
+  WRITE: "ACTION_TYPE_WRITE",
+  JOB: "JOB_TYPE",
+  SUBSCRIBER: "SUBSCRIBER_TYPE",
+};
+
+module.exports.PROTO_ACTION_TYPES = PROTO_ACTION_TYPES;

package/src/database.js

@@ -0,0 +1,244 @@
+const { Kysely, PostgresDialect } = require("kysely");
+const neonserverless = require("@neondatabase/serverless");
+const { AsyncLocalStorage } = require("async_hooks");
+const { AuditContextPlugin } = require("./auditing");
+const { KeelCamelCasePlugin } = require("./camelCasePlugin");
+const pg = require("pg");
+const { withSpan } = require("./tracing");
+const ws = require("ws");
+const fs = require("node:fs");
+const { Duration } = require("./Duration");
+
+// withDatabase is responsible for setting the correct database client in our AsyncLocalStorage
+// so that the the code in a custom function uses the correct client.
+// For GET and LIST action types, no transaction is used, but for
+// actions that mutate data such as CREATE, DELETE & UPDATE, all of the code inside
+// the user's custom function is wrapped in a transaction so we can rollback
+// the transaction if something goes wrong.
+// withDatabase shouldn't be exposed in the public api of the sdk
+async function withDatabase(db, requiresTransaction, cb) {
+  // db.transaction() provides a kysely instance bound to a transaction.
+  if (requiresTransaction) {
+    return db.transaction().execute(async (transaction) => {
+      return dbInstance.run(transaction, async () => {
+        return cb({ transaction });
+      });
+    });
+  }
+
+  // db.connection() provides a kysely instance bound to a single database connection.
+  return db.connection().execute(async (sDb) => {
+    return dbInstance.run(sDb, async () => {
+      return cb({ sDb });
+    });
+  });
+}
+
+const dbInstance = new AsyncLocalStorage();
+
+// used to establish a singleton for our vitest environment
+let vitestDb = null;
+
+// useDatabase will retrieve the database client set by withDatabase from the local storage
+function useDatabase() {
+  // retrieve the instance of the database client from the store which is aware of
+  // which context the current connection to the db is running in - e.g does the context
+  // require a transaction or not?
+  let fromStore = dbInstance.getStore();
+  if (fromStore) {
+    return fromStore;
+  }
+
+  // if the NODE_ENV is 'test' then we know we are inside of the vitest environment
+  // which covers any test files ending in *.test.ts. Custom function code runs in a different node process which will not have this environment variable. Tests written using our testing
+  // framework call actions (and in turn custom function code) over http using the ActionExecutor class
+  if ("NODE_ENV" in process.env && process.env.NODE_ENV == "test") {
+    if (!vitestDb) {
+      vitestDb = createDatabaseClient();
+    }
+    return vitestDb;
+  }
+
+  // If we've gotten to this point, then we know that we are in a custom function runtime server
+  // context and we haven't been able to retrieve the in-context instance of Kysely, which means we should throw an error.
+  console.trace();
+  throw new Error("useDatabase must be called within a function");
+}
+
+// createDatabaseClient will return a brand new instance of Kysely. Every instance of Kysely
+// represents an individual connection to the database.
+// not to be exported externally from our sdk - consumers should use useDatabase
+function createDatabaseClient({ connString } = {}) {
+  const db = new Kysely({
+    dialect: getDialect(connString),
+    plugins: [
+      // ensures that the audit context data is written to Postgres configuration parameters
+      new AuditContextPlugin(),
+      // allows users to query using camelCased versions of the database column names, which
+      // should match the names we use in our schema.
+      // We're using an extended version of Kysely's CamelCasePlugin which avoids changing keys of objects that represent
+      // rich data formats, specific to Keel (e.g. Duration)
+      new KeelCamelCasePlugin(),
+    ],
+    log(event) {
+      if ("DEBUG" in process.env) {
+        if (event.level === "query") {
+          console.log(event.query.sql);
+          console.log(event.query.parameters);
+        }
+      }
+    },
+  });
+
+  return db;
+}
+
+class InstrumentedPool extends pg.Pool {
+  async connect(...args) {
+    const _super = super.connect.bind(this);
+    return withSpan("Database Connect", function (span) {
+      span.setAttribute("dialect", process.env["KEEL_DB_CONN_TYPE"]);
+      return _super(...args);
+    });
+  }
+}
+
+class InstrumentedNeonServerlessPool extends neonserverless.Pool {
+  async connect(...args) {
+    const _super = super.connect.bind(this);
+    return withSpan("Database Connect", function (span) {
+      span.setAttribute("dialect", process.env["KEEL_DB_CONN_TYPE"]);
+      return _super(...args);
+    });
+  }
+}
+
+const txStatements = {
+  begin: "Transaction Begin",
+  commit: "Transaction Commit",
+  rollback: "Transaction Rollback",
+};
+
+class InstrumentedClient extends pg.Client {
+  async query(...args) {
+    const _super = super.query.bind(this);
+    const sql = args[0];
+
+    let sqlAttribute = false;
+    let spanName = txStatements[sql.toLowerCase()];
+    if (!spanName) {
+      spanName = "Database Query";
+      sqlAttribute = true;
+    }
+
+    return withSpan(spanName, function (span) {
+      if (sqlAttribute) {
+        span.setAttribute("sql", args[0]);
+        span.setAttribute("dialect", process.env["KEEL_DB_CONN_TYPE"]);
+      }
+      return _super(...args);
+    });
+  }
+}
+
+function getDialect(connString) {
+  const dbConnType = process.env.KEEL_DB_CONN_TYPE;
+  switch (dbConnType) {
+    case "pg":
+      // Adding a custom type parser for numeric fields: see https://kysely.dev/docs/recipes/data-types#configuring-runtime-javascript-types
+      // 1700 = type for NUMERIC
+      pg.types.setTypeParser(pg.types.builtins.NUMERIC, function (val) {
+        return parseFloat(val);
+      });
+      // Adding a custom type parser for interval fields: see https://kysely.dev/docs/recipes/data-types#configuring-runtime-javascript-types
+      // 1186 = type for INTERVAL
+      pg.types.setTypeParser(pg.types.builtins.INTERVAL, function (val) {
+        return new Duration(val);
+      });
+
+      return new PostgresDialect({
+        pool: new InstrumentedPool({
+          Client: InstrumentedClient,
+          // Increased idle time before closing a connection in the local pool (from 10s default).
+          // Establising a new connection on (almost) every functions query can be expensive, so this
+          // will reduce having to open connections as regularly. https://node-postgres.com/apis/pool
+          //
+          // NOTE: We should consider setting this to 0 (i.e. never pool locally) and open and close
+          // connections with each invocation. This is because the freeze/thaw nature of lambdas can cause problems
+          // with long-lived connections - see https://github.com/brianc/node-postgres/issues/2718
+          // Once we're "fully regional" this should not be a performance problem anymore.
+          //
+          // Although I doubt we will run into these freeze/thaw issues if idleTimeoutMillis is always shorter than the
+          // time is takes for a lambda to freeze (which is not a constant, but could be as short as several minutes,
+          // https://www.pluralsight.com/resources/blog/cloud/how-long-does-aws-lambda-keep-your-idle-functions-around-before-a-cold-start)
+          idleTimeoutMillis: 50000,
+
+          // If connString is not passed fall back to reading from env var
+          connectionString: connString || process.env.KEEL_DB_CONN,
+
+          // Allow the setting of a cert (.pem) file. RDS requires this to enforce SSL.
+          ...(process.env.KEEL_DB_CERT
+            ? { ssl: { ca: fs.readFileSync(process.env.KEEL_DB_CERT) } }
+            : undefined),
+        }),
+      });
+    case "neon":
+      // Adding a custom type parser for numeric fields: see https://kysely.dev/docs/recipes/data-types#configuring-runtime-javascript-types
+      // 1700 = type for NUMERIC
+      neonserverless.types.setTypeParser(
+        pg.types.builtins.NUMERIC,
+        function (val) {
+          return parseFloat(val);
+        }
+      );
+      // Adding a custom type parser for interval fields: see https://kysely.dev/docs/recipes/data-types#configuring-runtime-javascript-types
+      // 1186 = type for INTERVAL
+      neonserverless.types.setTypeParser(
+        pg.types.builtins.INTERVAL,
+        function (val) {
+          return new Duration(val);
+        }
+      );
+
+      neonserverless.neonConfig.webSocketConstructor = ws;
+
+      const pool = new InstrumentedNeonServerlessPool({
+        // If connString is not passed fall back to reading from env var
+        connectionString: connString || process.env.KEEL_DB_CONN,
+      });
+
+      pool.on("connect", (client) => {
+        const originalQuery = client.query;
+        client.query = function (...args) {
+          const sql = args[0];
+
+          let sqlAttribute = false;
+          let spanName = txStatements[sql.toLowerCase()];
+          if (!spanName) {
+            spanName = "Database Query";
+            sqlAttribute = true;
+          }
+
+          return withSpan(spanName, function (span) {
+            if (sqlAttribute) {
+              span.setAttribute("sql", args[0]);
+              span.setAttribute("dialect", dbConnType);
+            }
+            return originalQuery.apply(client, args);
+          });
+        };
+      });
+
+      return new PostgresDialect({
+        pool: pool,
+      });
+    default:
+      throw Error("unexpected KEEL_DB_CONN_TYPE: " + dbConnType);
+  }
+}
+
+module.exports = {
+  createDatabaseClient,
+  useDatabase,
+  withDatabase,
+};

package/src/errors.js

@@ -0,0 +1,160 @@
+const { createJSONRPCErrorResponse } = require("json-rpc-2.0");
+
+const RuntimeErrors = {
+  // Catchall error type for unhandled execution errors during custom function
+  UnknownError: -32001,
+  // DatabaseError represents any error at pg level that isn't handled explicitly below
+  DatabaseError: -32002,
+  // No result returned from custom function by user
+  NoResultError: -32003,
+  // When trying to delete/update a non existent record in the db
+  RecordNotFoundError: -32004,
+  ForeignKeyConstraintError: -32005,
+  NotNullConstraintError: -32006,
+  UniqueConstraintError: -32007,
+  PermissionError: -32008,
+  BadRequestError: -32009,
+};
+
+// Error presets
+class PermissionError extends Error {}
+
+class DatabaseError extends Error {
+  constructor(error) {
+    super(error.message);
+    this.error = error;
+  }
+}
+
+class NotFoundError extends Error {
+  errorCode = RuntimeErrors.RecordNotFoundError;
+  constructor(message) {
+    super(message); // Default message is handled by the runtime for consistency with built in actions
+  }
+}
+
+class BadRequestError extends Error {
+  errorCode = RuntimeErrors.BadRequestError;
+  constructor(message = "bad request") {
+    super(message);
+  }
+}
+
+class UnknownError extends Error {
+  errorCode = RuntimeErrors.UnknownError;
+  constructor(message = "unknown error") {
+    super(message);
+  }
+}
+
+const ErrorPresets = {
+  NotFound: NotFoundError,
+  BadRequest: BadRequestError,
+  Unknown: UnknownError,
+};
+
+// errorToJSONRPCResponse transforms a JavaScript Error instance (or derivative) into a valid JSONRPC response object to pass back to the Keel runtime.
+function errorToJSONRPCResponse(request, e) {
+  switch (e.constructor.name) {
+    case "PermissionError":
+      return createJSONRPCErrorResponse(
+        request.id,
+        RuntimeErrors.PermissionError,
+        e.message
+      );
+    // Any error thrown in the ModelAPI class is
+    // wrapped in a DatabaseError in order to differentiate 'our code' vs the user's own code.
+    case "NoResultError":
+      return createJSONRPCErrorResponse(
+        request.id,
+
+        // to be matched to https://github.com/teamkeel/keel/blob/e3115ffe381bfc371d4f45bbf96a15072a994ce5/runtime/actions/update.go#L54-L54
+        RuntimeErrors.RecordNotFoundError,
+        "" // Don't pass on the message as we want to normalise these at the runtime layer but still support custom messages in other NotFound errors
+      );
+    case "DatabaseError":
+      let err = e;
+
+      // If wrapped error then unwrap
+      if (e instanceof DatabaseError) {
+        err = e.error;
+      }
+
+      if (err.constructor.name == "NoResultError") {
+        return createJSONRPCErrorResponse(
+          request.id,
+
+          // to be matched to https://github.com/teamkeel/keel/blob/e3115ffe381bfc371d4f45bbf96a15072a994ce5/runtime/actions/update.go#L54-L54
+          RuntimeErrors.RecordNotFoundError,
+          "" // Don't pass on the message as we want to normalise these at the runtime layer but still support custom messages in other NotFound errors
+        );
+      }
+
+      // if the error contains 'code' then assume it has other pg error message keys
+      // todo: make this more ironclad.
+      // when using lib-pq, should match https://github.com/brianc/node-postgres/blob/master/packages/pg-protocol/src/parser.ts#L371-L386
+      if ("code" in err) {
+        const { code, detail, table } = err;
+
+        let rpcErrorCode, column, value;
+        const [col, val] = parseKeyMessage(err.detail);
+        column = col;
+        value = val;
+
+        switch (code) {
+          case "23502":
+            rpcErrorCode = RuntimeErrors.NotNullConstraintError;
+            column = err.column;
+            break;
+          case "23503":
+            rpcErrorCode = RuntimeErrors.ForeignKeyConstraintError;
+            break;
+          case "23505":
+            rpcErrorCode = RuntimeErrors.UniqueConstraintError;
+            break;
+          default:
+            rpcErrorCode = RuntimeErrors.DatabaseError;
+            break;
+        }
+
+        return createJSONRPCErrorResponse(request.id, rpcErrorCode, e.message, {
+          table,
+          column,
+          code,
+          detail,
+          value,
+        });
+      }
+
+      // we don't know what it is, but it's something else
+      return createJSONRPCErrorResponse(
+        request.id,
+        RuntimeErrors.DatabaseError,
+        e.message
+      );
+    default:
+      // Use the errorCode in the error if we have some from a preset
+      return createJSONRPCErrorResponse(
+        request.id,
+        e.errorCode ?? RuntimeErrors.UnknownError,
+        e.message
+      );
+  }
+}
+
+// example data:
+// Key (author_id)=(fake) is not present in table "author".
+const keyMessagePattern = /\Key\s[(](.*)[)][=][(](.*)[)]/;
+const parseKeyMessage = (msg) => {
+  const [, col, value] = keyMessagePattern.exec(msg) || [];
+
+  return [col, value];
+};
+
+module.exports = {
+  errorToJSONRPCResponse,
+  RuntimeErrors,
+  DatabaseError,
+  PermissionError,
+  ErrorPresets,
+};

package/src/handleJob.js

@@ -0,0 +1,110 @@
+const {
+  createJSONRPCErrorResponse,
+  createJSONRPCSuccessResponse,
+  JSONRPCErrorCode,
+} = require("json-rpc-2.0");
+const { createDatabaseClient } = require("./database");
+const { errorToJSONRPCResponse, RuntimeErrors } = require("./errors");
+const opentelemetry = require("@opentelemetry/api");
+const { withSpan } = require("./tracing");
+const { tryExecuteJob } = require("./tryExecuteJob");
+const { parseInputs } = require("./parsing");
+const { PROTO_ACTION_TYPES } = require("./consts");
+
+// Generic handler function that is agnostic to runtime environment (local or lambda)
+// to execute a job function based on the contents of a jsonrpc-2.0 payload object.
+// To read more about jsonrpc request and response shapes, please read https://www.jsonrpc.org/specification
+async function handleJob(request, config) {
+  // Try to extract trace context from caller
+  const activeContext = opentelemetry.propagation.extract(
+    opentelemetry.context.active(),
+    request.meta?.tracing
+  );
+
+  // Run the whole request with the extracted context
+  return opentelemetry.context.with(activeContext, () => {
+    // Wrapping span for the whole request
+    return withSpan(request.method, async (span) => {
+      let db = null;
+
+      try {
+        const { createJobContextAPI, jobs } = config;
+
+        if (!(request.method in jobs)) {
+          const message = `no corresponding job found for '${request.method}'`;
+          span.setStatus({
+            code: opentelemetry.SpanStatusCode.ERROR,
+            message: message,
+          });
+          return createJSONRPCErrorResponse(
+            request.id,
+            JSONRPCErrorCode.MethodNotFound,
+            message
+          );
+        }
+
+        // The ctx argument passed into the job function.
+        const ctx = createJobContextAPI({
+          meta: request.meta,
+        });
+
+        const permitted =
+          request.meta && request.meta.permissionState.status === "granted"
+            ? true
+            : null;
+
+        db = createDatabaseClient({
+          connString: request.meta?.secrets?.KEEL_DB_CONN,
+        });
+        const jobFunction = jobs[request.method];
+        const actionType = PROTO_ACTION_TYPES.JOB;
+
+        const functionConfig = jobFunction?.config ?? {};
+
+        await tryExecuteJob(
+          { request, permitted, db, actionType, functionConfig },
+          async () => {
+            // parse request params to convert objects into rich field types (e.g. InlineFile)
+            const inputs = parseInputs(request.params);
+
+            // Return the job function to the containing tryExecuteJob block
+            return jobFunction(ctx, inputs);
+          }
+        );
+
+        return createJSONRPCSuccessResponse(request.id, null);
+      } catch (e) {
+        if (e instanceof Error) {
+          span.recordException(e);
+          span.setStatus({
+            code: opentelemetry.SpanStatusCode.ERROR,
+            message: e.message,
+          });
+          return errorToJSONRPCResponse(request, e);
+        }
+
+        const message = JSON.stringify(e);
+
+        span.setStatus({
+          code: opentelemetry.SpanStatusCode.ERROR,
+          message: message,
+        });
+
+        return createJSONRPCErrorResponse(
+          request.id,
+          RuntimeErrors.UnknownError,
+          message
+        );
+      } finally {
+        if (db) {
+          await db.destroy();
+        }
+      }
+    });
+  });
+}
+
+module.exports = {
+  handleJob,
+  RuntimeErrors,
+};