@teamkeel/functions-runtime 0.365.18 → 0.366.0-audit-logs0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@teamkeel/functions-runtime",
-  "version": "0.365.18",
+  "version": "0.366.0-audit-logs0",
   "description": "Internal package used by @teamkeel/sdk",
   "main": "src/index.js",
   "scripts": {
@@ -28,6 +28,7 @@
     "json-rpc-2.0": "^1.4.1",
     "ksuid": "^3.0.0",
     "kysely": "^0.23.4",
-    "pg": "^8.8.0"
+    "pg": "^8.8.0",
+    "traceparent": "^1.0.0"
   }
 }

package/src/ModelAPI.js

@@ -3,6 +3,7 @@ const { QueryBuilder } = require("./QueryBuilder");
 const { QueryContext } = require("./QueryContext");
 const { applyWhereConditions } = require("./applyWhereConditions");
 const { applyJoins } = require("./applyJoins");
+
 const {
   applyLimit,
   applyOffset,

package/src/auditing.js

@@ -0,0 +1,119 @@
+const { AsyncLocalStorage } = require("async_hooks");
+const TraceParent = require("traceparent");
+const { sql, SelectionNode } = require("kysely");
+
+const auditContextStorage = new AsyncLocalStorage();
+
+// withAuditContext creates the audit context from the runtime request body
+// and sets it to in AsyncLocalStorage so that this data is available to the
+// ModelAPI during the execution of actions, jobs and subscribers.
+async function withAuditContext(request, cb) {
+  let audit = {};
+
+  if (request.meta?.identity) {
+    audit.identityId = request.meta.identity.id;
+  }
+  if (request.meta?.tracing?.traceparent) {
+    audit.traceId = TraceParent.fromString(
+      request.meta.tracing.traceparent
+    )?.traceId;
+  }
+
+  return await auditContextStorage.run(audit, () => {
+    return cb();
+  });
+}
+
+// getAuditContext retrieves the audit context from AsyncLocalStorage.
+function getAuditContext() {
+  let auditStore = auditContextStorage.getStore();
+  return {
+    identityId: auditStore?.identityId,
+    traceId: auditStore?.traceId,
+  };
+}
+
+// AuditContextPlugin is a Kysely plugin which ensures that the audit context data
+// is written to Postgres configuration parameters in the same execution as a query.
+// It does this by calling the set_identity_id() and set_trace_id() functions as a
+// clause in the returning statement. It then subsequently drops these from the actual result.
+// This ensures that these parameters are set when the tables' AFTER trigger function executes.
+class AuditContextPlugin {
+  constructor() {
+    this.identityIdAlias = "__keel_identity_id";
+    this.traceIdAlias = "__keel_trace_id";
+  }
+
+  #setIdentityClause(value) {
+    return `set_identity_id('${value}')`;
+  }
+
+  #setTraceIdClause(value) {
+    return `set_trace_id('${value}')`;
+  }
+
+  // Appends set_identity_id() and set_trace_id() function calls to the returning statement
+  // of INSERT, UPDATE and DELETE operations.
+  transformQuery(args) {
+    switch (args.node.kind) {
+      case "InsertQueryNode":
+      case "UpdateQueryNode":
+      case "DeleteQueryNode":
+        const returning = {
+          kind: "ReturningNode",
+          selections: [],
+        };
+        if (args.node.returning) {
+          returning.selections.push(...args.node.returning.selections);
+        }
+
+        // Retrieve the audit context from async storage.
+        const audit = getAuditContext();
+
+        if (audit.identityId) {
+          const rawNode = sql
+            .raw(
+              this.#setIdentityClause(audit.identityId, this.identityIdAlias)
+            )
+            .as(this.identityIdAlias)
+            .toOperationNode();
+
+          returning.selections.push(SelectionNode.create(rawNode));
+        }
+
+        if (audit.traceId) {
+          const rawNode = sql
+            .raw(this.#setTraceIdClause(audit.traceId))
+            .as(this.traceIdAlias)
+            .toOperationNode();
+
+          returning.selections.push(SelectionNode.create(rawNode));
+        }
+
+        return {
+          ...args.node,
+          returning: returning,
+        };
+    }
+
+    return {
+      ...args.node,
+    };
+  }
+
+  // Drops the set_identity_id() and set_trace_id() fields from the result.
+  transformResult(args) {
+    if (args.result?.rows) {
+      for (let i = 0; i < args.result.rows.length; i++) {
+        delete args.result.rows[i][this.identityIdAlias];
+        delete args.result.rows[i][this.traceIdAlias];
+      }
+    }
+
+    return args.result;
+  }
+}
+
+module.exports.withAuditContext = withAuditContext;
+module.exports.getAuditContext = getAuditContext;
+module.exports.AuditContextPlugin = AuditContextPlugin;

package/src/auditing.test.js

@@ -0,0 +1,362 @@
+import { test, expect, beforeEach } from "vitest";
+const { ModelAPI } = require("./ModelAPI");
+const { PROTO_ACTION_TYPES } = require("./consts");
+const { sql } = require("kysely");
+const { useDatabase, withDatabase } = require("./database");
+const KSUID = require("ksuid");
+const TraceParent = require("traceparent");
+const { withAuditContext } = require("./auditing");
+
+let personAPI;
+const db = useDatabase();
+
+beforeEach(async () => {
+  await sql`
+ 
+  DROP TABLE IF EXISTS post;
+  DROP TABLE IF EXISTS person;
+  DROP TABLE IF EXISTS author;
+
+  CREATE TABLE person(
+      id               text PRIMARY KEY,
+      name             text UNIQUE
+  );
+
+  CREATE OR REPLACE FUNCTION set_identity_id(id VARCHAR)
+  RETURNS TEXT AS $$
+  BEGIN
+      RETURN set_config('audit.identity_id', id, true);
+  END
+  $$ LANGUAGE plpgsql;
+
+  CREATE OR REPLACE FUNCTION set_trace_id(id VARCHAR)
+  RETURNS TEXT AS $$
+  BEGIN
+      RETURN set_config('audit.trace_id', id, true);
+  END
+  $$ LANGUAGE plpgsql;
+  `.execute(db);
+
+  personAPI = new ModelAPI("person", undefined, {});
+});
+
+async function identityIdFromConfigParam(database, nonLocal = true) {
+  const result =
+    await sql`SELECT NULLIF(current_setting('audit.identity_id', ${sql.literal(
+      nonLocal
+    )}), '') AS id`.execute(database);
+  return result.rows[0].id;
+}
+
+async function traceIdFromConfigParam(database, nonLocal = true) {
+  const result =
+    await sql`SELECT NULLIF(current_setting('audit.trace_id', ${sql.literal(
+      nonLocal
+    )}), '') AS id`.execute(database);
+  return result.rows[0].id;
+}
+
+test("auditing - capturing identity id in transaction", async () => {
+  const request = {
+    meta: {
+      identity: { id: KSUID.randomSync().string },
+    },
+  };
+
+  const identityId = request.meta.identity.id;
+
+  const row = await withDatabase(
+    db,
+    PROTO_ACTION_TYPES.CREATE, // CREATE will ensure a transaction is opened
+    async ({ transaction }) => {
+      const row = withAuditContext(request, async () => {
+        return await personAPI.create({
+          id: KSUID.randomSync().string,
+          name: "James",
+        });
+      });
+
+      expect(await identityIdFromConfigParam(transaction)).toEqual(identityId);
+      expect(await identityIdFromConfigParam(db)).toBeNull();
+
+      return row;
+    }
+  );
+
+  expect(row.name).toEqual("James");
+  expect(await identityIdFromConfigParam(db)).toBeNull();
+  expect(await identityIdFromConfigParam(db, false)).toBeNull();
+});
+
+test("auditing - capturing tracing in transaction", async () => {
+  const request = {
+    meta: {
+      tracing: {
+        traceparent: "00-80e1afed08e019fc1110464cfa66635c-7a085853722dc6d2-01",
+      },
+    },
+  };
+
+  const traceId = TraceParent.fromString(
+    request.meta.tracing.traceparent
+  ).traceId;
+
+  const row = await withDatabase(
+    db,
+    PROTO_ACTION_TYPES.CREATE, // CREATE will ensure a transaction is opened
+    async ({ transaction }) => {
+      const row = withAuditContext(request, async () => {
+        return await personAPI.create({
+          id: KSUID.randomSync().string,
+          name: "Jim",
+        });
+      });
+
+      expect(await traceIdFromConfigParam(transaction)).toEqual(traceId);
+      expect(await traceIdFromConfigParam(db)).toBeNull();
+
+      return row;
+    }
+  );
+
+  expect(row.name).toEqual("Jim");
+  expect(await traceIdFromConfigParam(db)).toBeNull();
+  expect(await traceIdFromConfigParam(db, false)).toBeNull();
+});
+
+test("auditing - capturing identity id without transaction", async () => {
+  const request = {
+    meta: {
+      identity: { id: KSUID.randomSync().string },
+    },
+  };
+
+  const row = await withDatabase(
+    db,
+    PROTO_ACTION_TYPES.GET, // GET will _not_ open a transaction
+    async ({ sDb }) => {
+      const row = withAuditContext(request, async () => {
+        return await personAPI.create({
+          id: KSUID.randomSync().string,
+          name: "James",
+        });
+      });
+
+      expect(await identityIdFromConfigParam(sDb)).toBeNull();
+      expect(await identityIdFromConfigParam(db)).toBeNull();
+
+      return row;
+    }
+  );
+
+  expect(row.name).toEqual("James");
+  expect(await identityIdFromConfigParam(db)).toBeNull();
+  expect(await identityIdFromConfigParam(db, false)).toBeNull();
+});
+
+test("auditing - capturing tracing without transaction", async () => {
+  const request = {
+    meta: {
+      tracing: {
+        traceparent: "00-80e1afed08e019fc1110464cfa66635c-7a085853722dc6d2-01",
+      },
+    },
+  };
+
+  const row = await withDatabase(
+    db,
+    PROTO_ACTION_TYPES.GET, // GET will _not_ open a transaction
+    async ({ sDb }) => {
+      const row = withAuditContext(request, async () => {
+        return await personAPI.create({
+          id: KSUID.randomSync().string,
+          name: "Jim",
+        });
+      });
+
+      expect(await traceIdFromConfigParam(sDb)).toBeNull();
+      expect(await traceIdFromConfigParam(db)).toBeNull();
+
+      return row;
+    }
+  );
+
+  expect(row.name).toEqual("Jim");
+  expect(await traceIdFromConfigParam(db)).toBeNull();
+  expect(await traceIdFromConfigParam(db, false)).toBeNull();
+});
+
+test("auditing - no audit context", async () => {
+  const row = await withDatabase(
+    db,
+    PROTO_ACTION_TYPES.CREATE,
+    async ({ transaction }) => {
+      const row = withAuditContext({}, async () => {
+        return await personAPI.create({
+          id: KSUID.randomSync().string,
+          name: "Jake",
+        });
+      });
+
+      expect(await identityIdFromConfigParam(transaction)).toBeNull();
+      expect(await identityIdFromConfigParam(db)).toBeNull();
+      expect(await traceIdFromConfigParam(transaction)).toBeNull();
+      expect(await traceIdFromConfigParam(db)).toBeNull();
+      return row;
+    }
+  );
+
+  expect(KSUID.parse(row.id).string).toEqual(row.id);
+  expect(row.name).toEqual("Jake");
+});
+
+test("auditing - ModelAPI.create", async () => {
+  const request = {
+    meta: {
+      identity: { id: KSUID.randomSync().string },
+      tracing: {
+        traceparent: "00-80e1afed08e019fc1110464cfa66635c-7a085853722dc6d2-01",
+      },
+    },
+  };
+
+  const identityId = request.meta.identity.id;
+  const traceId = TraceParent.fromString(
+    request.meta.tracing.traceparent
+  ).traceId;
+
+  const row = await withDatabase(
+    db,
+    PROTO_ACTION_TYPES.CREATE,
+    async ({ transaction }) => {
+      const row = withAuditContext(request, async () => {
+        return await personAPI.create({
+          id: KSUID.randomSync().string,
+          name: "Jake",
+        });
+      });
+
+      expect(await identityIdFromConfigParam(transaction)).toEqual(identityId);
+      expect(await traceIdFromConfigParam(transaction)).toEqual(traceId);
+
+      return row;
+    }
+  );
+
+  expect(row.name).toEqual("Jake");
+});
+
+test("auditing - ModelAPI.update", async () => {
+  const request = {
+    meta: {
+      identity: { id: KSUID.randomSync().string },
+      tracing: {
+        traceparent: "00-80e1afed08e019fc1110464cfa66635c-7a085853722dc6d2-01",
+      },
+    },
+  };
+
+  const identityId = request.meta.identity.id;
+  const traceId = TraceParent.fromString(
+    request.meta.tracing.traceparent
+  ).traceId;
+
+  const created = await personAPI.create({
+    id: KSUID.randomSync().string,
+    name: "Jake",
+  });
+
+  const row = await withDatabase(
+    db,
+    PROTO_ACTION_TYPES.CREATE,
+    async ({ transaction }) => {
+      const row = withAuditContext(request, async () => {
+        return await personAPI.update({ id: created.id }, { name: "Jim" });
+      });
+
+      expect(await identityIdFromConfigParam(transaction)).toEqual(identityId);
+      expect(await traceIdFromConfigParam(transaction)).toEqual(traceId);
+
+      return row;
+    }
+  );
+
+  expect(row.name).toEqual("Jim");
+});
+
+test("auditing - ModelAPI.delete", async () => {
+  const request = {
+    meta: {
+      identity: { id: KSUID.randomSync().string },
+      tracing: {
+        traceparent: "00-80e1afed08e019fc1110464cfa66635c-7a085853722dc6d2-01",
+      },
+    },
+  };
+
+  const identityId = request.meta.identity.id;
+  const traceId = TraceParent.fromString(
+    request.meta.tracing.traceparent
+  ).traceId;
+
+  const created = await personAPI.create({
+    id: KSUID.randomSync().string,
+    name: "Jake",
+  });
+
+  const row = await withDatabase(
+    db,
+    PROTO_ACTION_TYPES.CREATE,
+    async ({ transaction }) => {
+      const row = withAuditContext(request, async () => {
+        return await personAPI.delete({ id: created.id });
+      });
+
+      expect(await identityIdFromConfigParam(transaction)).toEqual(identityId);
+      expect(await traceIdFromConfigParam(transaction)).toEqual(traceId);
+
+      return row;
+    }
+  );
+
+  expect(row).toEqual(created.id);
+});
+
+test("auditing - identity id and trace id fields dropped from result", async () => {
+  const request = {
+    meta: {
+      identity: { id: KSUID.randomSync().string },
+      tracing: {
+        traceparent: "00-80e1afed08e019fc1110464cfa66635c-7a085853722dc6d2-01",
+      },
+    },
+  };
+
+  const identityId = request.meta.identity.id;
+  const traceId = TraceParent.fromString(
+    request.meta.tracing.traceparent
+  ).traceId;
+
+  const row = await withDatabase(
+    db,
+    PROTO_ACTION_TYPES.CREATE,
+    async ({ transaction }) => {
+      const row = withAuditContext(request, async () => {
+        return await personAPI.create({
+          id: KSUID.randomSync().string,
+          name: "Jake",
+        });
+      });
+
+      expect(await identityIdFromConfigParam(transaction)).toEqual(identityId);
+      expect(await traceIdFromConfigParam(transaction)).toEqual(traceId);
+
+      return row;
+    }
+  );
+
+  expect(row.name).toEqual("Jake");
+  expect(row.keelIdentityId).toBeUndefined();
+  expect(row.keelTraceId).toBeUndefined();
+  expect(Object.keys(row).length).toEqual(2);
+});

package/src/database.js

@@ -1,5 +1,6 @@
 const { Kysely, PostgresDialect, CamelCasePlugin } = require("kysely");
 const { AsyncLocalStorage } = require("async_hooks");
+const { AuditContextPlugin } = require("./auditing");
 const pg = require("pg");
 const { PROTO_ACTION_TYPES } = require("./consts");
 const { withSpan } = require("./tracing");
@@ -15,6 +16,7 @@ async function withDatabase(db, actionType, cb) {
   let requiresTransaction = true;
 
   switch (actionType) {
+    case PROTO_ACTION_TYPES.SUBSCRIBER:
     case PROTO_ACTION_TYPES.JOB:
     case PROTO_ACTION_TYPES.GET:
     case PROTO_ACTION_TYPES.LIST:
@@ -78,6 +80,8 @@ function getDatabaseClient() {
   db = new Kysely({
     dialect: getDialect(),
     plugins: [
+      // ensures that the audit context data is written to Postgres configuration parameters
+      new AuditContextPlugin(),
       // allows users to query using camelCased versions of the database column names, which
       // should match the names we use in our schema.
       // https://kysely-org.github.io/kysely/classes/CamelCasePlugin.html
@@ -119,14 +123,13 @@ class InstrumentedClient extends pg.Client {
     const sql = args[0];
 
     let sqlAttribute = false;
-
     let spanName = txStatements[sql.toLowerCase()];
     if (!spanName) {
       spanName = "Database Query";
       sqlAttribute = true;
     }
 
-    return withSpan(spanName, function (span) {
+    return await withSpan(spanName, function (span) {
       if (sqlAttribute) {
         span.setAttribute("sql", args[0]);
       }

package/src/handleRequest.test.js

@@ -282,79 +282,4 @@ describe("ModelAPI error handling", () => {
       },
     });
   });
-
-  test("when there is a uniqueness constraint error", async () => {
-    await sql`
-
-    INSERT INTO post (id, title, author_id) values(${
-      KSUID.randomSync().string
-    }, 'hello', 'adam')
-    `.execute(db);
-
-    const rpcReq = createJSONRPCRequest("123", "createPost", {
-      title: "hello",
-      author_id: "something",
-    });
-
-    expect(await handleRequest(rpcReq, functionConfig)).toEqual({
-      id: "123",
-      jsonrpc: "2.0",
-      error: {
-        code: RuntimeErrors.UniqueConstraintError,
-        message:
-          'duplicate key value violates unique constraint "post_title_key"',
-        data: {
-          code: "23505",
-          column: "title",
-          detail: "Key (title)=(hello) already exists.",
-          table: "post",
-          value: "hello",
-        },
-      },
-    });
-  });
-
-  test("when there is a null value in a foreign key column", async () => {
-    const rpcReq = createJSONRPCRequest("123", "createPost", { title: "123" });
-
-    expect(await handleRequest(rpcReq, functionConfig)).toEqual({
-      id: "123",
-      jsonrpc: "2.0",
-      error: {
-        code: RuntimeErrors.NotNullConstraintError,
-        message:
-          'null value in column "author_id" violates not-null constraint',
-        data: {
-          code: "23502",
-          column: "author_id",
-          detail: expect.stringContaining("Failing row contains"),
-          table: "post",
-        },
-      },
-    });
-  });
-
-  test("when there is a foreign key constraint violation", async () => {
-    const rpcReq2 = createJSONRPCRequest("123", "createPost", {
-      title: "123",
-      author_id: "fake",
-    });
-
-    expect(await handleRequest(rpcReq2, functionConfig)).toEqual({
-      id: "123",
-      jsonrpc: "2.0",
-      error: {
-        code: RuntimeErrors.ForeignKeyConstraintError,
-        message:
-          'insert or update on table "post" violates foreign key constraint "post_author_id_fkey"',
-        data: {
-          code: "23503",
-          column: "author_id",
-          detail: 'Key (author_id)=(fake) is not present in table "author".',
-          table: "post",
-          value: "fake",
-        },
-      },
-    });
-  });
 });

package/src/handleSubscriber.js

@@ -49,7 +49,7 @@ async function handleSubscriber(request, config) {
         const subscriberFunction = subscribers[request.method];
         const actionType = PROTO_ACTION_TYPES.SUBSCRIBER;
 
-        await tryExecuteSubscriber({ db, actionType }, async () => {
+        await tryExecuteSubscriber({ request, db, actionType }, async () => {
           // Return the subscriber function to the containing tryExecuteSubscriber block
           return subscriberFunction(ctx, request.params);
         });

package/src/tryExecuteFunction.js

@@ -1,4 +1,5 @@
 const { withDatabase } = require("./database");
+const { withAuditContext } = require("./auditing");
 const {
   withPermissions,
   PERMISSION_STATE,
@@ -10,12 +11,15 @@ const { PROTO_ACTION_TYPES } = require("./consts");
 // tryExecuteFunction will create a new database transaction around a function call
 // and handle any permissions checks. If a permission check fails, then an Error will be thrown and the catch block will be hit.
 function tryExecuteFunction(
-  { db, permitted, permissionFns, actionType, request, ctx },
+  { request, db, permitted, permissionFns, actionType, ctx },
   cb
 ) {
   return withPermissions(permitted, async ({ getPermissionState }) => {
     return withDatabase(db, actionType, async ({ transaction }) => {
-      const fnResult = await cb();
+      const fnResult = await withAuditContext(request, async () => {
+        return cb();
+      });
+
       // api.permissions maintains an internal state of whether the current function has been *explicitly* permitted/denied by the user in the course of their custom function, or if execution has already been permitted by a role based permission (evaluated in the main runtime).
       // we need to check that the final state is permitted or unpermitted. if it's not, then it means that the user has taken no explicit action to permit/deny
       // and therefore we default to checking the permissions defined in the schema automatically.

package/src/tryExecuteJob.js

@@ -1,14 +1,17 @@
 const { withDatabase } = require("./database");
+const { withAuditContext } = require("./auditing");
 const { withPermissions, PERMISSION_STATE } = require("./permissions");
-
 const { PermissionError } = require("./errors");
 
 // tryExecuteJob will create a new database transaction around a function call
 // and handle any permissions checks. If a permission check fails, then an Error will be thrown and the catch block will be hit.
 function tryExecuteJob({ db, permitted, actionType, request }, cb) {
   return withPermissions(permitted, async ({ getPermissionState }) => {
-    return withDatabase(db, actionType, async ({ transaction }) => {
-      await cb();
+    return withDatabase(db, actionType, async () => {
+      await withAuditContext(request, async () => {
+        return cb();
+      });
+
       // api.permissions maintains an internal state of whether the current operation has been *explicitly* permitted/denied by the user in the course of their custom function, or if execution has already been permitted by a role based permission (evaluated in the main runtime).
       // we need to check that the final state is permitted or unpermitted. if it's not, then it means that the user has taken no explicit action to permit/deny
       // and therefore we default to checking the permissions defined in the schema automatically.

package/src/tryExecuteSubscriber.js

@@ -1,9 +1,12 @@
 const { withDatabase } = require("./database");
+const { withAuditContext } = require("./auditing");
 
 // tryExecuteSubscriber will create a new database connection and execute the function call.
-function tryExecuteSubscriber({ db, actionType }, cb) {
+function tryExecuteSubscriber({ request, db, actionType }, cb) {
   return withDatabase(db, actionType, async () => {
-    await cb();
+    await withAuditContext(request, async () => {
+      return cb();
+    });
   });
 }