db-model-router 1.0.6 → 1.0.8

package/src/commons/kafka.js

@@ -0,0 +1,139 @@
+/**
+ * Kafka Producer Service
+ *
+ * Produces events to Kafka when database write operations (insert, update, upsert, delete) are performed.
+ * Enabled only when KAFKA_BROKER is set in environment variables.
+ *
+ * Event format:
+ * {
+ *   table_name: string,
+ *   operation_type: "insert" | "update" | "upsert" | "delete",
+ *   data: object,
+ *   timestamp: string (ISO 8601)
+ * }
+ *
+ * Each row produces its own event. Bulk operations produce one event per entry.
+ */
+
+let producer = null;
+let kafka = null;
+let isConnected = false;
+let isEnabled = false;
+let topicPrefix = "";
+
+/**
+ * Initialize Kafka producer if KAFKA_BROKER env variable is set.
+ * @param {object} [options] - Optional configuration overrides
+ * @param {string} [options.broker] - Kafka broker URL (default: process.env.KAFKA_BROKER)
+ * @param {string} [options.clientId] - Kafka client ID (default: process.env.KAFKA_CLIENT_ID || "db-model-router")
+ * @param {string} [options.topicPrefix] - Topic prefix (default: process.env.KAFKA_TOPIC_PREFIX || "dbmr")
+ * @returns {Promise<boolean>} Whether Kafka was successfully initialized
+ */
+async function init(options = {}) {
+  const broker = options.broker || process.env.KAFKA_BROKER;
+  if (!broker) {
+    isEnabled = false;
+    return false;
+  }
+
+  const clientId =
+    options.clientId || process.env.KAFKA_CLIENT_ID || "db-model-router";
+  topicPrefix = options.topicPrefix || process.env.KAFKA_TOPIC_PREFIX || "dbmr";
+
+  try {
+    const { Kafka } = require("kafkajs");
+    kafka = new Kafka({
+      clientId,
+      brokers: broker.split(",").map((b) => b.trim()),
+    });
+
+    producer = kafka.producer();
+    await producer.connect();
+    isConnected = true;
+    isEnabled = true;
+    return true;
+  } catch (err) {
+    console.error(
+      "[db-model-router] Kafka initialization failed:",
+      err.message,
+    );
+    isEnabled = false;
+    isConnected = false;
+    return false;
+  }
+}
+
+/**
+ * Produce Kafka event(s) for a database operation.
+ * If data is an array, one event is produced per entry (batched to avoid exceeding message size limits).
+ * If data is a single object, one event is produced.
+ *
+ * @param {string} tableName - The database table name
+ * @param {string} operationType - One of: "insert", "update", "upsert", "delete"
+ * @param {object|object[]} data - The data involved in the operation
+ * @returns {Promise<void>}
+ */
+async function produce(tableName, operationType, data) {
+  if (!isEnabled || !isConnected || !producer) {
+    return;
+  }
+
+  const topic = `${topicPrefix}.${tableName}`;
+  const entries = Array.isArray(data) ? data : [data];
+  const timestamp = new Date().toISOString();
+
+  const messages = entries.map((entry) => ({
+    key: tableName,
+    value: JSON.stringify({
+      table_name: tableName,
+      operation_type: operationType,
+      data: entry,
+      timestamp,
+    }),
+  }));
+
+  // Batch messages to avoid exceeding Kafka's max request size
+  const BATCH_SIZE = 500;
+  try {
+    for (let i = 0; i < messages.length; i += BATCH_SIZE) {
+      const batch = messages.slice(i, i + BATCH_SIZE);
+      await producer.send({ topic, messages: batch });
+    }
+  } catch (err) {
+    console.error(
+      `[db-model-router] Kafka produce failed for ${topic}:`,
+      err.message,
+    );
+  }
+}
+
+/**
+ * Disconnect the Kafka producer gracefully.
+ * @returns {Promise<void>}
+ */
+async function disconnect() {
+  if (producer && isConnected) {
+    try {
+      await producer.disconnect();
+    } catch (_) {
+      // ignore disconnect errors
+    }
+    isConnected = false;
+    isEnabled = false;
+  }
+}
+
+/**
+ * Check if Kafka is currently enabled and connected.
+ * @returns {boolean}
+ */
+function status() {
+  return isEnabled && isConnected;
+}
+
+module.exports = {
+  init,
+  produce,
+  disconnect,
+  status,
+};

package/src/commons/model.js

@@ -6,6 +6,7 @@ const {
   RemoveUnknownData,
 } = require("./validator");
 const { getType, jsonStringify, jsonSafeParse } = require("./function");
+const { produce } = require("./kafka");
 
 /**
  * Extract and remove reserved params from a data/payload object.
@@ -215,9 +216,12 @@ module.exports = function model(
         const getResult = await db.get(table, [
           [[primary_key, "=", insertResult.id]],
         ]);
-        return getResult.count > 0 ? getResult["data"][0] : null;
+        const record = getResult.count > 0 ? getResult["data"][0] : null;
+        if (record) produce(table, "insert", record);
+        return record;
       }
       //TODO: Bulk Insert -> Return inserted objects
+      produce(table, "insert", data);
       return insertResult;
     },
     update: async (data) => {
@@ -233,6 +237,7 @@ module.exports = function model(
         data = jsonStringify(data);
         updateResult = await db.upsert(table, data, unique);
         //TODO: Bulk Update -> Return updated objects
+        produce(table, "update", data);
       } else {
         stripTimestampFields(data, allTimestampKeys);
         await validateInput(
@@ -246,7 +251,9 @@ module.exports = function model(
           const getResult = await db.get(table, [
             [[primary_key, "=", updateResult.id]],
           ]);
-          return getResult.count > 0 ? getResult["data"][0] : null;
+          const record = getResult.count > 0 ? getResult["data"][0] : null;
+          if (record) produce(table, "update", record);
+          return record;
         } else if (data[0].hasOwnProperty(primary_key)) {
           const result = await db.get(
             table,
@@ -254,8 +261,10 @@ module.exports = function model(
             [],
             option.safeDelete,
           );
-          if (result.count > 0) return result["data"][0];
-          else return null;
+          if (result.count > 0) {
+            produce(table, "update", result["data"][0]);
+            return result["data"][0];
+          } else return null;
         }
       }
       return updateResult;
@@ -274,6 +283,7 @@ module.exports = function model(
         data = jsonStringify(data);
         updateResult = await db.upsert(table, data, unique);
         //TODO: Bulk Upsert -> Return Inserted/Updated objects
+        produce(table, "upsert", data);
       } else {
         stripTimestampFields(data, allTimestampKeys);
         await validateInput(
@@ -288,20 +298,27 @@ module.exports = function model(
           const getResult = await db.get(table, [
             [[primary_key, "=", updateResult.id]],
           ]);
-          return getResult.count > 0 ? getResult["data"][0] : null;
+          const record = getResult.count > 0 ? getResult["data"][0] : null;
+          if (record) produce(table, "upsert", record);
+          return record;
         } else if (originalData.hasOwnProperty(primary_key)) {
           const result = await db.get(table, [
             [[primary_key, "=", originalData[primary_key]]],
           ]);
-          if (result.count > 0) return result["data"][0];
-          else return null;
+          if (result.count > 0) {
+            produce(table, "upsert", result["data"][0]);
+            return result["data"][0];
+          } else return null;
         }
       }
       return updateResult;
     },
     remove: async (data) => {
+      const originalData = Array.isArray(data) ? [...data] : { ...data };
       let filter = dataToFilter(jsonSafeParse(data), primary_key);
-      return await db.remove(table, filter, option.safeDelete);
+      const removeResult = await db.remove(table, filter, option.safeDelete);
+      produce(table, "delete", originalData);
+      return removeResult;
     },
     byId: async (id, options = {}) => {
       let type = getType(id);
@@ -427,7 +444,10 @@ module.exports = function model(
         [],
         option.safeDelete,
       );
-      if (result.count > 0) return result["data"][0];
+      if (result.count > 0) {
+        produce(table, "update", result["data"][0]);
+        return result["data"][0];
+      }
       return null;
     },
     pk: primary_key,

package/src/commons/route.js

@@ -31,7 +31,7 @@ module.exports = function route(model, override = {}) {
     .Router({ mergeParams: true })
     .get("/:" + model.pk, (req, res) => {
       let payload = payloadOverride(
-        { ...req.query, ...req.params },
+        { ...(req.query || {}), ...(req.params || {}) },
         req,
         override,
       );
@@ -52,7 +52,7 @@ module.exports = function route(model, override = {}) {
         });
     })
     .post("/:id", (req, res) => {
-      let payload = payloadOverride(req.body, req, override);
+      let payload = payloadOverride(req.body || {}, req, override);
       delete payload[model.pk];
       model
         .insert(payload)
@@ -64,7 +64,7 @@ module.exports = function route(model, override = {}) {
         });
     })
     .put("/:id", (req, res) => {
-      let payload = payloadOverride(req.body, req, override);
+      let payload = payloadOverride(req.body || {}, req, override);
       payload[model.pk] = req.params.id;
       let validateAccessPayload = payloadOverride({}, req, override);
       validateAccessPayload[model.pk] = req.params.id;
@@ -89,7 +89,7 @@ module.exports = function route(model, override = {}) {
         });
     })
     .patch("/:id", (req, res) => {
-      let payload = payloadOverride(req.body, req, override);
+      let payload = payloadOverride(req.body || {}, req, override);
       payload[model.pk] = req.params.id;
       let validateAccessPayload = payloadOverride({}, req, override);
       validateAccessPayload[model.pk] = req.params.id;
@@ -114,7 +114,7 @@ module.exports = function route(model, override = {}) {
         });
     })
     .delete("/:id", (req, res) => {
-      let payload = payloadOverride(req.body, req, override);
+      let payload = payloadOverride(req.body || {}, req, override);
       payload[model.pk] = req.params.id;
       let validateAccessPayload = payloadOverride({}, req, override);
       validateAccessPayload[model.pk] = req.params.id;
@@ -140,7 +140,7 @@ module.exports = function route(model, override = {}) {
     })
     .get("/", (req, res) => {
       let payload = payloadOverride(
-        { ...req.query, ...req.params },
+        { ...(req.query || {}), ...(req.params || {}) },
         req,
         override,
       );

package/src/index.js

@@ -1,5 +1,6 @@
 const model = require("./commons/model.js");
 const route = require("./commons/route.js");
+const kafka = require("./commons/kafka.js");
 const routers = {
   mysql: "./mysql/db.js",
   mariadb: "./mysql/db.js",
@@ -56,4 +57,5 @@ module.exports = {
   },
   model,
   route,
+  kafka,
 };

package/src/mssql/db.js

@@ -2,8 +2,39 @@ const sql = require("mssql");
 const { jsonSafeParse } = require("../commons/function");
 
 let pool = null;
+let dateStringsMode = false;
 const WHERE_INVALID = "Invalid filter object";
 
+/**
+ * Formats a Date object to YYYY-MM-DD HH:mm:ss string in UTC.
+ */
+function formatDate(d) {
+  var y = d.getUTCFullYear();
+  var m = String(d.getUTCMonth() + 1).padStart(2, "0");
+  var day = String(d.getUTCDate()).padStart(2, "0");
+  var h = String(d.getUTCHours()).padStart(2, "0");
+  var min = String(d.getUTCMinutes()).padStart(2, "0");
+  var s = String(d.getUTCSeconds()).padStart(2, "0");
+  return y + "-" + m + "-" + day + " " + h + ":" + min + ":" + s;
+}
+
+/**
+ * Maps recordset rows, converting Date objects to formatted strings if dateStringsMode is enabled.
+ */
+function mapRecordset(rows) {
+  if (!dateStringsMode || !Array.isArray(rows)) return rows;
+  return rows.map(function (row) {
+    var mapped = {};
+    for (var key in row) {
+      if (Object.prototype.hasOwnProperty.call(row, key)) {
+        mapped[key] =
+          row[key] instanceof Date ? formatDate(row[key]) : row[key];
+      }
+    }
+    return mapped;
+  });
+}
+
 const RETRYABLE_ERRORS = [
   "ECONNREFUSED",
   "ECONNRESET",
@@ -34,6 +65,9 @@ function isRetryable(err) {
 }
 
 async function connect(config) {
+  dateStringsMode =
+    config.dateStrings === true || process.env.DB_DATE_STRINGS === "true";
+
   const mssqlConfig = {
     server: config.server || config.host || process.env.DB_HOST || "localhost",
     port: parseInt(config.port || process.env.DB_PORT || 1433),
@@ -75,7 +109,11 @@ async function query(sqlStr, parameter = []) {
     }
     // Replace positional @paramN placeholders if not already present
     const result = await request.query(sqlStr);
-    return result.recordset || { affectedRows: result.rowsAffected?.[0] || 0 };
+    return (
+      mapRecordset(result.recordset) || {
+        affectedRows: result.rowsAffected?.[0] || 0,
+      }
+    );
   });
 }
 
@@ -189,7 +227,7 @@ async function get(table, filter = [], sort = [], safeDelete = null) {
     request.input("param" + i, whereData.value[i]);
   }
   const result = await request.query(sqlStr);
-  const rows = jsonSafeParse(result.recordset || []);
+  const rows = jsonSafeParse(mapRecordset(result.recordset || []));
   const count = await qcount(table, filter, safeDelete);
   return { data: rows, count };
 }
@@ -216,7 +254,7 @@ async function list(
     request.input("param" + i, whereData.value[i]);
   }
   const result = await request.query(sqlStr);
-  const rows = jsonSafeParse(result.recordset || []);
+  const rows = jsonSafeParse(mapRecordset(result.recordset || []));
   const count = await qcount(table, filter, safeDelete);
   return { data: rows, count };
 }

package/src/mysql/db.js

@@ -4,6 +4,9 @@ let pool = null;
 const WHERE_INVALID = "Invalid filter object";
 
 function connect(credentails) {
+  // Force UTC timezone so timestamps are consistent
+  // dateStrings: true returns raw strings without JS Date conversion
+  credentails.timezone = "+00:00";
   pool = mysql.createPool(credentails);
   return pool;
 }

package/src/postgres/db.js

@@ -13,6 +13,7 @@ function sanitizeValue(v) {
     return v
       .replace("T", " ")
       .replace(/[+-]\d{2}:\d{2}$/, "")
+      .replace(/Z$/, "")
       .slice(0, 19);
   }
   return v;
@@ -75,6 +76,11 @@ function connect(config) {
 
   pool = new Pool(poolConfig);
 
+  // Set session timezone to UTC for consistent timestamp handling
+  pool.on("connect", (client) => {
+    client.query("SET timezone = 'UTC'");
+  });
+
   pool.on("error", (err) => {
     console.error("Unexpected PG pool error:", err.message);
   });