@ditjenpajakri/elasticsearch-logging 2.0.0 → 2.2.0

package/apm.env

@@ -0,0 +1,6 @@
+LOG_ES_HOST=https://10.244.66.34:9200
+LOG_ES_PASS=jkljkl2020
+LOG_ES_VAR_A_AA=aaa
+LOG_ES_VAR_A_BB=bbb
+LOG_ES_VAR_B=ccc
+LOG_ES_INDEX=test-index-%{DATE}

package/lib.js

@@ -1,172 +1,249 @@
 #! /usr/bin/env node
-'use strict'
+"use strict";
+
+const { Client } = require("@elastic/elasticsearch");
+const split = require("split2");
+const pump = require("pump");
+const fs = require("fs");
+const { configDotenv } = require("dotenv");
+
+/**
+ * Performs a deep merge of objects and returns new object. Does not modify
+ * objects (immutable) and merges arrays via concatenation.
+ *
+ * @param {...object} objects - Objects to merge
+ * @returns {object} New object with merged key/values
+ */
+function mergeDeep(...objects) {
+  const isObject = (obj) => obj && typeof obj === "object";
+
+  return objects.reduce((prev, obj) => {
+    Object.keys(obj).forEach((key) => {
+      const pVal = prev[key];
+      const oVal = obj[key];
+
+      if (Array.isArray(pVal) && Array.isArray(oVal)) {
+        prev[key] = pVal.concat(...oVal);
+      } else if (isObject(pVal) && isObject(oVal)) {
+        prev[key] = mergeDeep(pVal, oVal);
+      } else {
+        prev[key] = oVal;
+      }
+    });
 
-const { Client } = require('@elastic/elasticsearch')
-const split = require('split2')
-const pump = require('pump')
+    return prev;
+  }, {});
+}
 
 function setDateTimeString(value) {
-    if (typeof value === 'object' && value.hasOwnProperty('time')) {
-        if (
-            (typeof value.time === 'string' && value.time.length) ||
-            (typeof value.time === 'number' && value.time >= 0)
-        ) {
-            return new Date(value.time).toISOString()
-        }
+  if (typeof value === "object" && value.hasOwnProperty("time")) {
+    if (
+      (typeof value.time === "string" && value.time.length) ||
+      (typeof value.time === "number" && value.time >= 0)
+    ) {
+      return new Date(value.time).toISOString();
     }
-    return new Date().toISOString()
+  }
+  return new Date().toISOString();
 }
 
-function initializeBulkHandler (opts, client, splitter) {
-    const esVersion = Number(opts.esVersion || opts['es-version']) || 7
-    const index = opts.index || 'pino'
-    const buildIndexName = typeof index === 'function' ? index : null
-    const type = esVersion >= 7 ? undefined : (opts.type || 'log')
-    const opType = esVersion >= 7 ? (opts.opType || opts.op_type) : undefined
-  
-    // Resurrect connection pool on destroy
-    splitter.destroy = () => {
-      if (typeof client.connectionPool.resurrect === 'function') {
-        client.connectionPool.resurrect({ name: 'elasticsearch-js' })
-      }
+function initializeBulkHandler(opts, client, splitter) {
+  const esVersion = Number(opts.esVersion || opts["es-version"]) || 7;
+  const index = opts.index || "pino";
+  const buildIndexName = typeof index === "function" ? index : null;
+//   const type = esVersion >= 7 ? undefined : opts.type || "log";
+  const opType = esVersion >= 7 ? opts.opType || opts.op_type : undefined;
+
+  // Resurrect connection pool on destroy
+  splitter.destroy = () => {
+    if (typeof client.connectionPool.resurrect === "function") {
+      client.connectionPool.resurrect({ name: "elasticsearch-js" });
     }
-
-    console.log(client)
-  
-    const bulkInsert = client.helpers.bulk({
-      datasource: splitter,
-      flushBytes: opts.flushBytes || opts['flush-bytes'] || 1000,
-      flushInterval: opts.flushInterval || opts['flush-interval'] || 30000,
-      refreshOnCompletion: getIndexName(),
-      onDocument (doc) {
-        const date = doc.time || doc['@timestamp']
-        if (opType === 'create') {
-          doc['@timestamp'] = date
-        }
-  
-        return {
-          create: {
-            _index: getIndexName(date),
-            // _type: type,
-          }
-        }
-      },
-      onDrop (doc) {
-        const error = new Error('Dropped document')
-        error.document = doc
-        splitter.emit('insertError', error)
-      }
-    })
-  
-    bulkInsert.then(
-      (stats) => splitter.emit('insert', stats),
-      (err) => splitter.emit('error', err)
-    )
-  
-    function getIndexName (time = new Date().toISOString()) {
-      if (buildIndexName) {
-        return buildIndexName(time)
+  };
+
+  const bulkInsert = client.helpers.bulk({
+    datasource: splitter,
+    flushBytes: opts.flushBytes || opts["flush-bytes"] || 1000,
+    flushInterval: opts.flushInterval || opts["flush-interval"] || 30000,
+    refreshOnCompletion: getIndexName(),
+    onDocument(doc) {
+      const date = doc.time || doc["@timestamp"];
+      if (opType === "create") {
+        doc["@timestamp"] = date;
       }
-      return index.replace('%{DATE}', time.substring ? time.substring(0, 10) : "")
+
+      return {
+        create: {
+          _index: getIndexName(date),
+          // _type: type,
+        },
+      };
+    },
+    onDrop(doc) {
+      const error = new Error("Dropped document");
+      error.document = doc;
+      splitter.emit("insertError", error);
+    },
+  });
+
+  bulkInsert.then(
+    (stats) => splitter.emit("insert", stats),
+    (err) => splitter.emit("error", err)
+  );
+
+  function getIndexName(time = new Date().toISOString()) {
+    if (buildIndexName) {
+      return buildIndexName(time);
     }
+    return index.replace(
+      "%{DATE}",
+      time.substring ? time.substring(0, 10) : ""
+    );
   }
-  
+}
 
 function pinoElasticSearch(opts = {}) {
-    const splitter = split(function (line) {
-        let value
-        try {
-            value = JSON.parse(line)
-        } catch (error) {
-            console.log(line);
-            // this.emit('unknown', line, error)
-            return
-        }
-
-        if (typeof value === 'boolean') {
-            this.emit('unknown', line, 'Boolean value ignored')
-            return
-        }
-        if (value === null) {
-            this.emit('unknown', line, 'Null value ignored')
-            return
-        }
-        if (typeof value !== 'object') {
-            value = {
-                data: value,
-                ['@timestamp']: setDateTimeString(value)
-            }
-        } else {
-            if (value['@timestamp'] === undefined) {
-                value['@timestamp'] = setDateTimeString(value)
-            }
-        }
-        return value
-    }, { autoDestroy: true })
-
-    const clientOpts = {
-        node: opts.node,
-        auth: opts.auth,
-        // cloud: opts.cloud,
-        tls: {
-            rejectUnauthorized: false
-          }
-    }
+  const splitter = split(
+    function (line) {
+      let value;
+      try {
+        value = JSON.parse(line);
+      } catch (error) {
+        console.log(line);
+        // this.emit('unknown', line, error)
+        return;
+      }
 
-    if (opts.caFingerprint) {
-        clientOpts.caFingerprint = opts.caFingerprint
-    }
+      if (typeof value === "boolean") {
+        this.emit("unknown", line, "Boolean value ignored");
+        return;
+      }
+      if (value === null) {
+        this.emit("unknown", line, "Null value ignored");
+        return;
+      }
+      if (typeof value !== "object") {
+        value = {
+          data: value,
+          ["@timestamp"]: setDateTimeString(value),
+        };
+      } else {
+        value["message"] = value.msg || "-";
 
-    if (opts.Connection) {
-        clientOpts.Connection = opts.Connection
-    }
+        if (value.msg) delete value.msg;
 
-    if (opts.ConnectionPool) {
-        clientOpts.ConnectionPool = opts.ConnectionPool
-    }
+        value["@timestamp"] = setDateTimeString(value);
 
-    const client = new Client(clientOpts)
-    console.log("Logging to elasticsearch...");
-    console.log("Any line showing up here means that the line was not logged to elasticsearch.");
-    client.diagnostic.on('resurrect', () => {
-        initializeBulkHandler(opts, client, splitter)
-    })
+        if (value.time) delete value.time;
+      }
+      return mergeDeep(value, opts.additionalObject);
+    },
+    { autoDestroy: true }
+  );
+
+  const clientOpts = {
+    node: opts.node,
+    auth: opts.auth,
+    // cloud: opts.cloud,
+    tls: {
+      rejectUnauthorized: false,
+    },
+  };
+
+  if (opts.caFingerprint) {
+    clientOpts.caFingerprint = opts.caFingerprint;
+  }
+
+  if (opts.Connection) {
+    clientOpts.Connection = opts.Connection;
+  }
 
-    initializeBulkHandler(opts, client, splitter)
+  if (opts.ConnectionPool) {
+    clientOpts.ConnectionPool = opts.ConnectionPool;
+  }
+
+  const client = new Client(clientOpts);
+  console.log("Logging to elasticsearch...");
+  console.log(
+    "Any line showing up here means that the line was not logged to elasticsearch."
+  );
+  client.diagnostic.on("resurrect", () => {
+    initializeBulkHandler(opts, client, splitter);
+  });
 
-    return splitter
+  initializeBulkHandler(opts, client, splitter);
+
+  return splitter;
 }
 
 function start(opts) {
-    console.log('opts', opts)
-    const stream = pinoElasticSearch(opts)
-
-    stream.on('unknown', (line, error) => {
-        console.error('Elasticsearch client json error in line:\n' + line + '\nError:', error)
-    })
-    stream.on('error', (error) => {
-        console.error('Elasticsearch client error:', error)
-    })
-    stream.on('insertError', (error) => {
-        console.log(JSON.stringify(error))
-        console.error('Elasticsearch server error:', error)
-    })
-
-    pump(process.stdin, stream)
+  console.log("opts", opts);
+  const stream = pinoElasticSearch(opts);
+
+  stream.on("unknown", (line, error) => {
+    console.error(
+      "Elasticsearch client json error in line:\n" + line + "\nError:",
+      error
+    );
+  });
+  stream.on("error", (error) => {
+    console.error("Elasticsearch client error:", error);
+  });
+  stream.on("insertError", (error) => {
+    console.log(JSON.stringify(error));
+    console.error("Elasticsearch server error:", error);
+  });
+
+  pump(process.stdin, stream);
 }
 
-module.exports = pinoElasticSearch
+module.exports = pinoElasticSearch;
 
 if (require.main === module) {
-    start({
-        node: process.env.LOG_ES_HOST || 'http://localhost:9200',
-        auth: {
-            username: process.env.LOG_ES_USER || 'elastic',
-            password: process.env.LOG_ES_PASS || 'changeme'
-        },
-        index: process.env.LOG_ES_INDEX || 'DJP-%{DATE}',
-        flushBytes: process.env.LOG_ES_FLUSH_BYTES ? parseInt(process.env.LOG_ES_FLUSH_BYTES) : undefined,
-        flushInterval: process.env.LOG_ES_FLUSH_INTERVAL ? parseInt(process.env.LOG_ES_FLUSH_INTERVAL) : undefined,
-        rejectUnauthorized: process.env.LOG_ES_REJECT_UNAUTHORIZED === 'true' ? true : false,
-    })
-}
+  const fileEnv = process.argv[2];
+
+  if (fileEnv && fs.statSync(fileEnv)) {
+    configDotenv({
+      path: fileEnv,
+    });
+  }
+
+  const additionalFields = Object.keys(process.env).filter((key) =>
+    key.startsWith("LOG_ES_VAR_")
+  ).map(s => s.toLowerCase())
+
+  const additionalObject = additionalFields.reduce((obj, field) => {
+    const value = process.env[field.toLocaleUpperCase()];
+    function processKey(f, value) {
+      const parts = f ? f.split("_") : [];
+      if (parts.length) {
+        return processKey(parts.slice(0, parts.length - 1).join("_"), {
+          [parts[parts.length - 1]]: value,
+        });
+      } else {
+        if (f) return { [f]: value };
+        else return value;
+      }
+    }
+
+    return mergeDeep(obj, processKey(field, value));
+  }, {});
+
+  start({
+    node: process.env.LOG_ES_HOST || "http://localhost:9200",
+    additionalObject: additionalObject?.log?.es?.var ? additionalObject.log.es.var : {},
+    auth: {
+      username: process.env.LOG_ES_USER || "elastic",
+      password: process.env.LOG_ES_PASS || "changeme",
+    },
+    index: process.env.LOG_ES_INDEX || "logs-apm-%{DATE}",
+    flushBytes: process.env.LOG_ES_FLUSH_BYTES
+      ? parseInt(process.env.LOG_ES_FLUSH_BYTES)
+      : undefined,
+    flushInterval: process.env.LOG_ES_FLUSH_INTERVAL
+      ? parseInt(process.env.LOG_ES_FLUSH_INTERVAL)
+      : undefined,
+    rejectUnauthorized:
+      process.env.LOG_ES_REJECT_UNAUTHORIZED === "true" ? true : false,
+  });
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ditjenpajakri/elasticsearch-logging",
-  "version": "2.0.0",
+  "version": "2.2.0",
   "description": "Elasticsearch ",
   "main": "lib.js",
   "bin": {
@@ -17,6 +17,7 @@
   "license": "MIT",
   "dependencies": {
     "@elastic/elasticsearch": "8",
+    "dotenv": "^16.4.5",
     "pump": "^3.0.0",
     "split2": "^4.2.0"
   },

package/test

@@ -0,0 +1 @@
+{"a": "1234"}