codefresh 0.78.2

package/lib/interface/cli/commands/offline-logs/base.cmd.js

@@ -0,0 +1,33 @@
+const yargs = require("yargs");
+const Command = require("../../Command");
+
+const command = new Command({
+  command: "offline-logs",
+  root: true,
+  description: "Manages offline logs",
+  webDocs: {
+    category: "Logs category",
+    subCategory: "Logs sub category",
+    title: "Archives old logs to file or collection.",
+  },
+  builder: (yargs) => {
+    // set options which are used in both sub-commands
+    return yargs
+      .env("RUNTIME_MONGO")
+      .option("uri", {
+        describe:
+          "Mongodb URI. If not provided, will be parsed from environment variable RUNTIME_MONGO_URI.",
+        type: "string",
+      })
+      .option("db", {
+        describe:
+          "Database name. If not provided, will be parsed from environment variable RUNTIME_MONGO_DB.",
+        type: "string",
+      });
+  },
+  handler: () => {
+    yargs.showHelp();
+  },
+});
+
+module.exports = command;

package/lib/interface/cli/commands/offline-logs/ensure-index.cmd.js

@@ -0,0 +1,44 @@
+const { MongoClient } = require("mongodb");
+const Command = require("../../Command");
+const cmd = require("./base.cmd");
+const utils = require('./utils')
+
+const command = new Command({
+  command: "ensure-index",
+  parent: cmd,
+  description:
+    "Checks whether a collection has indexes and gives the user an option to add them. Adding an index may improve performance of some of the read operations.",
+  webDocs: {
+    category: "Logs",
+    title: "Ensures one or more offline-logs collections has indexes",
+  },
+  builder: (yargs) =>
+    yargs.example(
+      'codefresh offline-logs ensure-index --uri "mongodb://192.168.99.100:27017" --db logs'
+    ),
+  handler: async (argv) => {
+    const { uri, db } = argv;
+    const client = new MongoClient( uri, { useUnifiedTopology: true } );
+    await client.connect();
+    const database = client.db(db);
+    const failedCollections = [];
+    const errors = [];
+    for (const collection of utils.defaultCollections) {
+      try {
+        await utils.ensureIndex(database, collection);
+      } catch (error) {
+        console.error(`failed to ensure index of collection '${collection}', error: ${error.message}`);
+        failedCollections.push(collection);
+        errors.push(error);
+      }
+    }
+    client.close();
+    if (failedCollections.length) {
+      throw new Error(
+        `failed to ensure indexes of ${failedCollections.join(', ')}, ${errors.join(', ')}`
+      );
+    }
+  },
+});
+
+module.exports = command;

package/lib/interface/cli/commands/offline-logs/ensure-index.spec.js

@@ -0,0 +1,76 @@
+const utils = require('./utils')
+
+describe("ensure-index", () => {
+  describe("ensureIndex", () => {
+    utils.getUserInput = jest.fn()
+    const mockCollection = {
+      indexExists: jest.fn(),
+      estimatedDocumentCount: jest.fn().mockReturnValue(700),
+      createIndex: jest.fn(),
+    };
+    const mockDatabase = {
+      collection() {
+        return mockCollection;
+      },
+    };
+
+    beforeEach(() => {
+      mockCollection.indexExists.mockClear();
+      mockCollection.estimatedDocumentCount.mockClear();
+      mockCollection.createIndex.mockClear();
+      utils.getUserInput.mockClear();
+    });
+
+    it("index does not exists and user says yes on prompt", async () => {
+      //setup
+      mockCollection.indexExists.mockReturnValue(false);
+      utils.getUserInput.mockReturnValue(true);
+      const collection = 'whatever';
+      const expectedIndexObj = { accountId: 1, jobId: 1 }
+      const expectedOptions = { background: true }
+
+      //execute
+      await utils.ensureIndex(mockDatabase, collection);
+
+      //check
+      expect(mockCollection.indexExists).toBeCalledTimes(1);
+      expect(mockCollection.indexExists).toBeCalledWith("accountId_1_jobId_1");
+      expect(mockCollection.estimatedDocumentCount).toBeCalledTimes(1);
+      expect(utils.getUserInput).toBeCalledTimes(1);
+      expect(mockCollection.createIndex).toBeCalledTimes(1);
+      expect(mockCollection.createIndex).toBeCalledWith(expectedIndexObj, expectedOptions)
+      });
+    it("index does not exists and user says no on prompt", async () => {
+      //setup
+      mockCollection.indexExists.mockReturnValue(false);
+      utils.getUserInput.mockReturnValue(false);
+      const collection = 'whatever';
+
+      //execute
+      await utils.ensureIndex(mockDatabase, collection);
+
+      //check
+      expect(mockCollection.indexExists).toBeCalledTimes(1);
+      expect(mockCollection.indexExists).toBeCalledWith("accountId_1_jobId_1");
+      expect(mockCollection.estimatedDocumentCount).toBeCalledTimes(1);
+      expect(utils.getUserInput).toBeCalledTimes(1);
+      expect(mockCollection.createIndex).toBeCalledTimes(0);
+      });
+    it("index exists", async () => {
+      //setup
+      mockCollection.indexExists.mockReturnValue(true);
+      const collection = 'whatever';
+
+      //execute
+      await utils.ensureIndex(mockDatabase, collection);
+
+      //check
+      expect(mockCollection.indexExists).toBeCalledTimes(1);
+      expect(mockCollection.indexExists).toBeCalledWith("accountId_1_jobId_1");
+      expect(mockCollection.estimatedDocumentCount).toBeCalledTimes(0);
+      expect(utils.getUserInput).toBeCalledTimes(0);
+      expect(mockCollection.createIndex).toBeCalledTimes(0);
+      });
+      
+    });
+  });

package/lib/interface/cli/commands/offline-logs/offload-to-collection.cmd.js

@@ -0,0 +1,188 @@
+const { MongoClient, ObjectId } = require("mongodb");
+const moment = require("moment");
+const semver = require("semver");
+const Command = require("../../Command");
+const cmd = require("./base.cmd");
+const {
+  objectIdFromDate,
+  defaultCollections,
+  findMinLog,
+  checkCursorState,
+  createRangeStr,
+  getMinDate,
+} = require("./utils");
+
+const MERGE_RELEASE_VERSION = "4.2.0";
+
+const offloadToCollection = async function (
+  sourceDBObj,
+  collection,
+  targetDB,
+  cutoffDate
+) {
+  const sourceCollectionObj = sourceDBObj.collection(collection);
+
+  const cutoffDateObj = moment(cutoffDate).add(1, "days").startOf("day");
+
+  if (!cutoffDateObj.isValid()) {
+    throw new Error("please enter a valid date in ISO 8601 format");
+  }
+
+  const cutoffDateId = objectIdFromDate(cutoffDateObj.toDate());
+
+  const minLog = await findMinLog(
+    sourceCollectionObj,
+    cutoffDateId,
+    collection
+  );
+
+  if (!minLog) {
+    console.info(
+      `No logs to archive in collection ${collection} from the given date.`
+    );
+    return;
+  }
+
+  const lowerBound = getMinDate(minLog);
+  const upperBound = moment(cutoffDateObj);
+  const targetCollection = `${createRangeStr(
+    lowerBound,
+    upperBound,
+    collection
+  )}-archive`;
+
+  const mongoInfo = await sourceDBObj.admin().serverInfo();
+  const mongoServerVersion = await mongoInfo.version;
+
+  if (semver.lte(MERGE_RELEASE_VERSION, mongoServerVersion)) {
+    await archiveWithMerge(
+      sourceCollectionObj,
+      cutoffDateId,
+      targetDB,
+      targetCollection,
+      collection
+    );
+  } else {
+    await archiveWithOut(
+      sourceCollectionObj,
+      cutoffDateId,
+      targetCollection,
+      collection
+    );
+  }
+};
+///////////////////////////////////////////////////////////////////////////////////
+const archiveWithMerge = async function (
+  sourceCollectionObj,
+  cutoffDateId,
+  targetDB,
+  targetCollection,
+  collection
+) {
+  var result = sourceCollectionObj.aggregate([
+    { $match: { _id: { $lte: ObjectId(cutoffDateId) } } },
+    {
+      $merge: {
+        into: { db: targetDB, coll: targetCollection },
+        on: "_id",
+        whenMatched: "keepExisting",
+        whenNotMatched: "insert",
+      },
+    },
+  ]);
+
+  await result.toArray();
+
+  checkCursorState(result, collection, targetCollection);
+
+  await sourceCollectionObj.deleteMany({
+    _id: { $lte: ObjectId(cutoffDateId) },
+  });
+};
+///////////////////////////////////////////////////////////////////////////////////
+const archiveWithOut = async function (
+  sourceCollectionObj,
+  cutoffDateId,
+  targetCollection,
+  collection
+) {
+  var result = sourceCollectionObj.aggregate([
+    { $match: { _id: { $lte: ObjectId(cutoffDateId) } } },
+    { $out: targetCollection },
+  ]);
+
+  await result.toArray();
+
+  checkCursorState(result, collection, targetCollection);
+
+  await sourceCollectionObj.deleteMany({
+    _id: { $lte: ObjectId(cutoffDateId) },
+  });
+};
+///////////////////////////////////////////////////////////////////////////////////
+const command = new Command({
+  command: "offload-to-collection",
+  parent: cmd,
+  description:
+    "Archiving logs from one or more source collections to target collections.",
+  webDocs: {
+    category: "Logs",
+    title: "Offload To Collection",
+  },
+  builder: (yargs) =>
+    yargs
+      .option("targetDB", {
+        alias: "tdb",
+        describe:
+          "This option is available only for mongodb version 4.2 and up. for older versions it will be ignored. \
+          Target database name, if none inserted, db will be defined as target.",
+        type: "string",
+      })
+      .option("cutoffDate", {
+        alias: "cod",
+        describe:
+          "The date in ISO format (yyyy-MM-dd) from which logs will be archived (going backwards, including logs from that day).",
+        demandOption: true,
+        type: "string",
+      })
+      .example(
+        'codefresh offline-logs offload-to-collection --uri "mongodb://192.168.99.100:27017" --db logs --cod "2021-07-08" '
+      ),
+  handler: async (argv) => {
+    const { uri, db, targetDB, cutoffDate } = argv;
+
+    const client = new MongoClient(uri, { useUnifiedTopology: true });
+    try {
+      await client.connect();
+      const failedCollections = [];
+      const errors = [];
+      const sourceDBObj = client.db(db);
+      const promises = defaultCollections.map(async (collection) => {
+        try {
+          await offloadToCollection(
+            sourceDBObj,
+            collection,
+            targetDB || db,
+            cutoffDate
+          );
+        } catch (error) {
+          failedCollections.push(collection);
+          errors.push(error);
+        }
+      });
+      await Promise.all(promises);
+
+      if (failedCollections.length) {
+        throw new Error(
+          `failed to offload from collections: ${failedCollections.join(
+            ", "
+          )}. ${errors.join(", ")}`
+        );
+      }
+    } finally {
+      client.close();
+    }
+  },
+});
+
+module.exports = command;

package/lib/interface/cli/commands/offline-logs/offload-to-file.cmd.js

@@ -0,0 +1,175 @@
+const { MongoClient, ObjectId } = require("mongodb");
+const moment = require("moment");
+const Command = require("../../Command");
+const cmd = require("./base.cmd");
+const {
+  objectIdFromDate,
+  writeLogsToFile,
+  checkRemnant,
+  findMinLog,
+  getMinDate,
+  defaultCollections,
+} = require("./utils");
+
+const offloadToFile = async function (
+  database,
+  collection,
+  chunkDuration,
+  cutoffDate,
+  path
+) {
+  const collectionObj = database.collection(collection);
+
+  if (chunkDuration <= 0) {
+    throw new Error("please enter a valid chunkDuration ( > 0)");
+  }
+  let chunkSize = chunkDuration;
+
+  // the cutoff date will be the start of the next day from user input (archive will include the given cutoff date)
+  const cutoffDateObj = moment(cutoffDate, moment.ISO_8601, true)
+    .add(1, "days")
+    .startOf("day");
+  if (!cutoffDateObj.isValid()) {
+    throw new Error("please enter a valid date in ISO 8601 format");
+  }
+  const cutoffDateId = objectIdFromDate(cutoffDateObj.toDate());
+
+  const minLog = await findMinLog(collectionObj, cutoffDateId, collection);
+
+  if (!minLog) {
+    console.info(
+      `No logs to archive in Collection ${collection} from the given date.`
+    );
+    return;
+  }
+
+  const minDate = getMinDate(minLog);
+
+  for (
+    let lowerBound = moment(minDate);
+    lowerBound < cutoffDateObj;
+    lowerBound.add(chunkSize, "days")
+  ) {
+    //in case total days period is not devided with chunkSize, the last chunk size needs to be smaller.
+    const remnant = checkRemnant(lowerBound, cutoffDateObj);
+
+    if (remnant < chunkSize && remnant > 0) {
+      chunkSize = remnant;
+    }
+    const upperBound = moment(lowerBound).add(chunkSize, "days");
+
+    const lowerBoundId = objectIdFromDate(lowerBound.toDate());
+    const upperBoundId = objectIdFromDate(upperBound.toDate());
+
+    const logsToArchive = await collectionObj
+      .find({
+        _id: {
+          $gte: ObjectId(lowerBoundId),
+          $lt: ObjectId(upperBoundId),
+        },
+      })
+      .toArray();
+
+    if (logsToArchive.length > 0) {
+      writeLogsToFile(upperBound, lowerBound, collection, logsToArchive, path);
+
+      await collectionObj.deleteMany({
+        _id: {
+          $gte: ObjectId(lowerBoundId),
+          $lt: ObjectId(upperBoundId),
+        },
+      });
+    }
+  }
+};
+
+const command = new Command({
+  command: "offload-to-file",
+  parent: cmd,
+  description:
+    "Archiving logs from one or more source collections to files by chunks of days.",
+  webDocs: {
+    category: "Logs",
+    title: "Offload To File",
+  },
+  builder: (yargs) =>
+    yargs
+      .option("chunkDuration", {
+        alias: "chdur",
+        describe:
+          "Chunk size in days, each chunk will be archived into a different file.",
+        default: 1,
+        type: "number",
+      })
+      .option("cutoffDate", {
+        alias: "cod",
+        describe:
+          "The date in ISO format (yyyy-MM-dd) from which logs will be archived (going backwards, including logs from that day).",
+        demandOption: true,
+        type: "string",
+      })
+      .option("path", {
+        describe: "Directory path to which archive files will be saved.",
+        default: ".",
+        type: "string",
+      })
+      .option("collections", {
+        alias: "c",
+        describe: "Source collections names",
+        default: defaultCollections,
+        array: true,
+        type: "string",
+        coerce(arg) {
+          for (const collection of arg) {
+            if (
+              !collection.includes("logs") &&
+              !collection.includes("metadata")
+            ) {
+              throw new Error("invalid collection name");
+            }
+            return arg;
+          }
+        },
+      })
+      .example(
+        'codefresh offline-logs offload-to-file --uri "mongodb://192.168.99.100:27017" --db logs --collections archive-logs --cod "2021-07-08" --chdur 3 --path "./" '
+      ),
+  handler: async (argv) => {
+    const { uri, cutoffDate, chunkDuration, path, db, collections } = argv;
+    const client = new MongoClient(uri, { useUnifiedTopology: true });
+    try {
+      await client.connect();
+      const failedCollections = [];
+      const errors = [];
+      const database = client.db(db);
+      const promises = collections.map(async (collection) => {
+        try {
+          await offloadToFile(
+            database,
+            collection,
+            chunkDuration,
+            cutoffDate,
+            path
+          );
+        } catch (e) {
+          console.log(e);
+          failedCollections.push(collection);
+          errors.push(error);
+        }
+      });
+      await Promise.all(promises);
+
+      if (failedCollections.length) {
+        throw new Error(
+          `failed to offload from collections: ${failedCollections.join(
+            ", "
+          )}. ${errors.join(", ")}`
+        );
+      }
+    } finally {
+      client.close();
+    }
+  },
+});
+
+module.exports = command;

package/lib/interface/cli/commands/offline-logs/offload-to-file.spec.js

@@ -0,0 +1,59 @@
+const moment = require('moment')
+const { writeLogsToFile, checkRemnant } = require('./utils');
+const fs = require('fs')
+
+describe('offload-to-file', () => {
+  describe('writeLogsToFile', () => {
+    it('writes the input data to file', () => {
+      fs.writeFileSync = jest.fn((path, data) => {
+        return {path, data};
+      });
+      const upperBound = '2021-08-10';
+      const lowerBound = '2021-08-01';
+      const collection = 'logs';
+      const logsToArchive = [1, 2, 3, 4];
+      const expectedStr = JSON.stringify(logsToArchive)
+      const path = 'some/path';
+      writeLogsToFile(upperBound, lowerBound, collection, logsToArchive, path);
+      expect(fs.writeFileSync.mock.calls.length).toBe(1);
+      expect(fs.writeFileSync.mock.calls[0][0]).toContain(path);
+      expect(fs.writeFileSync.mock.calls[0][1]).toBe(expectedStr);
+    });
+    it('creates the correct file name', () => {
+      fs.writeFileSync = jest.fn((path, data) => {
+        return {path, data};
+      });
+      const upperBound = '2021-08-10';
+      const lowerBound = '2021-08-01';
+      const collection = 'logs';
+      const logsToArchive = [1, 2, 3, 4];
+      const expectedPath = 'some/path/2021-08-01-2021-08-09-logs.json'
+      const path = 'some/path';
+      writeLogsToFile(upperBound, lowerBound, collection, logsToArchive, path);
+      expect(fs.writeFileSync.mock.calls.length).toBe(1);
+      expect(fs.writeFileSync.mock.calls[0][0]).toBe(expectedPath);
+    });
+    it('checks leap year date calculation', () => {
+      fs.writeFileSync = jest.fn((path, data) => {
+        return {path, data};
+      });
+      const upperBound = '2020-03-01';
+      const lowerBound = '2020-02-01';
+      const collection = 'logs';
+      const logsToArchive = [1, 2, 3, 4];
+      const expectedPath = 'some/path/2020-02-01-2020-02-29-logs.json'
+      const path = 'some/path';
+      writeLogsToFile(upperBound, lowerBound, collection, logsToArchive, path);
+      expect(fs.writeFileSync.mock.calls.length).toBe(1);
+      expect(fs.writeFileSync.mock.calls[0][0]).toBe(expectedPath);
+    });
+  });
+  describe('checkRemnant', () => {
+    it('checkes days remnant to make sure time range is not going off boundaries', () => {
+      const lowerBound = '2020-08-01';
+      const cutoffDateObj = moment('2020-08-01').add(1, 'days').startOf("day");
+      const result = checkRemnant(lowerBound, cutoffDateObj);
+      expect(result).toBe(1);
+    });
+  });
+});