codefresh 0.78.0 → 0.78.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/interface/cli/commands/hybrid/helper.js +3 -3
- package/lib/interface/cli/commands/offline-logs/base.cmd.js +2 -2
- package/lib/interface/cli/commands/offline-logs/ensure-index.cmd.js +4 -2
- package/lib/interface/cli/commands/offline-logs/ensure-index.spec.js +2 -3
- package/lib/interface/cli/commands/offline-logs/offload-to-collection.cmd.js +177 -85
- package/lib/interface/cli/commands/offline-logs/offload-to-file.cmd.js +102 -83
- package/lib/interface/cli/commands/offline-logs/utils.js +59 -5
- package/lib/interface/cli/helpers/validation.js +4 -0
- package/package.json +4 -4
- package/yarn.lock +11 -12
|
@@ -40,10 +40,10 @@ const INSTALLATION_DEFAULTS = {
|
|
|
40
40
|
const RUNTIME_IMAGES = {
|
|
41
41
|
DOCKER_PUSHER_IMAGE: 'codefresh/cf-docker-pusher:6.0.3',
|
|
42
42
|
DOCKER_PULLER_IMAGE: 'codefresh/cf-docker-puller:8.0.2',
|
|
43
|
-
DOCKER_TAG_PUSHER_IMAGE: 'codefresh/cf-docker-tag-pusher:
|
|
43
|
+
DOCKER_TAG_PUSHER_IMAGE: 'codefresh/cf-docker-tag-pusher:1.3.0',
|
|
44
44
|
DOCKER_BUILDER_IMAGE: 'codefresh/cf-docker-builder:1.1.12',
|
|
45
|
-
GC_BUILDER_IMAGE: 'codefresh/cf-gc-builder:0.4.
|
|
46
|
-
CONTAINER_LOGGER_IMAGE: 'codefresh/cf-container-logger:1.
|
|
45
|
+
GC_BUILDER_IMAGE: 'codefresh/cf-gc-builder:0.4.4',
|
|
46
|
+
CONTAINER_LOGGER_IMAGE: 'codefresh/cf-container-logger:1.6.1',
|
|
47
47
|
GIT_CLONE_IMAGE: 'codefresh/cf-git-cloner:10.0.3',
|
|
48
48
|
COMPOSE_IMAGE: 'docker/compose:1.11.2',
|
|
49
49
|
KUBE_DEPLOY: 'codefresh/cf-deploy-kubernetes:latest',
|
|
@@ -16,12 +16,12 @@ const command = new Command({
|
|
|
16
16
|
.env("RUNTIME_MONGO")
|
|
17
17
|
.option("uri", {
|
|
18
18
|
describe:
|
|
19
|
-
"Mongodb URI. If not provided, will be parsed from environment
|
|
19
|
+
"Mongodb URI. If not provided, will be parsed from environment variable RUNTIME_MONGO_URI.",
|
|
20
20
|
type: "string",
|
|
21
21
|
})
|
|
22
22
|
.option("db", {
|
|
23
23
|
describe:
|
|
24
|
-
"Database name. If not provided, will be parsed from environment
|
|
24
|
+
"Database name. If not provided, will be parsed from environment variable RUNTIME_MONGO_DB.",
|
|
25
25
|
type: "string",
|
|
26
26
|
});
|
|
27
27
|
},
|
|
@@ -18,22 +18,24 @@ const command = new Command({
|
|
|
18
18
|
),
|
|
19
19
|
handler: async (argv) => {
|
|
20
20
|
const { uri, db } = argv;
|
|
21
|
-
const client = new MongoClient(uri);
|
|
21
|
+
const client = new MongoClient( uri, { useUnifiedTopology: true } );
|
|
22
22
|
await client.connect();
|
|
23
23
|
const database = client.db(db);
|
|
24
24
|
const failedCollections = [];
|
|
25
|
+
const errors = [];
|
|
25
26
|
for (const collection of utils.defaultCollections) {
|
|
26
27
|
try {
|
|
27
28
|
await utils.ensureIndex(database, collection);
|
|
28
29
|
} catch (error) {
|
|
29
30
|
console.error(`failed to ensure index of collection '${collection}', error: ${error.message}`);
|
|
30
31
|
failedCollections.push(collection);
|
|
32
|
+
errors.push(error);
|
|
31
33
|
}
|
|
32
34
|
}
|
|
33
35
|
client.close();
|
|
34
36
|
if (failedCollections.length) {
|
|
35
37
|
throw new Error(
|
|
36
|
-
`failed to ensure indexes of ${failedCollections.join(
|
|
38
|
+
`failed to ensure indexes of ${failedCollections.join(', ')}, ${errors.join(', ')}`
|
|
37
39
|
);
|
|
38
40
|
}
|
|
39
41
|
},
|
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
// const {Db, Collection} = require('mongodb');
|
|
2
|
-
const { Collection } = require('yaml/types');
|
|
3
1
|
const utils = require('./utils')
|
|
4
2
|
|
|
5
3
|
describe("ensure-index", () => {
|
|
@@ -29,6 +27,7 @@ describe("ensure-index", () => {
|
|
|
29
27
|
utils.getUserInput.mockReturnValue(true);
|
|
30
28
|
const collection = 'whatever';
|
|
31
29
|
const expectedIndexObj = { accountId: 1, jobId: 1 }
|
|
30
|
+
const expectedOptions = { background: true }
|
|
32
31
|
|
|
33
32
|
//execute
|
|
34
33
|
await utils.ensureIndex(mockDatabase, collection);
|
|
@@ -39,7 +38,7 @@ describe("ensure-index", () => {
|
|
|
39
38
|
expect(mockCollection.estimatedDocumentCount).toBeCalledTimes(1);
|
|
40
39
|
expect(utils.getUserInput).toBeCalledTimes(1);
|
|
41
40
|
expect(mockCollection.createIndex).toBeCalledTimes(1);
|
|
42
|
-
expect(mockCollection.createIndex).toBeCalledWith(expectedIndexObj)
|
|
41
|
+
expect(mockCollection.createIndex).toBeCalledWith(expectedIndexObj, expectedOptions)
|
|
43
42
|
});
|
|
44
43
|
it("index does not exists and user says no on prompt", async () => {
|
|
45
44
|
//setup
|
|
@@ -1,96 +1,188 @@
|
|
|
1
1
|
const { MongoClient, ObjectId } = require("mongodb");
|
|
2
|
-
const moment = require(
|
|
3
|
-
const
|
|
4
|
-
const
|
|
5
|
-
const
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
2
|
+
const moment = require("moment");
|
|
3
|
+
const semver = require("semver");
|
|
4
|
+
const Command = require("../../Command");
|
|
5
|
+
const cmd = require("./base.cmd");
|
|
6
|
+
const {
|
|
7
|
+
objectIdFromDate,
|
|
8
|
+
defaultCollections,
|
|
9
|
+
findMinLog,
|
|
10
|
+
checkCursorState,
|
|
11
|
+
createRangeStr,
|
|
12
|
+
getMinDate,
|
|
13
|
+
} = require("./utils");
|
|
14
|
+
|
|
15
|
+
const MERGE_RELEASE_VERSION = "4.2.0";
|
|
16
|
+
|
|
17
|
+
const offloadToCollection = async function (
|
|
18
|
+
sourceDBObj,
|
|
19
|
+
collection,
|
|
20
|
+
targetDB,
|
|
21
|
+
cutoffDate
|
|
22
|
+
) {
|
|
23
|
+
const sourceCollectionObj = sourceDBObj.collection(collection);
|
|
24
|
+
|
|
25
|
+
const cutoffDateObj = moment(cutoffDate).add(1, "days").startOf("day");
|
|
26
|
+
|
|
27
|
+
if (!cutoffDateObj.isValid()) {
|
|
28
|
+
throw new Error("please enter a valid date in ISO 8601 format");
|
|
17
29
|
}
|
|
18
30
|
|
|
19
|
-
const cutoffDateId = objectIdFromDate(cutoffDateObj.toDate())
|
|
31
|
+
const cutoffDateId = objectIdFromDate(cutoffDateObj.toDate());
|
|
20
32
|
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
if (!result.cursorState.killed){
|
|
34
|
-
console.info(`logs from '${collection}' were archived to '${targetCollection}'`)
|
|
35
|
-
await sourceCollectionObj.deleteMany({_id: {$lte: ObjectId(cutoffDateId)}})
|
|
33
|
+
const minLog = await findMinLog(
|
|
34
|
+
sourceCollectionObj,
|
|
35
|
+
cutoffDateId,
|
|
36
|
+
collection
|
|
37
|
+
);
|
|
38
|
+
|
|
39
|
+
if (!minLog) {
|
|
40
|
+
console.info(
|
|
41
|
+
`No logs to archive in collection ${collection} from the given date.`
|
|
42
|
+
);
|
|
43
|
+
return;
|
|
36
44
|
}
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
45
|
+
|
|
46
|
+
const lowerBound = getMinDate(minLog);
|
|
47
|
+
const upperBound = moment(cutoffDateObj);
|
|
48
|
+
const targetCollection = `${createRangeStr(
|
|
49
|
+
lowerBound,
|
|
50
|
+
upperBound,
|
|
51
|
+
collection
|
|
52
|
+
)}-archive`;
|
|
53
|
+
|
|
54
|
+
const mongoInfo = await sourceDBObj.admin().serverInfo();
|
|
55
|
+
const mongoServerVersion = await mongoInfo.version;
|
|
56
|
+
|
|
57
|
+
if (semver.lte(MERGE_RELEASE_VERSION, mongoServerVersion)) {
|
|
58
|
+
await archiveWithMerge(
|
|
59
|
+
sourceCollectionObj,
|
|
60
|
+
cutoffDateId,
|
|
61
|
+
targetDB,
|
|
62
|
+
targetCollection,
|
|
63
|
+
collection
|
|
64
|
+
);
|
|
65
|
+
} else {
|
|
66
|
+
await archiveWithOut(
|
|
67
|
+
sourceCollectionObj,
|
|
68
|
+
cutoffDateId,
|
|
69
|
+
targetCollection,
|
|
70
|
+
collection
|
|
71
|
+
);
|
|
40
72
|
}
|
|
41
|
-
}
|
|
73
|
+
};
|
|
74
|
+
///////////////////////////////////////////////////////////////////////////////////
|
|
75
|
+
const archiveWithMerge = async function (
|
|
76
|
+
sourceCollectionObj,
|
|
77
|
+
cutoffDateId,
|
|
78
|
+
targetDB,
|
|
79
|
+
targetCollection,
|
|
80
|
+
collection
|
|
81
|
+
) {
|
|
82
|
+
var result = sourceCollectionObj.aggregate([
|
|
83
|
+
{ $match: { _id: { $lte: ObjectId(cutoffDateId) } } },
|
|
84
|
+
{
|
|
85
|
+
$merge: {
|
|
86
|
+
into: { db: targetDB, coll: targetCollection },
|
|
87
|
+
on: "_id",
|
|
88
|
+
whenMatched: "keepExisting",
|
|
89
|
+
whenNotMatched: "insert",
|
|
90
|
+
},
|
|
91
|
+
},
|
|
92
|
+
]);
|
|
42
93
|
|
|
94
|
+
await result.toArray();
|
|
95
|
+
|
|
96
|
+
checkCursorState(result, collection, targetCollection);
|
|
97
|
+
|
|
98
|
+
await sourceCollectionObj.deleteMany({
|
|
99
|
+
_id: { $lte: ObjectId(cutoffDateId) },
|
|
100
|
+
});
|
|
101
|
+
};
|
|
102
|
+
///////////////////////////////////////////////////////////////////////////////////
|
|
103
|
+
const archiveWithOut = async function (
|
|
104
|
+
sourceCollectionObj,
|
|
105
|
+
cutoffDateId,
|
|
106
|
+
targetCollection,
|
|
107
|
+
collection
|
|
108
|
+
) {
|
|
109
|
+
var result = sourceCollectionObj.aggregate([
|
|
110
|
+
{ $match: { _id: { $lte: ObjectId(cutoffDateId) } } },
|
|
111
|
+
{ $out: targetCollection },
|
|
112
|
+
]);
|
|
113
|
+
|
|
114
|
+
await result.toArray();
|
|
115
|
+
|
|
116
|
+
checkCursorState(result, collection, targetCollection);
|
|
117
|
+
|
|
118
|
+
await sourceCollectionObj.deleteMany({
|
|
119
|
+
_id: { $lte: ObjectId(cutoffDateId) },
|
|
120
|
+
});
|
|
121
|
+
};
|
|
122
|
+
///////////////////////////////////////////////////////////////////////////////////
|
|
43
123
|
const command = new Command({
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
124
|
+
command: "offload-to-collection",
|
|
125
|
+
parent: cmd,
|
|
126
|
+
description:
|
|
127
|
+
"Archiving logs from one or more source collections to target collections.",
|
|
128
|
+
webDocs: {
|
|
129
|
+
category: "Logs",
|
|
130
|
+
title: "Offload To Collection",
|
|
131
|
+
},
|
|
132
|
+
builder: (yargs) =>
|
|
133
|
+
yargs
|
|
134
|
+
.option("targetDB", {
|
|
135
|
+
alias: "tdb",
|
|
136
|
+
describe:
|
|
137
|
+
"This option is available only for mongodb version 4.2 and up. for older versions it will be ignored. \
|
|
138
|
+
Target database name, if none inserted, db will be defined as target.",
|
|
139
|
+
type: "string",
|
|
140
|
+
})
|
|
141
|
+
.option("cutoffDate", {
|
|
142
|
+
alias: "cod",
|
|
143
|
+
describe:
|
|
144
|
+
"The date in ISO format (yyyy-MM-dd) from which logs will be archived (going backwards, including logs from that day).",
|
|
145
|
+
demandOption: true,
|
|
146
|
+
type: "string",
|
|
147
|
+
})
|
|
148
|
+
.example(
|
|
149
|
+
'codefresh offline-logs offload-to-collection --uri "mongodb://192.168.99.100:27017" --db logs --cod "2021-07-08" '
|
|
150
|
+
),
|
|
151
|
+
handler: async (argv) => {
|
|
152
|
+
const { uri, db, targetDB, cutoffDate } = argv;
|
|
153
|
+
|
|
154
|
+
const client = new MongoClient(uri, { useUnifiedTopology: true });
|
|
155
|
+
try {
|
|
156
|
+
await client.connect();
|
|
157
|
+
const failedCollections = [];
|
|
158
|
+
const errors = [];
|
|
159
|
+
const sourceDBObj = client.db(db);
|
|
160
|
+
const promises = defaultCollections.map(async (collection) => {
|
|
161
|
+
try {
|
|
162
|
+
await offloadToCollection(
|
|
163
|
+
sourceDBObj,
|
|
164
|
+
collection,
|
|
165
|
+
targetDB || db,
|
|
166
|
+
cutoffDate
|
|
167
|
+
);
|
|
168
|
+
} catch (error) {
|
|
169
|
+
failedCollections.push(collection);
|
|
170
|
+
errors.push(error);
|
|
89
171
|
}
|
|
90
|
-
}
|
|
91
|
-
|
|
172
|
+
});
|
|
173
|
+
await Promise.all(promises);
|
|
174
|
+
|
|
175
|
+
if (failedCollections.length) {
|
|
176
|
+
throw new Error(
|
|
177
|
+
`failed to offload from collections: ${failedCollections.join(
|
|
178
|
+
", "
|
|
179
|
+
)}. ${errors.join(", ")}`
|
|
180
|
+
);
|
|
92
181
|
}
|
|
93
|
-
}
|
|
94
|
-
|
|
182
|
+
} finally {
|
|
183
|
+
client.close();
|
|
184
|
+
}
|
|
185
|
+
},
|
|
186
|
+
});
|
|
95
187
|
|
|
96
|
-
module.exports = command;
|
|
188
|
+
module.exports = command;
|
|
@@ -1,59 +1,66 @@
|
|
|
1
|
-
|
|
2
1
|
const { MongoClient, ObjectId } = require("mongodb");
|
|
3
|
-
const moment = require(
|
|
4
|
-
const Command = require(
|
|
5
|
-
const cmd = require(
|
|
6
|
-
const {
|
|
2
|
+
const moment = require("moment");
|
|
3
|
+
const Command = require("../../Command");
|
|
4
|
+
const cmd = require("./base.cmd");
|
|
5
|
+
const {
|
|
6
|
+
objectIdFromDate,
|
|
7
|
+
writeLogsToFile,
|
|
8
|
+
checkRemnant,
|
|
9
|
+
findMinLog,
|
|
10
|
+
getMinDate,
|
|
11
|
+
defaultCollections,
|
|
12
|
+
} = require("./utils");
|
|
7
13
|
|
|
8
|
-
const offloadToFile = async function(
|
|
14
|
+
const offloadToFile = async function (
|
|
15
|
+
database,
|
|
16
|
+
collection,
|
|
17
|
+
chunkDuration,
|
|
18
|
+
cutoffDate,
|
|
19
|
+
path
|
|
20
|
+
) {
|
|
9
21
|
const collectionObj = database.collection(collection);
|
|
10
22
|
|
|
11
|
-
if (chunkDuration <= 0){
|
|
12
|
-
throw new Error(
|
|
23
|
+
if (chunkDuration <= 0) {
|
|
24
|
+
throw new Error("please enter a valid chunkDuration ( > 0)");
|
|
13
25
|
}
|
|
14
|
-
let chunkSize = chunkDuration
|
|
26
|
+
let chunkSize = chunkDuration;
|
|
15
27
|
|
|
16
28
|
// the cutoff date will be the start of the next day from user input (archive will include the given cutoff date)
|
|
17
29
|
const cutoffDateObj = moment(cutoffDate, moment.ISO_8601, true)
|
|
18
|
-
.add(1,
|
|
30
|
+
.add(1, "days")
|
|
19
31
|
.startOf("day");
|
|
20
|
-
if(!cutoffDateObj.isValid()){
|
|
21
|
-
throw new Error(
|
|
32
|
+
if (!cutoffDateObj.isValid()) {
|
|
33
|
+
throw new Error("please enter a valid date in ISO 8601 format");
|
|
22
34
|
}
|
|
23
|
-
const cutoffDateId = objectIdFromDate(cutoffDateObj.toDate())
|
|
35
|
+
const cutoffDateId = objectIdFromDate(cutoffDateObj.toDate());
|
|
36
|
+
|
|
37
|
+
const minLog = await findMinLog(collectionObj, cutoffDateId, collection);
|
|
24
38
|
|
|
25
|
-
|
|
26
|
-
.
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
if (minLog.length === 0) {
|
|
32
|
-
console.info(`No logs to archive in Collection ${collection} from the given date.`);
|
|
33
|
-
return;
|
|
39
|
+
if (!minLog) {
|
|
40
|
+
console.info(
|
|
41
|
+
`No logs to archive in Collection ${collection} from the given date.`
|
|
42
|
+
);
|
|
43
|
+
return;
|
|
34
44
|
}
|
|
35
|
-
|
|
36
|
-
const
|
|
37
|
-
const minDate = moment(
|
|
38
|
-
minLogId.getTimestamp()
|
|
39
|
-
).startOf("day").toDate();
|
|
45
|
+
|
|
46
|
+
const minDate = getMinDate(minLog);
|
|
40
47
|
|
|
41
48
|
for (
|
|
42
49
|
let lowerBound = moment(minDate);
|
|
43
50
|
lowerBound < cutoffDateObj;
|
|
44
|
-
lowerBound.add(
|
|
51
|
+
lowerBound.add(chunkSize, "days")
|
|
45
52
|
) {
|
|
46
53
|
//in case total days period is not devided with chunkSize, the last chunk size needs to be smaller.
|
|
47
|
-
const remnant = checkRemnant(lowerBound, cutoffDateObj)
|
|
54
|
+
const remnant = checkRemnant(lowerBound, cutoffDateObj);
|
|
48
55
|
|
|
49
56
|
if (remnant < chunkSize && remnant > 0) {
|
|
50
57
|
chunkSize = remnant;
|
|
51
58
|
}
|
|
52
|
-
const upperBound = moment(lowerBound).add(
|
|
59
|
+
const upperBound = moment(lowerBound).add(chunkSize, "days");
|
|
60
|
+
|
|
61
|
+
const lowerBoundId = objectIdFromDate(lowerBound.toDate());
|
|
62
|
+
const upperBoundId = objectIdFromDate(upperBound.toDate());
|
|
53
63
|
|
|
54
|
-
const lowerBoundId = objectIdFromDate(lowerBound.toDate())
|
|
55
|
-
const upperBoundId = objectIdFromDate(upperBound.toDate())
|
|
56
|
-
|
|
57
64
|
const logsToArchive = await collectionObj
|
|
58
65
|
.find({
|
|
59
66
|
_id: {
|
|
@@ -64,7 +71,7 @@ const offloadToFile = async function(database, collection, chunkDuration, cutoff
|
|
|
64
71
|
.toArray();
|
|
65
72
|
|
|
66
73
|
if (logsToArchive.length > 0) {
|
|
67
|
-
writeLogsToFile(upperBound, lowerBound, collection, logsToArchive, path)
|
|
74
|
+
writeLogsToFile(upperBound, lowerBound, collection, logsToArchive, path);
|
|
68
75
|
|
|
69
76
|
await collectionObj.deleteMany({
|
|
70
77
|
_id: {
|
|
@@ -74,83 +81,95 @@ const offloadToFile = async function(database, collection, chunkDuration, cutoff
|
|
|
74
81
|
});
|
|
75
82
|
}
|
|
76
83
|
}
|
|
77
|
-
}
|
|
84
|
+
};
|
|
78
85
|
|
|
79
86
|
const command = new Command({
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
87
|
+
command: "offload-to-file",
|
|
88
|
+
parent: cmd,
|
|
89
|
+
description:
|
|
90
|
+
"Archiving logs from one or more source collections to files by chunks of days.",
|
|
91
|
+
webDocs: {
|
|
92
|
+
category: "Logs",
|
|
93
|
+
title: "Offload To File",
|
|
94
|
+
},
|
|
95
|
+
builder: (yargs) =>
|
|
96
|
+
yargs
|
|
97
|
+
.option("chunkDuration", {
|
|
89
98
|
alias: "chdur",
|
|
90
99
|
describe:
|
|
91
100
|
"Chunk size in days, each chunk will be archived into a different file.",
|
|
92
101
|
default: 1,
|
|
93
102
|
type: "number",
|
|
94
103
|
})
|
|
95
|
-
.option(
|
|
104
|
+
.option("cutoffDate", {
|
|
96
105
|
alias: "cod",
|
|
97
106
|
describe:
|
|
98
107
|
"The date in ISO format (yyyy-MM-dd) from which logs will be archived (going backwards, including logs from that day).",
|
|
99
108
|
demandOption: true,
|
|
100
109
|
type: "string",
|
|
101
110
|
})
|
|
102
|
-
.option(
|
|
111
|
+
.option("path", {
|
|
103
112
|
describe: "Directory path to which archive files will be saved.",
|
|
104
113
|
default: ".",
|
|
105
114
|
type: "string",
|
|
106
115
|
})
|
|
107
|
-
.option(
|
|
108
|
-
alias:
|
|
116
|
+
.option("collections", {
|
|
117
|
+
alias: "c",
|
|
109
118
|
describe: "Source collections names",
|
|
110
119
|
default: defaultCollections,
|
|
111
120
|
array: true,
|
|
112
121
|
type: "string",
|
|
113
122
|
coerce(arg) {
|
|
114
123
|
for (const collection of arg) {
|
|
115
|
-
if (
|
|
116
|
-
|
|
124
|
+
if (
|
|
125
|
+
!collection.includes("logs") &&
|
|
126
|
+
!collection.includes("metadata")
|
|
127
|
+
) {
|
|
128
|
+
throw new Error("invalid collection name");
|
|
117
129
|
}
|
|
118
|
-
return arg
|
|
119
|
-
}
|
|
120
|
-
}
|
|
121
|
-
})
|
|
122
|
-
.example('codefresh offline-logs offload-to-file --uri "mongodb://192.168.99.100:27017" --db logs --collections archive-logs --cod "2021-07-08" --chdur 3 --path "./" '),
|
|
123
|
-
handler: async (argv) => {
|
|
124
|
-
const {
|
|
125
|
-
uri,
|
|
126
|
-
cutoffDate,
|
|
127
|
-
chunkDuration,
|
|
128
|
-
path,
|
|
129
|
-
db,
|
|
130
|
-
collections,
|
|
131
|
-
} = argv;
|
|
132
|
-
const client = new MongoClient(uri);
|
|
133
|
-
try {
|
|
134
|
-
await client.connect();
|
|
135
|
-
const failedCollections = [];
|
|
136
|
-
const database = client.db(db);
|
|
137
|
-
const promises = collections.map( async ( collection ) => {
|
|
138
|
-
try {
|
|
139
|
-
await offloadToFile( database, collection, chunkDuration, cutoffDate, path );
|
|
140
|
-
} catch (e) {
|
|
141
|
-
console.log(e)
|
|
142
|
-
failedCollections.push(collection);
|
|
130
|
+
return arg;
|
|
143
131
|
}
|
|
144
|
-
}
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
132
|
+
},
|
|
133
|
+
})
|
|
134
|
+
.example(
|
|
135
|
+
'codefresh offline-logs offload-to-file --uri "mongodb://192.168.99.100:27017" --db logs --collections archive-logs --cod "2021-07-08" --chdur 3 --path "./" '
|
|
136
|
+
),
|
|
137
|
+
handler: async (argv) => {
|
|
138
|
+
const { uri, cutoffDate, chunkDuration, path, db, collections } = argv;
|
|
139
|
+
const client = new MongoClient(uri, { useUnifiedTopology: true });
|
|
140
|
+
try {
|
|
141
|
+
await client.connect();
|
|
142
|
+
const failedCollections = [];
|
|
143
|
+
const errors = [];
|
|
144
|
+
const database = client.db(db);
|
|
145
|
+
const promises = collections.map(async (collection) => {
|
|
146
|
+
try {
|
|
147
|
+
await offloadToFile(
|
|
148
|
+
database,
|
|
149
|
+
collection,
|
|
150
|
+
chunkDuration,
|
|
151
|
+
cutoffDate,
|
|
152
|
+
path
|
|
153
|
+
);
|
|
154
|
+
} catch (e) {
|
|
155
|
+
console.log(e);
|
|
156
|
+
failedCollections.push(collection);
|
|
157
|
+
errors.push(error);
|
|
149
158
|
}
|
|
150
|
-
}
|
|
151
|
-
|
|
159
|
+
});
|
|
160
|
+
await Promise.all(promises);
|
|
161
|
+
|
|
162
|
+
if (failedCollections.length) {
|
|
163
|
+
throw new Error(
|
|
164
|
+
`failed to offload from collections: ${failedCollections.join(
|
|
165
|
+
", "
|
|
166
|
+
)}. ${errors.join(", ")}`
|
|
167
|
+
);
|
|
152
168
|
}
|
|
153
|
-
}
|
|
169
|
+
} finally {
|
|
170
|
+
client.close();
|
|
171
|
+
}
|
|
172
|
+
},
|
|
154
173
|
});
|
|
155
174
|
|
|
156
175
|
module.exports = command;
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
const readline = require("readline");
|
|
2
2
|
const moment = require("moment");
|
|
3
|
+
const { ObjectId } = require("mongodb")
|
|
3
4
|
|
|
4
5
|
const { join } = require("path");
|
|
5
6
|
const fs = require("fs");
|
|
@@ -18,17 +19,28 @@ const writeLogsToFile = function (
|
|
|
18
19
|
logsToArchive,
|
|
19
20
|
path
|
|
20
21
|
) {
|
|
21
|
-
const
|
|
22
|
-
const fileDateRange = `${moment(lowerBound).format(
|
|
23
|
-
"YYYY-MM-DD"
|
|
24
|
-
)}-${date.format("YYYY-MM-DD")}`;
|
|
25
|
-
const fileName = `${fileDateRange}-${collection}.json`;
|
|
22
|
+
const fileName = `${createRangeStr(lowerBound, upperBound, collection)}.json`
|
|
26
23
|
const absPath = join(path, fileName);
|
|
27
24
|
const data = JSON.stringify(logsToArchive);
|
|
28
25
|
fs.writeFileSync(absPath, data);
|
|
29
26
|
console.info(`logs from collection '${collection}' were offloaded to '${absPath}'`)
|
|
30
27
|
};
|
|
31
28
|
|
|
29
|
+
const createRangeStr = function(lowerBound, upperBound, collection) {
|
|
30
|
+
const upperBoundDate = moment(upperBound).subtract(1, "days");
|
|
31
|
+
const fileDateRange = `${moment(lowerBound).format(
|
|
32
|
+
"YYYY-MM-DD"
|
|
33
|
+
)}-${upperBoundDate.format("YYYY-MM-DD")}`;
|
|
34
|
+
return `${fileDateRange}-${collection}`;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const getMinDate = function(minLog) {
|
|
38
|
+
const minLogId = ObjectId(minLog._id)
|
|
39
|
+
return moment(
|
|
40
|
+
minLogId.getTimestamp()
|
|
41
|
+
).startOf("day");
|
|
42
|
+
}
|
|
43
|
+
|
|
32
44
|
const checkRemnant = function (lowerBound, cutoffDateObj) {
|
|
33
45
|
return Math.ceil(cutoffDateObj.diff(lowerBound, "days"));
|
|
34
46
|
};
|
|
@@ -61,6 +73,7 @@ const ensureIndex = async function (database, collection) {
|
|
|
61
73
|
|
|
62
74
|
await collectionObj.createIndex(
|
|
63
75
|
{ accountId: 1, jobId: 1 },
|
|
76
|
+
{ background: true },
|
|
64
77
|
);
|
|
65
78
|
console.info(
|
|
66
79
|
`index '${indexName}' was created for collection '${collection}'`
|
|
@@ -95,11 +108,52 @@ const getUserInput = async function (queryStr) {
|
|
|
95
108
|
}
|
|
96
109
|
};
|
|
97
110
|
|
|
111
|
+
const collectionExists = async function(dbObject, targetCollection) {
|
|
112
|
+
const collectionsObjectList = await dbObject.listCollections().toArray();
|
|
113
|
+
const collectionsList = collectionsObjectList.map((object) => {
|
|
114
|
+
return object.name;
|
|
115
|
+
})
|
|
116
|
+
return (collectionsList.indexOf(targetCollection) >= 0);
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
const findMinLog = async function(collectionObj, UpperBoundDateId, collection) {
|
|
120
|
+
const minLog = await collectionObj
|
|
121
|
+
.find({ _id: { $lt: ObjectId(UpperBoundDateId) } })
|
|
122
|
+
.sort({ _id: 1 })
|
|
123
|
+
.limit(1)
|
|
124
|
+
.toArray();
|
|
125
|
+
|
|
126
|
+
if (minLog.length === 0) {
|
|
127
|
+
return undefined;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
return minLog[0];
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
const checkCursorState = function (cursor, collection, targetCollection) {
|
|
134
|
+
if (!cursor.cursorState.killed) {
|
|
135
|
+
console.info(
|
|
136
|
+
`logs from '${collection}' were archived to '${targetCollection}'`
|
|
137
|
+
);
|
|
138
|
+
return;
|
|
139
|
+
} else {
|
|
140
|
+
throw new Error(
|
|
141
|
+
"Cursor error. Archiving operation may not be completed.\
|
|
142
|
+
The old logs were not deleted from the source collection."
|
|
143
|
+
);
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
98
147
|
module.exports = {
|
|
99
148
|
objectIdFromDate,
|
|
100
149
|
writeLogsToFile,
|
|
101
150
|
checkRemnant,
|
|
102
151
|
ensureIndex,
|
|
103
152
|
getUserInput,
|
|
153
|
+
collectionExists,
|
|
154
|
+
findMinLog,
|
|
155
|
+
checkCursorState,
|
|
156
|
+
createRangeStr,
|
|
157
|
+
getMinDate,
|
|
104
158
|
defaultCollections,
|
|
105
159
|
};
|
|
@@ -24,6 +24,7 @@ function _getPipelineName(filename) {
|
|
|
24
24
|
}
|
|
25
25
|
|
|
26
26
|
async function validatePipelineSpec(data) {
|
|
27
|
+
const specTemplate = _.get(data, 'spec.specTemplate');
|
|
27
28
|
const steps = _.get(data, 'spec.steps');
|
|
28
29
|
const stages = _.get(data, 'spec.stages');
|
|
29
30
|
const yamlObj = {};
|
|
@@ -36,6 +37,9 @@ async function validatePipelineSpec(data) {
|
|
|
36
37
|
if (stages) {
|
|
37
38
|
yamlObj.stages = stages;
|
|
38
39
|
}
|
|
40
|
+
if (specTemplate) { // CR-6414 Using specTemplate - skip check spec/stages - they are not used
|
|
41
|
+
return { valid: true, message: 'Using specTemplate' };
|
|
42
|
+
}
|
|
39
43
|
const validatedYaml = yaml.safeDump(yamlObj);
|
|
40
44
|
const result = await sdk.pipelines.validateYaml({ yaml: validatedYaml, outputFormat: 'lint' });
|
|
41
45
|
let message;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "codefresh",
|
|
3
|
-
"version": "0.78.
|
|
3
|
+
"version": "0.78.4",
|
|
4
4
|
"description": "Codefresh command line utility",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"preferGlobal": true,
|
|
@@ -36,9 +36,9 @@
|
|
|
36
36
|
"bluebird": "^3.5.1",
|
|
37
37
|
"cf-errors": "^0.1.16",
|
|
38
38
|
"chalk": "^4.1.0",
|
|
39
|
-
"cli-progress": "3.
|
|
39
|
+
"cli-progress": "3.10.0",
|
|
40
40
|
"codefresh-sdk": "^1.9.22",
|
|
41
|
-
"colors": "
|
|
41
|
+
"colors": "1.4.0",
|
|
42
42
|
"columnify": "^1.5.4",
|
|
43
43
|
"compare-versions": "^3.4.0",
|
|
44
44
|
"copy-dir": "^0.3.0",
|
|
@@ -64,7 +64,7 @@
|
|
|
64
64
|
"mongodb": "^3.0.1",
|
|
65
65
|
"node-forge": "^0.10.0",
|
|
66
66
|
"ora": "^3.0.0",
|
|
67
|
-
"prettyjson": "^1.2.
|
|
67
|
+
"prettyjson": "^1.2.5",
|
|
68
68
|
"promise-retry": "^2.0.1",
|
|
69
69
|
"recursive-readdir": "^2.2.1",
|
|
70
70
|
"request": "^2.88.0",
|
package/yarn.lock
CHANGED
|
@@ -1010,13 +1010,12 @@ cli-cursor@^3.1.0:
|
|
|
1010
1010
|
dependencies:
|
|
1011
1011
|
restore-cursor "^3.1.0"
|
|
1012
1012
|
|
|
1013
|
-
cli-progress@3.
|
|
1014
|
-
version "3.
|
|
1015
|
-
resolved "https://registry.yarnpkg.com/cli-progress/-/cli-progress-3.
|
|
1016
|
-
integrity sha512-
|
|
1013
|
+
cli-progress@3.10.0:
|
|
1014
|
+
version "3.10.0"
|
|
1015
|
+
resolved "https://registry.yarnpkg.com/cli-progress/-/cli-progress-3.10.0.tgz#63fd9d6343c598c93542fdfa3563a8b59887d78a"
|
|
1016
|
+
integrity sha512-kLORQrhYCAtUPLZxqsAt2YJGOvRdt34+O6jl5cQGb7iF3dM55FQZlTR+rQyIK9JUcO9bBMwZsTlND+3dmFU2Cw==
|
|
1017
1017
|
dependencies:
|
|
1018
|
-
|
|
1019
|
-
string-width "^2.1.1"
|
|
1018
|
+
string-width "^4.2.0"
|
|
1020
1019
|
|
|
1021
1020
|
cli-spinners@^2.0.0:
|
|
1022
1021
|
version "2.3.0"
|
|
@@ -1127,7 +1126,7 @@ color-name@~1.1.4:
|
|
|
1127
1126
|
resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2"
|
|
1128
1127
|
integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==
|
|
1129
1128
|
|
|
1130
|
-
colors
|
|
1129
|
+
colors@1.4.0:
|
|
1131
1130
|
version "1.4.0"
|
|
1132
1131
|
resolved "https://registry.yarnpkg.com/colors/-/colors-1.4.0.tgz#c50491479d4c1bdaed2c9ced32cf7c7dc2360f78"
|
|
1133
1132
|
integrity sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA==
|
|
@@ -4525,12 +4524,12 @@ pretty-format@^23.6.0:
|
|
|
4525
4524
|
ansi-regex "^3.0.0"
|
|
4526
4525
|
ansi-styles "^3.2.0"
|
|
4527
4526
|
|
|
4528
|
-
prettyjson@^1.2.
|
|
4529
|
-
version "1.2.
|
|
4530
|
-
resolved "https://registry.yarnpkg.com/prettyjson/-/prettyjson-1.2.
|
|
4531
|
-
integrity
|
|
4527
|
+
prettyjson@^1.2.5:
|
|
4528
|
+
version "1.2.5"
|
|
4529
|
+
resolved "https://registry.yarnpkg.com/prettyjson/-/prettyjson-1.2.5.tgz#ef3cfffcc70505c032abc59785884b4027031835"
|
|
4530
|
+
integrity sha512-rksPWtoZb2ZpT5OVgtmy0KHVM+Dca3iVwWY9ifwhcexfjebtgjg3wmrUt9PvJ59XIYBcknQeYHD8IAnVlh9lAw==
|
|
4532
4531
|
dependencies:
|
|
4533
|
-
colors "
|
|
4532
|
+
colors "1.4.0"
|
|
4534
4533
|
minimist "^1.2.0"
|
|
4535
4534
|
|
|
4536
4535
|
private@^0.1.8:
|