@contentstack/cli-cm-export-query 1.0.0-beta.1 → 1.0.0-beta.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/lib/commands/cm/stacks/export-query.d.ts +0 -1
- package/lib/commands/cm/stacks/export-query.js +23 -6
- package/lib/core/module-exporter.js +7 -5
- package/lib/core/query-executor.js +127 -41
- package/lib/types/index.d.ts +18 -0
- package/lib/utils/branch-helper.d.ts +10 -0
- package/lib/utils/branch-helper.js +68 -0
- package/lib/utils/config-handler.js +16 -9
- package/lib/utils/content-type-helper.js +3 -3
- package/lib/utils/dependency-resolver.js +10 -11
- package/lib/utils/index.d.ts +3 -1
- package/lib/utils/index.js +4 -1
- package/lib/utils/logger.d.ts +5 -1
- package/lib/utils/logger.js +16 -1
- package/lib/utils/query-parser.js +5 -5
- package/lib/utils/referenced-asset-handler.js +20 -10
- package/oclif.manifest.json +15 -4
- package/package.json +19 -16
package/LICENSE
CHANGED
|
@@ -16,16 +16,29 @@ class ExportQueryCommand extends cli_command_1.Command {
|
|
|
16
16
|
exportQueryConfig.developerHubBaseUrl = this.developerHubUrl;
|
|
17
17
|
}
|
|
18
18
|
this.exportDir = (0, cli_utilities_1.sanitizePath)(exportQueryConfig.exportDir);
|
|
19
|
-
//
|
|
19
|
+
// Create base context without module name - module field is set dynamically during each module export
|
|
20
|
+
exportQueryConfig.context = (0, utils_1.createLogContext)(exportQueryConfig);
|
|
21
|
+
cli_utilities_1.log.debug('Export configuration setup completed', exportQueryConfig.context);
|
|
22
|
+
// Initialize management API client
|
|
20
23
|
const managementAPIClient = await (0, cli_utilities_1.managementSDKClient)(exportQueryConfig);
|
|
24
|
+
// Setup and validate branch configuration
|
|
25
|
+
const stackAPIClient = managementAPIClient.stack({
|
|
26
|
+
api_key: exportQueryConfig.stackApiKey,
|
|
27
|
+
management_token: exportQueryConfig.managementToken,
|
|
28
|
+
});
|
|
29
|
+
// Setup branches (validate branch or set default to 'main')
|
|
30
|
+
await (0, utils_1.setupBranches)(exportQueryConfig, stackAPIClient);
|
|
31
|
+
cli_utilities_1.log.debug('Branch configuration setup completed', exportQueryConfig.context);
|
|
32
|
+
// Initialize and run query export
|
|
33
|
+
cli_utilities_1.log.debug('Starting query exporter', exportQueryConfig.context);
|
|
21
34
|
const queryExporter = new query_executor_1.QueryExporter(managementAPIClient, exportQueryConfig);
|
|
22
35
|
await queryExporter.execute();
|
|
23
|
-
|
|
24
|
-
|
|
36
|
+
cli_utilities_1.log.debug('Query exporter completed successfully', exportQueryConfig.context);
|
|
37
|
+
cli_utilities_1.log.success('Query-based export completed successfully!', exportQueryConfig.context);
|
|
38
|
+
cli_utilities_1.log.info(`Export files saved to: ${this.exportDir}`, exportQueryConfig.context);
|
|
25
39
|
}
|
|
26
40
|
catch (error) {
|
|
27
|
-
(0,
|
|
28
|
-
throw error;
|
|
41
|
+
(0, cli_utilities_1.handleAndLogError)(error);
|
|
29
42
|
}
|
|
30
43
|
}
|
|
31
44
|
}
|
|
@@ -57,6 +70,11 @@ ExportQueryCommand.flags = {
|
|
|
57
70
|
}),
|
|
58
71
|
branch: cli_utilities_1.flags.string({
|
|
59
72
|
description: 'Branch name to export from',
|
|
73
|
+
exclusive: ['branch-alias'],
|
|
74
|
+
}),
|
|
75
|
+
'branch-alias': cli_utilities_1.flags.string({
|
|
76
|
+
description: 'Alias of Branch to export from',
|
|
77
|
+
exclusive: ['branch'],
|
|
60
78
|
}),
|
|
61
79
|
query: cli_utilities_1.flags.string({
|
|
62
80
|
required: true,
|
|
@@ -76,4 +94,3 @@ ExportQueryCommand.flags = {
|
|
|
76
94
|
description: 'Skip confirmation prompts',
|
|
77
95
|
}),
|
|
78
96
|
};
|
|
79
|
-
ExportQueryCommand.aliases = ['cm:export-query'];
|
|
@@ -4,7 +4,6 @@ exports.ModuleExporter = void 0;
|
|
|
4
4
|
const tslib_1 = require("tslib");
|
|
5
5
|
const cli_utilities_1 = require("@contentstack/cli-utilities");
|
|
6
6
|
const cli_cm_export_1 = tslib_1.__importDefault(require("@contentstack/cli-cm-export"));
|
|
7
|
-
const logger_1 = require("../utils/logger");
|
|
8
7
|
class ModuleExporter {
|
|
9
8
|
constructor(exportQueryConfig) {
|
|
10
9
|
this.exportedModules = [];
|
|
@@ -12,25 +11,28 @@ class ModuleExporter {
|
|
|
12
11
|
}
|
|
13
12
|
async exportModule(moduleName, options = {}) {
|
|
14
13
|
try {
|
|
15
|
-
(
|
|
14
|
+
const moduleLogContext = Object.assign(Object.assign({}, this.exportQueryConfig.context), { module: moduleName });
|
|
15
|
+
cli_utilities_1.log.info(`Exporting module: ${moduleName}`, moduleLogContext);
|
|
16
|
+
cli_utilities_1.log.debug(`Building export command for module: ${moduleName}`, moduleLogContext);
|
|
16
17
|
// Build command arguments
|
|
17
18
|
const cmd = this.buildExportCommand(moduleName, options);
|
|
18
|
-
(0, logger_1.log)(this.exportQueryConfig, `Running export command: ${cmd.join(' ')}`, 'debug');
|
|
19
19
|
// Configurable delay
|
|
20
20
|
const delay = this.exportQueryConfig.exportDelayMs || 2000;
|
|
21
21
|
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
22
22
|
// Create export command instance
|
|
23
23
|
await cli_cm_export_1.default.run(cmd);
|
|
24
|
+
cli_utilities_1.log.debug(`Export command completed for module: ${moduleName}`, moduleLogContext);
|
|
24
25
|
// Read the exported data
|
|
25
26
|
// const data = await this.readExportedData(moduleName, options);
|
|
26
27
|
if (!this.exportedModules.includes(moduleName)) {
|
|
27
28
|
this.exportedModules.push(moduleName);
|
|
28
29
|
}
|
|
29
30
|
// success message
|
|
30
|
-
|
|
31
|
+
cli_utilities_1.log.success(`Successfully exported ${moduleName}`, moduleLogContext);
|
|
31
32
|
}
|
|
32
33
|
catch (error) {
|
|
33
|
-
(
|
|
34
|
+
const moduleLogContext = Object.assign(Object.assign({}, this.exportQueryConfig.context), { module: moduleName });
|
|
35
|
+
(0, cli_utilities_1.handleAndLogError)(error, moduleLogContext, `Failed to export ${moduleName}`);
|
|
34
36
|
throw error;
|
|
35
37
|
}
|
|
36
38
|
}
|
|
@@ -6,7 +6,6 @@ const cli_utilities_1 = require("@contentstack/cli-utilities");
|
|
|
6
6
|
const path = tslib_1.__importStar(require("path"));
|
|
7
7
|
const query_parser_1 = require("../utils/query-parser");
|
|
8
8
|
const module_exporter_1 = require("./module-exporter");
|
|
9
|
-
const logger_1 = require("../utils/logger");
|
|
10
9
|
const utils_1 = require("../utils");
|
|
11
10
|
const utils_2 = require("../utils");
|
|
12
11
|
const utils_3 = require("../utils");
|
|
@@ -23,10 +22,11 @@ class QueryExporter {
|
|
|
23
22
|
this.moduleExporter = new module_exporter_1.ModuleExporter(exportQueryConfig);
|
|
24
23
|
}
|
|
25
24
|
async execute() {
|
|
26
|
-
|
|
25
|
+
cli_utilities_1.log.info('Starting query-based export...', this.exportQueryConfig.context);
|
|
27
26
|
// Step 1: Parse and validate query
|
|
27
|
+
cli_utilities_1.log.debug('Parsing and validating query', this.exportQueryConfig.context);
|
|
28
28
|
const parsedQuery = await this.queryParser.parse(this.exportQueryConfig.query);
|
|
29
|
-
|
|
29
|
+
cli_utilities_1.log.success('Query parsed and validated successfully', this.exportQueryConfig.context);
|
|
30
30
|
// Step 2: Always export general modules
|
|
31
31
|
await this.exportGeneralModules();
|
|
32
32
|
// Step 4: Export queried modules
|
|
@@ -35,39 +35,44 @@ class QueryExporter {
|
|
|
35
35
|
const contentTypesFilePath = path.join((0, cli_utilities_1.sanitizePath)(this.exportQueryConfig.exportDir), (0, cli_utilities_1.sanitizePath)(this.exportQueryConfig.branchName || ''), 'content_types', 'schema.json');
|
|
36
36
|
const contentTypes = utils_2.fsUtil.readFile((0, cli_utilities_1.sanitizePath)(contentTypesFilePath)) || [];
|
|
37
37
|
if (contentTypes.length === 0) {
|
|
38
|
-
|
|
38
|
+
cli_utilities_1.log.info('No content types found, skipping export', this.exportQueryConfig.context);
|
|
39
39
|
process.exit(0);
|
|
40
40
|
}
|
|
41
41
|
// Step 5: export other content types which are referenced in previous step
|
|
42
|
+
cli_utilities_1.log.debug('Starting referenced content types export', this.exportQueryConfig.context);
|
|
42
43
|
await this.exportReferencedContentTypes();
|
|
43
44
|
// Step 6: export dependent modules global fields, extensions, taxonomies
|
|
45
|
+
cli_utilities_1.log.debug('Starting dependent modules export', this.exportQueryConfig.context);
|
|
44
46
|
await this.exportDependentModules();
|
|
45
47
|
// Step 7: export content modules entries, assets
|
|
48
|
+
cli_utilities_1.log.debug('Starting content modules export', this.exportQueryConfig.context);
|
|
46
49
|
await this.exportContentModules();
|
|
47
50
|
// Step 9: export all other modules
|
|
48
|
-
|
|
51
|
+
cli_utilities_1.log.success('Query-based export completed successfully!', this.exportQueryConfig.context);
|
|
49
52
|
}
|
|
50
53
|
// export general modules
|
|
51
54
|
async exportGeneralModules() {
|
|
52
|
-
|
|
55
|
+
cli_utilities_1.log.info('Exporting general modules...', this.exportQueryConfig.context);
|
|
53
56
|
for (const module of this.exportQueryConfig.modules.general) {
|
|
54
57
|
await this.moduleExporter.exportModule(module);
|
|
55
58
|
}
|
|
56
59
|
}
|
|
57
60
|
async exportQueriedModule(parsedQuery) {
|
|
61
|
+
cli_utilities_1.log.debug('Starting queried module export', this.exportQueryConfig.context);
|
|
58
62
|
for (const [moduleName] of Object.entries(parsedQuery.modules)) {
|
|
59
63
|
const module = moduleName;
|
|
60
64
|
if (!this.exportQueryConfig.modules.queryable.includes(module)) {
|
|
61
|
-
|
|
65
|
+
cli_utilities_1.log.error(`Module "${module}" is not queryable`, this.exportQueryConfig.context);
|
|
62
66
|
continue;
|
|
63
67
|
}
|
|
64
|
-
|
|
68
|
+
cli_utilities_1.log.info(`Exporting ${moduleName} with query...`, this.exportQueryConfig.context);
|
|
65
69
|
// Export the queried module
|
|
66
70
|
await this.moduleExporter.exportModule(module, { query: parsedQuery });
|
|
67
71
|
}
|
|
72
|
+
cli_utilities_1.log.debug('Queried module export completed', this.exportQueryConfig.context);
|
|
68
73
|
}
|
|
69
74
|
async exportReferencedContentTypes() {
|
|
70
|
-
|
|
75
|
+
cli_utilities_1.log.info('Starting export of referenced content types...', this.exportQueryConfig.context);
|
|
71
76
|
try {
|
|
72
77
|
const referencedHandler = new utils_1.ReferencedContentTypesHandler(this.exportQueryConfig);
|
|
73
78
|
const exportedContentTypeUIDs = new Set();
|
|
@@ -75,24 +80,25 @@ class QueryExporter {
|
|
|
75
80
|
const contentTypesFilePath = path.join((0, cli_utilities_1.sanitizePath)(this.exportQueryConfig.exportDir), (0, cli_utilities_1.sanitizePath)(this.exportQueryConfig.branchName || ''), 'content_types', 'schema.json');
|
|
76
81
|
const contentTypes = utils_2.fsUtil.readFile((0, cli_utilities_1.sanitizePath)(contentTypesFilePath)) || [];
|
|
77
82
|
if (contentTypes.length === 0) {
|
|
78
|
-
|
|
83
|
+
cli_utilities_1.log.info('No content types found, skipping referenced content types export', this.exportQueryConfig.context);
|
|
79
84
|
return;
|
|
80
85
|
}
|
|
81
86
|
// Step 2: Start with initial batch (all currently exported content types)
|
|
82
87
|
let currentBatch = [...contentTypes];
|
|
83
|
-
|
|
88
|
+
cli_utilities_1.log.info(`Starting with ${currentBatch.length} initial content types`, this.exportQueryConfig.context);
|
|
84
89
|
// track reference depth
|
|
85
90
|
let iterationCount = 0;
|
|
86
91
|
// Step 3: Process batches until no new references are found
|
|
87
92
|
while (currentBatch.length > 0 && iterationCount < this.exportQueryConfig.maxCTReferenceDepth) {
|
|
88
93
|
iterationCount++;
|
|
94
|
+
cli_utilities_1.log.debug(`Processing referenced content types iteration ${iterationCount}`, this.exportQueryConfig.context);
|
|
89
95
|
currentBatch.forEach((ct) => exportedContentTypeUIDs.add(ct.uid));
|
|
90
96
|
// Extract referenced content types from current batch
|
|
91
97
|
const referencedUIDs = await referencedHandler.extractReferencedContentTypes(currentBatch);
|
|
92
98
|
// Filter out already exported content types
|
|
93
99
|
const newReferencedUIDs = referencedUIDs.filter((uid) => !exportedContentTypeUIDs.has(uid));
|
|
94
100
|
if (newReferencedUIDs.length > 0) {
|
|
95
|
-
|
|
101
|
+
cli_utilities_1.log.info(`Found ${newReferencedUIDs.length} new referenced content types to fetch`, this.exportQueryConfig.context);
|
|
96
102
|
// // Add to exported set to avoid duplicates in future iterations
|
|
97
103
|
// newReferencedUIDs.forEach((uid) => exportedContentTypeUIDs.add(uid));
|
|
98
104
|
// Step 4: Fetch new content types using moduleExporter
|
|
@@ -110,31 +116,32 @@ class QueryExporter {
|
|
|
110
116
|
currentBatch = [...newContentTypes];
|
|
111
117
|
// Push new content types to main array
|
|
112
118
|
contentTypes.push(...newContentTypes);
|
|
113
|
-
|
|
119
|
+
cli_utilities_1.log.info(`Fetched ${currentBatch.length} new content types for next iteration`, this.exportQueryConfig.context);
|
|
114
120
|
}
|
|
115
121
|
else {
|
|
116
|
-
|
|
122
|
+
cli_utilities_1.log.info('No new referenced content types found, stopping recursion', this.exportQueryConfig.context);
|
|
117
123
|
break;
|
|
118
124
|
}
|
|
119
125
|
}
|
|
120
126
|
utils_2.fsUtil.writeFile((0, cli_utilities_1.sanitizePath)(contentTypesFilePath), contentTypes);
|
|
121
|
-
|
|
127
|
+
cli_utilities_1.log.success('Referenced content types export completed successfully', this.exportQueryConfig.context);
|
|
122
128
|
}
|
|
123
129
|
catch (error) {
|
|
124
|
-
(0,
|
|
130
|
+
(0, cli_utilities_1.handleAndLogError)(error, this.exportQueryConfig.context, 'Error exporting referenced content types');
|
|
125
131
|
throw error;
|
|
126
132
|
}
|
|
127
133
|
}
|
|
128
134
|
async exportDependentModules() {
|
|
129
|
-
|
|
135
|
+
cli_utilities_1.log.info('Starting export of dependent modules...', this.exportQueryConfig.context);
|
|
130
136
|
try {
|
|
131
137
|
const dependenciesHandler = new utils_3.ContentTypeDependenciesHandler(this.stackAPIClient, this.exportQueryConfig);
|
|
132
138
|
// Extract dependencies from all exported content types
|
|
133
139
|
const dependencies = await dependenciesHandler.extractDependencies();
|
|
140
|
+
cli_utilities_1.log.debug('Dependencies extracted successfully', this.exportQueryConfig.context);
|
|
134
141
|
// Export Global Fields
|
|
135
142
|
if (dependencies.globalFields.size > 0) {
|
|
136
143
|
const globalFieldUIDs = Array.from(dependencies.globalFields);
|
|
137
|
-
|
|
144
|
+
cli_utilities_1.log.info(`Exporting ${globalFieldUIDs.length} global fields...`, this.exportQueryConfig.context);
|
|
138
145
|
const query = {
|
|
139
146
|
modules: {
|
|
140
147
|
'global-fields': {
|
|
@@ -147,7 +154,7 @@ class QueryExporter {
|
|
|
147
154
|
// Export Extensions
|
|
148
155
|
if (dependencies.extensions.size > 0) {
|
|
149
156
|
const extensionUIDs = Array.from(dependencies.extensions);
|
|
150
|
-
|
|
157
|
+
cli_utilities_1.log.info(`Exporting ${extensionUIDs.length} extensions...`, this.exportQueryConfig.context);
|
|
151
158
|
const query = {
|
|
152
159
|
modules: {
|
|
153
160
|
extensions: {
|
|
@@ -160,7 +167,7 @@ class QueryExporter {
|
|
|
160
167
|
// export marketplace apps
|
|
161
168
|
if (dependencies.marketplaceApps.size > 0) {
|
|
162
169
|
const marketplaceAppInstallationUIDs = Array.from(dependencies.marketplaceApps);
|
|
163
|
-
|
|
170
|
+
cli_utilities_1.log.info(`Exporting ${marketplaceAppInstallationUIDs.length} marketplace apps...`, this.exportQueryConfig.context);
|
|
164
171
|
const query = {
|
|
165
172
|
modules: {
|
|
166
173
|
'marketplace-apps': {
|
|
@@ -173,7 +180,7 @@ class QueryExporter {
|
|
|
173
180
|
// Export Taxonomies
|
|
174
181
|
if (dependencies.taxonomies.size > 0) {
|
|
175
182
|
const taxonomyUIDs = Array.from(dependencies.taxonomies);
|
|
176
|
-
|
|
183
|
+
cli_utilities_1.log.info(`Exporting ${taxonomyUIDs.length} taxonomies...`, this.exportQueryConfig.context);
|
|
177
184
|
const query = {
|
|
178
185
|
modules: {
|
|
179
186
|
taxonomies: {
|
|
@@ -185,15 +192,15 @@ class QueryExporter {
|
|
|
185
192
|
}
|
|
186
193
|
// export personalize
|
|
187
194
|
await this.moduleExporter.exportModule('personalize');
|
|
188
|
-
|
|
195
|
+
cli_utilities_1.log.success('Dependent modules export completed successfully', this.exportQueryConfig.context);
|
|
189
196
|
}
|
|
190
197
|
catch (error) {
|
|
191
|
-
(0,
|
|
198
|
+
(0, cli_utilities_1.handleAndLogError)(error, this.exportQueryConfig.context, 'Error exporting dependent modules');
|
|
192
199
|
throw error;
|
|
193
200
|
}
|
|
194
201
|
}
|
|
195
202
|
async exportContentModules() {
|
|
196
|
-
|
|
203
|
+
cli_utilities_1.log.info('Starting export of content modules...', this.exportQueryConfig.context);
|
|
197
204
|
try {
|
|
198
205
|
// Step 1: Export entries for all exported content types
|
|
199
206
|
await this.exportEntries();
|
|
@@ -202,50 +209,129 @@ class QueryExporter {
|
|
|
202
209
|
const delay = this.exportQueryConfig.exportDelayMs || 5000;
|
|
203
210
|
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
204
211
|
await this.exportReferencedAssets();
|
|
205
|
-
|
|
212
|
+
cli_utilities_1.log.success('Content modules export completed successfully', this.exportQueryConfig.context);
|
|
206
213
|
}
|
|
207
214
|
catch (error) {
|
|
208
|
-
(0,
|
|
215
|
+
(0, cli_utilities_1.handleAndLogError)(error, this.exportQueryConfig.context, 'Error exporting content modules');
|
|
209
216
|
throw error;
|
|
210
217
|
}
|
|
211
218
|
}
|
|
212
219
|
async exportEntries() {
|
|
213
|
-
|
|
220
|
+
cli_utilities_1.log.info('Exporting entries...', this.exportQueryConfig.context);
|
|
214
221
|
try {
|
|
215
222
|
// Export entries - module exporter will automatically read exported content types
|
|
216
223
|
// and export entries for all of them
|
|
217
224
|
await this.moduleExporter.exportModule('entries');
|
|
218
|
-
|
|
225
|
+
cli_utilities_1.log.success('Entries export completed successfully', this.exportQueryConfig.context);
|
|
219
226
|
}
|
|
220
227
|
catch (error) {
|
|
221
|
-
(0,
|
|
228
|
+
(0, cli_utilities_1.handleAndLogError)(error, this.exportQueryConfig.context, 'Error exporting entries');
|
|
222
229
|
throw error;
|
|
223
230
|
}
|
|
224
231
|
}
|
|
225
232
|
async exportReferencedAssets() {
|
|
226
|
-
|
|
233
|
+
cli_utilities_1.log.info('Starting export of referenced assets...', this.exportQueryConfig.context);
|
|
227
234
|
try {
|
|
235
|
+
const assetsDir = path.join((0, cli_utilities_1.sanitizePath)(this.exportQueryConfig.exportDir), (0, cli_utilities_1.sanitizePath)(this.exportQueryConfig.branchName || ''), 'assets');
|
|
236
|
+
const metadataFilePath = path.join(assetsDir, 'metadata.json');
|
|
237
|
+
const assetFilePath = path.join(assetsDir, 'assets.json');
|
|
238
|
+
// Define temp file paths
|
|
239
|
+
const tempMetadataFilePath = path.join(assetsDir, 'metadata_temp.json');
|
|
240
|
+
const tempAssetFilePath = path.join(assetsDir, 'assets_temp.json');
|
|
228
241
|
const assetHandler = new utils_4.AssetReferenceHandler(this.exportQueryConfig);
|
|
229
242
|
// Extract referenced asset UIDs from all entries
|
|
243
|
+
cli_utilities_1.log.debug('Extracting referenced assets from entries', this.exportQueryConfig.context);
|
|
230
244
|
const assetUIDs = assetHandler.extractReferencedAssets();
|
|
231
245
|
if (assetUIDs.length > 0) {
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
246
|
+
cli_utilities_1.log.info(`Found ${assetUIDs.length} referenced assets to export`, this.exportQueryConfig.context);
|
|
247
|
+
// Define batch size - can be configurable through exportQueryConfig
|
|
248
|
+
const batchSize = this.exportQueryConfig.assetBatchSize || 100;
|
|
249
|
+
if (assetUIDs.length <= batchSize) {
|
|
250
|
+
const query = {
|
|
251
|
+
modules: {
|
|
252
|
+
assets: {
|
|
253
|
+
uid: { $in: assetUIDs },
|
|
254
|
+
},
|
|
237
255
|
},
|
|
238
|
-
}
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
256
|
+
};
|
|
257
|
+
await this.moduleExporter.exportModule('assets', { query });
|
|
258
|
+
}
|
|
259
|
+
// if asset size is bigger than batch size, then we need to export in batches
|
|
260
|
+
// Calculate number of batches
|
|
261
|
+
const totalBatches = Math.ceil(assetUIDs.length / batchSize);
|
|
262
|
+
cli_utilities_1.log.info(`Processing assets in ${totalBatches} batches of ${batchSize}`, this.exportQueryConfig.context);
|
|
263
|
+
// Process assets in batches
|
|
264
|
+
for (let i = 0; i < totalBatches; i++) {
|
|
265
|
+
const start = i * batchSize;
|
|
266
|
+
const end = Math.min(start + batchSize, assetUIDs.length);
|
|
267
|
+
const batchAssetUIDs = assetUIDs.slice(start, end);
|
|
268
|
+
cli_utilities_1.log.info(`Exporting batch ${i + 1}/${totalBatches} (${batchAssetUIDs.length} assets)...`, this.exportQueryConfig.context);
|
|
269
|
+
const query = {
|
|
270
|
+
modules: {
|
|
271
|
+
assets: {
|
|
272
|
+
uid: { $in: batchAssetUIDs },
|
|
273
|
+
},
|
|
274
|
+
},
|
|
275
|
+
};
|
|
276
|
+
await this.moduleExporter.exportModule('assets', { query });
|
|
277
|
+
// Read the current batch's metadata.json and assets.json files
|
|
278
|
+
const currentMetadata = utils_2.fsUtil.readFile((0, cli_utilities_1.sanitizePath)(metadataFilePath));
|
|
279
|
+
const currentAssets = utils_2.fsUtil.readFile((0, cli_utilities_1.sanitizePath)(assetFilePath));
|
|
280
|
+
// Check if this is the first batch
|
|
281
|
+
if (i === 0) {
|
|
282
|
+
// For first batch, initialize temp files with current content
|
|
283
|
+
utils_2.fsUtil.writeFile((0, cli_utilities_1.sanitizePath)(tempMetadataFilePath), currentMetadata);
|
|
284
|
+
utils_2.fsUtil.writeFile((0, cli_utilities_1.sanitizePath)(tempAssetFilePath), currentAssets);
|
|
285
|
+
cli_utilities_1.log.info(`Initialized temporary files with first batch data`, this.exportQueryConfig.context);
|
|
286
|
+
}
|
|
287
|
+
else {
|
|
288
|
+
// For subsequent batches, append to temp files with incremented keys
|
|
289
|
+
// Handle metadata (which contains arrays of asset info)
|
|
290
|
+
const tempMetadata = utils_2.fsUtil.readFile((0, cli_utilities_1.sanitizePath)(tempMetadataFilePath)) || {};
|
|
291
|
+
// Merge metadata by combining arrays
|
|
292
|
+
if (currentMetadata) {
|
|
293
|
+
Object.keys(currentMetadata).forEach((key) => {
|
|
294
|
+
if (!tempMetadata[key]) {
|
|
295
|
+
tempMetadata[key] = currentMetadata[key];
|
|
296
|
+
}
|
|
297
|
+
});
|
|
298
|
+
}
|
|
299
|
+
// Write updated metadata back to temp file
|
|
300
|
+
utils_2.fsUtil.writeFile((0, cli_utilities_1.sanitizePath)(tempMetadataFilePath), tempMetadata);
|
|
301
|
+
// Handle assets (which is an object with numeric keys)
|
|
302
|
+
const tempAssets = utils_2.fsUtil.readFile((0, cli_utilities_1.sanitizePath)(tempAssetFilePath)) || {};
|
|
303
|
+
let nextIndex = Object.keys(tempAssets).length + 1;
|
|
304
|
+
// Add current assets with incremented keys
|
|
305
|
+
Object.values(currentAssets).forEach((value) => {
|
|
306
|
+
tempAssets[nextIndex.toString()] = value;
|
|
307
|
+
nextIndex++;
|
|
308
|
+
});
|
|
309
|
+
utils_2.fsUtil.writeFile((0, cli_utilities_1.sanitizePath)(tempAssetFilePath), tempAssets);
|
|
310
|
+
cli_utilities_1.log.info(`Updated temporary files with batch ${i + 1} data`, this.exportQueryConfig.context);
|
|
311
|
+
}
|
|
312
|
+
// Optional: Add delay between batches to avoid rate limiting
|
|
313
|
+
if (i < totalBatches - 1 && this.exportQueryConfig.batchDelayMs) {
|
|
314
|
+
await new Promise((resolve) => setTimeout(resolve, this.exportQueryConfig.batchDelayMs));
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
// After all batches are processed, copy temp files back to original files
|
|
318
|
+
const finalMetadata = utils_2.fsUtil.readFile((0, cli_utilities_1.sanitizePath)(tempMetadataFilePath));
|
|
319
|
+
const finalAssets = utils_2.fsUtil.readFile((0, cli_utilities_1.sanitizePath)(tempAssetFilePath));
|
|
320
|
+
utils_2.fsUtil.writeFile((0, cli_utilities_1.sanitizePath)(metadataFilePath), finalMetadata);
|
|
321
|
+
utils_2.fsUtil.writeFile((0, cli_utilities_1.sanitizePath)(assetFilePath), finalAssets);
|
|
322
|
+
cli_utilities_1.log.info(`Final data written back to original files`, this.exportQueryConfig.context);
|
|
323
|
+
// Clean up temp files
|
|
324
|
+
utils_2.fsUtil.removeFile((0, cli_utilities_1.sanitizePath)(tempMetadataFilePath));
|
|
325
|
+
utils_2.fsUtil.removeFile((0, cli_utilities_1.sanitizePath)(tempAssetFilePath));
|
|
326
|
+
cli_utilities_1.log.info(`Temporary files cleaned up`, this.exportQueryConfig.context);
|
|
327
|
+
cli_utilities_1.log.success('Referenced assets exported successfully', this.exportQueryConfig.context);
|
|
242
328
|
}
|
|
243
329
|
else {
|
|
244
|
-
|
|
330
|
+
cli_utilities_1.log.info('No referenced assets found in entries', this.exportQueryConfig.context);
|
|
245
331
|
}
|
|
246
332
|
}
|
|
247
333
|
catch (error) {
|
|
248
|
-
(0,
|
|
334
|
+
(0, cli_utilities_1.handleAndLogError)(error, this.exportQueryConfig.context, 'Error exporting referenced assets');
|
|
249
335
|
throw error;
|
|
250
336
|
}
|
|
251
337
|
}
|
package/lib/types/index.d.ts
CHANGED
|
@@ -129,6 +129,19 @@ export interface DefaultConfig {
|
|
|
129
129
|
externalConfigPath?: string;
|
|
130
130
|
maxCTReferenceDepth: number;
|
|
131
131
|
}
|
|
132
|
+
/**
|
|
133
|
+
* Log context interface for centralized logging
|
|
134
|
+
*/
|
|
135
|
+
export interface LogContext {
|
|
136
|
+
command: string;
|
|
137
|
+
module: string;
|
|
138
|
+
email: string;
|
|
139
|
+
sessionId: string;
|
|
140
|
+
apiKey: string;
|
|
141
|
+
orgId: string;
|
|
142
|
+
authenticationMethod: string;
|
|
143
|
+
[key: string]: unknown;
|
|
144
|
+
}
|
|
132
145
|
export interface QueryExportConfig extends DefaultConfig {
|
|
133
146
|
query: string;
|
|
134
147
|
skipReferences: boolean;
|
|
@@ -136,10 +149,15 @@ export interface QueryExportConfig extends DefaultConfig {
|
|
|
136
149
|
stackApiKey: string;
|
|
137
150
|
managementToken?: string;
|
|
138
151
|
branchName: string;
|
|
152
|
+
branchAlias?: string;
|
|
139
153
|
securedAssets: boolean;
|
|
140
154
|
logsPath: string;
|
|
141
155
|
dataPath: string;
|
|
142
156
|
exportDelayMs?: number;
|
|
157
|
+
batchDelayMs?: number;
|
|
158
|
+
assetBatchSize?: number;
|
|
159
|
+
assetBatchDelayMs?: number;
|
|
160
|
+
context?: LogContext;
|
|
143
161
|
}
|
|
144
162
|
export interface QueryMetadata {
|
|
145
163
|
query: any;
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { QueryExportConfig } from '../types';
|
|
2
|
+
/**
|
|
3
|
+
* Validates and sets up branch configuration for the stack
|
|
4
|
+
*
|
|
5
|
+
* @param config The export configuration
|
|
6
|
+
* @param stackAPIClient The stack API client
|
|
7
|
+
* @returns Promise that resolves when branch setup is complete
|
|
8
|
+
*/
|
|
9
|
+
export declare const setupBranches: (config: QueryExportConfig, stackAPIClient: any) => Promise<void>;
|
|
10
|
+
export default setupBranches;
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.setupBranches = void 0;
|
|
4
|
+
const cli_utilities_1 = require("@contentstack/cli-utilities");
|
|
5
|
+
/**
|
|
6
|
+
* Validates and sets up branch configuration for the stack
|
|
7
|
+
*
|
|
8
|
+
* @param config The export configuration
|
|
9
|
+
* @param stackAPIClient The stack API client
|
|
10
|
+
* @returns Promise that resolves when branch setup is complete
|
|
11
|
+
*/
|
|
12
|
+
const setupBranches = async (config, stackAPIClient) => {
|
|
13
|
+
if (typeof config !== 'object') {
|
|
14
|
+
throw new Error('The branch configuration is invalid.');
|
|
15
|
+
}
|
|
16
|
+
const context = config.context;
|
|
17
|
+
try {
|
|
18
|
+
if (config.branchAlias) {
|
|
19
|
+
config.branchName = await (0, cli_utilities_1.getBranchFromAlias)(stackAPIClient, config.branchAlias);
|
|
20
|
+
return;
|
|
21
|
+
}
|
|
22
|
+
if (config.branchName) {
|
|
23
|
+
// Check if the specified branch exists
|
|
24
|
+
cli_utilities_1.log.info(`Validating branch: ${config.branchName}`, context);
|
|
25
|
+
const result = await stackAPIClient
|
|
26
|
+
.branch(config.branchName)
|
|
27
|
+
.fetch()
|
|
28
|
+
.catch((err) => {
|
|
29
|
+
(0, cli_utilities_1.handleAndLogError)(err, context, 'Error fetching branch');
|
|
30
|
+
return null;
|
|
31
|
+
});
|
|
32
|
+
if (result && typeof result === 'object') {
|
|
33
|
+
cli_utilities_1.log.success(`Branch '${config.branchName}' found`, context);
|
|
34
|
+
}
|
|
35
|
+
else {
|
|
36
|
+
throw new Error(`No branch found named ${config.branchName}.`);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
else {
|
|
40
|
+
// If no branch name provided, check if the stack has branches
|
|
41
|
+
cli_utilities_1.log.info('No branch specified, checking if stack has branches', context);
|
|
42
|
+
const result = await stackAPIClient
|
|
43
|
+
.branch()
|
|
44
|
+
.query()
|
|
45
|
+
.find()
|
|
46
|
+
.catch(() => {
|
|
47
|
+
cli_utilities_1.log.info('Stack does not have branches', context);
|
|
48
|
+
return null;
|
|
49
|
+
});
|
|
50
|
+
if (result && result.items && Array.isArray(result.items) && result.items.length > 0) {
|
|
51
|
+
// Set default branch to 'main' if it exists
|
|
52
|
+
config.branchName = 'main';
|
|
53
|
+
}
|
|
54
|
+
else {
|
|
55
|
+
// Stack doesn't have branches
|
|
56
|
+
cli_utilities_1.log.info('Stack does not have branches', context);
|
|
57
|
+
return;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
config.branchEnabled = true;
|
|
61
|
+
}
|
|
62
|
+
catch (error) {
|
|
63
|
+
(0, cli_utilities_1.handleAndLogError)(error, context, 'Error setting up branches');
|
|
64
|
+
throw error;
|
|
65
|
+
}
|
|
66
|
+
};
|
|
67
|
+
exports.setupBranches = setupBranches;
|
|
68
|
+
exports.default = exports.setupBranches;
|
|
@@ -12,23 +12,30 @@ async function setupQueryExportConfig(flags) {
|
|
|
12
12
|
const exportQueryConfig = Object.assign(Object.assign({}, config_1.default), { exportDir, stackApiKey: flags['stack-api-key'] || '', managementToken: flags.alias ? (_a = cli_utilities_1.configHandler.get(`tokens.${flags.alias}`)) === null || _a === void 0 ? void 0 : _a.token : undefined, query: flags.query, skipReferences: flags['skip-references'] || false, skipDependencies: flags['skip-dependencies'] || false, branchName: flags.branch, securedAssets: flags['secured-assets'] || false, isQueryBasedExport: true, logsPath: exportDir, dataPath: exportDir,
|
|
13
13
|
// Todo: accept the path of the config file from the user
|
|
14
14
|
externalConfigPath: path.join(__dirname, '../config/export-config.json') });
|
|
15
|
+
if (flags['branch-alias']) {
|
|
16
|
+
exportQueryConfig.branchAlias = flags['branch-alias'];
|
|
17
|
+
}
|
|
18
|
+
// override the external config path if the user provides a config file
|
|
19
|
+
if (flags.config) {
|
|
20
|
+
exportQueryConfig.externalConfigPath = (0, cli_utilities_1.sanitizePath)(flags['config']);
|
|
21
|
+
}
|
|
15
22
|
// Handle authentication
|
|
16
23
|
if (flags.alias) {
|
|
17
24
|
const { token, apiKey } = cli_utilities_1.configHandler.get(`tokens.${flags.alias}`) || {};
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
if (!
|
|
21
|
-
throw new Error(`No management token found
|
|
25
|
+
exportQueryConfig.managementToken = token;
|
|
26
|
+
exportQueryConfig.stackApiKey = apiKey;
|
|
27
|
+
if (!exportQueryConfig.managementToken) {
|
|
28
|
+
throw new Error(`No management token found for alias ${flags.alias}.`);
|
|
22
29
|
}
|
|
23
30
|
}
|
|
24
|
-
if (!
|
|
31
|
+
if (!exportQueryConfig.managementToken) {
|
|
25
32
|
if (!(0, cli_utilities_1.isAuthenticated)()) {
|
|
26
|
-
throw new Error('
|
|
33
|
+
throw new Error('Log in or provide an alias for the management token.');
|
|
27
34
|
}
|
|
28
35
|
else {
|
|
29
|
-
|
|
30
|
-
if (typeof
|
|
31
|
-
throw new Error('
|
|
36
|
+
exportQueryConfig.stackApiKey = flags['stack-api-key'] || (await (0, common_helper_1.askAPIKey)());
|
|
37
|
+
if (typeof exportQueryConfig.stackApiKey !== 'string') {
|
|
38
|
+
throw new Error('The API key is invalid.');
|
|
32
39
|
}
|
|
33
40
|
}
|
|
34
41
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.ReferencedContentTypesHandler = void 0;
|
|
4
|
-
const
|
|
4
|
+
const cli_utilities_1 = require("@contentstack/cli-utilities");
|
|
5
5
|
class ReferencedContentTypesHandler {
|
|
6
6
|
constructor(exportQueryConfig) {
|
|
7
7
|
this.exportQueryConfig = exportQueryConfig;
|
|
@@ -12,7 +12,7 @@ class ReferencedContentTypesHandler {
|
|
|
12
12
|
*/
|
|
13
13
|
async extractReferencedContentTypes(contentTypeBatch) {
|
|
14
14
|
const allReferencedTypes = new Set();
|
|
15
|
-
|
|
15
|
+
cli_utilities_1.log.info(`Extracting references from ${contentTypeBatch.length} content types`, this.exportQueryConfig.context);
|
|
16
16
|
for (const contentType of contentTypeBatch) {
|
|
17
17
|
if (contentType.schema) {
|
|
18
18
|
const referencedTypes = this.getReferencedContentTypes(contentType.schema);
|
|
@@ -20,7 +20,7 @@ class ReferencedContentTypesHandler {
|
|
|
20
20
|
}
|
|
21
21
|
}
|
|
22
22
|
const result = Array.from(allReferencedTypes);
|
|
23
|
-
|
|
23
|
+
cli_utilities_1.log.info(`Found ${result.length} referenced content types`, this.exportQueryConfig.context);
|
|
24
24
|
return result;
|
|
25
25
|
}
|
|
26
26
|
/**
|
|
@@ -5,7 +5,6 @@ const tslib_1 = require("tslib");
|
|
|
5
5
|
const path = tslib_1.__importStar(require("path"));
|
|
6
6
|
const index_1 = require("./index");
|
|
7
7
|
const cli_utilities_1 = require("@contentstack/cli-utilities");
|
|
8
|
-
const logger_1 = require("./logger");
|
|
9
8
|
class ContentTypeDependenciesHandler {
|
|
10
9
|
constructor(stackAPIClient, exportQueryConfig) {
|
|
11
10
|
this.exportQueryConfig = exportQueryConfig;
|
|
@@ -15,7 +14,7 @@ class ContentTypeDependenciesHandler {
|
|
|
15
14
|
const contentTypesFilePath = path.join((0, cli_utilities_1.sanitizePath)(this.exportQueryConfig.exportDir), (0, cli_utilities_1.sanitizePath)(this.exportQueryConfig.branchName || ''), 'content_types', 'schema.json');
|
|
16
15
|
const allContentTypes = index_1.fsUtil.readFile((0, cli_utilities_1.sanitizePath)(contentTypesFilePath)) || [];
|
|
17
16
|
if (allContentTypes.length === 0) {
|
|
18
|
-
|
|
17
|
+
cli_utilities_1.log.info('No content types found, skipping dependency extraction', this.exportQueryConfig.context);
|
|
19
18
|
return {
|
|
20
19
|
globalFields: new Set(),
|
|
21
20
|
extensions: new Set(),
|
|
@@ -23,7 +22,7 @@ class ContentTypeDependenciesHandler {
|
|
|
23
22
|
marketplaceApps: new Set(),
|
|
24
23
|
};
|
|
25
24
|
}
|
|
26
|
-
|
|
25
|
+
cli_utilities_1.log.info(`Extracting dependencies from ${allContentTypes.length} content types`, this.exportQueryConfig.context);
|
|
27
26
|
const dependencies = {
|
|
28
27
|
globalFields: new Set(),
|
|
29
28
|
extensions: new Set(),
|
|
@@ -38,26 +37,26 @@ class ContentTypeDependenciesHandler {
|
|
|
38
37
|
// Separate extensions from marketplace apps using the extracted extension UIDs
|
|
39
38
|
if (dependencies.extensions.size > 0) {
|
|
40
39
|
const extensionUIDs = Array.from(dependencies.extensions);
|
|
41
|
-
|
|
40
|
+
cli_utilities_1.log.info(`Processing ${extensionUIDs.length} extensions to identify marketplace apps...`, this.exportQueryConfig.context);
|
|
42
41
|
try {
|
|
43
42
|
const { extensions, marketplaceApps } = await this.fetchExtensionsAndMarketplaceApps(extensionUIDs);
|
|
44
43
|
dependencies.extensions = new Set(extensions);
|
|
45
44
|
dependencies.marketplaceApps = new Set(marketplaceApps);
|
|
46
|
-
|
|
45
|
+
cli_utilities_1.log.info(`Dependencies separated - Global Fields: ${dependencies.globalFields.size}, Extensions: ${dependencies.extensions.size}, Taxonomies: ${dependencies.taxonomies.size}, Marketplace Apps: ${dependencies.marketplaceApps.size}`, this.exportQueryConfig.context);
|
|
47
46
|
}
|
|
48
47
|
catch (error) {
|
|
49
|
-
(0,
|
|
48
|
+
(0, cli_utilities_1.handleAndLogError)(error, this.exportQueryConfig.context, 'Failed to separate extensions and Marketplace apps');
|
|
50
49
|
// Keep original extensions if separation fails
|
|
51
50
|
}
|
|
52
51
|
}
|
|
53
52
|
else {
|
|
54
|
-
|
|
53
|
+
cli_utilities_1.log.info(`Found dependencies - Global Fields: ${dependencies.globalFields.size}, Extensions: ${dependencies.extensions.size}, Taxonomies: ${dependencies.taxonomies.size}, Marketplace Apps: ${dependencies.marketplaceApps.size}`, this.exportQueryConfig.context);
|
|
55
54
|
}
|
|
56
55
|
return dependencies;
|
|
57
56
|
}
|
|
58
57
|
// Update the fetchExtensionsAndMarketplaceApps method to only fetch specific extension UIDs
|
|
59
58
|
async fetchExtensionsAndMarketplaceApps(extensionUIDs) {
|
|
60
|
-
|
|
59
|
+
cli_utilities_1.log.info(`Fetching details for ${extensionUIDs.length} extensions to identify marketplace apps...`, this.exportQueryConfig.context);
|
|
61
60
|
try {
|
|
62
61
|
// Query parameters to include marketplace extensions
|
|
63
62
|
const queryParams = {
|
|
@@ -70,7 +69,7 @@ class ContentTypeDependenciesHandler {
|
|
|
70
69
|
// Fetch all extensions including marketplace apps
|
|
71
70
|
const response = await this.stackAPIClient.extension().query(queryParams).find();
|
|
72
71
|
if (!response || !response.items) {
|
|
73
|
-
|
|
72
|
+
cli_utilities_1.log.warn(`No extensions found`, this.exportQueryConfig.context);
|
|
74
73
|
return { extensions: extensionUIDs, marketplaceApps: [] };
|
|
75
74
|
}
|
|
76
75
|
const marketplaceApps = [];
|
|
@@ -83,11 +82,11 @@ class ContentTypeDependenciesHandler {
|
|
|
83
82
|
regularExtensions.push(item.uid);
|
|
84
83
|
}
|
|
85
84
|
});
|
|
86
|
-
|
|
85
|
+
cli_utilities_1.log.info(`Identified ${marketplaceApps.length} marketplace apps and ${regularExtensions.length} regular extensions from ${extensionUIDs.length} total extensions`, this.exportQueryConfig.context);
|
|
87
86
|
return { extensions: regularExtensions, marketplaceApps };
|
|
88
87
|
}
|
|
89
88
|
catch (error) {
|
|
90
|
-
(0,
|
|
89
|
+
(0, cli_utilities_1.handleAndLogError)(error, this.exportQueryConfig.context, 'Failed to fetch extensions and Marketplace apps');
|
|
91
90
|
return { extensions: extensionUIDs, marketplaceApps: [] };
|
|
92
91
|
}
|
|
93
92
|
}
|
package/lib/utils/index.d.ts
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
export * as fileHelper from './file-helper';
|
|
2
2
|
export { fsUtil } from './file-helper';
|
|
3
|
-
export { log, unlinkFileLogger } from './logger';
|
|
3
|
+
export { log, unlinkFileLogger, createLogContext } from './logger';
|
|
4
|
+
export { LogContext } from '../types';
|
|
4
5
|
export * from './common-helper';
|
|
5
6
|
export * from './config-handler';
|
|
6
7
|
export * from './content-type-helper';
|
|
7
8
|
export * from './dependency-resolver';
|
|
8
9
|
export * from './referenced-asset-handler';
|
|
10
|
+
export { setupBranches } from './branch-helper';
|
package/lib/utils/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.unlinkFileLogger = exports.log = exports.fsUtil = exports.fileHelper = void 0;
|
|
3
|
+
exports.setupBranches = exports.createLogContext = exports.unlinkFileLogger = exports.log = exports.fsUtil = exports.fileHelper = void 0;
|
|
4
4
|
const tslib_1 = require("tslib");
|
|
5
5
|
exports.fileHelper = tslib_1.__importStar(require("./file-helper"));
|
|
6
6
|
var file_helper_1 = require("./file-helper");
|
|
@@ -8,8 +8,11 @@ Object.defineProperty(exports, "fsUtil", { enumerable: true, get: function () {
|
|
|
8
8
|
var logger_1 = require("./logger");
|
|
9
9
|
Object.defineProperty(exports, "log", { enumerable: true, get: function () { return logger_1.log; } });
|
|
10
10
|
Object.defineProperty(exports, "unlinkFileLogger", { enumerable: true, get: function () { return logger_1.unlinkFileLogger; } });
|
|
11
|
+
Object.defineProperty(exports, "createLogContext", { enumerable: true, get: function () { return logger_1.createLogContext; } });
|
|
11
12
|
tslib_1.__exportStar(require("./common-helper"), exports);
|
|
12
13
|
tslib_1.__exportStar(require("./config-handler"), exports);
|
|
13
14
|
tslib_1.__exportStar(require("./content-type-helper"), exports);
|
|
14
15
|
tslib_1.__exportStar(require("./dependency-resolver"), exports);
|
|
15
16
|
tslib_1.__exportStar(require("./referenced-asset-handler"), exports);
|
|
17
|
+
var branch_helper_1 = require("./branch-helper");
|
|
18
|
+
Object.defineProperty(exports, "setupBranches", { enumerable: true, get: function () { return branch_helper_1.setupBranches; } });
|
package/lib/utils/logger.d.ts
CHANGED
|
@@ -3,6 +3,10 @@
|
|
|
3
3
|
* Copyright (c) 2024 Contentstack LLC
|
|
4
4
|
* MIT Licensed
|
|
5
5
|
*/
|
|
6
|
-
import { QueryExportConfig } from '../types';
|
|
6
|
+
import { QueryExportConfig, LogContext } from '../types';
|
|
7
7
|
export declare const log: (config: QueryExportConfig, message: any, type: string) => Promise<void>;
|
|
8
8
|
export declare const unlinkFileLogger: () => void;
|
|
9
|
+
/**
|
|
10
|
+
* Creates a context object for logging from QueryExportConfig
|
|
11
|
+
*/
|
|
12
|
+
export declare function createLogContext(config: QueryExportConfig, moduleName?: string): LogContext;
|
package/lib/utils/logger.js
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* MIT Licensed
|
|
6
6
|
*/
|
|
7
7
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8
|
-
exports.unlinkFileLogger = exports.log = void 0;
|
|
8
|
+
exports.createLogContext = exports.unlinkFileLogger = exports.log = void 0;
|
|
9
9
|
const tslib_1 = require("tslib");
|
|
10
10
|
const winston = tslib_1.__importStar(require("winston"));
|
|
11
11
|
const path = tslib_1.__importStar(require("path"));
|
|
@@ -156,3 +156,18 @@ const unlinkFileLogger = () => {
|
|
|
156
156
|
}
|
|
157
157
|
};
|
|
158
158
|
exports.unlinkFileLogger = unlinkFileLogger;
|
|
159
|
+
/**
|
|
160
|
+
* Creates a context object for logging from QueryExportConfig
|
|
161
|
+
*/
|
|
162
|
+
function createLogContext(config, moduleName) {
|
|
163
|
+
return {
|
|
164
|
+
command: 'cm:stacks:export-query',
|
|
165
|
+
module: moduleName || '',
|
|
166
|
+
email: cli_utilities_1.configHandler.get('email') || '',
|
|
167
|
+
sessionId: cli_utilities_1.configHandler.get('sessionId') || '',
|
|
168
|
+
apiKey: config.stackApiKey || '',
|
|
169
|
+
orgId: cli_utilities_1.configHandler.get('oauthOrgUid') || '',
|
|
170
|
+
authenticationMethod: config.managementToken ? 'Management Token' : 'Basic Auth',
|
|
171
|
+
};
|
|
172
|
+
}
|
|
173
|
+
exports.createLogContext = createLogContext;
|
|
@@ -26,7 +26,7 @@ class QueryParser {
|
|
|
26
26
|
return JSON.parse(content);
|
|
27
27
|
}
|
|
28
28
|
catch (error) {
|
|
29
|
-
|
|
29
|
+
(0, cli_utilities_1.handleAndLogError)(error, this.config.context, 'Failed to parse the query file');
|
|
30
30
|
}
|
|
31
31
|
}
|
|
32
32
|
parseFromString(queryString) {
|
|
@@ -34,19 +34,19 @@ class QueryParser {
|
|
|
34
34
|
return JSON.parse(queryString);
|
|
35
35
|
}
|
|
36
36
|
catch (error) {
|
|
37
|
-
|
|
37
|
+
(0, cli_utilities_1.handleAndLogError)(error, this.config.context, 'Invalid JSON query');
|
|
38
38
|
}
|
|
39
39
|
}
|
|
40
40
|
validate(query) {
|
|
41
41
|
if (!query || typeof query !== 'object') {
|
|
42
|
-
throw new cli_utilities_1.CLIError('
|
|
42
|
+
throw new cli_utilities_1.CLIError('The query must be a valid JSON object.');
|
|
43
43
|
}
|
|
44
44
|
if (!query.modules || typeof query.modules !== 'object') {
|
|
45
|
-
throw new cli_utilities_1.CLIError('
|
|
45
|
+
throw new cli_utilities_1.CLIError('The query must contain a "modules" object.');
|
|
46
46
|
}
|
|
47
47
|
const modules = Object.keys(query.modules);
|
|
48
48
|
if (modules.length === 0) {
|
|
49
|
-
throw new cli_utilities_1.CLIError('
|
|
49
|
+
throw new cli_utilities_1.CLIError('The query must contain at least one module.');
|
|
50
50
|
}
|
|
51
51
|
// Validate supported modules
|
|
52
52
|
const queryableModules = this.config.modules.queryable;
|
|
@@ -6,7 +6,6 @@ const path = tslib_1.__importStar(require("path"));
|
|
|
6
6
|
const fs = tslib_1.__importStar(require("fs"));
|
|
7
7
|
const index_1 = require("./index");
|
|
8
8
|
const cli_utilities_1 = require("@contentstack/cli-utilities");
|
|
9
|
-
const logger_1 = require("./logger");
|
|
10
9
|
class AssetReferenceHandler {
|
|
11
10
|
constructor(exportQueryConfig) {
|
|
12
11
|
this.exportQueryConfig = exportQueryConfig;
|
|
@@ -16,10 +15,10 @@ class AssetReferenceHandler {
|
|
|
16
15
|
* Extract all asset UIDs by processing entries file by file (memory efficient)
|
|
17
16
|
*/
|
|
18
17
|
extractReferencedAssets() {
|
|
19
|
-
|
|
18
|
+
cli_utilities_1.log.info('Extracting referenced assets from entries...', this.exportQueryConfig.context);
|
|
20
19
|
try {
|
|
21
20
|
if (!fs.existsSync(this.entriesDir)) {
|
|
22
|
-
|
|
21
|
+
cli_utilities_1.log.warn('Entries directory does not exist', this.exportQueryConfig.context);
|
|
23
22
|
return [];
|
|
24
23
|
}
|
|
25
24
|
// Global set to maintain unique asset UIDs across all files
|
|
@@ -33,11 +32,11 @@ class AssetReferenceHandler {
|
|
|
33
32
|
totalEntriesProcessed += entriesInFile;
|
|
34
33
|
}
|
|
35
34
|
const result = Array.from(globalAssetUIDs);
|
|
36
|
-
|
|
35
|
+
cli_utilities_1.log.info(`Found ${result.length} unique asset UIDs from ${totalEntriesProcessed} entries across ${jsonFiles.length} files`, this.exportQueryConfig.context);
|
|
37
36
|
return result;
|
|
38
37
|
}
|
|
39
38
|
catch (error) {
|
|
40
|
-
(0,
|
|
39
|
+
(0, cli_utilities_1.handleAndLogError)(error, this.exportQueryConfig.context, 'Failed to extract assets');
|
|
41
40
|
return [];
|
|
42
41
|
}
|
|
43
42
|
}
|
|
@@ -62,11 +61,11 @@ class AssetReferenceHandler {
|
|
|
62
61
|
assetUIDs.forEach((uid) => globalAssetUIDs.add(uid));
|
|
63
62
|
// Count entries for logging
|
|
64
63
|
const entriesCount = Object.keys(fileContent).length;
|
|
65
|
-
|
|
64
|
+
cli_utilities_1.log.debug(`Processed ${entriesCount} entries from ${path.basename(filePath)}`, this.exportQueryConfig.context);
|
|
66
65
|
return entriesCount;
|
|
67
66
|
}
|
|
68
67
|
catch (error) {
|
|
69
|
-
|
|
68
|
+
cli_utilities_1.log.warn(`Failed to process file ${filePath}: ${(0, cli_utilities_1.formatError)(error)}`, this.exportQueryConfig.context);
|
|
70
69
|
return 0;
|
|
71
70
|
}
|
|
72
71
|
}
|
|
@@ -83,10 +82,21 @@ class AssetReferenceHandler {
|
|
|
83
82
|
assetUIDs.add(match[1]);
|
|
84
83
|
}
|
|
85
84
|
}
|
|
85
|
+
let assetUrlRegex = '';
|
|
86
|
+
let assetUIDMatchIndex;
|
|
87
|
+
if (process.env.ENVIRONMENT === 'NON_PROD') {
|
|
88
|
+
assetUrlRegex = '(https://.*?/v3/assets/(.*?)/(.*?)/(.*?)/(.*?)(?="))';
|
|
89
|
+
assetUIDMatchIndex = 3;
|
|
90
|
+
}
|
|
91
|
+
else {
|
|
92
|
+
assetUrlRegex =
|
|
93
|
+
'(https://(assets|(eu-|azure-na-|azure-eu-|gcp-na-|gcp-eu-)?images).contentstack.(io|com)/v3/assets/(.*?)/(.*?)/(.*?)/(.*?)(?="))';
|
|
94
|
+
assetUIDMatchIndex = 6;
|
|
95
|
+
}
|
|
86
96
|
// Pattern 2: Contentstack asset URLs
|
|
87
|
-
const urlRegex = new RegExp(
|
|
97
|
+
const urlRegex = new RegExp(assetUrlRegex, 'g');
|
|
88
98
|
while ((match = urlRegex.exec(content)) !== null) {
|
|
89
|
-
const assetUID = match[
|
|
99
|
+
const assetUID = match[assetUIDMatchIndex]; // The asset UID is in the 6th capture group
|
|
90
100
|
if (assetUID) {
|
|
91
101
|
assetUIDs.add(assetUID);
|
|
92
102
|
}
|
|
@@ -112,7 +122,7 @@ class AssetReferenceHandler {
|
|
|
112
122
|
}
|
|
113
123
|
}
|
|
114
124
|
catch (error) {
|
|
115
|
-
|
|
125
|
+
cli_utilities_1.log.warn(`Failed to read directory ${dir}: ${(0, cli_utilities_1.formatError)(error)}`, this.exportQueryConfig.context);
|
|
116
126
|
}
|
|
117
127
|
return jsonFiles;
|
|
118
128
|
}
|
package/oclif.manifest.json
CHANGED
|
@@ -1,9 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"commands": {
|
|
3
3
|
"cm:stacks:export-query": {
|
|
4
|
-
"aliases": [
|
|
5
|
-
"cm:export-query"
|
|
6
|
-
],
|
|
4
|
+
"aliases": [],
|
|
7
5
|
"args": {},
|
|
8
6
|
"description": "Export content from a stack using query-based filtering",
|
|
9
7
|
"examples": [
|
|
@@ -47,11 +45,24 @@
|
|
|
47
45
|
},
|
|
48
46
|
"branch": {
|
|
49
47
|
"description": "Branch name to export from",
|
|
48
|
+
"exclusive": [
|
|
49
|
+
"branch-alias"
|
|
50
|
+
],
|
|
50
51
|
"name": "branch",
|
|
51
52
|
"hasDynamicHelp": false,
|
|
52
53
|
"multiple": false,
|
|
53
54
|
"type": "option"
|
|
54
55
|
},
|
|
56
|
+
"branch-alias": {
|
|
57
|
+
"description": "Alias of Branch to export from",
|
|
58
|
+
"exclusive": [
|
|
59
|
+
"branch"
|
|
60
|
+
],
|
|
61
|
+
"name": "branch-alias",
|
|
62
|
+
"hasDynamicHelp": false,
|
|
63
|
+
"multiple": false,
|
|
64
|
+
"type": "option"
|
|
65
|
+
},
|
|
55
66
|
"query": {
|
|
56
67
|
"description": "Query as JSON string or file path",
|
|
57
68
|
"name": "query",
|
|
@@ -104,5 +115,5 @@
|
|
|
104
115
|
]
|
|
105
116
|
}
|
|
106
117
|
},
|
|
107
|
-
"version": "1.0.0-beta.
|
|
118
|
+
"version": "1.0.0-beta.11"
|
|
108
119
|
}
|
package/package.json
CHANGED
|
@@ -1,51 +1,54 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@contentstack/cli-cm-export-query",
|
|
3
3
|
"description": "Contentstack CLI plugin to export content from stack",
|
|
4
|
-
"version": "1.0.0-beta.
|
|
4
|
+
"version": "1.0.0-beta.11",
|
|
5
5
|
"author": "Contentstack",
|
|
6
6
|
"bugs": "https://github.com/contentstack/cli/issues",
|
|
7
7
|
"dependencies": {
|
|
8
|
-
"@contentstack/cli-cm-export": "~1.
|
|
9
|
-
"@contentstack/cli-command": "~1.
|
|
10
|
-
"@contentstack/cli-utilities": "~1.
|
|
11
|
-
"@oclif/core": "^4.
|
|
8
|
+
"@contentstack/cli-cm-export": "~1.24.1",
|
|
9
|
+
"@contentstack/cli-command": "~1.8.0",
|
|
10
|
+
"@contentstack/cli-utilities": "~1.18.1",
|
|
11
|
+
"@oclif/core": "^4.10.5",
|
|
12
12
|
"async": "^3.2.6",
|
|
13
13
|
"big-json": "^3.2.0",
|
|
14
14
|
"bluebird": "^3.7.2",
|
|
15
|
-
"
|
|
16
|
-
"lodash": "^4.17.21",
|
|
15
|
+
"lodash": "^4.18.1",
|
|
17
16
|
"merge": "^2.1.1",
|
|
18
17
|
"mkdirp": "^1.0.4",
|
|
19
18
|
"progress-stream": "^2.0.0",
|
|
20
19
|
"promise-limit": "^2.7.0",
|
|
21
|
-
"
|
|
20
|
+
"tslib": "^2.8.1",
|
|
21
|
+
"winston": "^3.19.0"
|
|
22
|
+
},
|
|
23
|
+
"overrides": {
|
|
24
|
+
"brace-expansion": "^5.0.5"
|
|
22
25
|
},
|
|
23
26
|
"devDependencies": {
|
|
24
|
-
"@contentstack/cli-dev-dependencies": "~1.3.
|
|
25
|
-
"@oclif/plugin-help": "^6.2.
|
|
26
|
-
"@oclif/test": "^4.1.
|
|
27
|
+
"@contentstack/cli-dev-dependencies": "~1.3.1",
|
|
28
|
+
"@oclif/plugin-help": "^6.2.44",
|
|
29
|
+
"@oclif/test": "^4.1.18",
|
|
27
30
|
"@types/big-json": "^3.2.5",
|
|
28
31
|
"@types/chai": "^4.3.20",
|
|
29
32
|
"@types/mkdirp": "^1.0.2",
|
|
30
33
|
"@types/mocha": "^10.0.10",
|
|
31
|
-
"@types/node": "^20.19.
|
|
34
|
+
"@types/node": "^20.19.39",
|
|
32
35
|
"@types/progress-stream": "^2.0.5",
|
|
33
36
|
"@types/sinon": "^17.0.4",
|
|
34
37
|
"chai": "^4.5.0",
|
|
35
|
-
"dotenv": "^16.
|
|
38
|
+
"dotenv": "^16.6.1",
|
|
36
39
|
"dotenv-expand": "^9.0.0",
|
|
37
40
|
"eslint": "^8.57.1",
|
|
38
|
-
"eslint-config-oclif": "^6.0.
|
|
41
|
+
"eslint-config-oclif": "^6.0.157",
|
|
39
42
|
"husky": "^9.1.7",
|
|
40
43
|
"mocha": "10.8.2",
|
|
41
44
|
"nyc": "^15.1.0",
|
|
42
45
|
"oclif": "^4.17.46",
|
|
43
|
-
"sinon": "^17.0.
|
|
46
|
+
"sinon": "^17.0.2",
|
|
44
47
|
"ts-node": "^10.9.2",
|
|
45
48
|
"typescript": "^4.9.5"
|
|
46
49
|
},
|
|
47
50
|
"scripts": {
|
|
48
|
-
"build": "npm run clean && npm run compile && cp -r src/config lib/",
|
|
51
|
+
"build": "npm run clean && npm install && npm run compile && cp -r src/config lib/",
|
|
49
52
|
"clean": "rm -rf ./lib ./node_modules tsconfig.build.tsbuildinfo",
|
|
50
53
|
"compile": "tsc -b tsconfig.json",
|
|
51
54
|
"postpack": "rm -f oclif.manifest.json",
|