@contentstack/cli-cm-export-query 1.0.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +108 -0
- package/lib/commands/cm/stacks/export-query.d.ts +11 -0
- package/lib/commands/cm/stacks/export-query.js +79 -0
- package/lib/config/export-config.json +1 -0
- package/lib/config/index.d.ts +3 -0
- package/lib/config/index.js +57 -0
- package/lib/config/index.ts +59 -0
- package/lib/core/module-exporter.d.ts +11 -0
- package/lib/core/module-exporter.js +75 -0
- package/lib/core/query-executor.d.ts +17 -0
- package/lib/core/query-executor.js +253 -0
- package/lib/types/index.d.ts +161 -0
- package/lib/types/index.js +2 -0
- package/lib/utils/common-helper.d.ts +1 -0
- package/lib/utils/common-helper.js +12 -0
- package/lib/utils/config-handler.d.ts +2 -0
- package/lib/utils/config-handler.js +37 -0
- package/lib/utils/content-type-helper.d.ts +19 -0
- package/lib/utils/content-type-helper.js +90 -0
- package/lib/utils/dependency-resolver.d.ts +17 -0
- package/lib/utils/dependency-resolver.js +126 -0
- package/lib/utils/file-helper.d.ts +2 -0
- package/lib/utils/file-helper.js +5 -0
- package/lib/utils/index.d.ts +8 -0
- package/lib/utils/index.js +15 -0
- package/lib/utils/logger.d.ts +8 -0
- package/lib/utils/logger.js +158 -0
- package/lib/utils/query-parser.d.ts +9 -0
- package/lib/utils/query-parser.js +60 -0
- package/lib/utils/referenced-asset-handler.d.ts +22 -0
- package/lib/utils/referenced-asset-handler.js +120 -0
- package/messages/index.json +1 -0
- package/oclif.manifest.json +108 -0
- package/package.json +94 -0
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.ContentTypeDependenciesHandler = void 0;
|
|
4
|
+
const tslib_1 = require("tslib");
|
|
5
|
+
const path = tslib_1.__importStar(require("path"));
|
|
6
|
+
const index_1 = require("./index");
|
|
7
|
+
const cli_utilities_1 = require("@contentstack/cli-utilities");
|
|
8
|
+
const logger_1 = require("./logger");
|
|
9
|
+
class ContentTypeDependenciesHandler {
|
|
10
|
+
constructor(stackAPIClient, exportQueryConfig) {
|
|
11
|
+
this.exportQueryConfig = exportQueryConfig;
|
|
12
|
+
this.stackAPIClient = stackAPIClient;
|
|
13
|
+
}
|
|
14
|
+
async extractDependencies() {
|
|
15
|
+
const contentTypesFilePath = path.join((0, cli_utilities_1.sanitizePath)(this.exportQueryConfig.exportDir), (0, cli_utilities_1.sanitizePath)(this.exportQueryConfig.branchName || ''), 'content_types', 'schema.json');
|
|
16
|
+
const allContentTypes = index_1.fsUtil.readFile((0, cli_utilities_1.sanitizePath)(contentTypesFilePath)) || [];
|
|
17
|
+
if (allContentTypes.length === 0) {
|
|
18
|
+
(0, logger_1.log)(this.exportQueryConfig, 'No content types found, skipping dependency extraction', 'info');
|
|
19
|
+
return {
|
|
20
|
+
globalFields: new Set(),
|
|
21
|
+
extensions: new Set(),
|
|
22
|
+
taxonomies: new Set(),
|
|
23
|
+
marketplaceApps: new Set(),
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
(0, logger_1.log)(this.exportQueryConfig, `Extracting dependencies from ${allContentTypes.length} content types`, 'info');
|
|
27
|
+
const dependencies = {
|
|
28
|
+
globalFields: new Set(),
|
|
29
|
+
extensions: new Set(),
|
|
30
|
+
taxonomies: new Set(),
|
|
31
|
+
marketplaceApps: new Set(),
|
|
32
|
+
};
|
|
33
|
+
for (const contentType of allContentTypes) {
|
|
34
|
+
if (contentType.schema) {
|
|
35
|
+
this.traverseSchemaForDependencies(contentType.schema, dependencies);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
// Separate extensions from marketplace apps using the extracted extension UIDs
|
|
39
|
+
if (dependencies.extensions.size > 0) {
|
|
40
|
+
const extensionUIDs = Array.from(dependencies.extensions);
|
|
41
|
+
(0, logger_1.log)(this.exportQueryConfig, `Processing ${extensionUIDs.length} extensions to identify marketplace apps...`, 'info');
|
|
42
|
+
try {
|
|
43
|
+
const { extensions, marketplaceApps } = await this.fetchExtensionsAndMarketplaceApps(extensionUIDs);
|
|
44
|
+
dependencies.extensions = new Set(extensions);
|
|
45
|
+
dependencies.marketplaceApps = new Set(marketplaceApps);
|
|
46
|
+
(0, logger_1.log)(this.exportQueryConfig, `Dependencies separated - Global Fields: ${dependencies.globalFields.size}, Extensions: ${dependencies.extensions.size}, Taxonomies: ${dependencies.taxonomies.size}, Marketplace Apps: ${dependencies.marketplaceApps.size}`, 'info');
|
|
47
|
+
}
|
|
48
|
+
catch (error) {
|
|
49
|
+
(0, logger_1.log)(this.exportQueryConfig, `Error separating extensions and marketplace apps: ${error.message}`, 'error');
|
|
50
|
+
// Keep original extensions if separation fails
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
else {
|
|
54
|
+
(0, logger_1.log)(this.exportQueryConfig, `Found dependencies - Global Fields: ${dependencies.globalFields.size}, Extensions: ${dependencies.extensions.size}, Taxonomies: ${dependencies.taxonomies.size}, Marketplace Apps: ${dependencies.marketplaceApps.size}`, 'info');
|
|
55
|
+
}
|
|
56
|
+
return dependencies;
|
|
57
|
+
}
|
|
58
|
+
// Update the fetchExtensionsAndMarketplaceApps method to only fetch specific extension UIDs
|
|
59
|
+
async fetchExtensionsAndMarketplaceApps(extensionUIDs) {
|
|
60
|
+
(0, logger_1.log)(this.exportQueryConfig, `Fetching details for ${extensionUIDs.length} extensions to identify marketplace apps...`, 'info');
|
|
61
|
+
try {
|
|
62
|
+
// Query parameters to include marketplace extensions
|
|
63
|
+
const queryParams = {
|
|
64
|
+
include_count: true,
|
|
65
|
+
include_marketplace_extensions: true,
|
|
66
|
+
query: {
|
|
67
|
+
uid: { $in: extensionUIDs },
|
|
68
|
+
},
|
|
69
|
+
};
|
|
70
|
+
// Fetch all extensions including marketplace apps
|
|
71
|
+
const response = await this.stackAPIClient.extension().query(queryParams).find();
|
|
72
|
+
if (!response || !response.items) {
|
|
73
|
+
(0, logger_1.log)(this.exportQueryConfig, `No extensions found`, 'warn');
|
|
74
|
+
return { extensions: extensionUIDs, marketplaceApps: [] };
|
|
75
|
+
}
|
|
76
|
+
const marketplaceApps = [];
|
|
77
|
+
const regularExtensions = [];
|
|
78
|
+
response.items.forEach((item) => {
|
|
79
|
+
if (item.app_uid && item.app_installation_uid) {
|
|
80
|
+
marketplaceApps.push(item.app_installation_uid);
|
|
81
|
+
}
|
|
82
|
+
else {
|
|
83
|
+
regularExtensions.push(item.uid);
|
|
84
|
+
}
|
|
85
|
+
});
|
|
86
|
+
(0, logger_1.log)(this.exportQueryConfig, `Identified ${marketplaceApps.length} marketplace apps and ${regularExtensions.length} regular extensions from ${extensionUIDs.length} total extensions`, 'info');
|
|
87
|
+
return { extensions: regularExtensions, marketplaceApps };
|
|
88
|
+
}
|
|
89
|
+
catch (error) {
|
|
90
|
+
(0, logger_1.log)(this.exportQueryConfig, `Error fetching extensions and marketplace apps: ${error.message}`, 'error');
|
|
91
|
+
return { extensions: extensionUIDs, marketplaceApps: [] };
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
traverseSchemaForDependencies(schema, dependencies) {
|
|
95
|
+
for (const field of schema) {
|
|
96
|
+
// Global fields
|
|
97
|
+
if (field.data_type === 'global_field' && field.reference_to) {
|
|
98
|
+
dependencies.globalFields.add(field.reference_to);
|
|
99
|
+
}
|
|
100
|
+
// Extensions
|
|
101
|
+
if (field.extension_uid) {
|
|
102
|
+
dependencies.extensions.add(field.extension_uid);
|
|
103
|
+
}
|
|
104
|
+
// Taxonomies - UPDATED LOGIC
|
|
105
|
+
if (field.data_type === 'taxonomy' && field.taxonomies && Array.isArray(field.taxonomies)) {
|
|
106
|
+
field.taxonomies.forEach((tax) => {
|
|
107
|
+
if (tax.taxonomy_uid) {
|
|
108
|
+
dependencies.taxonomies.add(tax.taxonomy_uid);
|
|
109
|
+
}
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
// Recursive traversal for nested structures
|
|
113
|
+
if (field.data_type === 'group' && field.schema) {
|
|
114
|
+
this.traverseSchemaForDependencies(field.schema, dependencies);
|
|
115
|
+
}
|
|
116
|
+
if (field.data_type === 'blocks' && field.blocks) {
|
|
117
|
+
for (const blockKey in field.blocks) {
|
|
118
|
+
if (field.blocks[blockKey].schema) {
|
|
119
|
+
this.traverseSchemaForDependencies(field.blocks[blockKey].schema, dependencies);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
exports.ContentTypeDependenciesHandler = ContentTypeDependenciesHandler;
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export * as fileHelper from './file-helper';
|
|
2
|
+
export { fsUtil } from './file-helper';
|
|
3
|
+
export { log, unlinkFileLogger } from './logger';
|
|
4
|
+
export * from './common-helper';
|
|
5
|
+
export * from './config-handler';
|
|
6
|
+
export * from './content-type-helper';
|
|
7
|
+
export * from './dependency-resolver';
|
|
8
|
+
export * from './referenced-asset-handler';
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.unlinkFileLogger = exports.log = exports.fsUtil = exports.fileHelper = void 0;
|
|
4
|
+
const tslib_1 = require("tslib");
|
|
5
|
+
exports.fileHelper = tslib_1.__importStar(require("./file-helper"));
|
|
6
|
+
var file_helper_1 = require("./file-helper");
|
|
7
|
+
Object.defineProperty(exports, "fsUtil", { enumerable: true, get: function () { return file_helper_1.fsUtil; } });
|
|
8
|
+
var logger_1 = require("./logger");
|
|
9
|
+
Object.defineProperty(exports, "log", { enumerable: true, get: function () { return logger_1.log; } });
|
|
10
|
+
Object.defineProperty(exports, "unlinkFileLogger", { enumerable: true, get: function () { return logger_1.unlinkFileLogger; } });
|
|
11
|
+
tslib_1.__exportStar(require("./common-helper"), exports);
|
|
12
|
+
tslib_1.__exportStar(require("./config-handler"), exports);
|
|
13
|
+
tslib_1.__exportStar(require("./content-type-helper"), exports);
|
|
14
|
+
tslib_1.__exportStar(require("./dependency-resolver"), exports);
|
|
15
|
+
tslib_1.__exportStar(require("./referenced-asset-handler"), exports);
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/*!
|
|
2
|
+
* Contentstack Export
|
|
3
|
+
* Copyright (c) 2024 Contentstack LLC
|
|
4
|
+
* MIT Licensed
|
|
5
|
+
*/
|
|
6
|
+
import { QueryExportConfig } from '../types';
|
|
7
|
+
export declare const log: (config: QueryExportConfig, message: any, type: string) => Promise<void>;
|
|
8
|
+
export declare const unlinkFileLogger: () => void;
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/*!
|
|
3
|
+
* Contentstack Export
|
|
4
|
+
* Copyright (c) 2024 Contentstack LLC
|
|
5
|
+
* MIT Licensed
|
|
6
|
+
*/
|
|
7
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8
|
+
exports.unlinkFileLogger = exports.log = void 0;
|
|
9
|
+
const tslib_1 = require("tslib");
|
|
10
|
+
const winston = tslib_1.__importStar(require("winston"));
|
|
11
|
+
const path = tslib_1.__importStar(require("path"));
|
|
12
|
+
const mkdirp_1 = tslib_1.__importDefault(require("mkdirp"));
|
|
13
|
+
const cli_utilities_1 = require("@contentstack/cli-utilities");
|
|
14
|
+
const slice = Array.prototype.slice;
|
|
15
|
+
const ansiRegexPattern = [
|
|
16
|
+
'[\\u001B\\u009B][[\\]()#;?]*(?:(?:(?:(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]+)*|[a-zA-Z\\d]+(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]*)*)?\\u0007)',
|
|
17
|
+
'(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PR-TZcf-ntqry=><~]))',
|
|
18
|
+
].join('|');
|
|
19
|
+
function returnString(args) {
|
|
20
|
+
let returnStr = '';
|
|
21
|
+
if (args && args.length) {
|
|
22
|
+
returnStr = args
|
|
23
|
+
.map(function (item) {
|
|
24
|
+
if (item && typeof item === 'object') {
|
|
25
|
+
try {
|
|
26
|
+
const redactedObject = (0, cli_utilities_1.redactObject)(item);
|
|
27
|
+
if (redactedObject && typeof redactedObject === 'object') {
|
|
28
|
+
return JSON.stringify(redactedObject);
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
catch (error) { }
|
|
32
|
+
return item;
|
|
33
|
+
}
|
|
34
|
+
return item;
|
|
35
|
+
})
|
|
36
|
+
.join(' ')
|
|
37
|
+
.trim();
|
|
38
|
+
}
|
|
39
|
+
returnStr = returnStr.replace(new RegExp(ansiRegexPattern, 'g'), '').trim();
|
|
40
|
+
return returnStr;
|
|
41
|
+
}
|
|
42
|
+
const myCustomLevels = {
|
|
43
|
+
levels: {
|
|
44
|
+
warn: 1,
|
|
45
|
+
info: 2,
|
|
46
|
+
debug: 3,
|
|
47
|
+
},
|
|
48
|
+
colors: {
|
|
49
|
+
//colors aren't being used anywhere as of now, we're using chalk to add colors while logging
|
|
50
|
+
info: 'blue',
|
|
51
|
+
debug: 'green',
|
|
52
|
+
warn: 'yellow',
|
|
53
|
+
error: 'red',
|
|
54
|
+
},
|
|
55
|
+
};
|
|
56
|
+
let logger;
|
|
57
|
+
let errorLogger;
|
|
58
|
+
let successTransport;
|
|
59
|
+
let errorTransport;
|
|
60
|
+
function init(_logPath) {
|
|
61
|
+
if (!logger || !errorLogger) {
|
|
62
|
+
const logsDir = path.resolve((0, cli_utilities_1.sanitizePath)(_logPath), 'logs', 'export');
|
|
63
|
+
// Create dir if doesn't already exist
|
|
64
|
+
mkdirp_1.default.sync(logsDir);
|
|
65
|
+
successTransport = {
|
|
66
|
+
filename: path.join((0, cli_utilities_1.sanitizePath)(logsDir), 'success.log'),
|
|
67
|
+
maxFiles: 20,
|
|
68
|
+
maxsize: 1000000,
|
|
69
|
+
tailable: true,
|
|
70
|
+
level: 'info',
|
|
71
|
+
};
|
|
72
|
+
errorTransport = {
|
|
73
|
+
filename: path.join((0, cli_utilities_1.sanitizePath)(logsDir), 'error.log'),
|
|
74
|
+
maxFiles: 20,
|
|
75
|
+
maxsize: 1000000,
|
|
76
|
+
tailable: true,
|
|
77
|
+
level: 'error',
|
|
78
|
+
};
|
|
79
|
+
logger = winston.createLogger({
|
|
80
|
+
transports: [
|
|
81
|
+
new winston.transports.File(successTransport),
|
|
82
|
+
new winston.transports.Console({ format: winston.format.simple() }),
|
|
83
|
+
],
|
|
84
|
+
levels: myCustomLevels.levels,
|
|
85
|
+
});
|
|
86
|
+
errorLogger = winston.createLogger({
|
|
87
|
+
transports: [
|
|
88
|
+
new winston.transports.File(errorTransport),
|
|
89
|
+
new winston.transports.Console({
|
|
90
|
+
level: 'error',
|
|
91
|
+
format: winston.format.combine(winston.format.colorize({ all: true, colors: { error: 'red' } }), winston.format.simple()),
|
|
92
|
+
}),
|
|
93
|
+
],
|
|
94
|
+
levels: { error: 0 },
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
return {
|
|
98
|
+
log: function (message) {
|
|
99
|
+
const args = slice.call(arguments);
|
|
100
|
+
const logString = returnString(args);
|
|
101
|
+
if (logString) {
|
|
102
|
+
logger.log('info', logString);
|
|
103
|
+
}
|
|
104
|
+
},
|
|
105
|
+
warn: function () {
|
|
106
|
+
const args = slice.call(arguments);
|
|
107
|
+
const logString = returnString(args);
|
|
108
|
+
if (logString) {
|
|
109
|
+
logger.log('warn', logString);
|
|
110
|
+
}
|
|
111
|
+
},
|
|
112
|
+
error: function (message) {
|
|
113
|
+
const args = slice.call(arguments);
|
|
114
|
+
const logString = returnString(args);
|
|
115
|
+
if (logString) {
|
|
116
|
+
errorLogger.log('error', logString);
|
|
117
|
+
}
|
|
118
|
+
},
|
|
119
|
+
debug: function () {
|
|
120
|
+
const args = slice.call(arguments);
|
|
121
|
+
const logString = returnString(args);
|
|
122
|
+
if (logString) {
|
|
123
|
+
logger.log('debug', logString);
|
|
124
|
+
}
|
|
125
|
+
},
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
const log = async (config, message, type) => {
|
|
129
|
+
const logsPath = (0, cli_utilities_1.sanitizePath)(config.logsPath || config.dataPath);
|
|
130
|
+
// ignoring the type argument, as we are not using it to create a logfile anymore
|
|
131
|
+
if (type !== 'error') {
|
|
132
|
+
// removed type argument from init method
|
|
133
|
+
init(logsPath).log(message);
|
|
134
|
+
}
|
|
135
|
+
else {
|
|
136
|
+
init(logsPath).error(message);
|
|
137
|
+
}
|
|
138
|
+
};
|
|
139
|
+
exports.log = log;
|
|
140
|
+
const unlinkFileLogger = () => {
|
|
141
|
+
if (logger) {
|
|
142
|
+
const transports = logger.transports;
|
|
143
|
+
transports.forEach((transport) => {
|
|
144
|
+
if (transport.name === 'file') {
|
|
145
|
+
logger.remove(transport);
|
|
146
|
+
}
|
|
147
|
+
});
|
|
148
|
+
}
|
|
149
|
+
if (errorLogger) {
|
|
150
|
+
const transports = errorLogger.transports;
|
|
151
|
+
transports.forEach((transport) => {
|
|
152
|
+
if (transport.name === 'file') {
|
|
153
|
+
errorLogger.remove(transport);
|
|
154
|
+
}
|
|
155
|
+
});
|
|
156
|
+
}
|
|
157
|
+
};
|
|
158
|
+
exports.unlinkFileLogger = unlinkFileLogger;
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.QueryParser = void 0;
|
|
4
|
+
const tslib_1 = require("tslib");
|
|
5
|
+
const fs = tslib_1.__importStar(require("fs"));
|
|
6
|
+
const cli_utilities_1 = require("@contentstack/cli-utilities");
|
|
7
|
+
class QueryParser {
|
|
8
|
+
constructor(config) {
|
|
9
|
+
this.config = config;
|
|
10
|
+
}
|
|
11
|
+
async parse(queryInput) {
|
|
12
|
+
let query;
|
|
13
|
+
// Check if it's a file path
|
|
14
|
+
if (queryInput.endsWith('.json') && fs.existsSync(queryInput)) {
|
|
15
|
+
query = this.parseFromFile(queryInput);
|
|
16
|
+
}
|
|
17
|
+
else {
|
|
18
|
+
query = this.parseFromString(queryInput);
|
|
19
|
+
}
|
|
20
|
+
this.validate(query);
|
|
21
|
+
return query;
|
|
22
|
+
}
|
|
23
|
+
parseFromFile(filePath) {
|
|
24
|
+
try {
|
|
25
|
+
const content = fs.readFileSync(filePath, 'utf-8');
|
|
26
|
+
return JSON.parse(content);
|
|
27
|
+
}
|
|
28
|
+
catch (error) {
|
|
29
|
+
throw new cli_utilities_1.CLIError(`Failed to parse query file: ${error.message}`);
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
parseFromString(queryString) {
|
|
33
|
+
try {
|
|
34
|
+
return JSON.parse(queryString);
|
|
35
|
+
}
|
|
36
|
+
catch (error) {
|
|
37
|
+
throw new cli_utilities_1.CLIError(`Invalid JSON query: ${error.message}`);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
validate(query) {
|
|
41
|
+
if (!query || typeof query !== 'object') {
|
|
42
|
+
throw new cli_utilities_1.CLIError('Query must be a valid JSON object');
|
|
43
|
+
}
|
|
44
|
+
if (!query.modules || typeof query.modules !== 'object') {
|
|
45
|
+
throw new cli_utilities_1.CLIError('Query must contain a "modules" object');
|
|
46
|
+
}
|
|
47
|
+
const modules = Object.keys(query.modules);
|
|
48
|
+
if (modules.length === 0) {
|
|
49
|
+
throw new cli_utilities_1.CLIError('Query must contain at least one module');
|
|
50
|
+
}
|
|
51
|
+
// Validate supported modules
|
|
52
|
+
const queryableModules = this.config.modules.queryable;
|
|
53
|
+
for (const module of modules) {
|
|
54
|
+
if (!queryableModules.includes(module)) {
|
|
55
|
+
throw new cli_utilities_1.CLIError(`Module "${module}" is not queryable. Supported modules: ${queryableModules.join(', ')}`);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
exports.QueryParser = QueryParser;
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { QueryExportConfig } from '../types';
|
|
2
|
+
export declare class AssetReferenceHandler {
|
|
3
|
+
private exportQueryConfig;
|
|
4
|
+
private entriesDir;
|
|
5
|
+
constructor(exportQueryConfig: QueryExportConfig);
|
|
6
|
+
/**
|
|
7
|
+
* Extract all asset UIDs by processing entries file by file (memory efficient)
|
|
8
|
+
*/
|
|
9
|
+
extractReferencedAssets(): string[];
|
|
10
|
+
/**
|
|
11
|
+
* Process a single file and extract asset UIDs from all its entries
|
|
12
|
+
*/
|
|
13
|
+
private processSingleFile;
|
|
14
|
+
/**
|
|
15
|
+
* Extract asset UIDs from stringified content using multiple patterns
|
|
16
|
+
*/
|
|
17
|
+
private extractAssetUIDsFromString;
|
|
18
|
+
/**
|
|
19
|
+
* Recursively find all JSON files in the entries directory
|
|
20
|
+
*/
|
|
21
|
+
private findAllJsonFiles;
|
|
22
|
+
}
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.AssetReferenceHandler = void 0;
|
|
4
|
+
const tslib_1 = require("tslib");
|
|
5
|
+
const path = tslib_1.__importStar(require("path"));
|
|
6
|
+
const fs = tslib_1.__importStar(require("fs"));
|
|
7
|
+
const index_1 = require("./index");
|
|
8
|
+
const cli_utilities_1 = require("@contentstack/cli-utilities");
|
|
9
|
+
const logger_1 = require("./logger");
|
|
10
|
+
class AssetReferenceHandler {
|
|
11
|
+
constructor(exportQueryConfig) {
|
|
12
|
+
this.exportQueryConfig = exportQueryConfig;
|
|
13
|
+
this.entriesDir = path.join((0, cli_utilities_1.sanitizePath)(exportQueryConfig.exportDir), (0, cli_utilities_1.sanitizePath)(exportQueryConfig.branchName || ''), 'entries');
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Extract all asset UIDs by processing entries file by file (memory efficient)
|
|
17
|
+
*/
|
|
18
|
+
extractReferencedAssets() {
|
|
19
|
+
(0, logger_1.log)(this.exportQueryConfig, 'Extracting referenced assets from entries...', 'info');
|
|
20
|
+
try {
|
|
21
|
+
if (!fs.existsSync(this.entriesDir)) {
|
|
22
|
+
(0, logger_1.log)(this.exportQueryConfig, 'Entries directory does not exist', 'warn');
|
|
23
|
+
return [];
|
|
24
|
+
}
|
|
25
|
+
// Global set to maintain unique asset UIDs across all files
|
|
26
|
+
const globalAssetUIDs = new Set();
|
|
27
|
+
// Get all JSON files
|
|
28
|
+
const jsonFiles = this.findAllJsonFiles(this.entriesDir);
|
|
29
|
+
// Process files one by one
|
|
30
|
+
let totalEntriesProcessed = 0;
|
|
31
|
+
for (const jsonFile of jsonFiles) {
|
|
32
|
+
const entriesInFile = this.processSingleFile(jsonFile, globalAssetUIDs);
|
|
33
|
+
totalEntriesProcessed += entriesInFile;
|
|
34
|
+
}
|
|
35
|
+
const result = Array.from(globalAssetUIDs);
|
|
36
|
+
(0, logger_1.log)(this.exportQueryConfig, `Found ${result.length} unique asset UIDs from ${totalEntriesProcessed} entries across ${jsonFiles.length} files`, 'info');
|
|
37
|
+
return result;
|
|
38
|
+
}
|
|
39
|
+
catch (error) {
|
|
40
|
+
(0, logger_1.log)(this.exportQueryConfig, `Error extracting assets: ${error.message}`, 'error');
|
|
41
|
+
return [];
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Process a single file and extract asset UIDs from all its entries
|
|
46
|
+
*/
|
|
47
|
+
processSingleFile(filePath, globalAssetUIDs) {
|
|
48
|
+
// Skip index.json files
|
|
49
|
+
if (path.basename(filePath) === 'index.json') {
|
|
50
|
+
return 0;
|
|
51
|
+
}
|
|
52
|
+
try {
|
|
53
|
+
const fileContent = index_1.fsUtil.readFile((0, cli_utilities_1.sanitizePath)(filePath));
|
|
54
|
+
if (!fileContent || typeof fileContent !== 'object') {
|
|
55
|
+
return 0;
|
|
56
|
+
}
|
|
57
|
+
// Stringify the ENTIRE file content at once
|
|
58
|
+
const fileString = JSON.stringify(fileContent);
|
|
59
|
+
// Extract all asset UIDs from the entire file
|
|
60
|
+
const assetUIDs = this.extractAssetUIDsFromString(fileString);
|
|
61
|
+
// Add to global set
|
|
62
|
+
assetUIDs.forEach((uid) => globalAssetUIDs.add(uid));
|
|
63
|
+
// Count entries for logging
|
|
64
|
+
const entriesCount = Object.keys(fileContent).length;
|
|
65
|
+
(0, logger_1.log)(this.exportQueryConfig, `Processed ${entriesCount} entries from ${path.basename(filePath)}`, 'debug');
|
|
66
|
+
return entriesCount;
|
|
67
|
+
}
|
|
68
|
+
catch (error) {
|
|
69
|
+
(0, logger_1.log)(this.exportQueryConfig, `Error processing file ${filePath}: ${error.message}`, 'warn');
|
|
70
|
+
return 0;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Extract asset UIDs from stringified content using multiple patterns
|
|
75
|
+
*/
|
|
76
|
+
extractAssetUIDsFromString(content) {
|
|
77
|
+
const assetUIDs = new Set();
|
|
78
|
+
// Pattern 1: HTML img tags with asset_uid
|
|
79
|
+
const htmlAssetRegex = /<img asset_uid=\\"([^"]+)\\"/g;
|
|
80
|
+
let match;
|
|
81
|
+
while ((match = htmlAssetRegex.exec(content)) !== null) {
|
|
82
|
+
if (match[1]) {
|
|
83
|
+
assetUIDs.add(match[1]);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
// Pattern 2: Contentstack asset URLs
|
|
87
|
+
const urlRegex = new RegExp('(https://(assets|(eu-|azure-na-|azure-eu-|gcp-na-|gcp-eu-)?images).contentstack.(io|com)/v3/assets/(.*?)/(.*?)/(.*?)/(.*?)(?="))', 'g');
|
|
88
|
+
while ((match = urlRegex.exec(content)) !== null) {
|
|
89
|
+
const assetUID = match[6]; // The asset UID is in the 6th capture group
|
|
90
|
+
if (assetUID) {
|
|
91
|
+
assetUIDs.add(assetUID);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
return Array.from(assetUIDs);
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* Recursively find all JSON files in the entries directory
|
|
98
|
+
*/
|
|
99
|
+
findAllJsonFiles(dir) {
|
|
100
|
+
const jsonFiles = [];
|
|
101
|
+
try {
|
|
102
|
+
const items = fs.readdirSync(dir);
|
|
103
|
+
for (const item of items) {
|
|
104
|
+
const fullPath = path.join(dir, item);
|
|
105
|
+
const stat = fs.statSync(fullPath);
|
|
106
|
+
if (stat.isDirectory()) {
|
|
107
|
+
jsonFiles.push(...this.findAllJsonFiles(fullPath));
|
|
108
|
+
}
|
|
109
|
+
else if (stat.isFile() && item.endsWith('.json')) {
|
|
110
|
+
jsonFiles.push(fullPath);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
catch (error) {
|
|
115
|
+
(0, logger_1.log)(this.exportQueryConfig, `Error reading directory ${dir}: ${error.message}`, 'warn');
|
|
116
|
+
}
|
|
117
|
+
return jsonFiles;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
exports.AssetReferenceHandler = AssetReferenceHandler;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{}
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
{
|
|
2
|
+
"commands": {
|
|
3
|
+
"cm:stacks:export-query": {
|
|
4
|
+
"aliases": [
|
|
5
|
+
"cm:export-query"
|
|
6
|
+
],
|
|
7
|
+
"args": {},
|
|
8
|
+
"description": "Export content from a stack using query-based filtering",
|
|
9
|
+
"examples": [
|
|
10
|
+
"csdx cm:stacks:export-query --query '{\"modules\":{\"content-types\":{\"title\":{\"$in\":[\"Blog\",\"Author\"]}}}}'",
|
|
11
|
+
"csdx cm:stacks:export-query --query ./ct-query.json --skip-references",
|
|
12
|
+
"csdx cm:stacks:export-query --alias <alias> --query '{\"modules\":{\"entries\":{\"content_type_uid\":\"blog\"}}}'",
|
|
13
|
+
"csdx cm:stacks:export-query --query '{\"modules\":{\"assets\":{\"title\":{\"$regex\":\"image\"}}}}'"
|
|
14
|
+
],
|
|
15
|
+
"flags": {
|
|
16
|
+
"config": {
|
|
17
|
+
"char": "c",
|
|
18
|
+
"description": "Path to the configuration file",
|
|
19
|
+
"name": "config",
|
|
20
|
+
"hasDynamicHelp": false,
|
|
21
|
+
"multiple": false,
|
|
22
|
+
"type": "option"
|
|
23
|
+
},
|
|
24
|
+
"stack-api-key": {
|
|
25
|
+
"char": "k",
|
|
26
|
+
"description": "Stack API key",
|
|
27
|
+
"name": "stack-api-key",
|
|
28
|
+
"hasDynamicHelp": false,
|
|
29
|
+
"multiple": false,
|
|
30
|
+
"type": "option"
|
|
31
|
+
},
|
|
32
|
+
"data-dir": {
|
|
33
|
+
"char": "d",
|
|
34
|
+
"description": "Path to store exported content",
|
|
35
|
+
"name": "data-dir",
|
|
36
|
+
"hasDynamicHelp": false,
|
|
37
|
+
"multiple": false,
|
|
38
|
+
"type": "option"
|
|
39
|
+
},
|
|
40
|
+
"alias": {
|
|
41
|
+
"char": "a",
|
|
42
|
+
"description": "Management token alias",
|
|
43
|
+
"name": "alias",
|
|
44
|
+
"hasDynamicHelp": false,
|
|
45
|
+
"multiple": false,
|
|
46
|
+
"type": "option"
|
|
47
|
+
},
|
|
48
|
+
"branch": {
|
|
49
|
+
"description": "Branch name to export from",
|
|
50
|
+
"name": "branch",
|
|
51
|
+
"hasDynamicHelp": false,
|
|
52
|
+
"multiple": false,
|
|
53
|
+
"type": "option"
|
|
54
|
+
},
|
|
55
|
+
"query": {
|
|
56
|
+
"description": "Query as JSON string or file path",
|
|
57
|
+
"name": "query",
|
|
58
|
+
"required": true,
|
|
59
|
+
"hasDynamicHelp": false,
|
|
60
|
+
"multiple": false,
|
|
61
|
+
"type": "option"
|
|
62
|
+
},
|
|
63
|
+
"skip-references": {
|
|
64
|
+
"description": "Skip referenced content types",
|
|
65
|
+
"name": "skip-references",
|
|
66
|
+
"allowNo": false,
|
|
67
|
+
"type": "boolean"
|
|
68
|
+
},
|
|
69
|
+
"skip-dependencies": {
|
|
70
|
+
"description": "Skip dependent modules (global-fields, extensions, taxonomies)",
|
|
71
|
+
"name": "skip-dependencies",
|
|
72
|
+
"allowNo": false,
|
|
73
|
+
"type": "boolean"
|
|
74
|
+
},
|
|
75
|
+
"secured-assets": {
|
|
76
|
+
"description": "Export secured assets",
|
|
77
|
+
"name": "secured-assets",
|
|
78
|
+
"allowNo": false,
|
|
79
|
+
"type": "boolean"
|
|
80
|
+
},
|
|
81
|
+
"yes": {
|
|
82
|
+
"char": "y",
|
|
83
|
+
"description": "Skip confirmation prompts",
|
|
84
|
+
"name": "yes",
|
|
85
|
+
"allowNo": false,
|
|
86
|
+
"type": "boolean"
|
|
87
|
+
}
|
|
88
|
+
},
|
|
89
|
+
"hasDynamicHelp": false,
|
|
90
|
+
"hiddenAliases": [],
|
|
91
|
+
"id": "cm:stacks:export-query",
|
|
92
|
+
"pluginAlias": "@contentstack/cli-cm-export-query",
|
|
93
|
+
"pluginName": "@contentstack/cli-cm-export-query",
|
|
94
|
+
"pluginType": "core",
|
|
95
|
+
"strict": true,
|
|
96
|
+
"usage": "cm:stacks:export-query --query <value> [options]",
|
|
97
|
+
"isESM": false,
|
|
98
|
+
"relativePath": [
|
|
99
|
+
"lib",
|
|
100
|
+
"commands",
|
|
101
|
+
"cm",
|
|
102
|
+
"stacks",
|
|
103
|
+
"export-query.js"
|
|
104
|
+
]
|
|
105
|
+
}
|
|
106
|
+
},
|
|
107
|
+
"version": "1.0.0-beta.1"
|
|
108
|
+
}
|