@sw-tsdk/plugin-connector 3.13.2-next.f72064b → 3.13.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -18
- package/lib/commands/migrator/convert.d.ts +0 -1
- package/lib/commands/migrator/convert.js +15 -167
- package/lib/commands/migrator/convert.js.map +1 -1
- package/lib/templates/migrator-runners/plugin_override.txt +4 -69
- package/lib/templates/migrator-runners/runner_override.txt +0 -29
- package/lib/templates/migrator-runners/script_override.txt +5 -71
- package/lib/transformers/base-transformer.js +14 -61
- package/lib/transformers/base-transformer.js.map +1 -1
- package/lib/transformers/connector-generator.d.ts +2 -102
- package/lib/transformers/connector-generator.js +49 -1188
- package/lib/transformers/connector-generator.js.map +1 -1
- package/lib/types/migrator-types.d.ts +0 -22
- package/lib/types/migrator-types.js.map +1 -1
- package/oclif.manifest.json +1 -1
- package/package.json +6 -6
- package/lib/templates/swimlane/__init__.py +0 -18
- package/lib/templates/swimlane/core/__init__.py +0 -0
- package/lib/templates/swimlane/core/adapters/__init__.py +0 -10
- package/lib/templates/swimlane/core/adapters/app.py +0 -59
- package/lib/templates/swimlane/core/adapters/app_revision.py +0 -49
- package/lib/templates/swimlane/core/adapters/helper.py +0 -84
- package/lib/templates/swimlane/core/adapters/record.py +0 -468
- package/lib/templates/swimlane/core/adapters/record_revision.py +0 -43
- package/lib/templates/swimlane/core/adapters/report.py +0 -65
- package/lib/templates/swimlane/core/adapters/task.py +0 -54
- package/lib/templates/swimlane/core/adapters/usergroup.py +0 -183
- package/lib/templates/swimlane/core/bulk.py +0 -48
- package/lib/templates/swimlane/core/cache.py +0 -165
- package/lib/templates/swimlane/core/client.py +0 -466
- package/lib/templates/swimlane/core/cursor.py +0 -100
- package/lib/templates/swimlane/core/fields/__init__.py +0 -46
- package/lib/templates/swimlane/core/fields/attachment.py +0 -82
- package/lib/templates/swimlane/core/fields/base/__init__.py +0 -15
- package/lib/templates/swimlane/core/fields/base/cursor.py +0 -90
- package/lib/templates/swimlane/core/fields/base/field.py +0 -149
- package/lib/templates/swimlane/core/fields/base/multiselect.py +0 -116
- package/lib/templates/swimlane/core/fields/comment.py +0 -48
- package/lib/templates/swimlane/core/fields/datetime.py +0 -112
- package/lib/templates/swimlane/core/fields/history.py +0 -28
- package/lib/templates/swimlane/core/fields/list.py +0 -266
- package/lib/templates/swimlane/core/fields/number.py +0 -38
- package/lib/templates/swimlane/core/fields/reference.py +0 -169
- package/lib/templates/swimlane/core/fields/text.py +0 -30
- package/lib/templates/swimlane/core/fields/tracking.py +0 -10
- package/lib/templates/swimlane/core/fields/usergroup.py +0 -137
- package/lib/templates/swimlane/core/fields/valueslist.py +0 -70
- package/lib/templates/swimlane/core/resolver.py +0 -46
- package/lib/templates/swimlane/core/resources/__init__.py +0 -0
- package/lib/templates/swimlane/core/resources/app.py +0 -136
- package/lib/templates/swimlane/core/resources/app_revision.py +0 -43
- package/lib/templates/swimlane/core/resources/attachment.py +0 -64
- package/lib/templates/swimlane/core/resources/base.py +0 -55
- package/lib/templates/swimlane/core/resources/comment.py +0 -33
- package/lib/templates/swimlane/core/resources/record.py +0 -499
- package/lib/templates/swimlane/core/resources/record_revision.py +0 -44
- package/lib/templates/swimlane/core/resources/report.py +0 -259
- package/lib/templates/swimlane/core/resources/revision_base.py +0 -69
- package/lib/templates/swimlane/core/resources/task.py +0 -16
- package/lib/templates/swimlane/core/resources/usergroup.py +0 -166
- package/lib/templates/swimlane/core/search.py +0 -31
- package/lib/templates/swimlane/core/wrappedsession.py +0 -12
- package/lib/templates/swimlane/exceptions.py +0 -191
- package/lib/templates/swimlane/utils/__init__.py +0 -132
- package/lib/templates/swimlane/utils/date_validator.py +0 -4
- package/lib/templates/swimlane/utils/list_validator.py +0 -7
- package/lib/templates/swimlane/utils/str_validator.py +0 -10
- package/lib/templates/swimlane/utils/version.py +0 -101
|
@@ -4,225 +4,19 @@ exports.ConnectorGenerator = void 0;
|
|
|
4
4
|
const tslib_1 = require("tslib");
|
|
5
5
|
const connector_interfaces_1 = require("@swimlane/connector-interfaces");
|
|
6
6
|
const node_fs_1 = require("node:fs");
|
|
7
|
-
const promises_1 = require("node:fs/promises");
|
|
8
7
|
const node_path_1 = require("node:path");
|
|
9
|
-
const node_os_1 = require("node:os");
|
|
10
8
|
const js_yaml_1 = tslib_1.__importDefault(require("js-yaml"));
|
|
11
9
|
const adm_zip_1 = tslib_1.__importDefault(require("adm-zip"));
|
|
12
|
-
/**
|
|
13
|
-
* Map asset.json inputParameter type (numeric) to JSON Schema type.
|
|
14
|
-
* 1: text, 2: text area, 3: code, 4: password, 5: list, 6: number, 7: boolean
|
|
15
|
-
*/
|
|
16
|
-
function assetInputTypeToSchemaType(typeCode) {
|
|
17
|
-
switch (typeCode) {
|
|
18
|
-
case 5: {
|
|
19
|
-
return 'array';
|
|
20
|
-
}
|
|
21
|
-
case 6: {
|
|
22
|
-
return 'number';
|
|
23
|
-
}
|
|
24
|
-
case 7: {
|
|
25
|
-
return 'boolean';
|
|
26
|
-
}
|
|
27
|
-
default: {
|
|
28
|
-
return 'string';
|
|
29
|
-
}
|
|
30
|
-
}
|
|
31
|
-
}
|
|
32
|
-
/**
|
|
33
|
-
* Map applicationInfo field type (and selectionType for valuesList) to JSON Schema type.
|
|
34
|
-
* reference, attachment, list, multi-select, checkbox -> array
|
|
35
|
-
* text, single select -> string
|
|
36
|
-
* numeric -> number
|
|
37
|
-
*/
|
|
38
|
-
function applicationFieldTypeToSchemaType(field) {
|
|
39
|
-
const ft = (field.fieldType || '').toLowerCase();
|
|
40
|
-
if (ft === 'numeric')
|
|
41
|
-
return 'number';
|
|
42
|
-
if (ft === 'reference' || ft === 'attachment' || ft === 'list')
|
|
43
|
-
return 'array';
|
|
44
|
-
if (ft === 'valueslist') {
|
|
45
|
-
const st = (field.selectionType || '').toLowerCase();
|
|
46
|
-
return st === 'multi' ? 'array' : 'string';
|
|
47
|
-
}
|
|
48
|
-
if (ft === 'checkbox')
|
|
49
|
-
return 'array';
|
|
50
|
-
return 'string';
|
|
51
|
-
}
|
|
52
|
-
/**
|
|
53
|
-
* Packages that should be excluded from requirements.txt
|
|
54
|
-
* - Provided by the runtime (requests, swimlane)
|
|
55
|
-
* - Installed via other means (swimbundle_utils via compile.sh)
|
|
56
|
-
*/
|
|
57
|
-
const EXCLUDED_PACKAGES = new Set([
|
|
58
|
-
'requests',
|
|
59
|
-
'swimlane',
|
|
60
|
-
'swimbundle_utils',
|
|
61
|
-
'dominions',
|
|
62
|
-
'ssdeep',
|
|
63
|
-
'pymongo', // Unsupported
|
|
64
|
-
]);
|
|
65
|
-
/** Packages that when excluded from requirements.txt should be installed via runner.sh (apt/pip). */
|
|
66
|
-
const RUNNER_EXCLUDED_PACKAGES = new Set(['ssdeep']);
|
|
67
|
-
/**
|
|
68
|
-
* Packages that should have their version constraints stripped.
|
|
69
|
-
* These packages commonly have compatibility issues when migrating from Python 3.7 to 3.11+
|
|
70
|
-
* because older pinned versions don't have wheels for newer Python versions.
|
|
71
|
-
* By stripping the version, pip can resolve a compatible version automatically.
|
|
72
|
-
*
|
|
73
|
-
* IMPORTANT: All package names MUST be lowercase since deduplicateRequirements()
|
|
74
|
-
* converts package names to lowercase before checking against this set.
|
|
75
|
-
*/
|
|
76
|
-
const PACKAGES_TO_STRIP_VERSION = new Set([
|
|
77
|
-
// Core scientific/data packages with C extensions
|
|
78
|
-
'numpy',
|
|
79
|
-
'scipy',
|
|
80
|
-
'pandas',
|
|
81
|
-
// Cryptography packages with C/Rust extensions
|
|
82
|
-
'cryptography',
|
|
83
|
-
'cffi',
|
|
84
|
-
'pycryptodome',
|
|
85
|
-
'pycryptodomex',
|
|
86
|
-
// NLP/ML packages (spacy ecosystem)
|
|
87
|
-
'spacy',
|
|
88
|
-
'thinc',
|
|
89
|
-
'blis',
|
|
90
|
-
'cymem',
|
|
91
|
-
'preshed',
|
|
92
|
-
'murmurhash',
|
|
93
|
-
'srsly',
|
|
94
|
-
// Other packages with C extensions or version-specific wheels
|
|
95
|
-
'regex',
|
|
96
|
-
'lxml',
|
|
97
|
-
'pillow',
|
|
98
|
-
'psycopg2',
|
|
99
|
-
'psycopg2-binary',
|
|
100
|
-
'grpcio',
|
|
101
|
-
'protobuf',
|
|
102
|
-
'pyzmq',
|
|
103
|
-
'greenlet',
|
|
104
|
-
'gevent',
|
|
105
|
-
'markupsafe',
|
|
106
|
-
'pyyaml',
|
|
107
|
-
'ruamel.yaml',
|
|
108
|
-
'msgpack',
|
|
109
|
-
'ujson',
|
|
110
|
-
'orjson',
|
|
111
|
-
// Packages that may have compatibility issues
|
|
112
|
-
'typed-ast',
|
|
113
|
-
'dataclasses',
|
|
114
|
-
'importlib-metadata',
|
|
115
|
-
'importlib_metadata',
|
|
116
|
-
'zipp',
|
|
117
|
-
'typing-extensions',
|
|
118
|
-
'typing_extensions',
|
|
119
|
-
'python_magic',
|
|
120
|
-
'pgpy',
|
|
121
|
-
'aiohttp',
|
|
122
|
-
'yarl',
|
|
123
|
-
'frozenlist',
|
|
124
|
-
'geoip2',
|
|
125
|
-
'extract_msg',
|
|
126
|
-
'click',
|
|
127
|
-
'ioc_finder',
|
|
128
|
-
'tzlocal',
|
|
129
|
-
// Swimlane packages
|
|
130
|
-
'datetime_parser',
|
|
131
|
-
'email_master',
|
|
132
|
-
'sw_aqueduct',
|
|
133
|
-
]);
|
|
134
10
|
class ConnectorGenerator {
|
|
135
|
-
/**
|
|
136
|
-
* For inputs with Type "record", resolve ValueType from applicationInfo.fields (field id = input.Value).
|
|
137
|
-
* Call after transform() so action YAML and temp_inputs get correct types.
|
|
138
|
-
*/
|
|
139
|
-
static patchRecordInputTypes(transformationResult, applicationInfo) {
|
|
140
|
-
if (!applicationInfo?.fields?.length)
|
|
141
|
-
return;
|
|
142
|
-
const fieldById = new Map(applicationInfo.fields.filter(f => f.id).map(f => [f.id, f]));
|
|
143
|
-
for (const input of transformationResult.inputs) {
|
|
144
|
-
const typeRaw = input.Type ?? input.type;
|
|
145
|
-
if (String(typeRaw).toLowerCase() !== 'record')
|
|
146
|
-
continue;
|
|
147
|
-
const fieldId = input.Value ?? input.value;
|
|
148
|
-
if (typeof fieldId !== 'string')
|
|
149
|
-
continue;
|
|
150
|
-
const field = fieldById.get(fieldId);
|
|
151
|
-
if (field) {
|
|
152
|
-
input.ValueType = applicationFieldTypeToSchemaType(field);
|
|
153
|
-
if (input.ValueType === 'array') {
|
|
154
|
-
input.arrayItemType = (field.fieldType || '').toLowerCase();
|
|
155
|
-
input.arrayItemValueType = (field.inputType || '').toLowerCase();
|
|
156
|
-
}
|
|
157
|
-
}
|
|
158
|
-
}
|
|
159
|
-
}
|
|
160
|
-
/**
|
|
161
|
-
* For outputs that map to date fields (by Value = field id in applicationInfo), compute which keys need
|
|
162
|
-
* conversion to ISO8601 and the timetype/format for each. Sets transformationResult.outputDateConversions.
|
|
163
|
-
* Uses the application denoted by each output block's ApplicationId (same level as Mappings) when present,
|
|
164
|
-
* otherwise the current application (taskApplicationId).
|
|
165
|
-
* - DataFormat "Standard" -> no conversion.
|
|
166
|
-
* - DataFormat "Unix EPOCH" -> timetype = UnixEpochUnit (seconds or milliseconds).
|
|
167
|
-
* - DataFormat "custom" -> timetype = customDataFormat.
|
|
168
|
-
* - Otherwise -> timetype = DataFormat (treat as custom format string).
|
|
169
|
-
*/
|
|
170
|
-
static patchOutputDateConversions(transformationResult, applicationInfoMap, currentApplicationId) {
|
|
171
|
-
transformationResult.outputDateConversions = [];
|
|
172
|
-
if (!applicationInfoMap || !transformationResult.outputs?.length)
|
|
173
|
-
return;
|
|
174
|
-
const keyToTimetype = new Map();
|
|
175
|
-
for (const output of transformationResult.outputs) {
|
|
176
|
-
const outputBlockAppId = output.ApplicationId ?? output.applicationId;
|
|
177
|
-
const appId = typeof outputBlockAppId === 'string' ? outputBlockAppId : transformationResult.taskApplicationId ?? currentApplicationId;
|
|
178
|
-
if (typeof appId !== 'string')
|
|
179
|
-
continue;
|
|
180
|
-
const applicationInfo = applicationInfoMap[appId];
|
|
181
|
-
if (!applicationInfo?.fields?.length)
|
|
182
|
-
continue;
|
|
183
|
-
const fieldById = new Map(applicationInfo.fields.filter((f) => f.id).map((f) => [f.id, f]));
|
|
184
|
-
const fieldId = output.Value ?? output.value;
|
|
185
|
-
if (typeof fieldId !== 'string')
|
|
186
|
-
continue;
|
|
187
|
-
const field = fieldById.get(fieldId);
|
|
188
|
-
const fieldType = (field?.fieldType ?? '').toLowerCase();
|
|
189
|
-
if (fieldType !== 'date')
|
|
190
|
-
continue;
|
|
191
|
-
const dataFormat = String(output.DataFormat ?? output.dataFormat ?? '').trim();
|
|
192
|
-
if (dataFormat === '' || dataFormat.toLowerCase() === 'standard')
|
|
193
|
-
continue;
|
|
194
|
-
let timetype;
|
|
195
|
-
if (dataFormat.toLowerCase() === 'unix epoch') {
|
|
196
|
-
timetype = String(output.UnixEpochUnit ?? output.unixEpochUnit ?? 'seconds').toLowerCase();
|
|
197
|
-
}
|
|
198
|
-
else if (dataFormat.toLowerCase() === 'custom') {
|
|
199
|
-
timetype = String(output.customDataFormat ?? output.CustomDataFormat ?? '').trim();
|
|
200
|
-
if (!timetype)
|
|
201
|
-
continue;
|
|
202
|
-
}
|
|
203
|
-
else {
|
|
204
|
-
timetype = dataFormat;
|
|
205
|
-
}
|
|
206
|
-
const key = output.Key;
|
|
207
|
-
if (!keyToTimetype.has(key))
|
|
208
|
-
keyToTimetype.set(key, timetype);
|
|
209
|
-
}
|
|
210
|
-
transformationResult.outputDateConversions = [...keyToTimetype.entries()].map(([key, timetype]) => ({ key, timetype }));
|
|
211
|
-
}
|
|
212
|
-
/**
|
|
213
|
-
* Initializes a forked plugin (copy base code, requirements, asset).
|
|
214
|
-
* Returns the set of runner-excluded package names that were skipped (e.g. ssdeep) so runner.sh can be updated.
|
|
215
|
-
*/
|
|
216
11
|
static async initializeForkedPlugin(transformedExport, fromDirectory, toDirectory) {
|
|
217
12
|
const { forkedName } = transformedExport;
|
|
218
13
|
console.log(`Initializing forked plugin: ${forkedName}`);
|
|
219
|
-
const excludedRunnerPackages = await this.generateRequirements(fromDirectory, toDirectory, forkedName);
|
|
220
14
|
await Promise.all([
|
|
221
15
|
this.createBaseCode(fromDirectory, toDirectory, forkedName),
|
|
16
|
+
this.generateRequirements(fromDirectory, toDirectory, forkedName),
|
|
222
17
|
this.generateAsset(fromDirectory, toDirectory, forkedName),
|
|
223
18
|
]);
|
|
224
19
|
console.log(`Forked plugin initialized: ${forkedName}`);
|
|
225
|
-
return excludedRunnerPackages;
|
|
226
20
|
}
|
|
227
21
|
static async generateLogo(toDirectory) {
|
|
228
22
|
const templatePath = (0, node_path_1.join)(__dirname, '../templates/migrator-runners/image.png');
|
|
@@ -232,37 +26,13 @@ class ConnectorGenerator {
|
|
|
232
26
|
}
|
|
233
27
|
static async generateBaseStructure(toDirectory) {
|
|
234
28
|
await node_fs_1.promises.mkdir((0, node_path_1.join)(toDirectory, 'connector', 'config', 'actions'), { recursive: true });
|
|
235
|
-
await node_fs_1.promises.mkdir((0, node_path_1.join)(toDirectory, 'image')
|
|
236
|
-
await node_fs_1.promises.mkdir((0, node_path_1.join)(toDirectory, 'docs')
|
|
237
|
-
await node_fs_1.promises.mkdir((0, node_path_1.join)(toDirectory, 'data')
|
|
238
|
-
await node_fs_1.promises.mkdir((0, node_path_1.join)(toDirectory, 'doc_images')
|
|
239
|
-
await node_fs_1.promises.mkdir((0, node_path_1.join)(toDirectory, 'connector', 'config', 'assets')
|
|
240
|
-
await node_fs_1.promises.mkdir((0, node_path_1.join)(toDirectory, 'connector', 'src')
|
|
241
|
-
|
|
242
|
-
const defaultRequirements = [
|
|
243
|
-
'cachetools>=4.2.4',
|
|
244
|
-
'certifi==2024.7.4',
|
|
245
|
-
'pendulum==3.0.0',
|
|
246
|
-
'pyjwt>=2.4.0',
|
|
247
|
-
'pyuri>=0.3,<0.4',
|
|
248
|
-
// 'requests[security]>=2,<3',
|
|
249
|
-
'six>=1.12.0',
|
|
250
|
-
'sortedcontainers==2.4.0',
|
|
251
|
-
'shortid==0.1.2',
|
|
252
|
-
'beautifulsoup4',
|
|
253
|
-
'pandas',
|
|
254
|
-
].join('\n') + '\n';
|
|
255
|
-
await this.createFile((0, node_path_1.join)(toDirectory, 'requirements.txt'), defaultRequirements);
|
|
256
|
-
// Copy swimlane template to connector/swimlane
|
|
257
|
-
try {
|
|
258
|
-
const swimlaneTemplatePath = (0, node_path_1.join)(__dirname, '../templates/swimlane');
|
|
259
|
-
const swimlaneDestinationPath = (0, node_path_1.join)(toDirectory, 'connector', 'swimlane');
|
|
260
|
-
await this.copyDirectoryRecursive(swimlaneTemplatePath, swimlaneDestinationPath);
|
|
261
|
-
}
|
|
262
|
-
catch (error) {
|
|
263
|
-
console.error(`Error copying swimlane template: ${error}`);
|
|
264
|
-
// Continue even if swimlane template copy fails
|
|
265
|
-
}
|
|
29
|
+
await node_fs_1.promises.mkdir((0, node_path_1.join)(toDirectory, 'image'));
|
|
30
|
+
await node_fs_1.promises.mkdir((0, node_path_1.join)(toDirectory, 'docs'));
|
|
31
|
+
await node_fs_1.promises.mkdir((0, node_path_1.join)(toDirectory, 'data'));
|
|
32
|
+
await node_fs_1.promises.mkdir((0, node_path_1.join)(toDirectory, 'doc_images'));
|
|
33
|
+
await node_fs_1.promises.mkdir((0, node_path_1.join)(toDirectory, 'connector', 'config', 'assets'));
|
|
34
|
+
await node_fs_1.promises.mkdir((0, node_path_1.join)(toDirectory, 'connector', 'src'));
|
|
35
|
+
await this.createFile((0, node_path_1.join)(toDirectory, 'requirements.txt'), '');
|
|
266
36
|
await this.createFile((0, node_path_1.join)(toDirectory, 'docs', 'CHANGELOG.md'), '');
|
|
267
37
|
await this.createFile((0, node_path_1.join)(toDirectory, 'docs', 'README.md'), '# Example Readme');
|
|
268
38
|
await this.createFile((0, node_path_1.join)(toDirectory, 'docs', 'EXTERNAL_README.md'), '# Example External Readme');
|
|
@@ -282,25 +52,18 @@ class ConnectorGenerator {
|
|
|
282
52
|
}
|
|
283
53
|
static async generateAction(transformedExport, toDirectory) {
|
|
284
54
|
let content;
|
|
285
|
-
const outputDateConversions = transformedExport.outputDateConversions;
|
|
286
55
|
if (transformedExport.type === 'script') {
|
|
287
|
-
content = transformedExport.error ? `Error: ${transformedExport.error}` : await this.getActionContentScript(transformedExport.script
|
|
56
|
+
content = transformedExport.error ? `Error: ${transformedExport.error}` : await this.getActionContentScript(transformedExport.script);
|
|
288
57
|
}
|
|
289
58
|
else {
|
|
290
|
-
content = transformedExport.error ? `Error: ${transformedExport.error}` : await this.getActionContentFork(transformedExport.script
|
|
59
|
+
content = transformedExport.error ? `Error: ${transformedExport.error}` : await this.getActionContentFork(transformedExport.script);
|
|
291
60
|
}
|
|
292
|
-
content = this.replaceTaskExecuteRequestCall(content);
|
|
293
61
|
const outputPath = (0, node_path_1.join)(toDirectory, 'connector', 'src', `${transformedExport.exportUid}.py`);
|
|
294
62
|
await this.createFile(outputPath, content);
|
|
295
63
|
}
|
|
296
64
|
static async generateAsset(fromDirectory, toDirectory, packageName) {
|
|
297
65
|
const assetPath = (0, node_path_1.join)(fromDirectory, 'packages', packageName, 'imports', 'asset.json');
|
|
298
|
-
|
|
299
|
-
// e.g., sw_google_gsuite -> google_gsuite_asset.yaml with name: google_gsuite_asset
|
|
300
|
-
const assetNameBase = packageName.replace(/^sw_/, '').toLowerCase();
|
|
301
|
-
const assetFileName = `${assetNameBase}_asset.yaml`;
|
|
302
|
-
const assetName = `${assetNameBase}_asset`;
|
|
303
|
-
const destinationPath = (0, node_path_1.join)(toDirectory, 'connector', 'config', 'assets', assetFileName);
|
|
66
|
+
const destinationPath = (0, node_path_1.join)(toDirectory, 'connector', 'config', 'assets', 'asset.yaml');
|
|
304
67
|
try {
|
|
305
68
|
await node_fs_1.promises.access(assetPath);
|
|
306
69
|
const assetContent = await node_fs_1.promises.readFile(assetPath, 'utf8');
|
|
@@ -309,15 +72,17 @@ class ConnectorGenerator {
|
|
|
309
72
|
const requiredInputs = [];
|
|
310
73
|
for (const [key, rawValue] of Object.entries(assetJson.inputParameters || {})) {
|
|
311
74
|
const value = rawValue;
|
|
312
|
-
const schemaType = assetInputTypeToSchemaType(value.type);
|
|
313
75
|
inputProperties[key] = {
|
|
314
76
|
title: value.name || key,
|
|
315
77
|
description: value.description || '',
|
|
316
|
-
type:
|
|
78
|
+
type: 'string',
|
|
317
79
|
};
|
|
318
80
|
if (value.type === 4) {
|
|
319
81
|
inputProperties[key].format = 'password';
|
|
320
82
|
}
|
|
83
|
+
if (value.example !== undefined) {
|
|
84
|
+
inputProperties[key].example = value.example;
|
|
85
|
+
}
|
|
321
86
|
if (value.default !== undefined) {
|
|
322
87
|
inputProperties[key].default = value.default;
|
|
323
88
|
}
|
|
@@ -325,163 +90,22 @@ class ConnectorGenerator {
|
|
|
325
90
|
requiredInputs.push(key);
|
|
326
91
|
}
|
|
327
92
|
}
|
|
328
|
-
const
|
|
93
|
+
const assetYaml = js_yaml_1.default.dump({
|
|
329
94
|
schema: 'asset/1',
|
|
330
|
-
name:
|
|
331
|
-
title: assetJson.name
|
|
332
|
-
description: assetJson.description
|
|
95
|
+
name: 'asset',
|
|
96
|
+
title: assetJson.name,
|
|
97
|
+
description: assetJson.description,
|
|
333
98
|
inputs: {
|
|
334
99
|
type: 'object',
|
|
335
100
|
properties: inputProperties,
|
|
336
101
|
required: requiredInputs.length > 0 ? requiredInputs : undefined,
|
|
337
102
|
},
|
|
338
103
|
meta: {},
|
|
339
|
-
};
|
|
340
|
-
|
|
341
|
-
if (!assetData.title || typeof assetData.title !== 'string' || assetData.title.trim() === '') {
|
|
342
|
-
assetData.title = 'Asset';
|
|
343
|
-
}
|
|
344
|
-
else {
|
|
345
|
-
assetData.title = assetData.title.trim();
|
|
346
|
-
if (assetData.title.length > 50) {
|
|
347
|
-
assetData.title = assetData.title.slice(0, 50);
|
|
348
|
-
}
|
|
349
|
-
}
|
|
350
|
-
const assetYaml = js_yaml_1.default.dump(assetData, { lineWidth: -1, noRefs: true });
|
|
351
|
-
// Validate the YAML can be parsed back and has required fields
|
|
352
|
-
let parsed;
|
|
353
|
-
try {
|
|
354
|
-
parsed = js_yaml_1.default.load(assetYaml);
|
|
355
|
-
if (!parsed || !parsed.title) {
|
|
356
|
-
throw new Error('Generated asset YAML is missing required title field');
|
|
357
|
-
}
|
|
358
|
-
}
|
|
359
|
-
catch (parseError) {
|
|
360
|
-
console.error(`Generated invalid asset YAML for ${packageName}:`, parseError);
|
|
361
|
-
console.error('Asset data:', JSON.stringify(assetData, null, 2));
|
|
362
|
-
throw new Error(`Failed to generate valid asset YAML for ${packageName}: ${parseError}`);
|
|
363
|
-
}
|
|
364
|
-
// Write atomically using a temporary file to prevent corruption
|
|
365
|
-
const dir = (0, node_path_1.join)(destinationPath, '..');
|
|
366
|
-
await node_fs_1.promises.mkdir(dir, { recursive: true });
|
|
367
|
-
// Write to a temporary file first, then rename (atomic operation)
|
|
368
|
-
const tempFile = (0, node_path_1.join)((0, node_os_1.tmpdir)(), `${assetName}-${Date.now()}-${Math.random().toString(36).slice(7)}.yaml`);
|
|
369
|
-
try {
|
|
370
|
-
await node_fs_1.promises.writeFile(tempFile, assetYaml, 'utf8');
|
|
371
|
-
// Atomically move the temp file to the final location
|
|
372
|
-
await node_fs_1.promises.rename(tempFile, destinationPath);
|
|
373
|
-
console.log(`Generated asset file: ${assetFileName} (name: ${assetName}) for package: ${packageName}`);
|
|
374
|
-
}
|
|
375
|
-
catch (writeError) {
|
|
376
|
-
// Clean up temp file if it exists
|
|
377
|
-
await node_fs_1.promises.unlink(tempFile).catch(() => { });
|
|
378
|
-
throw writeError;
|
|
379
|
-
}
|
|
380
|
-
}
|
|
381
|
-
catch (error) {
|
|
382
|
-
console.error(`Error generating asset ${assetFileName} for ${packageName}:`, error);
|
|
383
|
-
}
|
|
384
|
-
}
|
|
385
|
-
/**
|
|
386
|
-
* Adds asset parameters from task input mappings to a separate input asset file.
|
|
387
|
-
* Creates input_asset.yaml to avoid overwriting existing asset.yaml from forked plugins.
|
|
388
|
-
*/
|
|
389
|
-
static async addAssetParameters(toDirectory, assetParameters, applicationName) {
|
|
390
|
-
if (assetParameters.length === 0) {
|
|
391
|
-
return;
|
|
392
|
-
}
|
|
393
|
-
const destinationPath = (0, node_path_1.join)(toDirectory, 'connector', 'config', 'assets', 'input_asset.yaml');
|
|
394
|
-
try {
|
|
395
|
-
// Create title and truncate to 50 characters
|
|
396
|
-
const fullTitle = applicationName ? `${applicationName} Key Store` : 'Input Asset';
|
|
397
|
-
const assetTitle = fullTitle.length > 50 ? fullTitle.slice(0, 50) : fullTitle;
|
|
398
|
-
let assetData = {
|
|
399
|
-
schema: 'asset/1',
|
|
400
|
-
name: 'input_asset',
|
|
401
|
-
title: assetTitle,
|
|
402
|
-
description: '',
|
|
403
|
-
inputs: {
|
|
404
|
-
type: 'object',
|
|
405
|
-
properties: {},
|
|
406
|
-
required: [],
|
|
407
|
-
},
|
|
408
|
-
meta: {},
|
|
409
|
-
};
|
|
410
|
-
// Try to read existing input_asset.yaml (in case we're adding more parameters)
|
|
411
|
-
try {
|
|
412
|
-
const existingContent = await node_fs_1.promises.readFile(destinationPath, 'utf8');
|
|
413
|
-
assetData = js_yaml_1.default.load(existingContent);
|
|
414
|
-
// Update title if application name is provided and truncate to 50 characters
|
|
415
|
-
if (applicationName) {
|
|
416
|
-
assetData.title = assetTitle;
|
|
417
|
-
}
|
|
418
|
-
else if (!assetData.title || assetData.title.trim() === '') {
|
|
419
|
-
assetData.title = 'Input Asset';
|
|
420
|
-
}
|
|
421
|
-
else {
|
|
422
|
-
// Ensure existing title is also truncated
|
|
423
|
-
assetData.title = assetData.title.trim();
|
|
424
|
-
if (assetData.title.length > 50) {
|
|
425
|
-
assetData.title = assetData.title.slice(0, 50);
|
|
426
|
-
}
|
|
427
|
-
}
|
|
428
|
-
// Ensure structure exists
|
|
429
|
-
if (!assetData.inputs) {
|
|
430
|
-
assetData.inputs = { type: 'object', properties: {}, required: [] };
|
|
431
|
-
}
|
|
432
|
-
if (!assetData.inputs.properties) {
|
|
433
|
-
assetData.inputs.properties = {};
|
|
434
|
-
}
|
|
435
|
-
if (!assetData.inputs.required) {
|
|
436
|
-
assetData.inputs.required = [];
|
|
437
|
-
}
|
|
438
|
-
}
|
|
439
|
-
catch {
|
|
440
|
-
// Asset doesn't exist, use default structure
|
|
441
|
-
// Ensure directory exists
|
|
442
|
-
await node_fs_1.promises.mkdir((0, node_path_1.join)(toDirectory, 'connector', 'config', 'assets'), { recursive: true });
|
|
443
|
-
}
|
|
444
|
-
// Add asset parameters from tasks
|
|
445
|
-
for (const param of assetParameters) {
|
|
446
|
-
const key = param.Key;
|
|
447
|
-
if (!assetData.inputs.properties[key]) {
|
|
448
|
-
assetData.inputs.properties[key] = {
|
|
449
|
-
title: key,
|
|
450
|
-
type: 'string',
|
|
451
|
-
};
|
|
452
|
-
// If it's a credentials type, mark as password format
|
|
453
|
-
if (param.Type === 'credentials' || param.Type === 'asset') {
|
|
454
|
-
assetData.inputs.properties[key].format = 'password';
|
|
455
|
-
}
|
|
456
|
-
// Add example if provided
|
|
457
|
-
if (param.Example !== undefined && param.Example !== null) {
|
|
458
|
-
assetData.inputs.properties[key].examples = Array.isArray(param.Example) ? param.Example : [param.Example];
|
|
459
|
-
}
|
|
460
|
-
}
|
|
461
|
-
}
|
|
462
|
-
// Title is already set above, just ensure it's truncated if needed
|
|
463
|
-
if (assetData.title && assetData.title.length > 50) {
|
|
464
|
-
assetData.title = assetData.title.slice(0, 50);
|
|
465
|
-
}
|
|
466
|
-
// Write atomically using a temporary file to prevent corruption
|
|
467
|
-
const dir = (0, node_path_1.join)(destinationPath, '..');
|
|
468
|
-
await node_fs_1.promises.mkdir(dir, { recursive: true });
|
|
469
|
-
const assetYaml = js_yaml_1.default.dump(assetData, { lineWidth: -1, noRefs: true });
|
|
470
|
-
// Write to a temporary file first, then rename (atomic operation)
|
|
471
|
-
const tempFile = (0, node_path_1.join)((0, node_os_1.tmpdir)(), `input_asset-${Date.now()}-${Math.random().toString(36).slice(7)}.yaml`);
|
|
472
|
-
try {
|
|
473
|
-
await node_fs_1.promises.writeFile(tempFile, assetYaml, 'utf8');
|
|
474
|
-
// Atomically move the temp file to the final location
|
|
475
|
-
await node_fs_1.promises.rename(tempFile, destinationPath);
|
|
476
|
-
}
|
|
477
|
-
catch (writeError) {
|
|
478
|
-
// Clean up temp file if it exists
|
|
479
|
-
await node_fs_1.promises.unlink(tempFile).catch(() => { });
|
|
480
|
-
throw writeError;
|
|
481
|
-
}
|
|
104
|
+
});
|
|
105
|
+
await node_fs_1.promises.writeFile(destinationPath, assetYaml, 'utf8');
|
|
482
106
|
}
|
|
483
107
|
catch (error) {
|
|
484
|
-
console.error(
|
|
108
|
+
console.error(`Error generating asset for ${packageName}:`, error);
|
|
485
109
|
}
|
|
486
110
|
}
|
|
487
111
|
static async extractZip(fromDirectory, packageName) {
|
|
@@ -500,250 +124,7 @@ class ConnectorGenerator {
|
|
|
500
124
|
return null;
|
|
501
125
|
}
|
|
502
126
|
}
|
|
503
|
-
static async addExtractedImportsToRequirements(toDirectory, taskImports) {
|
|
504
|
-
const requirementsPath = (0, node_path_1.join)(toDirectory, 'requirements.txt');
|
|
505
|
-
if (taskImports.size === 0) {
|
|
506
|
-
return;
|
|
507
|
-
}
|
|
508
|
-
try {
|
|
509
|
-
// Filter out standard library modules that don't need to be in requirements.txt
|
|
510
|
-
const standardLibraryModules = new Set([
|
|
511
|
-
'json',
|
|
512
|
-
'os',
|
|
513
|
-
'sys',
|
|
514
|
-
'time',
|
|
515
|
-
'datetime',
|
|
516
|
-
're',
|
|
517
|
-
'math',
|
|
518
|
-
'random',
|
|
519
|
-
'string',
|
|
520
|
-
'collections',
|
|
521
|
-
'itertools',
|
|
522
|
-
'functools',
|
|
523
|
-
'operator',
|
|
524
|
-
'copy',
|
|
525
|
-
'pickle',
|
|
526
|
-
'base64',
|
|
527
|
-
'hashlib',
|
|
528
|
-
'urllib',
|
|
529
|
-
'http',
|
|
530
|
-
'email',
|
|
531
|
-
'csv',
|
|
532
|
-
'xml',
|
|
533
|
-
'html',
|
|
534
|
-
'logging',
|
|
535
|
-
'threading',
|
|
536
|
-
'multiprocessing',
|
|
537
|
-
'subprocess',
|
|
538
|
-
'socket',
|
|
539
|
-
'ssl',
|
|
540
|
-
'pathlib',
|
|
541
|
-
'shutil',
|
|
542
|
-
'tempfile',
|
|
543
|
-
'io',
|
|
544
|
-
'codecs',
|
|
545
|
-
'unicodedata',
|
|
546
|
-
'struct',
|
|
547
|
-
'array',
|
|
548
|
-
'queue',
|
|
549
|
-
'heapq',
|
|
550
|
-
'bisect',
|
|
551
|
-
'weakref',
|
|
552
|
-
'types',
|
|
553
|
-
'inspect',
|
|
554
|
-
'traceback',
|
|
555
|
-
'warnings',
|
|
556
|
-
'contextlib',
|
|
557
|
-
'abc',
|
|
558
|
-
'atexit',
|
|
559
|
-
'gc',
|
|
560
|
-
'locale',
|
|
561
|
-
'gettext',
|
|
562
|
-
'argparse',
|
|
563
|
-
'configparser',
|
|
564
|
-
'fileinput',
|
|
565
|
-
'glob',
|
|
566
|
-
'fnmatch',
|
|
567
|
-
'linecache',
|
|
568
|
-
'stat',
|
|
569
|
-
'errno',
|
|
570
|
-
'ctypes',
|
|
571
|
-
'mmap',
|
|
572
|
-
'select',
|
|
573
|
-
'signal',
|
|
574
|
-
'pwd',
|
|
575
|
-
'grp',
|
|
576
|
-
'termios',
|
|
577
|
-
'tty',
|
|
578
|
-
'pty',
|
|
579
|
-
'fcntl',
|
|
580
|
-
'resource',
|
|
581
|
-
'syslog',
|
|
582
|
-
'platform',
|
|
583
|
-
'pipes',
|
|
584
|
-
'sched',
|
|
585
|
-
'asyncio',
|
|
586
|
-
'concurrent',
|
|
587
|
-
'dbm',
|
|
588
|
-
'sqlite3',
|
|
589
|
-
'zlib',
|
|
590
|
-
'gzip',
|
|
591
|
-
'bz2',
|
|
592
|
-
'lzma',
|
|
593
|
-
'zipfile',
|
|
594
|
-
'tarfile',
|
|
595
|
-
'shlex',
|
|
596
|
-
'readline',
|
|
597
|
-
'rlcompleter',
|
|
598
|
-
'cmd',
|
|
599
|
-
'doctest',
|
|
600
|
-
'unittest',
|
|
601
|
-
'pdb',
|
|
602
|
-
'profile',
|
|
603
|
-
'pstats',
|
|
604
|
-
'timeit',
|
|
605
|
-
'trace',
|
|
606
|
-
'cgitb',
|
|
607
|
-
'pydoc',
|
|
608
|
-
'dis',
|
|
609
|
-
'pickletools',
|
|
610
|
-
'formatter',
|
|
611
|
-
'msilib',
|
|
612
|
-
'msvcrt',
|
|
613
|
-
'nt',
|
|
614
|
-
'ntpath',
|
|
615
|
-
'nturl2path',
|
|
616
|
-
'winreg',
|
|
617
|
-
'winsound',
|
|
618
|
-
'posix',
|
|
619
|
-
'posixpath',
|
|
620
|
-
'pwd',
|
|
621
|
-
'spwd',
|
|
622
|
-
'grp',
|
|
623
|
-
'crypt',
|
|
624
|
-
'termios',
|
|
625
|
-
'tty',
|
|
626
|
-
'pty',
|
|
627
|
-
'fcntl',
|
|
628
|
-
'pipes',
|
|
629
|
-
'resource',
|
|
630
|
-
'nis',
|
|
631
|
-
'syslog',
|
|
632
|
-
'optparse',
|
|
633
|
-
'imp',
|
|
634
|
-
'importlib',
|
|
635
|
-
'keyword',
|
|
636
|
-
'parser',
|
|
637
|
-
'ast',
|
|
638
|
-
'symtable',
|
|
639
|
-
'symbol',
|
|
640
|
-
'token',
|
|
641
|
-
'tokenize',
|
|
642
|
-
'tabnanny',
|
|
643
|
-
'py_compile',
|
|
644
|
-
'compileall',
|
|
645
|
-
'pyclbr',
|
|
646
|
-
'bdb',
|
|
647
|
-
'pdb',
|
|
648
|
-
'profile',
|
|
649
|
-
'pstats',
|
|
650
|
-
'timeit',
|
|
651
|
-
'trace',
|
|
652
|
-
'cgitb',
|
|
653
|
-
'pydoc',
|
|
654
|
-
'doctest',
|
|
655
|
-
'unittest',
|
|
656
|
-
'test',
|
|
657
|
-
'lib2to3',
|
|
658
|
-
'distutils',
|
|
659
|
-
'ensurepip',
|
|
660
|
-
'venv',
|
|
661
|
-
'wsgiref',
|
|
662
|
-
'html',
|
|
663
|
-
'http',
|
|
664
|
-
'urllib',
|
|
665
|
-
'xmlrpc',
|
|
666
|
-
'ipaddress',
|
|
667
|
-
'secrets',
|
|
668
|
-
'statistics',
|
|
669
|
-
'pathlib',
|
|
670
|
-
'enum',
|
|
671
|
-
'numbers',
|
|
672
|
-
'fractions',
|
|
673
|
-
'decimal',
|
|
674
|
-
'cmath',
|
|
675
|
-
'array',
|
|
676
|
-
'memoryview',
|
|
677
|
-
'collections',
|
|
678
|
-
'heapq',
|
|
679
|
-
'bisect',
|
|
680
|
-
'array',
|
|
681
|
-
'weakref',
|
|
682
|
-
'types',
|
|
683
|
-
'copy',
|
|
684
|
-
'pprint',
|
|
685
|
-
'reprlib',
|
|
686
|
-
'dataclasses',
|
|
687
|
-
'dataclasses_json',
|
|
688
|
-
'typing',
|
|
689
|
-
'typing_extensions',
|
|
690
|
-
'backports',
|
|
691
|
-
'builtins',
|
|
692
|
-
'__builtin__',
|
|
693
|
-
'__future__',
|
|
694
|
-
]);
|
|
695
|
-
// Filter out standard library, excluded packages, and already included packages
|
|
696
|
-
const importsToAdd = [...taskImports]
|
|
697
|
-
.filter(importName => {
|
|
698
|
-
// Skip standard library modules
|
|
699
|
-
if (standardLibraryModules.has(importName)) {
|
|
700
|
-
return false;
|
|
701
|
-
}
|
|
702
|
-
// Skip excluded packages (provided by runtime or installed via other means)
|
|
703
|
-
if (EXCLUDED_PACKAGES.has(importName.toLowerCase())) {
|
|
704
|
-
return false;
|
|
705
|
-
}
|
|
706
|
-
// Skip if it's already in requirements (we'll check this by reading the file)
|
|
707
|
-
return true;
|
|
708
|
-
})
|
|
709
|
-
.filter(Boolean);
|
|
710
|
-
if (importsToAdd.length === 0) {
|
|
711
|
-
return;
|
|
712
|
-
}
|
|
713
|
-
// Read existing requirements to avoid duplicates
|
|
714
|
-
let existingRequirements = '';
|
|
715
|
-
try {
|
|
716
|
-
existingRequirements = await node_fs_1.promises.readFile(requirementsPath, 'utf8');
|
|
717
|
-
}
|
|
718
|
-
catch {
|
|
719
|
-
// File might not exist yet, that's okay
|
|
720
|
-
}
|
|
721
|
-
const existingPackages = new Set(existingRequirements
|
|
722
|
-
.split('\n')
|
|
723
|
-
.map(line => line.trim().split(/[!<=>]/)[0].toLowerCase())
|
|
724
|
-
.filter(Boolean));
|
|
725
|
-
// Add imports that aren't already in requirements.txt
|
|
726
|
-
const newPackages = importsToAdd
|
|
727
|
-
.filter(importName => {
|
|
728
|
-
const packageName = importName.toLowerCase();
|
|
729
|
-
return !existingPackages.has(packageName);
|
|
730
|
-
})
|
|
731
|
-
.map(importName => importName.toLowerCase());
|
|
732
|
-
if (newPackages.length > 0) {
|
|
733
|
-
await node_fs_1.promises.appendFile(requirementsPath, newPackages.join('\n') + '\n');
|
|
734
|
-
console.log(`Added ${newPackages.length} extracted import(s) to requirements.txt`);
|
|
735
|
-
}
|
|
736
|
-
}
|
|
737
|
-
catch (error) {
|
|
738
|
-
console.error('Error adding extracted imports to requirements.txt:', error);
|
|
739
|
-
}
|
|
740
|
-
}
|
|
741
|
-
/**
|
|
742
|
-
* Appends forked plugin dependencies to requirements.txt.
|
|
743
|
-
* Returns the set of package names that were excluded but need runner.sh (e.g. ssdeep).
|
|
744
|
-
*/
|
|
745
127
|
static async generateRequirements(fromDirectory, toDirectory, packageName) {
|
|
746
|
-
const excludedRunner = new Set();
|
|
747
128
|
const packageExtractedDir = (0, node_path_1.join)(fromDirectory, 'packages', packageName);
|
|
748
129
|
const requirementsPath = (0, node_path_1.join)(toDirectory, 'requirements.txt');
|
|
749
130
|
try {
|
|
@@ -751,7 +132,7 @@ class ConnectorGenerator {
|
|
|
751
132
|
const whlFiles = files.filter(file => file.endsWith('.whl'));
|
|
752
133
|
if (whlFiles.length === 0) {
|
|
753
134
|
console.warn(`No .whl files found in ${packageExtractedDir}`);
|
|
754
|
-
return
|
|
135
|
+
return;
|
|
755
136
|
}
|
|
756
137
|
const dependencies = whlFiles
|
|
757
138
|
.map(whlFile => {
|
|
@@ -764,253 +145,14 @@ class ConnectorGenerator {
|
|
|
764
145
|
if (packageNameFromWhl === packageName) {
|
|
765
146
|
return null;
|
|
766
147
|
}
|
|
767
|
-
// Skip excluded packages (provided by runtime or installed via other means)
|
|
768
|
-
if (EXCLUDED_PACKAGES.has(packageNameFromWhl.toLowerCase())) {
|
|
769
|
-
if (RUNNER_EXCLUDED_PACKAGES.has(packageNameFromWhl.toLowerCase())) {
|
|
770
|
-
excludedRunner.add(packageNameFromWhl.toLowerCase());
|
|
771
|
-
}
|
|
772
|
-
return null;
|
|
773
|
-
}
|
|
774
148
|
return `${packageNameFromWhl}==${packageVersion}`;
|
|
775
149
|
})
|
|
776
150
|
.filter(Boolean);
|
|
777
151
|
await node_fs_1.promises.appendFile(requirementsPath, dependencies.join('\n') + '\n');
|
|
778
152
|
console.log(`requirements.txt generated at: ${requirementsPath}`);
|
|
779
|
-
return excludedRunner;
|
|
780
153
|
}
|
|
781
154
|
catch (error) {
|
|
782
155
|
console.error('Error generating requirements.txt:', error);
|
|
783
|
-
return excludedRunner;
|
|
784
|
-
}
|
|
785
|
-
}
|
|
786
|
-
/**
|
|
787
|
-
* Resolves version conflicts by taking the highest version requirement.
|
|
788
|
-
* Handles common cases like ==, >=, <=, >, < constraints.
|
|
789
|
-
*/
|
|
790
|
-
static resolveVersionConflict(packageName, requirements) {
|
|
791
|
-
if (requirements.length === 1) {
|
|
792
|
-
return requirements[0];
|
|
793
|
-
}
|
|
794
|
-
const versionInfo = requirements.map(req => {
|
|
795
|
-
// Match version specifiers: ==2.0, >=2.0, <=2.0, >2.0, <2.0, ~=2.0
|
|
796
|
-
const match = req.match(/([!<=>~]+)\s*([\d.]+(?:[\dA-Za-z]*)?)/);
|
|
797
|
-
if (!match) {
|
|
798
|
-
return { req, operator: null, version: null, versionParts: null };
|
|
799
|
-
}
|
|
800
|
-
const operator = match[1];
|
|
801
|
-
const versionStr = match[2];
|
|
802
|
-
// Parse version into parts for comparison
|
|
803
|
-
const versionParts = versionStr.split('.').map(part => {
|
|
804
|
-
const numMatch = part.match(/^(\d+)/);
|
|
805
|
-
return numMatch ? Number.parseInt(numMatch[1], 10) : 0;
|
|
806
|
-
});
|
|
807
|
-
return { req, operator, version: versionStr, versionParts };
|
|
808
|
-
});
|
|
809
|
-
// Filter out requirements without version info
|
|
810
|
-
const withVersions = versionInfo.filter((v) => v.version !== null && v.operator !== null && v.versionParts !== null);
|
|
811
|
-
if (withVersions.length === 0) {
|
|
812
|
-
// No version info, return first
|
|
813
|
-
console.warn(`Multiple requirements for ${packageName} without version info: ${requirements.join(', ')}. Using: ${requirements[0]}`);
|
|
814
|
-
return requirements[0];
|
|
815
|
-
}
|
|
816
|
-
// Find the highest version
|
|
817
|
-
let highest = withVersions[0];
|
|
818
|
-
for (let i = 1; i < withVersions.length; i++) {
|
|
819
|
-
const current = withVersions[i];
|
|
820
|
-
// Compare versions
|
|
821
|
-
const comparison = this.compareVersions(current.versionParts, highest.versionParts);
|
|
822
|
-
if (comparison > 0) {
|
|
823
|
-
// Current is higher
|
|
824
|
-
highest = current;
|
|
825
|
-
}
|
|
826
|
-
else if (comparison === 0) {
|
|
827
|
-
// Same version, prefer == over >= or other operators
|
|
828
|
-
if (current.operator === '==' && highest.operator !== '==') {
|
|
829
|
-
highest = current;
|
|
830
|
-
}
|
|
831
|
-
else if (current.operator === '>=' && highest.operator === '>=') {
|
|
832
|
-
// Both are >= with same version, keep current (they're equivalent)
|
|
833
|
-
// No change needed
|
|
834
|
-
}
|
|
835
|
-
}
|
|
836
|
-
}
|
|
837
|
-
// Log if we're resolving a conflict
|
|
838
|
-
if (withVersions.length > 1) {
|
|
839
|
-
const allReqs = requirements.join(', ');
|
|
840
|
-
console.log(`Resolved version conflict for ${packageName}: ${allReqs} -> ${highest.req}`);
|
|
841
|
-
}
|
|
842
|
-
return highest.req;
|
|
843
|
-
}
|
|
844
|
-
/**
|
|
845
|
-
* Compares two version arrays (e.g., [2, 5, 0] vs [2, 4, 1]).
|
|
846
|
-
* Returns: positive if v1 > v2, negative if v1 < v2, 0 if equal.
|
|
847
|
-
*/
|
|
848
|
-
static compareVersions(v1, v2) {
|
|
849
|
-
const maxLength = Math.max(v1.length, v2.length);
|
|
850
|
-
for (let i = 0; i < maxLength; i++) {
|
|
851
|
-
const part1 = v1[i] || 0;
|
|
852
|
-
const part2 = v2[i] || 0;
|
|
853
|
-
if (part1 > part2)
|
|
854
|
-
return 1;
|
|
855
|
-
if (part1 < part2)
|
|
856
|
-
return -1;
|
|
857
|
-
}
|
|
858
|
-
return 0;
|
|
859
|
-
}
|
|
860
|
-
/**
|
|
861
|
-
* Deduplicates requirements.txt and resolves version conflicts.
|
|
862
|
-
* When multiple versions of the same package are specified, keeps the highest version.
|
|
863
|
-
*/
|
|
864
|
-
static async deduplicateRequirements(toDirectory) {
|
|
865
|
-
const requirementsPath = (0, node_path_1.join)(toDirectory, 'requirements.txt');
|
|
866
|
-
try {
|
|
867
|
-
let content = '';
|
|
868
|
-
try {
|
|
869
|
-
content = await node_fs_1.promises.readFile(requirementsPath, 'utf8');
|
|
870
|
-
}
|
|
871
|
-
catch {
|
|
872
|
-
// File might not exist, nothing to deduplicate
|
|
873
|
-
return;
|
|
874
|
-
}
|
|
875
|
-
const lines = content
|
|
876
|
-
.split('\n')
|
|
877
|
-
.map(line => line.trim())
|
|
878
|
-
.filter(line => line && !line.startsWith('#'));
|
|
879
|
-
if (lines.length === 0) {
|
|
880
|
-
return;
|
|
881
|
-
}
|
|
882
|
-
// Parse requirements into a map: packageName -> requirements[]
|
|
883
|
-
const requirementsMap = new Map();
|
|
884
|
-
for (const line of lines) {
|
|
885
|
-
// Extract package name (everything before version specifiers like ==, >=, <=, >, <, !=)
|
|
886
|
-
// Handle packages with extras like requests[security]>=2.0
|
|
887
|
-
const packageMatch = line.match(/^([\w[\]-]+?)(?:\s*[!<=>]|$)/);
|
|
888
|
-
if (!packageMatch)
|
|
889
|
-
continue;
|
|
890
|
-
// Extract base package name (remove extras like [security])
|
|
891
|
-
const fullPackageName = packageMatch[1];
|
|
892
|
-
const basePackageName = fullPackageName.split('[')[0].toLowerCase();
|
|
893
|
-
const requirement = line.trim();
|
|
894
|
-
if (!requirementsMap.has(basePackageName)) {
|
|
895
|
-
requirementsMap.set(basePackageName, []);
|
|
896
|
-
}
|
|
897
|
-
requirementsMap.get(basePackageName).push(requirement);
|
|
898
|
-
}
|
|
899
|
-
// Deduplicate and resolve conflicts
|
|
900
|
-
const deduplicated = [];
|
|
901
|
-
const strippedPackages = [];
|
|
902
|
-
for (const [packageName, reqs] of requirementsMap.entries()) {
|
|
903
|
-
// Skip excluded packages (provided by runtime or installed via other means)
|
|
904
|
-
if (EXCLUDED_PACKAGES.has(packageName)) {
|
|
905
|
-
continue;
|
|
906
|
-
}
|
|
907
|
-
// Strip version for packages known to have Python 3.11+ compatibility issues
|
|
908
|
-
if (PACKAGES_TO_STRIP_VERSION.has(packageName)) {
|
|
909
|
-
// Extract the package name with extras (e.g., requests[security] -> requests[security])
|
|
910
|
-
const reqWithExtras = reqs[0].match(/^([\w-]+(?:\[[^\]]+])?)/);
|
|
911
|
-
const packageWithExtras = reqWithExtras ? reqWithExtras[1] : packageName;
|
|
912
|
-
deduplicated.push(packageWithExtras);
|
|
913
|
-
strippedPackages.push(packageName);
|
|
914
|
-
continue;
|
|
915
|
-
}
|
|
916
|
-
if (reqs.length === 1) {
|
|
917
|
-
// Single requirement, use it as-is
|
|
918
|
-
deduplicated.push(reqs[0]);
|
|
919
|
-
}
|
|
920
|
-
else {
|
|
921
|
-
// Multiple requirements for same package - need to resolve
|
|
922
|
-
const uniqueReqs = [...new Set(reqs)];
|
|
923
|
-
if (uniqueReqs.length === 1) {
|
|
924
|
-
// All are identical, use one
|
|
925
|
-
deduplicated.push(uniqueReqs[0]);
|
|
926
|
-
}
|
|
927
|
-
else {
|
|
928
|
-
// Different versions/constraints - resolve by taking the higher version
|
|
929
|
-
const resolved = this.resolveVersionConflict(packageName, uniqueReqs);
|
|
930
|
-
deduplicated.push(resolved);
|
|
931
|
-
}
|
|
932
|
-
}
|
|
933
|
-
}
|
|
934
|
-
// Log stripped packages
|
|
935
|
-
if (strippedPackages.length > 0) {
|
|
936
|
-
console.log(`Stripped version constraints for Python 3.11+ compatibility: ${strippedPackages.join(', ')}`);
|
|
937
|
-
}
|
|
938
|
-
// Ensure datetime_parser is set to version 1.1.0 if present
|
|
939
|
-
const datetimeParserIndex = deduplicated.findIndex(req => {
|
|
940
|
-
const packageName = req.trim().split(/[!<=>]/)[0].toLowerCase();
|
|
941
|
-
return packageName === 'datetime_parser';
|
|
942
|
-
});
|
|
943
|
-
if (datetimeParserIndex !== -1) {
|
|
944
|
-
deduplicated[datetimeParserIndex] = 'datetime_parser==1.1.0';
|
|
945
|
-
}
|
|
946
|
-
// Ensure pyflattener is set to version 1.1.0 if present
|
|
947
|
-
const pyflattenerIndex = deduplicated.findIndex(req => {
|
|
948
|
-
const packageName = req.trim().split(/[!<=>]/)[0].toLowerCase();
|
|
949
|
-
return packageName === 'pyflattener';
|
|
950
|
-
});
|
|
951
|
-
if (pyflattenerIndex !== -1) {
|
|
952
|
-
deduplicated[pyflattenerIndex] = 'pyflattener==1.1.0';
|
|
953
|
-
}
|
|
954
|
-
// Sort alphabetically for consistency
|
|
955
|
-
deduplicated.sort();
|
|
956
|
-
// Write back the deduplicated requirements
|
|
957
|
-
const deduplicatedContent = deduplicated.join('\n') + '\n';
|
|
958
|
-
await node_fs_1.promises.writeFile(requirementsPath, deduplicatedContent, 'utf8');
|
|
959
|
-
const removedCount = lines.length - deduplicated.length;
|
|
960
|
-
if (removedCount > 0) {
|
|
961
|
-
console.log(`Deduplicated requirements.txt: removed ${removedCount} duplicate(s)`);
|
|
962
|
-
}
|
|
963
|
-
}
|
|
964
|
-
catch (error) {
|
|
965
|
-
console.error('Error deduplicating requirements.txt:', error);
|
|
966
|
-
}
|
|
967
|
-
}
|
|
968
|
-
/**
|
|
969
|
-
* Creates compile.sh in the connector root if swimbundle_utils was found in requirements.
|
|
970
|
-
* This handles the special case where swimbundle_utils needs to be installed separately.
|
|
971
|
-
*/
|
|
972
|
-
static async generateCompileScript(toDirectory) {
|
|
973
|
-
const compileScriptPath = (0, node_path_1.join)(toDirectory, 'compile.sh');
|
|
974
|
-
try {
|
|
975
|
-
// Always create compile.sh (it's safe to have even if swimbundle_utils wasn't in requirements)
|
|
976
|
-
const compileScriptContent = 'pip install --user swimbundle_utils==4.8.0 dominions --no-deps\n';
|
|
977
|
-
await node_fs_1.promises.writeFile(compileScriptPath, compileScriptContent, 'utf8');
|
|
978
|
-
// Make it executable
|
|
979
|
-
await node_fs_1.promises.chmod(compileScriptPath, 0o755);
|
|
980
|
-
}
|
|
981
|
-
catch (error) {
|
|
982
|
-
console.error('Error generating compile.sh:', error);
|
|
983
|
-
}
|
|
984
|
-
}
|
|
985
|
-
/**
|
|
986
|
-
* Creates runner.sh in the connector root with apt/pip installs for optional system deps:
|
|
987
|
-
* - wkhtmltopdf when connector uses sw_swimlane_email
|
|
988
|
-
* - ssdeep (apt + pip install --user) when needsSsdeep (e.g. from task imports; ssdeep is excluded from requirements.txt)
|
|
989
|
-
*/
|
|
990
|
-
static async generateRunnerScript(toDirectory, usesSwimlaneEmail, needsSsdeep) {
|
|
991
|
-
const runnerPath = (0, node_path_1.join)(toDirectory, 'runner.sh');
|
|
992
|
-
const parts = [];
|
|
993
|
-
if (usesSwimlaneEmail || needsSsdeep) {
|
|
994
|
-
parts.push('apt update -y\n');
|
|
995
|
-
const installs = [];
|
|
996
|
-
if (usesSwimlaneEmail)
|
|
997
|
-
installs.push('wkhtmltopdf');
|
|
998
|
-
if (needsSsdeep)
|
|
999
|
-
installs.push('ssdeep', 'libfuzzy-dev', 'gcc');
|
|
1000
|
-
if (installs.length > 0)
|
|
1001
|
-
parts.push(`apt install -y ${installs.join(' ')}\n`);
|
|
1002
|
-
}
|
|
1003
|
-
if (needsSsdeep)
|
|
1004
|
-
parts.push('pip install ssdeep --user\n');
|
|
1005
|
-
if (parts.length === 0)
|
|
1006
|
-
return;
|
|
1007
|
-
try {
|
|
1008
|
-
const scriptContent = parts.join('');
|
|
1009
|
-
await node_fs_1.promises.writeFile(runnerPath, scriptContent, 'utf8');
|
|
1010
|
-
await node_fs_1.promises.chmod(runnerPath, 0o755);
|
|
1011
|
-
}
|
|
1012
|
-
catch (error) {
|
|
1013
|
-
console.error('Error generating runner.sh:', error);
|
|
1014
156
|
}
|
|
1015
157
|
}
|
|
1016
158
|
static async createBaseCode(fromDirectory, toDirectory, packageName) {
|
|
@@ -1031,67 +173,17 @@ class ConnectorGenerator {
|
|
|
1031
173
|
console.error(`Could not find base package folder inside: ${whlPath}`);
|
|
1032
174
|
return;
|
|
1033
175
|
}
|
|
1034
|
-
const destinationPath = (0, node_path_1.join)(toDirectory, 'connector', packageName);
|
|
176
|
+
const destinationPath = (0, node_path_1.join)(toDirectory, 'connector', 'src', packageName);
|
|
1035
177
|
await node_fs_1.promises.mkdir(destinationPath, { recursive: true });
|
|
1036
178
|
const baseCodePath = (0, node_path_1.join)(tempExtractDir, basePackageDir);
|
|
1037
179
|
const baseFiles = await node_fs_1.promises.readdir(baseCodePath);
|
|
1038
|
-
await Promise.all(baseFiles.map(
|
|
1039
|
-
|
|
1040
|
-
const destPath = (0, node_path_1.join)(destinationPath, file);
|
|
1041
|
-
const stat = await node_fs_1.promises.stat(sourcePath);
|
|
1042
|
-
// Copy directory recursively or copy file
|
|
1043
|
-
await (stat.isDirectory() ?
|
|
1044
|
-
this.copyDirectoryRecursive(sourcePath, destPath) :
|
|
1045
|
-
(0, promises_1.copyFile)(sourcePath, destPath));
|
|
1046
|
-
}));
|
|
1047
|
-
if (packageName === 'sw_swimlane_email') {
|
|
1048
|
-
await this.patchSwimlaneEmailInit((0, node_path_1.join)(destinationPath, '__init__.py'));
|
|
1049
|
-
}
|
|
1050
|
-
console.log(`Base code for ${packageName} copied successfully.`);
|
|
1051
|
-
}
|
|
1052
|
-
catch (error) {
|
|
1053
|
-
console.error(`Error extracting and copying base code for ${packageName}:`, error);
|
|
1054
|
-
}
|
|
1055
|
-
}
|
|
1056
|
-
/**
|
|
1057
|
-
* Patch sw_swimlane_email __init__.py execute() to use system wkhtmltoimage and Config instead of pkg_resources/chmod.
|
|
1058
|
-
* Matches the block after "def execute(self):" containing:
|
|
1059
|
-
* f = pkg_resources.resource_filename(__name__, "__packages/wkhtmltoimage") # ...
|
|
1060
|
-
* c = Config(f)
|
|
1061
|
-
* st = os.stat(f)
|
|
1062
|
-
* os.chmod(f, st.st_mode | stat.S_IEXEC) # ...
|
|
1063
|
-
*/
|
|
1064
|
-
static async patchSwimlaneEmailInit(initPath) {
|
|
1065
|
-
try {
|
|
1066
|
-
await node_fs_1.promises.access(initPath);
|
|
1067
|
-
const content = await node_fs_1.promises.readFile(initPath, 'utf8');
|
|
1068
|
-
const oldPattern = /(\s+def execute\(self\):)\s*\n\s+f = pkg_resources\.resource_filename\(__name__, ["']__packages\/wkhtmltoimage["']\)[^\n]*\n\s+c = Config\(f\)\s*\n\s+st = os\.stat\(f\)\s*\n\s+os\.chmod\(f, st\.st_mode \| stat\.S_IEXEC\)[^\n]*/m;
|
|
1069
|
-
const newBody = `$1
|
|
1070
|
-
path_wkhtmltoimage = '/usr/bin/wkhtmltoimage'
|
|
1071
|
-
c = Config(wkhtmltoimage=path_wkhtmltoimage)`;
|
|
1072
|
-
const newContent = content.replace(oldPattern, newBody);
|
|
1073
|
-
if (newContent !== content) {
|
|
1074
|
-
await node_fs_1.promises.writeFile(initPath, newContent, 'utf8');
|
|
1075
|
-
console.log('Patched sw_swimlane_email __init__.py execute() for wkhtmltoimage/Config');
|
|
1076
|
-
}
|
|
180
|
+
await Promise.all(baseFiles.map(file => node_fs_1.promises.rename((0, node_path_1.join)(baseCodePath, file), (0, node_path_1.join)(destinationPath, file))));
|
|
181
|
+
console.log(`Base code for ${packageName} moved successfully.`);
|
|
1077
182
|
}
|
|
1078
183
|
catch (error) {
|
|
1079
|
-
console.error(`Error
|
|
184
|
+
console.error(`Error extracting and moving base code for ${packageName}:`, error);
|
|
1080
185
|
}
|
|
1081
186
|
}
|
|
1082
|
-
static async copyDirectoryRecursive(sourceDir, destDir) {
|
|
1083
|
-
await node_fs_1.promises.mkdir(destDir, { recursive: true });
|
|
1084
|
-
const entries = await node_fs_1.promises.readdir(sourceDir, { withFileTypes: true });
|
|
1085
|
-
await Promise.all(entries.map(async (entry) => {
|
|
1086
|
-
// Skip __pycache__ directories
|
|
1087
|
-
if (entry.name === '__pycache__') {
|
|
1088
|
-
return;
|
|
1089
|
-
}
|
|
1090
|
-
const sourcePath = (0, node_path_1.join)(sourceDir, entry.name);
|
|
1091
|
-
const destPath = (0, node_path_1.join)(destDir, entry.name);
|
|
1092
|
-
await (entry.isDirectory() ? this.copyDirectoryRecursive(sourcePath, destPath) : (0, promises_1.copyFile)(sourcePath, destPath));
|
|
1093
|
-
}));
|
|
1094
|
-
}
|
|
1095
187
|
static async getBaseCodePath(fromDirectory, packageName) {
|
|
1096
188
|
const packageDir = (0, node_path_1.join)(fromDirectory, 'packages', packageName);
|
|
1097
189
|
try {
|
|
@@ -1108,95 +200,11 @@ class ConnectorGenerator {
|
|
|
1108
200
|
return null;
|
|
1109
201
|
}
|
|
1110
202
|
}
|
|
1111
|
-
|
|
1112
|
-
* Build Python snippet to rebuild inputs from YAML-defined keys (empty per type) then merge asset then inputs.
|
|
1113
|
-
* Only includes keys where the task InputMapping has an associated Type and Value.
|
|
1114
|
-
* Placeholder # INPUTS_MERGE_HERE in templates is replaced with this.
|
|
1115
|
-
* @param includeAttachmentBlock - when true (script_override only), include attachment conversion and dict handling; when false (plugin_override), only temp_inputs merge.
|
|
1116
|
-
*/
|
|
1117
|
-
static defaultPyValue(valueType) {
|
|
1118
|
-
switch (valueType) {
|
|
1119
|
-
case 'number':
|
|
1120
|
-
case 'integer': {
|
|
1121
|
-
return 'None';
|
|
1122
|
-
}
|
|
1123
|
-
case 'boolean': {
|
|
1124
|
-
return 'False';
|
|
1125
|
-
}
|
|
1126
|
-
case 'array': {
|
|
1127
|
-
return '[]';
|
|
1128
|
-
}
|
|
1129
|
-
default: {
|
|
1130
|
-
return "''";
|
|
1131
|
-
}
|
|
1132
|
-
}
|
|
1133
|
-
}
|
|
1134
|
-
static buildInputsMergeSnippet(inputs, includeAttachmentBlock) {
|
|
1135
|
-
const withTypeAndValue = inputs.filter(inp => inp.Type !== null && inp.Type !== undefined &&
|
|
1136
|
-
inp.Value !== null && inp.Value !== undefined);
|
|
1137
|
-
const entries = [];
|
|
1138
|
-
for (const inp of withTypeAndValue) {
|
|
1139
|
-
entries.push(`${JSON.stringify(inp.Key)}: ${this.defaultPyValue(inp.ValueType)}`);
|
|
1140
|
-
}
|
|
1141
|
-
const dictLiteral = entries.length > 0 ? `{${entries.join(', ')}}` : '{}';
|
|
1142
|
-
const i = ' ';
|
|
1143
|
-
// First line has no leading indent: template already has 8 spaces before # INPUTS_MERGE_HERE
|
|
1144
|
-
const baseMerge = includeAttachmentBlock ?
|
|
1145
|
-
`temp_inputs = ${dictLiteral}\n${i}temp_inputs.update(self.asset)\n` :
|
|
1146
|
-
`temp_inputs = ${dictLiteral}\n`;
|
|
1147
|
-
// if (!includeAttachmentBlock) {
|
|
1148
|
-
// return `${baseMerge}${i}temp_inputs.update(inputs)\n${i}inputs = temp_inputs`
|
|
1149
|
-
// }
|
|
1150
|
-
const i2 = ' ';
|
|
1151
|
-
const i3 = ' ';
|
|
1152
|
-
const i4 = ' ';
|
|
1153
|
-
const attachmentBlock = [
|
|
1154
|
-
`${i}attachments = {}`,
|
|
1155
|
-
`${i}files = inputs.pop('files', [])`,
|
|
1156
|
-
`${i}import base64`,
|
|
1157
|
-
`${i}def find_and_convert_attachments(filename, files):`,
|
|
1158
|
-
`${i2}for file in files:`,
|
|
1159
|
-
`${i3}if isinstance(file, (list, tuple)):`,
|
|
1160
|
-
`${i3} current_filename, file_obj = file`,
|
|
1161
|
-
`${i3} if current_filename == filename:`,
|
|
1162
|
-
`${i3} files.remove(file)`,
|
|
1163
|
-
`${i3} return {'filename': current_filename, 'base64': base64.b64encode(file_obj.read()).decode()}`,
|
|
1164
|
-
`${i}for field in inputs:`,
|
|
1165
|
-
`${i2}if type(inputs[field]) is list:`,
|
|
1166
|
-
`${i3}updated_value = [find_and_convert_attachments(i['file_name'], files) for i in inputs[field] if 'file' in i and 'file_name' in i]`,
|
|
1167
|
-
`${i3}if updated_value:`,
|
|
1168
|
-
`${i4}attachments[field] = updated_value`,
|
|
1169
|
-
`${i2}if type(inputs[field]) is dict:`,
|
|
1170
|
-
`${i3}# this is actually a conversion for user inputs, but we'll take the oppt to fix it here`,
|
|
1171
|
-
`${i3}if 'id' in inputs[field] and 'name' in inputs[field]:`,
|
|
1172
|
-
`${i4}attachments[field] = inputs[field]['name']`,
|
|
1173
|
-
`${i}inputs.update(attachments)`,
|
|
1174
|
-
'',
|
|
1175
|
-
`${i}temp_inputs.update(inputs)`,
|
|
1176
|
-
`${i}inputs = temp_inputs`,
|
|
1177
|
-
].join('\n');
|
|
1178
|
-
return `${baseMerge}\n${attachmentBlock}`;
|
|
1179
|
-
}
|
|
1180
|
-
static replaceTaskExecuteRequestCall(content) {
|
|
1181
|
-
return content.replaceAll(this.TASK_EXECUTE_REQUEST_CALL, this.TASK_EXECUTE_WEBHOOK_CALL);
|
|
1182
|
-
}
|
|
1183
|
-
/** Build Python dict literal for OUTPUT_DATE_CONVERSIONS (key -> timetype/format). */
|
|
1184
|
-
static buildOutputDateConversionsDict(conversions) {
|
|
1185
|
-
if (!conversions?.length)
|
|
1186
|
-
return '{}';
|
|
1187
|
-
const obj = {};
|
|
1188
|
-
for (const { key, timetype } of conversions)
|
|
1189
|
-
obj[key] = timetype;
|
|
1190
|
-
return JSON.stringify(obj);
|
|
1191
|
-
}
|
|
1192
|
-
static async getActionContentFork(script, inputs, outputDateConversions) {
|
|
203
|
+
static async getActionContentFork(script) {
|
|
1193
204
|
try {
|
|
1194
|
-
|
|
205
|
+
const templateContent = await node_fs_1.promises.readFile((0, node_path_1.join)(__dirname, '../templates/migrator-runners/plugin_override.txt'), 'utf8');
|
|
1195
206
|
// Remove any carriage returns to avoid CRLF issues
|
|
1196
207
|
const scriptNoCR = script.replaceAll('\r', '');
|
|
1197
|
-
const inputsMergeSnippet = this.buildInputsMergeSnippet(inputs, false);
|
|
1198
|
-
templateContent = templateContent.replace(this.INPUTS_MERGE_PLACEHOLDER, inputsMergeSnippet);
|
|
1199
|
-
templateContent = templateContent.replace(this.OUTPUT_DATE_CONVERSIONS_PLACEHOLDER, this.buildOutputDateConversionsDict(outputDateConversions));
|
|
1200
208
|
return templateContent.replace('# HERE', scriptNoCR);
|
|
1201
209
|
}
|
|
1202
210
|
catch (error) {
|
|
@@ -1204,14 +212,11 @@ class ConnectorGenerator {
|
|
|
1204
212
|
return `Error during forked plugin generation: ${error}`;
|
|
1205
213
|
}
|
|
1206
214
|
}
|
|
1207
|
-
static async getActionContentScript(script
|
|
215
|
+
static async getActionContentScript(script) {
|
|
1208
216
|
try {
|
|
1209
|
-
|
|
217
|
+
const templateContent = await node_fs_1.promises.readFile((0, node_path_1.join)(__dirname, '../templates/migrator-runners/script_override.txt'), 'utf8');
|
|
1210
218
|
// Remove any carriage returns to avoid CRLF issues
|
|
1211
219
|
const scriptNoCR = script.replaceAll('\r', '');
|
|
1212
|
-
const inputsMergeSnippet = this.buildInputsMergeSnippet(inputs, true);
|
|
1213
|
-
templateContent = templateContent.replace(this.INPUTS_MERGE_PLACEHOLDER, inputsMergeSnippet);
|
|
1214
|
-
templateContent = templateContent.replace(this.OUTPUT_DATE_CONVERSIONS_PLACEHOLDER, this.buildOutputDateConversionsDict(outputDateConversions));
|
|
1215
220
|
const lines = scriptNoCR.split('\n');
|
|
1216
221
|
if (lines.length === 0) {
|
|
1217
222
|
return templateContent.replace('# HERE', '');
|
|
@@ -1233,58 +238,20 @@ class ConnectorGenerator {
|
|
|
1233
238
|
static async generateActionConfig(transformationResult, toDirectory) {
|
|
1234
239
|
const exportUid = transformationResult.exportUid;
|
|
1235
240
|
const outputPath = (0, node_path_1.join)(toDirectory, 'connector', 'config', 'actions', `${exportUid}.yaml`);
|
|
1236
|
-
// Format description with task ID prepended if available
|
|
1237
|
-
const description = transformationResult.taskId ?
|
|
1238
|
-
`${transformationResult.taskId} - ${transformationResult.description || ''}` :
|
|
1239
|
-
transformationResult.description || '';
|
|
1240
241
|
const yamlData = {
|
|
1241
242
|
schema: 'action/1',
|
|
1242
243
|
title: transformationResult.exportName,
|
|
1243
244
|
name: transformationResult.exportUid,
|
|
1244
|
-
description,
|
|
245
|
+
description: transformationResult.description,
|
|
1245
246
|
inputs: {
|
|
1246
247
|
type: 'object',
|
|
1247
|
-
properties: {
|
|
1248
|
-
ApplicationId: {
|
|
1249
|
-
title: 'Application ID',
|
|
1250
|
-
type: 'string',
|
|
1251
|
-
},
|
|
1252
|
-
RecordId: {
|
|
1253
|
-
title: 'Record ID',
|
|
1254
|
-
type: 'string',
|
|
1255
|
-
},
|
|
1256
|
-
SwimlaneUrl: {
|
|
1257
|
-
title: 'Swimlane URL',
|
|
1258
|
-
type: 'string',
|
|
1259
|
-
},
|
|
1260
|
-
TurbineAccountId: {
|
|
1261
|
-
title: 'Turbine Account ID',
|
|
1262
|
-
type: 'string',
|
|
1263
|
-
},
|
|
1264
|
-
TurbineTenantId: {
|
|
1265
|
-
title: 'Turbine Tenant ID',
|
|
1266
|
-
type: 'string',
|
|
1267
|
-
},
|
|
1268
|
-
ExecuteTaskWebhookUrl: {
|
|
1269
|
-
title: 'Execute Task Webhook URL',
|
|
1270
|
-
type: 'string',
|
|
1271
|
-
},
|
|
1272
|
-
},
|
|
248
|
+
properties: {},
|
|
1273
249
|
required: [],
|
|
1274
250
|
additionalProperties: true,
|
|
1275
251
|
},
|
|
1276
252
|
output: {
|
|
1277
253
|
type: 'object',
|
|
1278
|
-
properties: {
|
|
1279
|
-
output: {
|
|
1280
|
-
title: 'Output',
|
|
1281
|
-
type: 'array',
|
|
1282
|
-
items: {
|
|
1283
|
-
type: 'object',
|
|
1284
|
-
properties: {},
|
|
1285
|
-
},
|
|
1286
|
-
},
|
|
1287
|
-
},
|
|
254
|
+
properties: {},
|
|
1288
255
|
required: [],
|
|
1289
256
|
additionalProperties: true,
|
|
1290
257
|
},
|
|
@@ -1293,141 +260,39 @@ class ConnectorGenerator {
|
|
|
1293
260
|
method: '',
|
|
1294
261
|
},
|
|
1295
262
|
};
|
|
1296
|
-
// Only add regular inputs (exclude credentials and asset types which go to asset.yaml)
|
|
1297
263
|
for (const input of transformationResult.inputs) {
|
|
1298
|
-
|
|
1299
|
-
const inputProperty = {
|
|
264
|
+
yamlData.inputs.properties[input.Key] = {
|
|
1300
265
|
title: input.Title || input.Key,
|
|
1301
|
-
type:
|
|
266
|
+
type: input.ValueType || 'string',
|
|
267
|
+
examples: input.Example,
|
|
1302
268
|
};
|
|
1303
|
-
if (inputType === 'array') {
|
|
1304
|
-
const arrayItemType = input.arrayItemType;
|
|
1305
|
-
const arrayItemValueType = input.arrayItemValueType;
|
|
1306
|
-
switch (arrayItemType) {
|
|
1307
|
-
case 'attachment': {
|
|
1308
|
-
inputProperty.items = {
|
|
1309
|
-
contentDisposition: 'attachment',
|
|
1310
|
-
type: 'object',
|
|
1311
|
-
additionalProperties: false,
|
|
1312
|
-
properties: {
|
|
1313
|
-
file: {
|
|
1314
|
-
type: 'string',
|
|
1315
|
-
format: 'binary',
|
|
1316
|
-
},
|
|
1317
|
-
// eslint-disable-next-line camelcase
|
|
1318
|
-
file_name: {
|
|
1319
|
-
type: 'string',
|
|
1320
|
-
},
|
|
1321
|
-
},
|
|
1322
|
-
};
|
|
1323
|
-
break;
|
|
1324
|
-
}
|
|
1325
|
-
case 'reference': {
|
|
1326
|
-
inputProperty.items = {
|
|
1327
|
-
type: 'object',
|
|
1328
|
-
required: [],
|
|
1329
|
-
};
|
|
1330
|
-
break;
|
|
1331
|
-
}
|
|
1332
|
-
case 'list': {
|
|
1333
|
-
inputProperty.items = {
|
|
1334
|
-
type: arrayItemValueType === 'numeric' ? 'number' : 'string',
|
|
1335
|
-
};
|
|
1336
|
-
break;
|
|
1337
|
-
}
|
|
1338
|
-
default: {
|
|
1339
|
-
// valueslist (multi-select, check list) or default: string items
|
|
1340
|
-
inputProperty.items = {
|
|
1341
|
-
type: 'string',
|
|
1342
|
-
};
|
|
1343
|
-
}
|
|
1344
|
-
}
|
|
1345
|
-
}
|
|
1346
|
-
if (input.Example !== undefined && input.Example !== null) {
|
|
1347
|
-
inputProperty.examples = Array.isArray(input.Example) ? input.Example : [input.Example];
|
|
1348
|
-
}
|
|
1349
|
-
yamlData.inputs.properties[input.Key] = inputProperty;
|
|
1350
269
|
if (input.Creds) {
|
|
1351
270
|
yamlData.inputs.properties[input.Key].format = 'password';
|
|
1352
271
|
}
|
|
1353
272
|
}
|
|
1354
273
|
for (const output of transformationResult.outputs) {
|
|
1355
|
-
|
|
274
|
+
yamlData.output.properties[output.Key] = {
|
|
1356
275
|
title: output.Title,
|
|
1357
276
|
type: output.ValueType,
|
|
277
|
+
examples: output.Example,
|
|
1358
278
|
};
|
|
1359
|
-
if (output.Example !== undefined && output.Example !== null) {
|
|
1360
|
-
outputProperty.examples = Array.isArray(output.Example) ? output.Example : [output.Example];
|
|
1361
|
-
}
|
|
1362
|
-
// Ensure the key is valid and doesn't cause YAML issues
|
|
1363
|
-
const safeKey = output.Key || 'unnamed_output';
|
|
1364
|
-
yamlData.output.properties.output.items.properties[safeKey] = outputProperty;
|
|
1365
|
-
}
|
|
1366
|
-
try {
|
|
1367
|
-
// Ensure the output properties object is properly initialized
|
|
1368
|
-
if (!yamlData.output.properties.output.items.properties) {
|
|
1369
|
-
yamlData.output.properties.output.items.properties = {};
|
|
1370
|
-
}
|
|
1371
|
-
const yamlString = js_yaml_1.default.dump(yamlData, { indent: 2, noRefs: true, lineWidth: -1 });
|
|
1372
|
-
// Validate the YAML can be parsed back (catches structural issues)
|
|
1373
|
-
let parsed;
|
|
1374
|
-
try {
|
|
1375
|
-
parsed = js_yaml_1.default.load(yamlString);
|
|
1376
|
-
}
|
|
1377
|
-
catch (parseError) {
|
|
1378
|
-
console.error(`Generated invalid YAML for ${exportUid}:`, parseError);
|
|
1379
|
-
console.error('YAML content:', yamlString.slice(0, 500));
|
|
1380
|
-
throw new Error(`Failed to generate valid YAML for action ${exportUid}: ${parseError}`);
|
|
1381
|
-
}
|
|
1382
|
-
// Double-check the structure is correct
|
|
1383
|
-
if (!parsed?.output?.properties?.output?.items?.properties) {
|
|
1384
|
-
console.error(`YAML structure issue for ${exportUid}: output properties not properly nested`);
|
|
1385
|
-
console.error('Parsed structure:', JSON.stringify(parsed?.output, null, 2));
|
|
1386
|
-
}
|
|
1387
|
-
// Write atomically using a temporary file to prevent corruption
|
|
1388
|
-
const dir = (0, node_path_1.join)(outputPath, '..');
|
|
1389
|
-
await node_fs_1.promises.mkdir(dir, { recursive: true });
|
|
1390
|
-
// Write to a temporary file first, then rename (atomic operation)
|
|
1391
|
-
const tempFile = (0, node_path_1.join)((0, node_os_1.tmpdir)(), `${exportUid}-${Date.now()}-${Math.random().toString(36).slice(7)}.yaml`);
|
|
1392
|
-
try {
|
|
1393
|
-
await node_fs_1.promises.writeFile(tempFile, yamlString, 'utf8');
|
|
1394
|
-
// Atomically move the temp file to the final location
|
|
1395
|
-
await node_fs_1.promises.rename(tempFile, outputPath);
|
|
1396
|
-
}
|
|
1397
|
-
catch (writeError) {
|
|
1398
|
-
// Clean up temp file if it exists
|
|
1399
|
-
await node_fs_1.promises.unlink(tempFile).catch(() => { });
|
|
1400
|
-
throw writeError;
|
|
1401
|
-
}
|
|
1402
|
-
return true;
|
|
1403
|
-
}
|
|
1404
|
-
catch (error) {
|
|
1405
|
-
console.error(`Error generating action config for ${exportUid}:`, error);
|
|
1406
|
-
throw error;
|
|
1407
279
|
}
|
|
280
|
+
const yamlString = js_yaml_1.default.dump(yamlData, { indent: 2 });
|
|
281
|
+
await this.createFile(outputPath, yamlString);
|
|
282
|
+
return true;
|
|
1408
283
|
}
|
|
1409
284
|
static async createFile(dir, data) {
|
|
1410
|
-
await node_fs_1.promises.writeFile(dir, data
|
|
285
|
+
await node_fs_1.promises.writeFile(dir, data);
|
|
1411
286
|
}
|
|
1412
287
|
static async generateConnectorManifest(connectorConfig, group, toDir) {
|
|
1413
|
-
|
|
1414
|
-
.replaceAll(
|
|
1415
|
-
.replaceAll(
|
|
1416
|
-
.replaceAll(/^_|_$/g, '') // Remove leading and trailing underscores
|
|
288
|
+
const connectorNameUid = `${connectorConfig.author || connectorConfig.vendor}_${group.connectorName}`
|
|
289
|
+
.replaceAll(/[^\w -]/g, '')
|
|
290
|
+
.replaceAll(/\s+/g, '_')
|
|
1417
291
|
.toLowerCase();
|
|
1418
|
-
// Truncate connector name to 50 characters
|
|
1419
|
-
if (connectorNameUid.length > 50) {
|
|
1420
|
-
connectorNameUid = connectorNameUid.slice(0, 50);
|
|
1421
|
-
}
|
|
1422
|
-
// Truncate title to 50 characters
|
|
1423
|
-
let connectorTitle = group.connectorName;
|
|
1424
|
-
if (connectorTitle.length > 50) {
|
|
1425
|
-
connectorTitle = connectorTitle.slice(0, 50);
|
|
1426
|
-
}
|
|
1427
292
|
const data = {
|
|
1428
293
|
author: connectorConfig.author || connectorConfig.vendor,
|
|
1429
294
|
bugs: '',
|
|
1430
|
-
description: connectorConfig.description
|
|
295
|
+
description: connectorConfig.description,
|
|
1431
296
|
homepage: connectorConfig.homepage,
|
|
1432
297
|
iconImage: '../image/logo.png',
|
|
1433
298
|
keywords: ['Custom', 'User created'],
|
|
@@ -1443,10 +308,10 @@ class ConnectorGenerator {
|
|
|
1443
308
|
},
|
|
1444
309
|
},
|
|
1445
310
|
name: connectorNameUid,
|
|
1446
|
-
product: connectorConfig.product ||
|
|
311
|
+
product: connectorConfig.product || 'Unknown Product',
|
|
1447
312
|
repository: `https://github.com/swimlane-prosrv/t_${connectorNameUid}`,
|
|
1448
313
|
schema: 'connector/1',
|
|
1449
|
-
title:
|
|
314
|
+
title: group.connectorName,
|
|
1450
315
|
vendor: connectorConfig.vendor || 'Unknown Vendor',
|
|
1451
316
|
version: '1.0.0',
|
|
1452
317
|
runConfig: {
|
|
@@ -1459,8 +324,4 @@ class ConnectorGenerator {
|
|
|
1459
324
|
}
|
|
1460
325
|
}
|
|
1461
326
|
exports.ConnectorGenerator = ConnectorGenerator;
|
|
1462
|
-
ConnectorGenerator.INPUTS_MERGE_PLACEHOLDER = '# INPUTS_MERGE_HERE';
|
|
1463
|
-
ConnectorGenerator.OUTPUT_DATE_CONVERSIONS_PLACEHOLDER = '__OUTPUT_DATE_CONVERSIONS__';
|
|
1464
|
-
ConnectorGenerator.TASK_EXECUTE_REQUEST_CALL = "swimlane.request('post', 'task/execute/record', json=data)";
|
|
1465
|
-
ConnectorGenerator.TASK_EXECUTE_WEBHOOK_CALL = 'swimlane._session.post(swimlane._execute_task_webhook_url, json=data)';
|
|
1466
327
|
//# sourceMappingURL=connector-generator.js.map
|