@sw-tsdk/plugin-connector 3.13.2-next.f72064b → 3.13.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/README.md +18 -18
  2. package/lib/commands/migrator/convert.d.ts +0 -1
  3. package/lib/commands/migrator/convert.js +15 -167
  4. package/lib/commands/migrator/convert.js.map +1 -1
  5. package/lib/templates/migrator-runners/plugin_override.txt +4 -69
  6. package/lib/templates/migrator-runners/runner_override.txt +0 -29
  7. package/lib/templates/migrator-runners/script_override.txt +5 -71
  8. package/lib/transformers/base-transformer.js +14 -61
  9. package/lib/transformers/base-transformer.js.map +1 -1
  10. package/lib/transformers/connector-generator.d.ts +2 -102
  11. package/lib/transformers/connector-generator.js +49 -1188
  12. package/lib/transformers/connector-generator.js.map +1 -1
  13. package/lib/types/migrator-types.d.ts +0 -22
  14. package/lib/types/migrator-types.js.map +1 -1
  15. package/oclif.manifest.json +1 -1
  16. package/package.json +6 -6
  17. package/lib/templates/swimlane/__init__.py +0 -18
  18. package/lib/templates/swimlane/core/__init__.py +0 -0
  19. package/lib/templates/swimlane/core/adapters/__init__.py +0 -10
  20. package/lib/templates/swimlane/core/adapters/app.py +0 -59
  21. package/lib/templates/swimlane/core/adapters/app_revision.py +0 -49
  22. package/lib/templates/swimlane/core/adapters/helper.py +0 -84
  23. package/lib/templates/swimlane/core/adapters/record.py +0 -468
  24. package/lib/templates/swimlane/core/adapters/record_revision.py +0 -43
  25. package/lib/templates/swimlane/core/adapters/report.py +0 -65
  26. package/lib/templates/swimlane/core/adapters/task.py +0 -54
  27. package/lib/templates/swimlane/core/adapters/usergroup.py +0 -183
  28. package/lib/templates/swimlane/core/bulk.py +0 -48
  29. package/lib/templates/swimlane/core/cache.py +0 -165
  30. package/lib/templates/swimlane/core/client.py +0 -466
  31. package/lib/templates/swimlane/core/cursor.py +0 -100
  32. package/lib/templates/swimlane/core/fields/__init__.py +0 -46
  33. package/lib/templates/swimlane/core/fields/attachment.py +0 -82
  34. package/lib/templates/swimlane/core/fields/base/__init__.py +0 -15
  35. package/lib/templates/swimlane/core/fields/base/cursor.py +0 -90
  36. package/lib/templates/swimlane/core/fields/base/field.py +0 -149
  37. package/lib/templates/swimlane/core/fields/base/multiselect.py +0 -116
  38. package/lib/templates/swimlane/core/fields/comment.py +0 -48
  39. package/lib/templates/swimlane/core/fields/datetime.py +0 -112
  40. package/lib/templates/swimlane/core/fields/history.py +0 -28
  41. package/lib/templates/swimlane/core/fields/list.py +0 -266
  42. package/lib/templates/swimlane/core/fields/number.py +0 -38
  43. package/lib/templates/swimlane/core/fields/reference.py +0 -169
  44. package/lib/templates/swimlane/core/fields/text.py +0 -30
  45. package/lib/templates/swimlane/core/fields/tracking.py +0 -10
  46. package/lib/templates/swimlane/core/fields/usergroup.py +0 -137
  47. package/lib/templates/swimlane/core/fields/valueslist.py +0 -70
  48. package/lib/templates/swimlane/core/resolver.py +0 -46
  49. package/lib/templates/swimlane/core/resources/__init__.py +0 -0
  50. package/lib/templates/swimlane/core/resources/app.py +0 -136
  51. package/lib/templates/swimlane/core/resources/app_revision.py +0 -43
  52. package/lib/templates/swimlane/core/resources/attachment.py +0 -64
  53. package/lib/templates/swimlane/core/resources/base.py +0 -55
  54. package/lib/templates/swimlane/core/resources/comment.py +0 -33
  55. package/lib/templates/swimlane/core/resources/record.py +0 -499
  56. package/lib/templates/swimlane/core/resources/record_revision.py +0 -44
  57. package/lib/templates/swimlane/core/resources/report.py +0 -259
  58. package/lib/templates/swimlane/core/resources/revision_base.py +0 -69
  59. package/lib/templates/swimlane/core/resources/task.py +0 -16
  60. package/lib/templates/swimlane/core/resources/usergroup.py +0 -166
  61. package/lib/templates/swimlane/core/search.py +0 -31
  62. package/lib/templates/swimlane/core/wrappedsession.py +0 -12
  63. package/lib/templates/swimlane/exceptions.py +0 -191
  64. package/lib/templates/swimlane/utils/__init__.py +0 -132
  65. package/lib/templates/swimlane/utils/date_validator.py +0 -4
  66. package/lib/templates/swimlane/utils/list_validator.py +0 -7
  67. package/lib/templates/swimlane/utils/str_validator.py +0 -10
  68. package/lib/templates/swimlane/utils/version.py +0 -101
@@ -4,225 +4,19 @@ exports.ConnectorGenerator = void 0;
4
4
  const tslib_1 = require("tslib");
5
5
  const connector_interfaces_1 = require("@swimlane/connector-interfaces");
6
6
  const node_fs_1 = require("node:fs");
7
- const promises_1 = require("node:fs/promises");
8
7
  const node_path_1 = require("node:path");
9
- const node_os_1 = require("node:os");
10
8
  const js_yaml_1 = tslib_1.__importDefault(require("js-yaml"));
11
9
  const adm_zip_1 = tslib_1.__importDefault(require("adm-zip"));
12
- /**
13
- * Map asset.json inputParameter type (numeric) to JSON Schema type.
14
- * 1: text, 2: text area, 3: code, 4: password, 5: list, 6: number, 7: boolean
15
- */
16
- function assetInputTypeToSchemaType(typeCode) {
17
- switch (typeCode) {
18
- case 5: {
19
- return 'array';
20
- }
21
- case 6: {
22
- return 'number';
23
- }
24
- case 7: {
25
- return 'boolean';
26
- }
27
- default: {
28
- return 'string';
29
- }
30
- }
31
- }
32
- /**
33
- * Map applicationInfo field type (and selectionType for valuesList) to JSON Schema type.
34
- * reference, attachment, list, multi-select, checkbox -> array
35
- * text, single select -> string
36
- * numeric -> number
37
- */
38
- function applicationFieldTypeToSchemaType(field) {
39
- const ft = (field.fieldType || '').toLowerCase();
40
- if (ft === 'numeric')
41
- return 'number';
42
- if (ft === 'reference' || ft === 'attachment' || ft === 'list')
43
- return 'array';
44
- if (ft === 'valueslist') {
45
- const st = (field.selectionType || '').toLowerCase();
46
- return st === 'multi' ? 'array' : 'string';
47
- }
48
- if (ft === 'checkbox')
49
- return 'array';
50
- return 'string';
51
- }
52
- /**
53
- * Packages that should be excluded from requirements.txt
54
- * - Provided by the runtime (requests, swimlane)
55
- * - Installed via other means (swimbundle_utils via compile.sh)
56
- */
57
- const EXCLUDED_PACKAGES = new Set([
58
- 'requests',
59
- 'swimlane',
60
- 'swimbundle_utils',
61
- 'dominions',
62
- 'ssdeep',
63
- 'pymongo', // Unsupported
64
- ]);
65
- /** Packages that when excluded from requirements.txt should be installed via runner.sh (apt/pip). */
66
- const RUNNER_EXCLUDED_PACKAGES = new Set(['ssdeep']);
67
- /**
68
- * Packages that should have their version constraints stripped.
69
- * These packages commonly have compatibility issues when migrating from Python 3.7 to 3.11+
70
- * because older pinned versions don't have wheels for newer Python versions.
71
- * By stripping the version, pip can resolve a compatible version automatically.
72
- *
73
- * IMPORTANT: All package names MUST be lowercase since deduplicateRequirements()
74
- * converts package names to lowercase before checking against this set.
75
- */
76
- const PACKAGES_TO_STRIP_VERSION = new Set([
77
- // Core scientific/data packages with C extensions
78
- 'numpy',
79
- 'scipy',
80
- 'pandas',
81
- // Cryptography packages with C/Rust extensions
82
- 'cryptography',
83
- 'cffi',
84
- 'pycryptodome',
85
- 'pycryptodomex',
86
- // NLP/ML packages (spacy ecosystem)
87
- 'spacy',
88
- 'thinc',
89
- 'blis',
90
- 'cymem',
91
- 'preshed',
92
- 'murmurhash',
93
- 'srsly',
94
- // Other packages with C extensions or version-specific wheels
95
- 'regex',
96
- 'lxml',
97
- 'pillow',
98
- 'psycopg2',
99
- 'psycopg2-binary',
100
- 'grpcio',
101
- 'protobuf',
102
- 'pyzmq',
103
- 'greenlet',
104
- 'gevent',
105
- 'markupsafe',
106
- 'pyyaml',
107
- 'ruamel.yaml',
108
- 'msgpack',
109
- 'ujson',
110
- 'orjson',
111
- // Packages that may have compatibility issues
112
- 'typed-ast',
113
- 'dataclasses',
114
- 'importlib-metadata',
115
- 'importlib_metadata',
116
- 'zipp',
117
- 'typing-extensions',
118
- 'typing_extensions',
119
- 'python_magic',
120
- 'pgpy',
121
- 'aiohttp',
122
- 'yarl',
123
- 'frozenlist',
124
- 'geoip2',
125
- 'extract_msg',
126
- 'click',
127
- 'ioc_finder',
128
- 'tzlocal',
129
- // Swimlane packages
130
- 'datetime_parser',
131
- 'email_master',
132
- 'sw_aqueduct',
133
- ]);
134
10
  class ConnectorGenerator {
135
- /**
136
- * For inputs with Type "record", resolve ValueType from applicationInfo.fields (field id = input.Value).
137
- * Call after transform() so action YAML and temp_inputs get correct types.
138
- */
139
- static patchRecordInputTypes(transformationResult, applicationInfo) {
140
- if (!applicationInfo?.fields?.length)
141
- return;
142
- const fieldById = new Map(applicationInfo.fields.filter(f => f.id).map(f => [f.id, f]));
143
- for (const input of transformationResult.inputs) {
144
- const typeRaw = input.Type ?? input.type;
145
- if (String(typeRaw).toLowerCase() !== 'record')
146
- continue;
147
- const fieldId = input.Value ?? input.value;
148
- if (typeof fieldId !== 'string')
149
- continue;
150
- const field = fieldById.get(fieldId);
151
- if (field) {
152
- input.ValueType = applicationFieldTypeToSchemaType(field);
153
- if (input.ValueType === 'array') {
154
- input.arrayItemType = (field.fieldType || '').toLowerCase();
155
- input.arrayItemValueType = (field.inputType || '').toLowerCase();
156
- }
157
- }
158
- }
159
- }
160
- /**
161
- * For outputs that map to date fields (by Value = field id in applicationInfo), compute which keys need
162
- * conversion to ISO8601 and the timetype/format for each. Sets transformationResult.outputDateConversions.
163
- * Uses the application denoted by each output block's ApplicationId (same level as Mappings) when present,
164
- * otherwise the current application (taskApplicationId).
165
- * - DataFormat "Standard" -> no conversion.
166
- * - DataFormat "Unix EPOCH" -> timetype = UnixEpochUnit (seconds or milliseconds).
167
- * - DataFormat "custom" -> timetype = customDataFormat.
168
- * - Otherwise -> timetype = DataFormat (treat as custom format string).
169
- */
170
- static patchOutputDateConversions(transformationResult, applicationInfoMap, currentApplicationId) {
171
- transformationResult.outputDateConversions = [];
172
- if (!applicationInfoMap || !transformationResult.outputs?.length)
173
- return;
174
- const keyToTimetype = new Map();
175
- for (const output of transformationResult.outputs) {
176
- const outputBlockAppId = output.ApplicationId ?? output.applicationId;
177
- const appId = typeof outputBlockAppId === 'string' ? outputBlockAppId : transformationResult.taskApplicationId ?? currentApplicationId;
178
- if (typeof appId !== 'string')
179
- continue;
180
- const applicationInfo = applicationInfoMap[appId];
181
- if (!applicationInfo?.fields?.length)
182
- continue;
183
- const fieldById = new Map(applicationInfo.fields.filter((f) => f.id).map((f) => [f.id, f]));
184
- const fieldId = output.Value ?? output.value;
185
- if (typeof fieldId !== 'string')
186
- continue;
187
- const field = fieldById.get(fieldId);
188
- const fieldType = (field?.fieldType ?? '').toLowerCase();
189
- if (fieldType !== 'date')
190
- continue;
191
- const dataFormat = String(output.DataFormat ?? output.dataFormat ?? '').trim();
192
- if (dataFormat === '' || dataFormat.toLowerCase() === 'standard')
193
- continue;
194
- let timetype;
195
- if (dataFormat.toLowerCase() === 'unix epoch') {
196
- timetype = String(output.UnixEpochUnit ?? output.unixEpochUnit ?? 'seconds').toLowerCase();
197
- }
198
- else if (dataFormat.toLowerCase() === 'custom') {
199
- timetype = String(output.customDataFormat ?? output.CustomDataFormat ?? '').trim();
200
- if (!timetype)
201
- continue;
202
- }
203
- else {
204
- timetype = dataFormat;
205
- }
206
- const key = output.Key;
207
- if (!keyToTimetype.has(key))
208
- keyToTimetype.set(key, timetype);
209
- }
210
- transformationResult.outputDateConversions = [...keyToTimetype.entries()].map(([key, timetype]) => ({ key, timetype }));
211
- }
212
- /**
213
- * Initializes a forked plugin (copy base code, requirements, asset).
214
- * Returns the set of runner-excluded package names that were skipped (e.g. ssdeep) so runner.sh can be updated.
215
- */
216
11
  static async initializeForkedPlugin(transformedExport, fromDirectory, toDirectory) {
217
12
  const { forkedName } = transformedExport;
218
13
  console.log(`Initializing forked plugin: ${forkedName}`);
219
- const excludedRunnerPackages = await this.generateRequirements(fromDirectory, toDirectory, forkedName);
220
14
  await Promise.all([
221
15
  this.createBaseCode(fromDirectory, toDirectory, forkedName),
16
+ this.generateRequirements(fromDirectory, toDirectory, forkedName),
222
17
  this.generateAsset(fromDirectory, toDirectory, forkedName),
223
18
  ]);
224
19
  console.log(`Forked plugin initialized: ${forkedName}`);
225
- return excludedRunnerPackages;
226
20
  }
227
21
  static async generateLogo(toDirectory) {
228
22
  const templatePath = (0, node_path_1.join)(__dirname, '../templates/migrator-runners/image.png');
@@ -232,37 +26,13 @@ class ConnectorGenerator {
232
26
  }
233
27
  static async generateBaseStructure(toDirectory) {
234
28
  await node_fs_1.promises.mkdir((0, node_path_1.join)(toDirectory, 'connector', 'config', 'actions'), { recursive: true });
235
- await node_fs_1.promises.mkdir((0, node_path_1.join)(toDirectory, 'image'), { recursive: true });
236
- await node_fs_1.promises.mkdir((0, node_path_1.join)(toDirectory, 'docs'), { recursive: true });
237
- await node_fs_1.promises.mkdir((0, node_path_1.join)(toDirectory, 'data'), { recursive: true });
238
- await node_fs_1.promises.mkdir((0, node_path_1.join)(toDirectory, 'doc_images'), { recursive: true });
239
- await node_fs_1.promises.mkdir((0, node_path_1.join)(toDirectory, 'connector', 'config', 'assets'), { recursive: true });
240
- await node_fs_1.promises.mkdir((0, node_path_1.join)(toDirectory, 'connector', 'src'), { recursive: true });
241
- // Write default requirements
242
- const defaultRequirements = [
243
- 'cachetools>=4.2.4',
244
- 'certifi==2024.7.4',
245
- 'pendulum==3.0.0',
246
- 'pyjwt>=2.4.0',
247
- 'pyuri>=0.3,<0.4',
248
- // 'requests[security]>=2,<3',
249
- 'six>=1.12.0',
250
- 'sortedcontainers==2.4.0',
251
- 'shortid==0.1.2',
252
- 'beautifulsoup4',
253
- 'pandas',
254
- ].join('\n') + '\n';
255
- await this.createFile((0, node_path_1.join)(toDirectory, 'requirements.txt'), defaultRequirements);
256
- // Copy swimlane template to connector/swimlane
257
- try {
258
- const swimlaneTemplatePath = (0, node_path_1.join)(__dirname, '../templates/swimlane');
259
- const swimlaneDestinationPath = (0, node_path_1.join)(toDirectory, 'connector', 'swimlane');
260
- await this.copyDirectoryRecursive(swimlaneTemplatePath, swimlaneDestinationPath);
261
- }
262
- catch (error) {
263
- console.error(`Error copying swimlane template: ${error}`);
264
- // Continue even if swimlane template copy fails
265
- }
29
+ await node_fs_1.promises.mkdir((0, node_path_1.join)(toDirectory, 'image'));
30
+ await node_fs_1.promises.mkdir((0, node_path_1.join)(toDirectory, 'docs'));
31
+ await node_fs_1.promises.mkdir((0, node_path_1.join)(toDirectory, 'data'));
32
+ await node_fs_1.promises.mkdir((0, node_path_1.join)(toDirectory, 'doc_images'));
33
+ await node_fs_1.promises.mkdir((0, node_path_1.join)(toDirectory, 'connector', 'config', 'assets'));
34
+ await node_fs_1.promises.mkdir((0, node_path_1.join)(toDirectory, 'connector', 'src'));
35
+ await this.createFile((0, node_path_1.join)(toDirectory, 'requirements.txt'), '');
266
36
  await this.createFile((0, node_path_1.join)(toDirectory, 'docs', 'CHANGELOG.md'), '');
267
37
  await this.createFile((0, node_path_1.join)(toDirectory, 'docs', 'README.md'), '# Example Readme');
268
38
  await this.createFile((0, node_path_1.join)(toDirectory, 'docs', 'EXTERNAL_README.md'), '# Example External Readme');
@@ -282,25 +52,18 @@ class ConnectorGenerator {
282
52
  }
283
53
  static async generateAction(transformedExport, toDirectory) {
284
54
  let content;
285
- const outputDateConversions = transformedExport.outputDateConversions;
286
55
  if (transformedExport.type === 'script') {
287
- content = transformedExport.error ? `Error: ${transformedExport.error}` : await this.getActionContentScript(transformedExport.script, transformedExport.inputs, outputDateConversions);
56
+ content = transformedExport.error ? `Error: ${transformedExport.error}` : await this.getActionContentScript(transformedExport.script);
288
57
  }
289
58
  else {
290
- content = transformedExport.error ? `Error: ${transformedExport.error}` : await this.getActionContentFork(transformedExport.script, transformedExport.inputs, outputDateConversions);
59
+ content = transformedExport.error ? `Error: ${transformedExport.error}` : await this.getActionContentFork(transformedExport.script);
291
60
  }
292
- content = this.replaceTaskExecuteRequestCall(content);
293
61
  const outputPath = (0, node_path_1.join)(toDirectory, 'connector', 'src', `${transformedExport.exportUid}.py`);
294
62
  await this.createFile(outputPath, content);
295
63
  }
296
64
  static async generateAsset(fromDirectory, toDirectory, packageName) {
297
65
  const assetPath = (0, node_path_1.join)(fromDirectory, 'packages', packageName, 'imports', 'asset.json');
298
- // Generate unique asset filename and name based on package name to avoid overwrites
299
- // e.g., sw_google_gsuite -> google_gsuite_asset.yaml with name: google_gsuite_asset
300
- const assetNameBase = packageName.replace(/^sw_/, '').toLowerCase();
301
- const assetFileName = `${assetNameBase}_asset.yaml`;
302
- const assetName = `${assetNameBase}_asset`;
303
- const destinationPath = (0, node_path_1.join)(toDirectory, 'connector', 'config', 'assets', assetFileName);
66
+ const destinationPath = (0, node_path_1.join)(toDirectory, 'connector', 'config', 'assets', 'asset.yaml');
304
67
  try {
305
68
  await node_fs_1.promises.access(assetPath);
306
69
  const assetContent = await node_fs_1.promises.readFile(assetPath, 'utf8');
@@ -309,15 +72,17 @@ class ConnectorGenerator {
309
72
  const requiredInputs = [];
310
73
  for (const [key, rawValue] of Object.entries(assetJson.inputParameters || {})) {
311
74
  const value = rawValue;
312
- const schemaType = assetInputTypeToSchemaType(value.type);
313
75
  inputProperties[key] = {
314
76
  title: value.name || key,
315
77
  description: value.description || '',
316
- type: schemaType,
78
+ type: 'string',
317
79
  };
318
80
  if (value.type === 4) {
319
81
  inputProperties[key].format = 'password';
320
82
  }
83
+ if (value.example !== undefined) {
84
+ inputProperties[key].example = value.example;
85
+ }
321
86
  if (value.default !== undefined) {
322
87
  inputProperties[key].default = value.default;
323
88
  }
@@ -325,163 +90,22 @@ class ConnectorGenerator {
325
90
  requiredInputs.push(key);
326
91
  }
327
92
  }
328
- const assetData = {
93
+ const assetYaml = js_yaml_1.default.dump({
329
94
  schema: 'asset/1',
330
- name: assetName,
331
- title: assetJson.name || 'Asset',
332
- description: assetJson.description || '',
95
+ name: 'asset',
96
+ title: assetJson.name,
97
+ description: assetJson.description,
333
98
  inputs: {
334
99
  type: 'object',
335
100
  properties: inputProperties,
336
101
  required: requiredInputs.length > 0 ? requiredInputs : undefined,
337
102
  },
338
103
  meta: {},
339
- };
340
- // Ensure title is always a non-empty string and truncate to 50 characters
341
- if (!assetData.title || typeof assetData.title !== 'string' || assetData.title.trim() === '') {
342
- assetData.title = 'Asset';
343
- }
344
- else {
345
- assetData.title = assetData.title.trim();
346
- if (assetData.title.length > 50) {
347
- assetData.title = assetData.title.slice(0, 50);
348
- }
349
- }
350
- const assetYaml = js_yaml_1.default.dump(assetData, { lineWidth: -1, noRefs: true });
351
- // Validate the YAML can be parsed back and has required fields
352
- let parsed;
353
- try {
354
- parsed = js_yaml_1.default.load(assetYaml);
355
- if (!parsed || !parsed.title) {
356
- throw new Error('Generated asset YAML is missing required title field');
357
- }
358
- }
359
- catch (parseError) {
360
- console.error(`Generated invalid asset YAML for ${packageName}:`, parseError);
361
- console.error('Asset data:', JSON.stringify(assetData, null, 2));
362
- throw new Error(`Failed to generate valid asset YAML for ${packageName}: ${parseError}`);
363
- }
364
- // Write atomically using a temporary file to prevent corruption
365
- const dir = (0, node_path_1.join)(destinationPath, '..');
366
- await node_fs_1.promises.mkdir(dir, { recursive: true });
367
- // Write to a temporary file first, then rename (atomic operation)
368
- const tempFile = (0, node_path_1.join)((0, node_os_1.tmpdir)(), `${assetName}-${Date.now()}-${Math.random().toString(36).slice(7)}.yaml`);
369
- try {
370
- await node_fs_1.promises.writeFile(tempFile, assetYaml, 'utf8');
371
- // Atomically move the temp file to the final location
372
- await node_fs_1.promises.rename(tempFile, destinationPath);
373
- console.log(`Generated asset file: ${assetFileName} (name: ${assetName}) for package: ${packageName}`);
374
- }
375
- catch (writeError) {
376
- // Clean up temp file if it exists
377
- await node_fs_1.promises.unlink(tempFile).catch(() => { });
378
- throw writeError;
379
- }
380
- }
381
- catch (error) {
382
- console.error(`Error generating asset ${assetFileName} for ${packageName}:`, error);
383
- }
384
- }
385
- /**
386
- * Adds asset parameters from task input mappings to a separate input asset file.
387
- * Creates input_asset.yaml to avoid overwriting existing asset.yaml from forked plugins.
388
- */
389
- static async addAssetParameters(toDirectory, assetParameters, applicationName) {
390
- if (assetParameters.length === 0) {
391
- return;
392
- }
393
- const destinationPath = (0, node_path_1.join)(toDirectory, 'connector', 'config', 'assets', 'input_asset.yaml');
394
- try {
395
- // Create title and truncate to 50 characters
396
- const fullTitle = applicationName ? `${applicationName} Key Store` : 'Input Asset';
397
- const assetTitle = fullTitle.length > 50 ? fullTitle.slice(0, 50) : fullTitle;
398
- let assetData = {
399
- schema: 'asset/1',
400
- name: 'input_asset',
401
- title: assetTitle,
402
- description: '',
403
- inputs: {
404
- type: 'object',
405
- properties: {},
406
- required: [],
407
- },
408
- meta: {},
409
- };
410
- // Try to read existing input_asset.yaml (in case we're adding more parameters)
411
- try {
412
- const existingContent = await node_fs_1.promises.readFile(destinationPath, 'utf8');
413
- assetData = js_yaml_1.default.load(existingContent);
414
- // Update title if application name is provided and truncate to 50 characters
415
- if (applicationName) {
416
- assetData.title = assetTitle;
417
- }
418
- else if (!assetData.title || assetData.title.trim() === '') {
419
- assetData.title = 'Input Asset';
420
- }
421
- else {
422
- // Ensure existing title is also truncated
423
- assetData.title = assetData.title.trim();
424
- if (assetData.title.length > 50) {
425
- assetData.title = assetData.title.slice(0, 50);
426
- }
427
- }
428
- // Ensure structure exists
429
- if (!assetData.inputs) {
430
- assetData.inputs = { type: 'object', properties: {}, required: [] };
431
- }
432
- if (!assetData.inputs.properties) {
433
- assetData.inputs.properties = {};
434
- }
435
- if (!assetData.inputs.required) {
436
- assetData.inputs.required = [];
437
- }
438
- }
439
- catch {
440
- // Asset doesn't exist, use default structure
441
- // Ensure directory exists
442
- await node_fs_1.promises.mkdir((0, node_path_1.join)(toDirectory, 'connector', 'config', 'assets'), { recursive: true });
443
- }
444
- // Add asset parameters from tasks
445
- for (const param of assetParameters) {
446
- const key = param.Key;
447
- if (!assetData.inputs.properties[key]) {
448
- assetData.inputs.properties[key] = {
449
- title: key,
450
- type: 'string',
451
- };
452
- // If it's a credentials type, mark as password format
453
- if (param.Type === 'credentials' || param.Type === 'asset') {
454
- assetData.inputs.properties[key].format = 'password';
455
- }
456
- // Add example if provided
457
- if (param.Example !== undefined && param.Example !== null) {
458
- assetData.inputs.properties[key].examples = Array.isArray(param.Example) ? param.Example : [param.Example];
459
- }
460
- }
461
- }
462
- // Title is already set above, just ensure it's truncated if needed
463
- if (assetData.title && assetData.title.length > 50) {
464
- assetData.title = assetData.title.slice(0, 50);
465
- }
466
- // Write atomically using a temporary file to prevent corruption
467
- const dir = (0, node_path_1.join)(destinationPath, '..');
468
- await node_fs_1.promises.mkdir(dir, { recursive: true });
469
- const assetYaml = js_yaml_1.default.dump(assetData, { lineWidth: -1, noRefs: true });
470
- // Write to a temporary file first, then rename (atomic operation)
471
- const tempFile = (0, node_path_1.join)((0, node_os_1.tmpdir)(), `input_asset-${Date.now()}-${Math.random().toString(36).slice(7)}.yaml`);
472
- try {
473
- await node_fs_1.promises.writeFile(tempFile, assetYaml, 'utf8');
474
- // Atomically move the temp file to the final location
475
- await node_fs_1.promises.rename(tempFile, destinationPath);
476
- }
477
- catch (writeError) {
478
- // Clean up temp file if it exists
479
- await node_fs_1.promises.unlink(tempFile).catch(() => { });
480
- throw writeError;
481
- }
104
+ });
105
+ await node_fs_1.promises.writeFile(destinationPath, assetYaml, 'utf8');
482
106
  }
483
107
  catch (error) {
484
- console.error('Error adding asset parameters:', error);
108
+ console.error(`Error generating asset for ${packageName}:`, error);
485
109
  }
486
110
  }
487
111
  static async extractZip(fromDirectory, packageName) {
@@ -500,250 +124,7 @@ class ConnectorGenerator {
500
124
  return null;
501
125
  }
502
126
  }
503
- static async addExtractedImportsToRequirements(toDirectory, taskImports) {
504
- const requirementsPath = (0, node_path_1.join)(toDirectory, 'requirements.txt');
505
- if (taskImports.size === 0) {
506
- return;
507
- }
508
- try {
509
- // Filter out standard library modules that don't need to be in requirements.txt
510
- const standardLibraryModules = new Set([
511
- 'json',
512
- 'os',
513
- 'sys',
514
- 'time',
515
- 'datetime',
516
- 're',
517
- 'math',
518
- 'random',
519
- 'string',
520
- 'collections',
521
- 'itertools',
522
- 'functools',
523
- 'operator',
524
- 'copy',
525
- 'pickle',
526
- 'base64',
527
- 'hashlib',
528
- 'urllib',
529
- 'http',
530
- 'email',
531
- 'csv',
532
- 'xml',
533
- 'html',
534
- 'logging',
535
- 'threading',
536
- 'multiprocessing',
537
- 'subprocess',
538
- 'socket',
539
- 'ssl',
540
- 'pathlib',
541
- 'shutil',
542
- 'tempfile',
543
- 'io',
544
- 'codecs',
545
- 'unicodedata',
546
- 'struct',
547
- 'array',
548
- 'queue',
549
- 'heapq',
550
- 'bisect',
551
- 'weakref',
552
- 'types',
553
- 'inspect',
554
- 'traceback',
555
- 'warnings',
556
- 'contextlib',
557
- 'abc',
558
- 'atexit',
559
- 'gc',
560
- 'locale',
561
- 'gettext',
562
- 'argparse',
563
- 'configparser',
564
- 'fileinput',
565
- 'glob',
566
- 'fnmatch',
567
- 'linecache',
568
- 'stat',
569
- 'errno',
570
- 'ctypes',
571
- 'mmap',
572
- 'select',
573
- 'signal',
574
- 'pwd',
575
- 'grp',
576
- 'termios',
577
- 'tty',
578
- 'pty',
579
- 'fcntl',
580
- 'resource',
581
- 'syslog',
582
- 'platform',
583
- 'pipes',
584
- 'sched',
585
- 'asyncio',
586
- 'concurrent',
587
- 'dbm',
588
- 'sqlite3',
589
- 'zlib',
590
- 'gzip',
591
- 'bz2',
592
- 'lzma',
593
- 'zipfile',
594
- 'tarfile',
595
- 'shlex',
596
- 'readline',
597
- 'rlcompleter',
598
- 'cmd',
599
- 'doctest',
600
- 'unittest',
601
- 'pdb',
602
- 'profile',
603
- 'pstats',
604
- 'timeit',
605
- 'trace',
606
- 'cgitb',
607
- 'pydoc',
608
- 'dis',
609
- 'pickletools',
610
- 'formatter',
611
- 'msilib',
612
- 'msvcrt',
613
- 'nt',
614
- 'ntpath',
615
- 'nturl2path',
616
- 'winreg',
617
- 'winsound',
618
- 'posix',
619
- 'posixpath',
620
- 'pwd',
621
- 'spwd',
622
- 'grp',
623
- 'crypt',
624
- 'termios',
625
- 'tty',
626
- 'pty',
627
- 'fcntl',
628
- 'pipes',
629
- 'resource',
630
- 'nis',
631
- 'syslog',
632
- 'optparse',
633
- 'imp',
634
- 'importlib',
635
- 'keyword',
636
- 'parser',
637
- 'ast',
638
- 'symtable',
639
- 'symbol',
640
- 'token',
641
- 'tokenize',
642
- 'tabnanny',
643
- 'py_compile',
644
- 'compileall',
645
- 'pyclbr',
646
- 'bdb',
647
- 'pdb',
648
- 'profile',
649
- 'pstats',
650
- 'timeit',
651
- 'trace',
652
- 'cgitb',
653
- 'pydoc',
654
- 'doctest',
655
- 'unittest',
656
- 'test',
657
- 'lib2to3',
658
- 'distutils',
659
- 'ensurepip',
660
- 'venv',
661
- 'wsgiref',
662
- 'html',
663
- 'http',
664
- 'urllib',
665
- 'xmlrpc',
666
- 'ipaddress',
667
- 'secrets',
668
- 'statistics',
669
- 'pathlib',
670
- 'enum',
671
- 'numbers',
672
- 'fractions',
673
- 'decimal',
674
- 'cmath',
675
- 'array',
676
- 'memoryview',
677
- 'collections',
678
- 'heapq',
679
- 'bisect',
680
- 'array',
681
- 'weakref',
682
- 'types',
683
- 'copy',
684
- 'pprint',
685
- 'reprlib',
686
- 'dataclasses',
687
- 'dataclasses_json',
688
- 'typing',
689
- 'typing_extensions',
690
- 'backports',
691
- 'builtins',
692
- '__builtin__',
693
- '__future__',
694
- ]);
695
- // Filter out standard library, excluded packages, and already included packages
696
- const importsToAdd = [...taskImports]
697
- .filter(importName => {
698
- // Skip standard library modules
699
- if (standardLibraryModules.has(importName)) {
700
- return false;
701
- }
702
- // Skip excluded packages (provided by runtime or installed via other means)
703
- if (EXCLUDED_PACKAGES.has(importName.toLowerCase())) {
704
- return false;
705
- }
706
- // Skip if it's already in requirements (we'll check this by reading the file)
707
- return true;
708
- })
709
- .filter(Boolean);
710
- if (importsToAdd.length === 0) {
711
- return;
712
- }
713
- // Read existing requirements to avoid duplicates
714
- let existingRequirements = '';
715
- try {
716
- existingRequirements = await node_fs_1.promises.readFile(requirementsPath, 'utf8');
717
- }
718
- catch {
719
- // File might not exist yet, that's okay
720
- }
721
- const existingPackages = new Set(existingRequirements
722
- .split('\n')
723
- .map(line => line.trim().split(/[!<=>]/)[0].toLowerCase())
724
- .filter(Boolean));
725
- // Add imports that aren't already in requirements.txt
726
- const newPackages = importsToAdd
727
- .filter(importName => {
728
- const packageName = importName.toLowerCase();
729
- return !existingPackages.has(packageName);
730
- })
731
- .map(importName => importName.toLowerCase());
732
- if (newPackages.length > 0) {
733
- await node_fs_1.promises.appendFile(requirementsPath, newPackages.join('\n') + '\n');
734
- console.log(`Added ${newPackages.length} extracted import(s) to requirements.txt`);
735
- }
736
- }
737
- catch (error) {
738
- console.error('Error adding extracted imports to requirements.txt:', error);
739
- }
740
- }
741
- /**
742
- * Appends forked plugin dependencies to requirements.txt.
743
- * Returns the set of package names that were excluded but need runner.sh (e.g. ssdeep).
744
- */
745
127
  static async generateRequirements(fromDirectory, toDirectory, packageName) {
746
- const excludedRunner = new Set();
747
128
  const packageExtractedDir = (0, node_path_1.join)(fromDirectory, 'packages', packageName);
748
129
  const requirementsPath = (0, node_path_1.join)(toDirectory, 'requirements.txt');
749
130
  try {
@@ -751,7 +132,7 @@ class ConnectorGenerator {
751
132
  const whlFiles = files.filter(file => file.endsWith('.whl'));
752
133
  if (whlFiles.length === 0) {
753
134
  console.warn(`No .whl files found in ${packageExtractedDir}`);
754
- return excludedRunner;
135
+ return;
755
136
  }
756
137
  const dependencies = whlFiles
757
138
  .map(whlFile => {
@@ -764,253 +145,14 @@ class ConnectorGenerator {
764
145
  if (packageNameFromWhl === packageName) {
765
146
  return null;
766
147
  }
767
- // Skip excluded packages (provided by runtime or installed via other means)
768
- if (EXCLUDED_PACKAGES.has(packageNameFromWhl.toLowerCase())) {
769
- if (RUNNER_EXCLUDED_PACKAGES.has(packageNameFromWhl.toLowerCase())) {
770
- excludedRunner.add(packageNameFromWhl.toLowerCase());
771
- }
772
- return null;
773
- }
774
148
  return `${packageNameFromWhl}==${packageVersion}`;
775
149
  })
776
150
  .filter(Boolean);
777
151
  await node_fs_1.promises.appendFile(requirementsPath, dependencies.join('\n') + '\n');
778
152
  console.log(`requirements.txt generated at: ${requirementsPath}`);
779
- return excludedRunner;
780
153
  }
781
154
  catch (error) {
782
155
  console.error('Error generating requirements.txt:', error);
783
- return excludedRunner;
784
- }
785
- }
786
- /**
787
- * Resolves version conflicts by taking the highest version requirement.
788
- * Handles common cases like ==, >=, <=, >, < constraints.
789
- */
790
- static resolveVersionConflict(packageName, requirements) {
791
- if (requirements.length === 1) {
792
- return requirements[0];
793
- }
794
- const versionInfo = requirements.map(req => {
795
- // Match version specifiers: ==2.0, >=2.0, <=2.0, >2.0, <2.0, ~=2.0
796
- const match = req.match(/([!<=>~]+)\s*([\d.]+(?:[\dA-Za-z]*)?)/);
797
- if (!match) {
798
- return { req, operator: null, version: null, versionParts: null };
799
- }
800
- const operator = match[1];
801
- const versionStr = match[2];
802
- // Parse version into parts for comparison
803
- const versionParts = versionStr.split('.').map(part => {
804
- const numMatch = part.match(/^(\d+)/);
805
- return numMatch ? Number.parseInt(numMatch[1], 10) : 0;
806
- });
807
- return { req, operator, version: versionStr, versionParts };
808
- });
809
- // Filter out requirements without version info
810
- const withVersions = versionInfo.filter((v) => v.version !== null && v.operator !== null && v.versionParts !== null);
811
- if (withVersions.length === 0) {
812
- // No version info, return first
813
- console.warn(`Multiple requirements for ${packageName} without version info: ${requirements.join(', ')}. Using: ${requirements[0]}`);
814
- return requirements[0];
815
- }
816
- // Find the highest version
817
- let highest = withVersions[0];
818
- for (let i = 1; i < withVersions.length; i++) {
819
- const current = withVersions[i];
820
- // Compare versions
821
- const comparison = this.compareVersions(current.versionParts, highest.versionParts);
822
- if (comparison > 0) {
823
- // Current is higher
824
- highest = current;
825
- }
826
- else if (comparison === 0) {
827
- // Same version, prefer == over >= or other operators
828
- if (current.operator === '==' && highest.operator !== '==') {
829
- highest = current;
830
- }
831
- else if (current.operator === '>=' && highest.operator === '>=') {
832
- // Both are >= with same version, keep current (they're equivalent)
833
- // No change needed
834
- }
835
- }
836
- }
837
- // Log if we're resolving a conflict
838
- if (withVersions.length > 1) {
839
- const allReqs = requirements.join(', ');
840
- console.log(`Resolved version conflict for ${packageName}: ${allReqs} -> ${highest.req}`);
841
- }
842
- return highest.req;
843
- }
844
- /**
845
- * Compares two version arrays (e.g., [2, 5, 0] vs [2, 4, 1]).
846
- * Returns: positive if v1 > v2, negative if v1 < v2, 0 if equal.
847
- */
848
- static compareVersions(v1, v2) {
849
- const maxLength = Math.max(v1.length, v2.length);
850
- for (let i = 0; i < maxLength; i++) {
851
- const part1 = v1[i] || 0;
852
- const part2 = v2[i] || 0;
853
- if (part1 > part2)
854
- return 1;
855
- if (part1 < part2)
856
- return -1;
857
- }
858
- return 0;
859
- }
860
- /**
861
- * Deduplicates requirements.txt and resolves version conflicts.
862
- * When multiple versions of the same package are specified, keeps the highest version.
863
- */
864
- static async deduplicateRequirements(toDirectory) {
865
- const requirementsPath = (0, node_path_1.join)(toDirectory, 'requirements.txt');
866
- try {
867
- let content = '';
868
- try {
869
- content = await node_fs_1.promises.readFile(requirementsPath, 'utf8');
870
- }
871
- catch {
872
- // File might not exist, nothing to deduplicate
873
- return;
874
- }
875
- const lines = content
876
- .split('\n')
877
- .map(line => line.trim())
878
- .filter(line => line && !line.startsWith('#'));
879
- if (lines.length === 0) {
880
- return;
881
- }
882
- // Parse requirements into a map: packageName -> requirements[]
883
- const requirementsMap = new Map();
884
- for (const line of lines) {
885
- // Extract package name (everything before version specifiers like ==, >=, <=, >, <, !=)
886
- // Handle packages with extras like requests[security]>=2.0
887
- const packageMatch = line.match(/^([\w[\]-]+?)(?:\s*[!<=>]|$)/);
888
- if (!packageMatch)
889
- continue;
890
- // Extract base package name (remove extras like [security])
891
- const fullPackageName = packageMatch[1];
892
- const basePackageName = fullPackageName.split('[')[0].toLowerCase();
893
- const requirement = line.trim();
894
- if (!requirementsMap.has(basePackageName)) {
895
- requirementsMap.set(basePackageName, []);
896
- }
897
- requirementsMap.get(basePackageName).push(requirement);
898
- }
899
- // Deduplicate and resolve conflicts
900
- const deduplicated = [];
901
- const strippedPackages = [];
902
- for (const [packageName, reqs] of requirementsMap.entries()) {
903
- // Skip excluded packages (provided by runtime or installed via other means)
904
- if (EXCLUDED_PACKAGES.has(packageName)) {
905
- continue;
906
- }
907
- // Strip version for packages known to have Python 3.11+ compatibility issues
908
- if (PACKAGES_TO_STRIP_VERSION.has(packageName)) {
909
- // Extract the package name with extras (e.g., requests[security] -> requests[security])
910
- const reqWithExtras = reqs[0].match(/^([\w-]+(?:\[[^\]]+])?)/);
911
- const packageWithExtras = reqWithExtras ? reqWithExtras[1] : packageName;
912
- deduplicated.push(packageWithExtras);
913
- strippedPackages.push(packageName);
914
- continue;
915
- }
916
- if (reqs.length === 1) {
917
- // Single requirement, use it as-is
918
- deduplicated.push(reqs[0]);
919
- }
920
- else {
921
- // Multiple requirements for same package - need to resolve
922
- const uniqueReqs = [...new Set(reqs)];
923
- if (uniqueReqs.length === 1) {
924
- // All are identical, use one
925
- deduplicated.push(uniqueReqs[0]);
926
- }
927
- else {
928
- // Different versions/constraints - resolve by taking the higher version
929
- const resolved = this.resolveVersionConflict(packageName, uniqueReqs);
930
- deduplicated.push(resolved);
931
- }
932
- }
933
- }
934
- // Log stripped packages
935
- if (strippedPackages.length > 0) {
936
- console.log(`Stripped version constraints for Python 3.11+ compatibility: ${strippedPackages.join(', ')}`);
937
- }
938
- // Ensure datetime_parser is set to version 1.1.0 if present
939
- const datetimeParserIndex = deduplicated.findIndex(req => {
940
- const packageName = req.trim().split(/[!<=>]/)[0].toLowerCase();
941
- return packageName === 'datetime_parser';
942
- });
943
- if (datetimeParserIndex !== -1) {
944
- deduplicated[datetimeParserIndex] = 'datetime_parser==1.1.0';
945
- }
946
- // Ensure pyflattener is set to version 1.1.0 if present
947
- const pyflattenerIndex = deduplicated.findIndex(req => {
948
- const packageName = req.trim().split(/[!<=>]/)[0].toLowerCase();
949
- return packageName === 'pyflattener';
950
- });
951
- if (pyflattenerIndex !== -1) {
952
- deduplicated[pyflattenerIndex] = 'pyflattener==1.1.0';
953
- }
954
- // Sort alphabetically for consistency
955
- deduplicated.sort();
956
- // Write back the deduplicated requirements
957
- const deduplicatedContent = deduplicated.join('\n') + '\n';
958
- await node_fs_1.promises.writeFile(requirementsPath, deduplicatedContent, 'utf8');
959
- const removedCount = lines.length - deduplicated.length;
960
- if (removedCount > 0) {
961
- console.log(`Deduplicated requirements.txt: removed ${removedCount} duplicate(s)`);
962
- }
963
- }
964
- catch (error) {
965
- console.error('Error deduplicating requirements.txt:', error);
966
- }
967
- }
968
- /**
969
- * Creates compile.sh in the connector root if swimbundle_utils was found in requirements.
970
- * This handles the special case where swimbundle_utils needs to be installed separately.
971
- */
972
- static async generateCompileScript(toDirectory) {
973
- const compileScriptPath = (0, node_path_1.join)(toDirectory, 'compile.sh');
974
- try {
975
- // Always create compile.sh (it's safe to have even if swimbundle_utils wasn't in requirements)
976
- const compileScriptContent = 'pip install --user swimbundle_utils==4.8.0 dominions --no-deps\n';
977
- await node_fs_1.promises.writeFile(compileScriptPath, compileScriptContent, 'utf8');
978
- // Make it executable
979
- await node_fs_1.promises.chmod(compileScriptPath, 0o755);
980
- }
981
- catch (error) {
982
- console.error('Error generating compile.sh:', error);
983
- }
984
- }
985
- /**
986
- * Creates runner.sh in the connector root with apt/pip installs for optional system deps:
987
- * - wkhtmltopdf when connector uses sw_swimlane_email
988
- * - ssdeep (apt + pip install --user) when needsSsdeep (e.g. from task imports; ssdeep is excluded from requirements.txt)
989
- */
990
- static async generateRunnerScript(toDirectory, usesSwimlaneEmail, needsSsdeep) {
991
- const runnerPath = (0, node_path_1.join)(toDirectory, 'runner.sh');
992
- const parts = [];
993
- if (usesSwimlaneEmail || needsSsdeep) {
994
- parts.push('apt update -y\n');
995
- const installs = [];
996
- if (usesSwimlaneEmail)
997
- installs.push('wkhtmltopdf');
998
- if (needsSsdeep)
999
- installs.push('ssdeep', 'libfuzzy-dev', 'gcc');
1000
- if (installs.length > 0)
1001
- parts.push(`apt install -y ${installs.join(' ')}\n`);
1002
- }
1003
- if (needsSsdeep)
1004
- parts.push('pip install ssdeep --user\n');
1005
- if (parts.length === 0)
1006
- return;
1007
- try {
1008
- const scriptContent = parts.join('');
1009
- await node_fs_1.promises.writeFile(runnerPath, scriptContent, 'utf8');
1010
- await node_fs_1.promises.chmod(runnerPath, 0o755);
1011
- }
1012
- catch (error) {
1013
- console.error('Error generating runner.sh:', error);
1014
156
  }
1015
157
  }
1016
158
  static async createBaseCode(fromDirectory, toDirectory, packageName) {
@@ -1031,67 +173,17 @@ class ConnectorGenerator {
1031
173
  console.error(`Could not find base package folder inside: ${whlPath}`);
1032
174
  return;
1033
175
  }
1034
- const destinationPath = (0, node_path_1.join)(toDirectory, 'connector', packageName);
176
+ const destinationPath = (0, node_path_1.join)(toDirectory, 'connector', 'src', packageName);
1035
177
  await node_fs_1.promises.mkdir(destinationPath, { recursive: true });
1036
178
  const baseCodePath = (0, node_path_1.join)(tempExtractDir, basePackageDir);
1037
179
  const baseFiles = await node_fs_1.promises.readdir(baseCodePath);
1038
- await Promise.all(baseFiles.map(async (file) => {
1039
- const sourcePath = (0, node_path_1.join)(baseCodePath, file);
1040
- const destPath = (0, node_path_1.join)(destinationPath, file);
1041
- const stat = await node_fs_1.promises.stat(sourcePath);
1042
- // Copy directory recursively or copy file
1043
- await (stat.isDirectory() ?
1044
- this.copyDirectoryRecursive(sourcePath, destPath) :
1045
- (0, promises_1.copyFile)(sourcePath, destPath));
1046
- }));
1047
- if (packageName === 'sw_swimlane_email') {
1048
- await this.patchSwimlaneEmailInit((0, node_path_1.join)(destinationPath, '__init__.py'));
1049
- }
1050
- console.log(`Base code for ${packageName} copied successfully.`);
1051
- }
1052
- catch (error) {
1053
- console.error(`Error extracting and copying base code for ${packageName}:`, error);
1054
- }
1055
- }
1056
- /**
1057
- * Patch sw_swimlane_email __init__.py execute() to use system wkhtmltoimage and Config instead of pkg_resources/chmod.
1058
- * Matches the block after "def execute(self):" containing:
1059
- * f = pkg_resources.resource_filename(__name__, "__packages/wkhtmltoimage") # ...
1060
- * c = Config(f)
1061
- * st = os.stat(f)
1062
- * os.chmod(f, st.st_mode | stat.S_IEXEC) # ...
1063
- */
1064
- static async patchSwimlaneEmailInit(initPath) {
1065
- try {
1066
- await node_fs_1.promises.access(initPath);
1067
- const content = await node_fs_1.promises.readFile(initPath, 'utf8');
1068
- const oldPattern = /(\s+def execute\(self\):)\s*\n\s+f = pkg_resources\.resource_filename\(__name__, ["']__packages\/wkhtmltoimage["']\)[^\n]*\n\s+c = Config\(f\)\s*\n\s+st = os\.stat\(f\)\s*\n\s+os\.chmod\(f, st\.st_mode \| stat\.S_IEXEC\)[^\n]*/m;
1069
- const newBody = `$1
1070
- path_wkhtmltoimage = '/usr/bin/wkhtmltoimage'
1071
- c = Config(wkhtmltoimage=path_wkhtmltoimage)`;
1072
- const newContent = content.replace(oldPattern, newBody);
1073
- if (newContent !== content) {
1074
- await node_fs_1.promises.writeFile(initPath, newContent, 'utf8');
1075
- console.log('Patched sw_swimlane_email __init__.py execute() for wkhtmltoimage/Config');
1076
- }
180
+ await Promise.all(baseFiles.map(file => node_fs_1.promises.rename((0, node_path_1.join)(baseCodePath, file), (0, node_path_1.join)(destinationPath, file))));
181
+ console.log(`Base code for ${packageName} moved successfully.`);
1077
182
  }
1078
183
  catch (error) {
1079
- console.error(`Error patching sw_swimlane_email __init__.py: ${initPath}`, error);
184
+ console.error(`Error extracting and moving base code for ${packageName}:`, error);
1080
185
  }
1081
186
  }
1082
- static async copyDirectoryRecursive(sourceDir, destDir) {
1083
- await node_fs_1.promises.mkdir(destDir, { recursive: true });
1084
- const entries = await node_fs_1.promises.readdir(sourceDir, { withFileTypes: true });
1085
- await Promise.all(entries.map(async (entry) => {
1086
- // Skip __pycache__ directories
1087
- if (entry.name === '__pycache__') {
1088
- return;
1089
- }
1090
- const sourcePath = (0, node_path_1.join)(sourceDir, entry.name);
1091
- const destPath = (0, node_path_1.join)(destDir, entry.name);
1092
- await (entry.isDirectory() ? this.copyDirectoryRecursive(sourcePath, destPath) : (0, promises_1.copyFile)(sourcePath, destPath));
1093
- }));
1094
- }
1095
187
  static async getBaseCodePath(fromDirectory, packageName) {
1096
188
  const packageDir = (0, node_path_1.join)(fromDirectory, 'packages', packageName);
1097
189
  try {
@@ -1108,95 +200,11 @@ class ConnectorGenerator {
1108
200
  return null;
1109
201
  }
1110
202
  }
1111
- /**
1112
- * Build Python snippet to rebuild inputs from YAML-defined keys (empty per type) then merge asset then inputs.
1113
- * Only includes keys where the task InputMapping has an associated Type and Value.
1114
- * Placeholder # INPUTS_MERGE_HERE in templates is replaced with this.
1115
- * @param includeAttachmentBlock - when true (script_override only), include attachment conversion and dict handling; when false (plugin_override), only temp_inputs merge.
1116
- */
1117
- static defaultPyValue(valueType) {
1118
- switch (valueType) {
1119
- case 'number':
1120
- case 'integer': {
1121
- return 'None';
1122
- }
1123
- case 'boolean': {
1124
- return 'False';
1125
- }
1126
- case 'array': {
1127
- return '[]';
1128
- }
1129
- default: {
1130
- return "''";
1131
- }
1132
- }
1133
- }
1134
- static buildInputsMergeSnippet(inputs, includeAttachmentBlock) {
1135
- const withTypeAndValue = inputs.filter(inp => inp.Type !== null && inp.Type !== undefined &&
1136
- inp.Value !== null && inp.Value !== undefined);
1137
- const entries = [];
1138
- for (const inp of withTypeAndValue) {
1139
- entries.push(`${JSON.stringify(inp.Key)}: ${this.defaultPyValue(inp.ValueType)}`);
1140
- }
1141
- const dictLiteral = entries.length > 0 ? `{${entries.join(', ')}}` : '{}';
1142
- const i = ' ';
1143
- // First line has no leading indent: template already has 8 spaces before # INPUTS_MERGE_HERE
1144
- const baseMerge = includeAttachmentBlock ?
1145
- `temp_inputs = ${dictLiteral}\n${i}temp_inputs.update(self.asset)\n` :
1146
- `temp_inputs = ${dictLiteral}\n`;
1147
- // if (!includeAttachmentBlock) {
1148
- // return `${baseMerge}${i}temp_inputs.update(inputs)\n${i}inputs = temp_inputs`
1149
- // }
1150
- const i2 = ' ';
1151
- const i3 = ' ';
1152
- const i4 = ' ';
1153
- const attachmentBlock = [
1154
- `${i}attachments = {}`,
1155
- `${i}files = inputs.pop('files', [])`,
1156
- `${i}import base64`,
1157
- `${i}def find_and_convert_attachments(filename, files):`,
1158
- `${i2}for file in files:`,
1159
- `${i3}if isinstance(file, (list, tuple)):`,
1160
- `${i3} current_filename, file_obj = file`,
1161
- `${i3} if current_filename == filename:`,
1162
- `${i3} files.remove(file)`,
1163
- `${i3} return {'filename': current_filename, 'base64': base64.b64encode(file_obj.read()).decode()}`,
1164
- `${i}for field in inputs:`,
1165
- `${i2}if type(inputs[field]) is list:`,
1166
- `${i3}updated_value = [find_and_convert_attachments(i['file_name'], files) for i in inputs[field] if 'file' in i and 'file_name' in i]`,
1167
- `${i3}if updated_value:`,
1168
- `${i4}attachments[field] = updated_value`,
1169
- `${i2}if type(inputs[field]) is dict:`,
1170
- `${i3}# this is actually a conversion for user inputs, but we'll take the oppt to fix it here`,
1171
- `${i3}if 'id' in inputs[field] and 'name' in inputs[field]:`,
1172
- `${i4}attachments[field] = inputs[field]['name']`,
1173
- `${i}inputs.update(attachments)`,
1174
- '',
1175
- `${i}temp_inputs.update(inputs)`,
1176
- `${i}inputs = temp_inputs`,
1177
- ].join('\n');
1178
- return `${baseMerge}\n${attachmentBlock}`;
1179
- }
1180
- static replaceTaskExecuteRequestCall(content) {
1181
- return content.replaceAll(this.TASK_EXECUTE_REQUEST_CALL, this.TASK_EXECUTE_WEBHOOK_CALL);
1182
- }
1183
- /** Build Python dict literal for OUTPUT_DATE_CONVERSIONS (key -> timetype/format). */
1184
- static buildOutputDateConversionsDict(conversions) {
1185
- if (!conversions?.length)
1186
- return '{}';
1187
- const obj = {};
1188
- for (const { key, timetype } of conversions)
1189
- obj[key] = timetype;
1190
- return JSON.stringify(obj);
1191
- }
1192
- static async getActionContentFork(script, inputs, outputDateConversions) {
203
+ static async getActionContentFork(script) {
1193
204
  try {
1194
- let templateContent = await node_fs_1.promises.readFile((0, node_path_1.join)(__dirname, '../templates/migrator-runners/plugin_override.txt'), 'utf8');
205
+ const templateContent = await node_fs_1.promises.readFile((0, node_path_1.join)(__dirname, '../templates/migrator-runners/plugin_override.txt'), 'utf8');
1195
206
  // Remove any carriage returns to avoid CRLF issues
1196
207
  const scriptNoCR = script.replaceAll('\r', '');
1197
- const inputsMergeSnippet = this.buildInputsMergeSnippet(inputs, false);
1198
- templateContent = templateContent.replace(this.INPUTS_MERGE_PLACEHOLDER, inputsMergeSnippet);
1199
- templateContent = templateContent.replace(this.OUTPUT_DATE_CONVERSIONS_PLACEHOLDER, this.buildOutputDateConversionsDict(outputDateConversions));
1200
208
  return templateContent.replace('# HERE', scriptNoCR);
1201
209
  }
1202
210
  catch (error) {
@@ -1204,14 +212,11 @@ class ConnectorGenerator {
1204
212
  return `Error during forked plugin generation: ${error}`;
1205
213
  }
1206
214
  }
1207
- static async getActionContentScript(script, inputs, outputDateConversions) {
215
+ static async getActionContentScript(script) {
1208
216
  try {
1209
- let templateContent = await node_fs_1.promises.readFile((0, node_path_1.join)(__dirname, '../templates/migrator-runners/script_override.txt'), 'utf8');
217
+ const templateContent = await node_fs_1.promises.readFile((0, node_path_1.join)(__dirname, '../templates/migrator-runners/script_override.txt'), 'utf8');
1210
218
  // Remove any carriage returns to avoid CRLF issues
1211
219
  const scriptNoCR = script.replaceAll('\r', '');
1212
- const inputsMergeSnippet = this.buildInputsMergeSnippet(inputs, true);
1213
- templateContent = templateContent.replace(this.INPUTS_MERGE_PLACEHOLDER, inputsMergeSnippet);
1214
- templateContent = templateContent.replace(this.OUTPUT_DATE_CONVERSIONS_PLACEHOLDER, this.buildOutputDateConversionsDict(outputDateConversions));
1215
220
  const lines = scriptNoCR.split('\n');
1216
221
  if (lines.length === 0) {
1217
222
  return templateContent.replace('# HERE', '');
@@ -1233,58 +238,20 @@ class ConnectorGenerator {
1233
238
  static async generateActionConfig(transformationResult, toDirectory) {
1234
239
  const exportUid = transformationResult.exportUid;
1235
240
  const outputPath = (0, node_path_1.join)(toDirectory, 'connector', 'config', 'actions', `${exportUid}.yaml`);
1236
- // Format description with task ID prepended if available
1237
- const description = transformationResult.taskId ?
1238
- `${transformationResult.taskId} - ${transformationResult.description || ''}` :
1239
- transformationResult.description || '';
1240
241
  const yamlData = {
1241
242
  schema: 'action/1',
1242
243
  title: transformationResult.exportName,
1243
244
  name: transformationResult.exportUid,
1244
- description,
245
+ description: transformationResult.description,
1245
246
  inputs: {
1246
247
  type: 'object',
1247
- properties: {
1248
- ApplicationId: {
1249
- title: 'Application ID',
1250
- type: 'string',
1251
- },
1252
- RecordId: {
1253
- title: 'Record ID',
1254
- type: 'string',
1255
- },
1256
- SwimlaneUrl: {
1257
- title: 'Swimlane URL',
1258
- type: 'string',
1259
- },
1260
- TurbineAccountId: {
1261
- title: 'Turbine Account ID',
1262
- type: 'string',
1263
- },
1264
- TurbineTenantId: {
1265
- title: 'Turbine Tenant ID',
1266
- type: 'string',
1267
- },
1268
- ExecuteTaskWebhookUrl: {
1269
- title: 'Execute Task Webhook URL',
1270
- type: 'string',
1271
- },
1272
- },
248
+ properties: {},
1273
249
  required: [],
1274
250
  additionalProperties: true,
1275
251
  },
1276
252
  output: {
1277
253
  type: 'object',
1278
- properties: {
1279
- output: {
1280
- title: 'Output',
1281
- type: 'array',
1282
- items: {
1283
- type: 'object',
1284
- properties: {},
1285
- },
1286
- },
1287
- },
254
+ properties: {},
1288
255
  required: [],
1289
256
  additionalProperties: true,
1290
257
  },
@@ -1293,141 +260,39 @@ class ConnectorGenerator {
1293
260
  method: '',
1294
261
  },
1295
262
  };
1296
- // Only add regular inputs (exclude credentials and asset types which go to asset.yaml)
1297
263
  for (const input of transformationResult.inputs) {
1298
- const inputType = input.ValueType || 'string';
1299
- const inputProperty = {
264
+ yamlData.inputs.properties[input.Key] = {
1300
265
  title: input.Title || input.Key,
1301
- type: inputType,
266
+ type: input.ValueType || 'string',
267
+ examples: input.Example,
1302
268
  };
1303
- if (inputType === 'array') {
1304
- const arrayItemType = input.arrayItemType;
1305
- const arrayItemValueType = input.arrayItemValueType;
1306
- switch (arrayItemType) {
1307
- case 'attachment': {
1308
- inputProperty.items = {
1309
- contentDisposition: 'attachment',
1310
- type: 'object',
1311
- additionalProperties: false,
1312
- properties: {
1313
- file: {
1314
- type: 'string',
1315
- format: 'binary',
1316
- },
1317
- // eslint-disable-next-line camelcase
1318
- file_name: {
1319
- type: 'string',
1320
- },
1321
- },
1322
- };
1323
- break;
1324
- }
1325
- case 'reference': {
1326
- inputProperty.items = {
1327
- type: 'object',
1328
- required: [],
1329
- };
1330
- break;
1331
- }
1332
- case 'list': {
1333
- inputProperty.items = {
1334
- type: arrayItemValueType === 'numeric' ? 'number' : 'string',
1335
- };
1336
- break;
1337
- }
1338
- default: {
1339
- // valueslist (multi-select, check list) or default: string items
1340
- inputProperty.items = {
1341
- type: 'string',
1342
- };
1343
- }
1344
- }
1345
- }
1346
- if (input.Example !== undefined && input.Example !== null) {
1347
- inputProperty.examples = Array.isArray(input.Example) ? input.Example : [input.Example];
1348
- }
1349
- yamlData.inputs.properties[input.Key] = inputProperty;
1350
269
  if (input.Creds) {
1351
270
  yamlData.inputs.properties[input.Key].format = 'password';
1352
271
  }
1353
272
  }
1354
273
  for (const output of transformationResult.outputs) {
1355
- const outputProperty = {
274
+ yamlData.output.properties[output.Key] = {
1356
275
  title: output.Title,
1357
276
  type: output.ValueType,
277
+ examples: output.Example,
1358
278
  };
1359
- if (output.Example !== undefined && output.Example !== null) {
1360
- outputProperty.examples = Array.isArray(output.Example) ? output.Example : [output.Example];
1361
- }
1362
- // Ensure the key is valid and doesn't cause YAML issues
1363
- const safeKey = output.Key || 'unnamed_output';
1364
- yamlData.output.properties.output.items.properties[safeKey] = outputProperty;
1365
- }
1366
- try {
1367
- // Ensure the output properties object is properly initialized
1368
- if (!yamlData.output.properties.output.items.properties) {
1369
- yamlData.output.properties.output.items.properties = {};
1370
- }
1371
- const yamlString = js_yaml_1.default.dump(yamlData, { indent: 2, noRefs: true, lineWidth: -1 });
1372
- // Validate the YAML can be parsed back (catches structural issues)
1373
- let parsed;
1374
- try {
1375
- parsed = js_yaml_1.default.load(yamlString);
1376
- }
1377
- catch (parseError) {
1378
- console.error(`Generated invalid YAML for ${exportUid}:`, parseError);
1379
- console.error('YAML content:', yamlString.slice(0, 500));
1380
- throw new Error(`Failed to generate valid YAML for action ${exportUid}: ${parseError}`);
1381
- }
1382
- // Double-check the structure is correct
1383
- if (!parsed?.output?.properties?.output?.items?.properties) {
1384
- console.error(`YAML structure issue for ${exportUid}: output properties not properly nested`);
1385
- console.error('Parsed structure:', JSON.stringify(parsed?.output, null, 2));
1386
- }
1387
- // Write atomically using a temporary file to prevent corruption
1388
- const dir = (0, node_path_1.join)(outputPath, '..');
1389
- await node_fs_1.promises.mkdir(dir, { recursive: true });
1390
- // Write to a temporary file first, then rename (atomic operation)
1391
- const tempFile = (0, node_path_1.join)((0, node_os_1.tmpdir)(), `${exportUid}-${Date.now()}-${Math.random().toString(36).slice(7)}.yaml`);
1392
- try {
1393
- await node_fs_1.promises.writeFile(tempFile, yamlString, 'utf8');
1394
- // Atomically move the temp file to the final location
1395
- await node_fs_1.promises.rename(tempFile, outputPath);
1396
- }
1397
- catch (writeError) {
1398
- // Clean up temp file if it exists
1399
- await node_fs_1.promises.unlink(tempFile).catch(() => { });
1400
- throw writeError;
1401
- }
1402
- return true;
1403
- }
1404
- catch (error) {
1405
- console.error(`Error generating action config for ${exportUid}:`, error);
1406
- throw error;
1407
279
  }
280
+ const yamlString = js_yaml_1.default.dump(yamlData, { indent: 2 });
281
+ await this.createFile(outputPath, yamlString);
282
+ return true;
1408
283
  }
1409
284
  static async createFile(dir, data) {
1410
- await node_fs_1.promises.writeFile(dir, data, 'utf8');
285
+ await node_fs_1.promises.writeFile(dir, data);
1411
286
  }
1412
287
  static async generateConnectorManifest(connectorConfig, group, toDir) {
1413
- let connectorNameUid = `${connectorConfig.author || connectorConfig.vendor}_${group.connectorName}`
1414
- .replaceAll(/\W/g, '_') // Replace all non-word characters (including dashes, spaces, colons, etc.) with underscores
1415
- .replaceAll(/_+/g, '_') // Replace multiple consecutive underscores with a single underscore
1416
- .replaceAll(/^_|_$/g, '') // Remove leading and trailing underscores
288
+ const connectorNameUid = `${connectorConfig.author || connectorConfig.vendor}_${group.connectorName}`
289
+ .replaceAll(/[^\w -]/g, '')
290
+ .replaceAll(/\s+/g, '_')
1417
291
  .toLowerCase();
1418
- // Truncate connector name to 50 characters
1419
- if (connectorNameUid.length > 50) {
1420
- connectorNameUid = connectorNameUid.slice(0, 50);
1421
- }
1422
- // Truncate title to 50 characters
1423
- let connectorTitle = group.connectorName;
1424
- if (connectorTitle.length > 50) {
1425
- connectorTitle = connectorTitle.slice(0, 50);
1426
- }
1427
292
  const data = {
1428
293
  author: connectorConfig.author || connectorConfig.vendor,
1429
294
  bugs: '',
1430
- description: connectorConfig.description || group.connectorName,
295
+ description: connectorConfig.description,
1431
296
  homepage: connectorConfig.homepage,
1432
297
  iconImage: '../image/logo.png',
1433
298
  keywords: ['Custom', 'User created'],
@@ -1443,10 +308,10 @@ class ConnectorGenerator {
1443
308
  },
1444
309
  },
1445
310
  name: connectorNameUid,
1446
- product: connectorConfig.product || group.connectorName,
311
+ product: connectorConfig.product || 'Unknown Product',
1447
312
  repository: `https://github.com/swimlane-prosrv/t_${connectorNameUid}`,
1448
313
  schema: 'connector/1',
1449
- title: connectorTitle,
314
+ title: group.connectorName,
1450
315
  vendor: connectorConfig.vendor || 'Unknown Vendor',
1451
316
  version: '1.0.0',
1452
317
  runConfig: {
@@ -1459,8 +324,4 @@ class ConnectorGenerator {
1459
324
  }
1460
325
  }
1461
326
  exports.ConnectorGenerator = ConnectorGenerator;
1462
- ConnectorGenerator.INPUTS_MERGE_PLACEHOLDER = '# INPUTS_MERGE_HERE';
1463
- ConnectorGenerator.OUTPUT_DATE_CONVERSIONS_PLACEHOLDER = '__OUTPUT_DATE_CONVERSIONS__';
1464
- ConnectorGenerator.TASK_EXECUTE_REQUEST_CALL = "swimlane.request('post', 'task/execute/record', json=data)";
1465
- ConnectorGenerator.TASK_EXECUTE_WEBHOOK_CALL = 'swimlane._session.post(swimlane._execute_task_webhook_url, json=data)';
1466
327
  //# sourceMappingURL=connector-generator.js.map