@atmyapp/cli 0.0.2 → 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +91 -27
- package/dist/cli/commands/migrate.js +55 -10
- package/dist/cli/utils/content-processor.js +1 -1
- package/dist/cli/utils/definition-processor.js +1 -1
- package/dist/cli/utils/index.d.ts +2 -0
- package/dist/cli/utils/index.js +2 -0
- package/dist/cli/utils/parallel-schema-processor.d.ts +5 -0
- package/dist/cli/utils/parallel-schema-processor.js +147 -0
- package/dist/cli/utils/schema-processor.js +108 -9
- package/dist/cli/utils/worker-file-processor.d.ts +7 -0
- package/dist/cli/utils/worker-file-processor.js +247 -0
- package/dist/cli/utils/worker-pool.d.ts +25 -0
- package/dist/cli/utils/worker-pool.js +126 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
[](https://opensource.org/licenses/ISC)
|
|
5
5
|
[](https://www.typescriptlang.org/)
|
|
6
6
|
|
|
7
|
-
> 🔧 **Migrate your TypeScript definitions seamlessly.** The official CLI tool for AtMyApp - AI-powered content management that migrates your type definitions to the AtMyApp platform with zero configuration.
|
|
7
|
+
> 🔧 **Migrate your TypeScript definitions seamlessly.** The official CLI tool for AtMyApp - AI-powered content management that migrates your type definitions to the AtMyApp platform with zero configuration and lightning-fast parallel processing.
|
|
8
8
|
|
|
9
9
|
## 📖 Table of Contents
|
|
10
10
|
|
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
- [📚 Commands](#-commands)
|
|
15
15
|
- [use Command](#use-command)
|
|
16
16
|
- [migrate Command](#migrate-command)
|
|
17
|
+
- [⚡ Performance Features](#-performance-features)
|
|
17
18
|
- [🎯 Type Definitions](#-type-definitions)
|
|
18
19
|
- [Content Definitions](#content-definitions)
|
|
19
20
|
- [Event Definitions](#event-definitions)
|
|
@@ -31,10 +32,12 @@
|
|
|
31
32
|
📊 **Event Analytics Support** - Built-in support for event tracking definitions
|
|
32
33
|
🖼️ **Media Type Support** - Handles image and file definitions with optimization configs
|
|
33
34
|
🔄 **Real-time Processing** - Process multiple definition files simultaneously
|
|
35
|
+
⚡ **Lightning Fast** - Multi-threaded parallel processing for large codebases
|
|
34
36
|
🎯 **Type-Safe** - Full TypeScript support with comprehensive validation
|
|
35
|
-
|
|
37
|
+
🚀 **Zero Configuration** - Works out of the box with smart defaults
|
|
36
38
|
🔐 **Secure** - API key authentication with session management
|
|
37
|
-
🌊 **Pipeline Architecture** - Extensible processing pipeline for custom transformations
|
|
39
|
+
🌊 **Pipeline Architecture** - Extensible processing pipeline for custom transformations
|
|
40
|
+
📊 **Performance Monitoring** - Built-in timing and performance metrics
|
|
38
41
|
|
|
39
42
|
## 📦 Installation
|
|
40
43
|
|
|
@@ -55,7 +58,7 @@ pnpm add -g @atmyapp/cli
|
|
|
55
58
|
# 1. Authenticate with your AtMyApp project
|
|
56
59
|
ama use --token your-api-token --url https://your-project.atmyapp.com
|
|
57
60
|
|
|
58
|
-
# 2. Migrate your definitions
|
|
61
|
+
# 2. Migrate your definitions with parallel processing
|
|
59
62
|
ama migrate
|
|
60
63
|
|
|
61
64
|
# 3. Or run in dry-run mode to preview changes
|
|
@@ -92,7 +95,7 @@ ama use --token "ama_pk_..." --url "https://edge.atmyapp.com/projects/your-proje
|
|
|
92
95
|
|
|
93
96
|
### migrate Command
|
|
94
97
|
|
|
95
|
-
Migrate TypeScript definitions to the AtMyApp platform.
|
|
98
|
+
Migrate TypeScript definitions to the AtMyApp platform with optimized parallel processing.
|
|
96
99
|
|
|
97
100
|
```bash
|
|
98
101
|
ama migrate [options]
|
|
@@ -104,20 +107,79 @@ ama migrate [options]
|
|
|
104
107
|
- `--verbose` - Enable verbose logging (default: false)
|
|
105
108
|
- `--tsconfig <path>` - Path to tsconfig.json (default: "tsconfig.json")
|
|
106
109
|
- `--continue-on-error` - Continue processing even if some files fail (default: false)
|
|
110
|
+
- `--parallel` - Enable parallel processing using worker threads (default: true)
|
|
111
|
+
- `--max-workers <number>` - Maximum number of worker threads (default: CPU cores, max 8)
|
|
112
|
+
- `--no-filtering` - Disable file pre-filtering optimization (default: false)
|
|
107
113
|
|
|
108
114
|
**Examples:**
|
|
109
115
|
|
|
110
116
|
```bash
|
|
111
|
-
# Basic migration
|
|
117
|
+
# Basic migration with parallel processing (default)
|
|
112
118
|
ama migrate
|
|
113
119
|
|
|
114
|
-
# Dry run with verbose output
|
|
120
|
+
# Dry run with verbose output and performance metrics
|
|
115
121
|
ama migrate --dry-run --verbose
|
|
116
122
|
|
|
117
123
|
# Use custom tsconfig and continue on errors
|
|
118
124
|
ama migrate --tsconfig ./custom-tsconfig.json --continue-on-error
|
|
125
|
+
|
|
126
|
+
# Force sequential processing (slower, for debugging)
|
|
127
|
+
ama migrate --no-parallel
|
|
128
|
+
|
|
129
|
+
# Use specific number of worker threads
|
|
130
|
+
ama migrate --max-workers 4
|
|
131
|
+
|
|
132
|
+
# Maximum performance for large codebases
|
|
133
|
+
ama migrate --max-workers 8 --verbose
|
|
119
134
|
```
|
|
120
135
|
|
|
136
|
+
## ⚡ Performance Features
|
|
137
|
+
|
|
138
|
+
### Multi-threaded Processing
|
|
139
|
+
|
|
140
|
+
The CLI uses Node.js worker threads to process TypeScript files in parallel, providing significant performance improvements for large codebases:
|
|
141
|
+
|
|
142
|
+
- **Automatic scaling**: Uses optimal number of workers based on CPU cores
|
|
143
|
+
- **Smart filtering**: Pre-filters files to only process those with ATMYAPP exports
|
|
144
|
+
- **Program caching**: Reuses TypeScript compilation results across workers
|
|
145
|
+
- **Batch processing**: Groups schema generation for maximum efficiency
|
|
146
|
+
|
|
147
|
+
### Performance Optimizations
|
|
148
|
+
|
|
149
|
+
1. **File Pre-filtering**: Quickly scans files for ATMYAPP exports before processing
|
|
150
|
+
2. **Worker Pool Management**: Efficiently distributes work across available CPU cores
|
|
151
|
+
3. **TypeScript Program Caching**: Avoids redundant compilation overhead
|
|
152
|
+
4. **Parallel Schema Generation**: Processes multiple definition types simultaneously
|
|
153
|
+
5. **Chunked Processing**: Handles large file sets in optimized chunks
|
|
154
|
+
|
|
155
|
+
### Performance Monitoring
|
|
156
|
+
|
|
157
|
+
Enable verbose mode to see detailed performance metrics:
|
|
158
|
+
|
|
159
|
+
```bash
|
|
160
|
+
ama migrate --verbose
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
**Sample Output:**
|
|
164
|
+
|
|
165
|
+
```
|
|
166
|
+
✅ Successfully processed 127 AMA contents in 2.34s
|
|
167
|
+
📊 Performance Summary:
|
|
168
|
+
Total time: 3.45s
|
|
169
|
+
Processing time: 2.34s
|
|
170
|
+
Files processed: 127
|
|
171
|
+
Processing mode: Parallel
|
|
172
|
+
Worker threads: 8
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
### Expected Performance Improvements
|
|
176
|
+
|
|
177
|
+
With parallel processing enabled (default), you can expect:
|
|
178
|
+
|
|
179
|
+
- **Small codebases** (< 50 files): 1.5-2x faster
|
|
180
|
+
- **Medium codebases** (50-200 files): 2-4x faster
|
|
181
|
+
- **Large codebases** (200+ files): 3-6x faster
|
|
182
|
+
|
|
121
183
|
## 🎯 Type Definitions
|
|
122
184
|
|
|
123
185
|
### Content Definitions
|
|
@@ -148,23 +210,23 @@ export type ATMYAPP = [BlogPostContent];
|
|
|
148
210
|
|
|
149
211
|
### Event Definitions
|
|
150
212
|
|
|
151
|
-
Define analytics events using `
|
|
213
|
+
Define analytics events using `AmaCustomEventDef` with ordered columns:
|
|
152
214
|
|
|
153
215
|
```typescript
|
|
154
|
-
import {
|
|
216
|
+
import { AmaCustomEventDef } from "@atmyapp/core";
|
|
155
217
|
|
|
156
218
|
// Define event types for analytics tracking
|
|
157
|
-
export type PageViewEvent =
|
|
219
|
+
export type PageViewEvent = AmaCustomEventDef<
|
|
158
220
|
"page_view",
|
|
159
221
|
["page", "referrer", "timestamp", "user_id"]
|
|
160
222
|
>;
|
|
161
223
|
|
|
162
|
-
export type PurchaseEvent =
|
|
224
|
+
export type PurchaseEvent = AmaCustomEventDef<
|
|
163
225
|
"purchase",
|
|
164
226
|
["product_id", "amount", "currency", "user_id", "timestamp"]
|
|
165
227
|
>;
|
|
166
228
|
|
|
167
|
-
export type ClickEvent =
|
|
229
|
+
export type ClickEvent = AmaCustomEventDef<
|
|
168
230
|
"button_click",
|
|
169
231
|
["element", "position", "timestamp"]
|
|
170
232
|
>;
|
|
@@ -219,7 +281,7 @@ export type ATMYAPP = [HeroImage, UserManual];
|
|
|
219
281
|
|
|
220
282
|
```typescript
|
|
221
283
|
// types/ecommerce.ts
|
|
222
|
-
import { AmaContentDef,
|
|
284
|
+
import { AmaContentDef, AmaCustomEventDef, AmaImageDef } from "@atmyapp/core";
|
|
223
285
|
|
|
224
286
|
// Product catalog
|
|
225
287
|
interface Product {
|
|
@@ -244,17 +306,17 @@ export type ProductImage = AmaImageDef<
|
|
|
244
306
|
>;
|
|
245
307
|
|
|
246
308
|
// E-commerce events
|
|
247
|
-
export type ProductViewEvent =
|
|
309
|
+
export type ProductViewEvent = AmaCustomEventDef<
|
|
248
310
|
"product_view",
|
|
249
311
|
["product_id", "category", "price", "user_id", "timestamp"]
|
|
250
312
|
>;
|
|
251
313
|
|
|
252
|
-
export type AddToCartEvent =
|
|
314
|
+
export type AddToCartEvent = AmaCustomEventDef<
|
|
253
315
|
"add_to_cart",
|
|
254
316
|
["product_id", "quantity", "price", "user_id", "timestamp"]
|
|
255
317
|
>;
|
|
256
318
|
|
|
257
|
-
export type PurchaseEvent =
|
|
319
|
+
export type PurchaseEvent = AmaCustomEventDef<
|
|
258
320
|
"purchase",
|
|
259
321
|
["order_id", "total_amount", "currency", "user_id", "timestamp"]
|
|
260
322
|
>;
|
|
@@ -274,7 +336,7 @@ export type ATMYAPP = [
|
|
|
274
336
|
|
|
275
337
|
```typescript
|
|
276
338
|
// types/blog.ts
|
|
277
|
-
import { AmaContentDef,
|
|
339
|
+
import { AmaContentDef, AmaCustomEventDef, AmaImageDef } from "@atmyapp/core";
|
|
278
340
|
|
|
279
341
|
// Blog content types
|
|
280
342
|
interface BlogPost {
|
|
@@ -314,17 +376,17 @@ export type BlogHeroImage = AmaImageDef<
|
|
|
314
376
|
>;
|
|
315
377
|
|
|
316
378
|
// Blog analytics events
|
|
317
|
-
export type ArticleReadEvent =
|
|
379
|
+
export type ArticleReadEvent = AmaCustomEventDef<
|
|
318
380
|
"article_read",
|
|
319
381
|
["article_id", "reading_time", "completion_rate", "referrer", "timestamp"]
|
|
320
382
|
>;
|
|
321
383
|
|
|
322
|
-
export type CommentEvent =
|
|
384
|
+
export type CommentEvent = AmaCustomEventDef<
|
|
323
385
|
"comment_posted",
|
|
324
386
|
["article_id", "comment_id", "user_id", "timestamp"]
|
|
325
387
|
>;
|
|
326
388
|
|
|
327
|
-
export type ShareEvent =
|
|
389
|
+
export type ShareEvent = AmaCustomEventDef<
|
|
328
390
|
"article_shared",
|
|
329
391
|
["article_id", "platform", "user_id", "timestamp"]
|
|
330
392
|
>;
|
|
@@ -344,36 +406,36 @@ export type ATMYAPP = [
|
|
|
344
406
|
|
|
345
407
|
```typescript
|
|
346
408
|
// types/analytics.ts
|
|
347
|
-
import {
|
|
409
|
+
import { AmaCustomEventDef } from "@atmyapp/core";
|
|
348
410
|
|
|
349
411
|
// User interaction events
|
|
350
|
-
export type PageViewEvent =
|
|
412
|
+
export type PageViewEvent = AmaCustomEventDef<
|
|
351
413
|
"page_view",
|
|
352
414
|
["page", "referrer", "user_agent", "session_id", "timestamp"]
|
|
353
415
|
>;
|
|
354
416
|
|
|
355
|
-
export type ClickEvent =
|
|
417
|
+
export type ClickEvent = AmaCustomEventDef<
|
|
356
418
|
"click",
|
|
357
419
|
["element", "element_text", "page", "position_x", "position_y", "timestamp"]
|
|
358
420
|
>;
|
|
359
421
|
|
|
360
|
-
export type FormSubmissionEvent =
|
|
422
|
+
export type FormSubmissionEvent = AmaCustomEventDef<
|
|
361
423
|
"form_submit",
|
|
362
424
|
["form_id", "form_name", "success", "validation_errors", "timestamp"]
|
|
363
425
|
>;
|
|
364
426
|
|
|
365
|
-
export type ScrollEvent =
|
|
427
|
+
export type ScrollEvent = AmaCustomEventDef<
|
|
366
428
|
"scroll",
|
|
367
429
|
["page", "scroll_depth", "session_id", "timestamp"]
|
|
368
430
|
>;
|
|
369
431
|
|
|
370
|
-
export type ErrorEvent =
|
|
432
|
+
export type ErrorEvent = AmaCustomEventDef<
|
|
371
433
|
"error",
|
|
372
434
|
["error_message", "error_stack", "page", "user_agent", "timestamp"]
|
|
373
435
|
>;
|
|
374
436
|
|
|
375
437
|
// Performance events
|
|
376
|
-
export type PerformanceEvent =
|
|
438
|
+
export type PerformanceEvent = AmaCustomEventDef<
|
|
377
439
|
"performance",
|
|
378
440
|
["page", "load_time", "dom_ready", "first_paint", "timestamp"]
|
|
379
441
|
>;
|
|
@@ -510,6 +572,8 @@ tests/
|
|
|
510
572
|
│ ├── content-processor.test.ts # Content processing tests
|
|
511
573
|
│ ├── definition-processor.test.ts # Pipeline tests
|
|
512
574
|
│ ├── schema-processor.test.ts # TypeScript processing tests
|
|
575
|
+
│ ├── parallel-processing.test.ts # Parallel processing tests
|
|
576
|
+
│ ├── definitions-examples.test.ts # Example definition tests
|
|
513
577
|
│ └── integration.test.ts # End-to-end tests
|
|
514
578
|
├── definitions/
|
|
515
579
|
│ ├── someFile.ts # Basic test definitions
|
|
@@ -15,6 +15,7 @@ const commander_1 = require("commander");
|
|
|
15
15
|
const utils_1 = require("../utils");
|
|
16
16
|
Object.defineProperty(exports, "registerTypeTransformer", { enumerable: true, get: function () { return utils_1.registerTypeTransformer; } });
|
|
17
17
|
Object.defineProperty(exports, "definitionPipeline", { enumerable: true, get: function () { return utils_1.definitionPipeline; } });
|
|
18
|
+
const parallel_schema_processor_1 = require("../utils/parallel-schema-processor");
|
|
18
19
|
// Main migrate command function
|
|
19
20
|
function migrateCommand() {
|
|
20
21
|
return new commander_1.Command("migrate")
|
|
@@ -23,22 +24,46 @@ function migrateCommand() {
|
|
|
23
24
|
.option("--verbose", "Enable verbose logging", false)
|
|
24
25
|
.option("--tsconfig <path>", "Path to tsconfig.json", "tsconfig.json")
|
|
25
26
|
.option("--continue-on-error", "Continue processing even if some files fail", false)
|
|
27
|
+
.option("--parallel", "Enable parallel processing using worker threads (default: true)", true)
|
|
28
|
+
.option("--max-workers <number>", "Maximum number of worker threads (default: CPU cores, max 8)", (value) => parseInt(value), undefined)
|
|
29
|
+
.option("--no-filtering", "Disable file pre-filtering optimization", false)
|
|
26
30
|
.action((options) => __awaiter(this, void 0, void 0, function* () {
|
|
31
|
+
const startTime = Date.now();
|
|
27
32
|
const logger = new utils_1.Logger(options.verbose);
|
|
28
33
|
try {
|
|
29
34
|
logger.info("🚀 Starting migration process");
|
|
30
35
|
logger.verbose_log(`Options: ${JSON.stringify(options)}`);
|
|
36
|
+
if (options.parallel) {
|
|
37
|
+
logger.info("⚡ Parallel processing enabled");
|
|
38
|
+
if (options.maxWorkers) {
|
|
39
|
+
logger.info(`👥 Using ${options.maxWorkers} worker threads`);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
else {
|
|
43
|
+
logger.info("🔄 Using sequential processing");
|
|
44
|
+
}
|
|
31
45
|
const config = (0, utils_1.getConfig)();
|
|
32
46
|
const patterns = config.include || ["**/*.ts", "**/*.tsx"];
|
|
33
47
|
// Create .ama directory if it doesn't exist
|
|
34
48
|
(0, utils_1.ensureAmaDirectory)(logger);
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
49
|
+
let processingResult;
|
|
50
|
+
if (options.parallel !== false) {
|
|
51
|
+
// Use optimized parallel processing pipeline
|
|
52
|
+
logger.info("🚀 Using optimized parallel processing pipeline");
|
|
53
|
+
processingResult = yield (0, parallel_schema_processor_1.optimizedMigrationPipeline)(patterns, options.tsconfig, options.continueOnError, logger, options.maxWorkers);
|
|
54
|
+
}
|
|
55
|
+
else {
|
|
56
|
+
// Fallback to original sequential processing
|
|
57
|
+
logger.info("🔄 Using original sequential processing");
|
|
58
|
+
const files = yield (0, utils_1.scanFiles)(patterns, logger);
|
|
59
|
+
logger.info(`📚 Found ${files.length} files to process`);
|
|
60
|
+
const project = (0, utils_1.createProject)(files, options.tsconfig, logger);
|
|
61
|
+
processingResult = (0, utils_1.processFiles)(project.getSourceFiles(), options.tsconfig, options.continueOnError, logger);
|
|
62
|
+
}
|
|
63
|
+
const { contents, errors, successCount, failureCount } = processingResult;
|
|
40
64
|
// Report processing results
|
|
41
|
-
|
|
65
|
+
const processingTime = ((Date.now() - startTime) / 1000).toFixed(2);
|
|
66
|
+
logger.success(`✅ Successfully processed ${successCount} AMA contents in ${processingTime}s`);
|
|
42
67
|
if (failureCount > 0) {
|
|
43
68
|
logger.warn(`⚠️ Failed to process ${failureCount} items`);
|
|
44
69
|
if (options.verbose && errors.length > 0) {
|
|
@@ -51,25 +76,45 @@ function migrateCommand() {
|
|
|
51
76
|
process.exit(1);
|
|
52
77
|
}
|
|
53
78
|
// Generate and save output
|
|
79
|
+
const outputStartTime = Date.now();
|
|
80
|
+
logger.info("🔧 Generating output definitions...");
|
|
54
81
|
const output = (0, utils_1.generateOutput)(contents, config, logger);
|
|
82
|
+
const outputTime = ((Date.now() - outputStartTime) / 1000).toFixed(2);
|
|
83
|
+
logger.verbose_log(`Output generation took ${outputTime}s`);
|
|
55
84
|
(0, utils_1.saveOutputToFile)(output, logger);
|
|
56
85
|
// Upload definitions unless dry-run is enabled
|
|
57
86
|
if (!options.dryRun) {
|
|
58
|
-
logger.info("Uploading definitions to AtMyApp platform");
|
|
87
|
+
logger.info("📤 Uploading definitions to AtMyApp platform");
|
|
88
|
+
const uploadStartTime = Date.now();
|
|
59
89
|
const uploadSuccess = yield (0, utils_1.uploadDefinitions)(output, config, logger);
|
|
90
|
+
const uploadTime = ((Date.now() - uploadStartTime) / 1000).toFixed(2);
|
|
91
|
+
logger.verbose_log(`Upload took ${uploadTime}s`);
|
|
60
92
|
if (!uploadSuccess) {
|
|
61
93
|
logger.warn("Upload failed, but definitions were generated successfully");
|
|
62
94
|
process.exit(1);
|
|
63
95
|
}
|
|
64
96
|
}
|
|
65
97
|
else {
|
|
66
|
-
logger.info("Dry run mode enabled. Skipping upload to server.");
|
|
98
|
+
logger.info("🏁 Dry run mode enabled. Skipping upload to server.");
|
|
99
|
+
}
|
|
100
|
+
const totalTime = ((Date.now() - startTime) / 1000).toFixed(2);
|
|
101
|
+
logger.success(`🎉 Migration completed successfully in ${totalTime}s`);
|
|
102
|
+
// Performance summary
|
|
103
|
+
if (options.verbose) {
|
|
104
|
+
logger.info("📊 Performance Summary:");
|
|
105
|
+
logger.info(` Total time: ${totalTime}s`);
|
|
106
|
+
logger.info(` Processing time: ${processingTime}s`);
|
|
107
|
+
logger.info(` Files processed: ${successCount}`);
|
|
108
|
+
logger.info(` Processing mode: ${options.parallel !== false ? "Parallel" : "Sequential"}`);
|
|
109
|
+
if (options.parallel !== false && options.maxWorkers) {
|
|
110
|
+
logger.info(` Worker threads: ${options.maxWorkers}`);
|
|
111
|
+
}
|
|
67
112
|
}
|
|
68
|
-
logger.success("Migration completed successfully");
|
|
69
113
|
}
|
|
70
114
|
catch (error) {
|
|
115
|
+
const totalTime = ((Date.now() - startTime) / 1000).toFixed(2);
|
|
71
116
|
const message = error instanceof Error ? error.message : "Unknown error";
|
|
72
|
-
logger.error(
|
|
117
|
+
logger.error(`💥 Fatal error after ${totalTime}s: ${message}`, error);
|
|
73
118
|
process.exit(1);
|
|
74
119
|
}
|
|
75
120
|
}));
|
|
@@ -21,7 +21,7 @@ function determineContentType(content) {
|
|
|
21
21
|
// Check for event types
|
|
22
22
|
if (((_b = content.structure) === null || _b === void 0 ? void 0 : _b.type) === "event" ||
|
|
23
23
|
((_e = (_d = (_c = content.structure) === null || _c === void 0 ? void 0 : _c.properties) === null || _d === void 0 ? void 0 : _d.type) === null || _e === void 0 ? void 0 : _e.const) === "event" ||
|
|
24
|
-
((_f = content.structure) === null || _f === void 0 ? void 0 : _f.__amatype) === "
|
|
24
|
+
((_f = content.structure) === null || _f === void 0 ? void 0 : _f.__amatype) === "AmaCustomEventDef") {
|
|
25
25
|
return "event";
|
|
26
26
|
}
|
|
27
27
|
// Check for image types based on structure or extension
|
|
@@ -140,7 +140,7 @@ exports.builtInProcessors = {
|
|
|
140
140
|
// Check for event types first
|
|
141
141
|
if (((_b = content.structure) === null || _b === void 0 ? void 0 : _b.type) === "event" ||
|
|
142
142
|
((_e = (_d = (_c = content.structure) === null || _c === void 0 ? void 0 : _c.properties) === null || _d === void 0 ? void 0 : _d.type) === null || _e === void 0 ? void 0 : _e.const) === "event" ||
|
|
143
|
-
((_f = content.structure) === null || _f === void 0 ? void 0 : _f.__amatype) === "
|
|
143
|
+
((_f = content.structure) === null || _f === void 0 ? void 0 : _f.__amatype) === "AmaCustomEventDef") {
|
|
144
144
|
content.type = "event";
|
|
145
145
|
}
|
|
146
146
|
else if (((_g = content.structure) === null || _g === void 0 ? void 0 : _g.__amatype) === "AmaImageDef") {
|
|
@@ -5,5 +5,7 @@ export * from "./content-processor";
|
|
|
5
5
|
export * from "./file-operations";
|
|
6
6
|
export * from "./upload";
|
|
7
7
|
export * from "./definition-processor";
|
|
8
|
+
export * from "./parallel-schema-processor";
|
|
9
|
+
export * from "./worker-pool";
|
|
8
10
|
export * from "../types/migrate";
|
|
9
11
|
export * from "../logger";
|
package/dist/cli/utils/index.js
CHANGED
|
@@ -22,6 +22,8 @@ __exportStar(require("./content-processor"), exports);
|
|
|
22
22
|
__exportStar(require("./file-operations"), exports);
|
|
23
23
|
__exportStar(require("./upload"), exports);
|
|
24
24
|
__exportStar(require("./definition-processor"), exports);
|
|
25
|
+
__exportStar(require("./parallel-schema-processor"), exports);
|
|
26
|
+
__exportStar(require("./worker-pool"), exports);
|
|
25
27
|
// Re-export types and logger
|
|
26
28
|
__exportStar(require("../types/migrate"), exports);
|
|
27
29
|
__exportStar(require("../logger"), exports);
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import { Logger } from "../logger";
|
|
2
|
+
import { ProcessingResult } from "../types/migrate";
|
|
3
|
+
export declare function scanFilesOptimized(patterns: string[], logger: Logger): Promise<string[]>;
|
|
4
|
+
export declare function processFilesParallel(files: string[], tsconfigPath: string, continueOnError: boolean, logger: Logger, maxWorkers?: number): Promise<ProcessingResult>;
|
|
5
|
+
export declare function optimizedMigrationPipeline(patterns: string[], tsconfigPath: string, continueOnError: boolean, logger: Logger, maxWorkers?: number): Promise<ProcessingResult>;
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
|
+
};
|
|
14
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
+
exports.scanFilesOptimized = scanFilesOptimized;
|
|
16
|
+
exports.processFilesParallel = processFilesParallel;
|
|
17
|
+
exports.optimizedMigrationPipeline = optimizedMigrationPipeline;
|
|
18
|
+
const fast_glob_1 = __importDefault(require("fast-glob"));
|
|
19
|
+
const worker_pool_1 = require("./worker-pool");
|
|
20
|
+
const path_1 = __importDefault(require("path"));
|
|
21
|
+
// Enhanced version of scanFiles with better performance
|
|
22
|
+
function scanFilesOptimized(patterns, logger) {
|
|
23
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
24
|
+
logger.info("🔍 Scanning files with optimized parallel processing...");
|
|
25
|
+
logger.verbose_log(`Using patterns: ${patterns.join(", ")}`);
|
|
26
|
+
const files = yield (0, fast_glob_1.default)(patterns, {
|
|
27
|
+
ignore: ["**/node_modules/**", "**/test/**", "**/dist/**", "**/.ama/**"],
|
|
28
|
+
absolute: true,
|
|
29
|
+
cwd: process.cwd(),
|
|
30
|
+
suppressErrors: true, // Don't fail on permission errors
|
|
31
|
+
followSymbolicLinks: false, // Skip symlinks for better performance
|
|
32
|
+
});
|
|
33
|
+
logger.verbose_log(`Found ${files.length} files matching patterns`);
|
|
34
|
+
return files;
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
// Parallel processing of files using worker threads
|
|
38
|
+
function processFilesParallel(files, tsconfigPath, continueOnError, logger, maxWorkers) {
|
|
39
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
40
|
+
const contents = [];
|
|
41
|
+
const errors = [];
|
|
42
|
+
let successCount = 0;
|
|
43
|
+
let failureCount = 0;
|
|
44
|
+
logger.info(`📚 Processing ${files.length} files in parallel...`);
|
|
45
|
+
// Filter files that likely contain ATMYAPP exports for better performance
|
|
46
|
+
const relevantFiles = yield filterRelevantFiles(files, logger);
|
|
47
|
+
if (relevantFiles.length === 0) {
|
|
48
|
+
logger.warn("No files with ATMYAPP exports found");
|
|
49
|
+
return { contents, errors, successCount, failureCount };
|
|
50
|
+
}
|
|
51
|
+
logger.info(`🎯 Processing ${relevantFiles.length} relevant files (filtered from ${files.length})`);
|
|
52
|
+
// In test environment, fall back to sequential processing
|
|
53
|
+
// to avoid worker thread module loading issues
|
|
54
|
+
if (process.env.NODE_ENV === "test" || process.env.JEST_WORKER_ID) {
|
|
55
|
+
logger.verbose_log("Using fallback sequential processing in test environment");
|
|
56
|
+
// Import sequential processing functions
|
|
57
|
+
const { scanFiles, createProject, processFiles, } = require("./schema-processor");
|
|
58
|
+
const project = createProject(relevantFiles, tsconfigPath, logger);
|
|
59
|
+
const result = processFiles(project.getSourceFiles(), tsconfigPath, continueOnError, logger);
|
|
60
|
+
return result;
|
|
61
|
+
}
|
|
62
|
+
// Create worker tasks
|
|
63
|
+
const tasks = relevantFiles.map((file, index) => ({
|
|
64
|
+
id: `task-${index}-${path_1.default.basename(file)}`,
|
|
65
|
+
filePath: file,
|
|
66
|
+
tsconfigPath,
|
|
67
|
+
}));
|
|
68
|
+
// Process files using worker pool
|
|
69
|
+
const workerPool = new worker_pool_1.WorkerPool(logger, maxWorkers);
|
|
70
|
+
try {
|
|
71
|
+
const results = yield workerPool.processFiles(tasks);
|
|
72
|
+
// Aggregate results
|
|
73
|
+
for (const result of results) {
|
|
74
|
+
if (result.success) {
|
|
75
|
+
contents.push(...result.contents);
|
|
76
|
+
successCount += result.contents.length;
|
|
77
|
+
if (result.contents.length > 0) {
|
|
78
|
+
logger.verbose_log(`✅ Processed ${result.contents.length} definitions from ${result.id}`);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
else {
|
|
82
|
+
failureCount++;
|
|
83
|
+
const errorMessage = `❌ ${result.id} - ${result.error}`;
|
|
84
|
+
errors.push(errorMessage);
|
|
85
|
+
if (!continueOnError) {
|
|
86
|
+
throw new Error(errorMessage);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
logger.success(`✅ Parallel processing completed: ${successCount} definitions from ${relevantFiles.length} files`);
|
|
91
|
+
}
|
|
92
|
+
catch (error) {
|
|
93
|
+
logger.error("Parallel processing failed:", error);
|
|
94
|
+
throw error;
|
|
95
|
+
}
|
|
96
|
+
return { contents, errors, successCount, failureCount };
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
// Pre-filter files to only process those likely to contain ATMYAPP exports
|
|
100
|
+
function filterRelevantFiles(files, logger) {
|
|
101
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
102
|
+
logger.verbose_log("🔍 Pre-filtering files for ATMYAPP exports...");
|
|
103
|
+
const fs = require("fs").promises;
|
|
104
|
+
const relevantFiles = [];
|
|
105
|
+
// Process files in chunks for better performance
|
|
106
|
+
const chunkSize = 50;
|
|
107
|
+
const chunks = [];
|
|
108
|
+
for (let i = 0; i < files.length; i += chunkSize) {
|
|
109
|
+
chunks.push(files.slice(i, i + chunkSize));
|
|
110
|
+
}
|
|
111
|
+
for (const chunk of chunks) {
|
|
112
|
+
const chunkPromises = chunk.map((file) => __awaiter(this, void 0, void 0, function* () {
|
|
113
|
+
try {
|
|
114
|
+
// Quick text search for ATMYAPP export
|
|
115
|
+
const content = yield fs.readFile(file, "utf8");
|
|
116
|
+
// Simple regex to check for ATMYAPP exports
|
|
117
|
+
if (/export\s+type\s+ATMYAPP\s*=/.test(content)) {
|
|
118
|
+
return file;
|
|
119
|
+
}
|
|
120
|
+
return null;
|
|
121
|
+
}
|
|
122
|
+
catch (error) {
|
|
123
|
+
// Skip files that can't be read
|
|
124
|
+
logger.verbose_log(`Skipping unreadable file: ${file}`);
|
|
125
|
+
return null;
|
|
126
|
+
}
|
|
127
|
+
}));
|
|
128
|
+
const chunkResults = yield Promise.all(chunkPromises);
|
|
129
|
+
relevantFiles.push(...chunkResults.filter(Boolean));
|
|
130
|
+
}
|
|
131
|
+
logger.verbose_log(`📊 Filtered to ${relevantFiles.length} relevant files from ${files.length} total`);
|
|
132
|
+
return relevantFiles;
|
|
133
|
+
});
|
|
134
|
+
}
|
|
135
|
+
// Optimized file processing pipeline
|
|
136
|
+
function optimizedMigrationPipeline(patterns, tsconfigPath, continueOnError, logger, maxWorkers) {
|
|
137
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
138
|
+
// Step 1: Scan files with optimization
|
|
139
|
+
const files = yield scanFilesOptimized(patterns, logger);
|
|
140
|
+
if (files.length === 0) {
|
|
141
|
+
logger.warn("No files found matching patterns");
|
|
142
|
+
return { contents: [], errors: [], successCount: 0, failureCount: 0 };
|
|
143
|
+
}
|
|
144
|
+
// Step 2: Process files in parallel
|
|
145
|
+
return yield processFilesParallel(files, tsconfigPath, continueOnError, logger, maxWorkers);
|
|
146
|
+
});
|
|
147
|
+
}
|
|
@@ -124,9 +124,68 @@ function extractDefinitionTypes(atmyappType, logger) {
|
|
|
124
124
|
}
|
|
125
125
|
return elementTypes;
|
|
126
126
|
}
|
|
127
|
+
// Extract event information directly from TypeScript AST
|
|
128
|
+
function extractEventInfoFromAST(file, definitionType, logger) {
|
|
129
|
+
try {
|
|
130
|
+
// Find the type alias declaration for this definition type
|
|
131
|
+
const typeAlias = file.getTypeAlias(definitionType);
|
|
132
|
+
if (!typeAlias) {
|
|
133
|
+
logger.verbose_log(`Could not find type alias for ${definitionType}`);
|
|
134
|
+
return null;
|
|
135
|
+
}
|
|
136
|
+
const typeNode = typeAlias.getTypeNode();
|
|
137
|
+
if (!typeNode) {
|
|
138
|
+
logger.verbose_log(`Type alias ${definitionType} has no type node`);
|
|
139
|
+
return null;
|
|
140
|
+
}
|
|
141
|
+
// Check if this is a type reference (like AmaCustomEventDef<...>)
|
|
142
|
+
if (ts_morph_1.Node.isTypeReference(typeNode)) {
|
|
143
|
+
const typeName = typeNode.getTypeName();
|
|
144
|
+
const typeArguments = typeNode.getTypeArguments();
|
|
145
|
+
// Check if this is AmaCustomEventDef
|
|
146
|
+
if (ts_morph_1.Node.isIdentifier(typeName) &&
|
|
147
|
+
typeName.getText() === "AmaCustomEventDef") {
|
|
148
|
+
if (typeArguments.length >= 2) {
|
|
149
|
+
// First argument should be the event ID (string literal)
|
|
150
|
+
const idArg = typeArguments[0];
|
|
151
|
+
let eventId = null;
|
|
152
|
+
if (ts_morph_1.Node.isLiteralTypeNode(idArg)) {
|
|
153
|
+
const literal = idArg.getLiteral();
|
|
154
|
+
if (ts_morph_1.Node.isStringLiteral(literal)) {
|
|
155
|
+
eventId = literal.getLiteralValue();
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
// Second argument should be the columns (tuple of string literals)
|
|
159
|
+
const columnsArg = typeArguments[1];
|
|
160
|
+
let columns = [];
|
|
161
|
+
if (ts_morph_1.Node.isTupleTypeNode(columnsArg)) {
|
|
162
|
+
columnsArg.getElements().forEach((element) => {
|
|
163
|
+
if (ts_morph_1.Node.isLiteralTypeNode(element)) {
|
|
164
|
+
const literal = element.getLiteral();
|
|
165
|
+
if (ts_morph_1.Node.isStringLiteral(literal)) {
|
|
166
|
+
columns.push(literal.getLiteralValue());
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
});
|
|
170
|
+
}
|
|
171
|
+
if (eventId && columns.length > 0) {
|
|
172
|
+
logger.verbose_log(`AST extraction successful for ${definitionType}: id=${eventId}, columns=[${columns.join(", ")}]`);
|
|
173
|
+
return { id: eventId, columns };
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
logger.verbose_log(`Failed to extract event info from AST for ${definitionType}`);
|
|
179
|
+
return null;
|
|
180
|
+
}
|
|
181
|
+
catch (error) {
|
|
182
|
+
logger.verbose_log(`Error during AST extraction for ${definitionType}: ${error}`);
|
|
183
|
+
return null;
|
|
184
|
+
}
|
|
185
|
+
}
|
|
127
186
|
// Processes an ATMYAPP export to extract content definitions
|
|
128
187
|
function processAtmyappExport(atmyappType, file, tsconfigPath, logger) {
|
|
129
|
-
var _a, _b, _c, _d, _e, _f, _g, _h, _j;
|
|
188
|
+
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q;
|
|
130
189
|
const contents = [];
|
|
131
190
|
logger.verbose_log(`Processing ATMYAPP export in ${file.getFilePath()}`);
|
|
132
191
|
// Extract individual definition types from the array
|
|
@@ -165,37 +224,77 @@ function processAtmyappExport(atmyappType, file, tsconfigPath, logger) {
|
|
|
165
224
|
continue;
|
|
166
225
|
}
|
|
167
226
|
if (!schema.properties) {
|
|
227
|
+
// For event definitions, the schema generator might fail due to generics
|
|
228
|
+
// Try to extract event information directly from the TypeScript AST
|
|
229
|
+
logger.verbose_log(`Schema has no properties. Attempting AST-based extraction for ${definitionType}`);
|
|
230
|
+
// Try to extract event definition from TypeScript AST
|
|
231
|
+
const eventInfo = extractEventInfoFromAST(file, definitionType, logger);
|
|
232
|
+
if (eventInfo) {
|
|
233
|
+
logger.verbose_log(`Successfully extracted event via AST: ${eventInfo.id} with columns: ${eventInfo.columns.join(", ")}`);
|
|
234
|
+
contents.push({
|
|
235
|
+
path: eventInfo.id,
|
|
236
|
+
structure: {
|
|
237
|
+
type: "event",
|
|
238
|
+
properties: {
|
|
239
|
+
id: { const: eventInfo.id },
|
|
240
|
+
columns: { const: eventInfo.columns },
|
|
241
|
+
type: { const: "event" },
|
|
242
|
+
},
|
|
243
|
+
},
|
|
244
|
+
});
|
|
245
|
+
continue;
|
|
246
|
+
}
|
|
168
247
|
logger.warn(`Invalid schema structure for ${definitionType}`);
|
|
169
248
|
continue;
|
|
170
249
|
}
|
|
171
250
|
const properties = schema.properties;
|
|
251
|
+
// Debug: Log the actual schema structure
|
|
252
|
+
logger.verbose_log(`Schema for ${definitionType}: ${JSON.stringify(properties, null, 2)}`);
|
|
172
253
|
// Check if this is an event definition
|
|
173
254
|
const isEventDef = ((_a = properties.type) === null || _a === void 0 ? void 0 : _a.const) === "event" ||
|
|
174
255
|
(((_b = properties.__is_ATMYAPP_Object) === null || _b === void 0 ? void 0 : _b.const) === true &&
|
|
175
256
|
properties.id &&
|
|
176
257
|
properties.columns);
|
|
177
258
|
if (isEventDef) {
|
|
178
|
-
// Handle
|
|
259
|
+
// Handle AmaCustomEventDef - use id as path and extract event structure
|
|
179
260
|
let eventId = null;
|
|
180
261
|
let columns = [];
|
|
181
|
-
// Extract event ID
|
|
262
|
+
// Extract event ID - try different possible structures
|
|
182
263
|
if ((_c = properties.id) === null || _c === void 0 ? void 0 : _c.const) {
|
|
183
264
|
eventId = properties.id.const;
|
|
184
265
|
}
|
|
185
|
-
|
|
186
|
-
|
|
266
|
+
else if (((_d = properties.id) === null || _d === void 0 ? void 0 : _d.enum) && properties.id.enum.length === 1) {
|
|
267
|
+
eventId = properties.id.enum[0];
|
|
268
|
+
}
|
|
269
|
+
else if (((_e = properties.id) === null || _e === void 0 ? void 0 : _e.type) === "string" && ((_f = properties.id) === null || _f === void 0 ? void 0 : _f.title)) {
|
|
270
|
+
// Fallback: try to extract from title or other metadata
|
|
271
|
+
eventId = properties.id.title;
|
|
272
|
+
}
|
|
273
|
+
// Extract columns - try different possible structures
|
|
274
|
+
if ((_g = properties.columns) === null || _g === void 0 ? void 0 : _g.const) {
|
|
187
275
|
columns = properties.columns.const;
|
|
188
276
|
}
|
|
189
|
-
else if ((
|
|
277
|
+
else if ((_j = (_h = properties.columns) === null || _h === void 0 ? void 0 : _h.items) === null || _j === void 0 ? void 0 : _j.const) {
|
|
190
278
|
columns = properties.columns.items.const;
|
|
191
279
|
}
|
|
192
|
-
else if (((
|
|
280
|
+
else if (((_k = properties.columns) === null || _k === void 0 ? void 0 : _k.items) &&
|
|
193
281
|
Array.isArray(properties.columns.items)) {
|
|
194
282
|
// Handle array of const items - extract const value from each item
|
|
195
283
|
columns = properties.columns.items
|
|
196
284
|
.map((item) => item.const)
|
|
197
285
|
.filter(Boolean);
|
|
198
286
|
}
|
|
287
|
+
else if ((_m = (_l = properties.columns) === null || _l === void 0 ? void 0 : _l.items) === null || _m === void 0 ? void 0 : _m.enum) {
|
|
288
|
+
// Handle tuple type where each position has enum with single value
|
|
289
|
+
columns = properties.columns.items.enum;
|
|
290
|
+
}
|
|
291
|
+
else if (((_o = properties.columns) === null || _o === void 0 ? void 0 : _o.enum) &&
|
|
292
|
+
Array.isArray(properties.columns.enum[0])) {
|
|
293
|
+
// Handle case where columns is an enum with array values
|
|
294
|
+
columns = properties.columns.enum[0];
|
|
295
|
+
}
|
|
296
|
+
// Debug: Log what we extracted
|
|
297
|
+
logger.verbose_log(`Extracted from ${definitionType}: eventId=${eventId}, columns=${JSON.stringify(columns)}`);
|
|
199
298
|
if (!eventId) {
|
|
200
299
|
logger.warn(`Could not extract event ID from ${definitionType}`);
|
|
201
300
|
continue;
|
|
@@ -223,10 +322,10 @@ function processAtmyappExport(atmyappType, file, tsconfigPath, logger) {
|
|
|
223
322
|
let path = null;
|
|
224
323
|
let structure = null;
|
|
225
324
|
// Look for path in different possible locations
|
|
226
|
-
if ((
|
|
325
|
+
if ((_p = properties.path) === null || _p === void 0 ? void 0 : _p.const) {
|
|
227
326
|
path = properties.path.const;
|
|
228
327
|
}
|
|
229
|
-
else if ((
|
|
328
|
+
else if ((_q = properties._path) === null || _q === void 0 ? void 0 : _q.const) {
|
|
230
329
|
path = properties._path.const;
|
|
231
330
|
}
|
|
232
331
|
// Look for structure/data in different possible locations
|
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
36
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
37
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
38
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
39
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
40
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
41
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
42
|
+
});
|
|
43
|
+
};
|
|
44
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
45
|
+
exports.processFileInWorker = processFileInWorker;
|
|
46
|
+
const ts_morph_1 = require("ts-morph");
|
|
47
|
+
const ts = __importStar(require("typescript"));
|
|
48
|
+
const path_1 = require("path");
|
|
49
|
+
const typescript_json_schema_1 = require("typescript-json-schema");
|
|
50
|
+
const fs_1 = require("fs");
|
|
51
|
+
// Cache for TypeScript programs to avoid recompilation
|
|
52
|
+
const programCache = new Map();
|
|
53
|
+
// Optimized function to process a single file in a worker
|
|
54
|
+
function processFileInWorker(filePath, tsconfigPath) {
|
|
55
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
56
|
+
const contents = [];
|
|
57
|
+
// Create or reuse TypeScript project
|
|
58
|
+
const resolvedTsConfigPath = (0, path_1.resolve)(process.cwd(), tsconfigPath);
|
|
59
|
+
const projectOptions = {
|
|
60
|
+
tsConfigFilePath: (0, fs_1.existsSync)(resolvedTsConfigPath)
|
|
61
|
+
? resolvedTsConfigPath
|
|
62
|
+
: undefined,
|
|
63
|
+
skipAddingFilesFromTsConfig: true,
|
|
64
|
+
compilerOptions: !(0, fs_1.existsSync)(resolvedTsConfigPath)
|
|
65
|
+
? {
|
|
66
|
+
target: ts.ScriptTarget.ESNext,
|
|
67
|
+
module: ts.ModuleKind.ESNext,
|
|
68
|
+
moduleResolution: ts.ModuleResolutionKind.NodeJs,
|
|
69
|
+
esModuleInterop: true,
|
|
70
|
+
jsx: ts.JsxEmit.React,
|
|
71
|
+
skipLibCheck: true,
|
|
72
|
+
}
|
|
73
|
+
: undefined,
|
|
74
|
+
};
|
|
75
|
+
const project = new ts_morph_1.Project(projectOptions);
|
|
76
|
+
const sourceFile = project.addSourceFileAtPath(filePath);
|
|
77
|
+
// Look for ATMYAPP exports
|
|
78
|
+
const atmyappExports = sourceFile.getTypeAliases().filter((alias) => {
|
|
79
|
+
const name = alias.getName();
|
|
80
|
+
const isExported = alias.isExported();
|
|
81
|
+
return name === "ATMYAPP" && isExported;
|
|
82
|
+
});
|
|
83
|
+
if (atmyappExports.length === 0) {
|
|
84
|
+
return contents;
|
|
85
|
+
}
|
|
86
|
+
// Process each ATMYAPP export
|
|
87
|
+
for (const atmyappExport of atmyappExports) {
|
|
88
|
+
const fileContents = yield processAtmyappExportOptimized(atmyappExport, sourceFile, tsconfigPath);
|
|
89
|
+
contents.push(...fileContents);
|
|
90
|
+
}
|
|
91
|
+
return contents;
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
// Optimized version that reuses TypeScript programs
|
|
95
|
+
function processAtmyappExportOptimized(atmyappType, file, tsconfigPath) {
|
|
96
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
97
|
+
var _a, _b;
|
|
98
|
+
const contents = [];
|
|
99
|
+
const filePath = file.getFilePath();
|
|
100
|
+
// Extract definition types
|
|
101
|
+
const definitionTypes = extractDefinitionTypes(atmyappType);
|
|
102
|
+
if (definitionTypes.length === 0) {
|
|
103
|
+
return contents;
|
|
104
|
+
}
|
|
105
|
+
// Create or reuse TypeScript program
|
|
106
|
+
const resolvedTsConfigPath = (0, path_1.resolve)(process.cwd(), tsconfigPath);
|
|
107
|
+
const cacheKey = `${filePath}:${resolvedTsConfigPath}`;
|
|
108
|
+
let program = programCache.get(cacheKey);
|
|
109
|
+
if (!program) {
|
|
110
|
+
const compilerOptions = (0, fs_1.existsSync)(resolvedTsConfigPath)
|
|
111
|
+
? { configFile: resolvedTsConfigPath }
|
|
112
|
+
: {
|
|
113
|
+
target: ts.ScriptTarget.ES2015,
|
|
114
|
+
module: ts.ModuleKind.ESNext,
|
|
115
|
+
strict: true,
|
|
116
|
+
esModuleInterop: true,
|
|
117
|
+
skipLibCheck: true,
|
|
118
|
+
jsx: ts.JsxEmit.Preserve,
|
|
119
|
+
};
|
|
120
|
+
program = (0, typescript_json_schema_1.getProgramFromFiles)([filePath], compilerOptions);
|
|
121
|
+
programCache.set(cacheKey, program);
|
|
122
|
+
}
|
|
123
|
+
// Batch process all definition types for this file
|
|
124
|
+
const schemaPromises = definitionTypes.map((definitionType) => __awaiter(this, void 0, void 0, function* () {
|
|
125
|
+
try {
|
|
126
|
+
const schema = (0, typescript_json_schema_1.generateSchema)(program, definitionType, {
|
|
127
|
+
required: true,
|
|
128
|
+
noExtraProps: true,
|
|
129
|
+
aliasRef: true,
|
|
130
|
+
ref: false,
|
|
131
|
+
defaultNumberType: "number",
|
|
132
|
+
ignoreErrors: true,
|
|
133
|
+
skipLibCheck: true,
|
|
134
|
+
});
|
|
135
|
+
return { definitionType, schema };
|
|
136
|
+
}
|
|
137
|
+
catch (error) {
|
|
138
|
+
return { definitionType, schema: null, error };
|
|
139
|
+
}
|
|
140
|
+
}));
|
|
141
|
+
const schemaResults = yield Promise.all(schemaPromises);
|
|
142
|
+
// Process schema results
|
|
143
|
+
for (const result of schemaResults) {
|
|
144
|
+
if (!result.schema || !result.schema.properties) {
|
|
145
|
+
continue;
|
|
146
|
+
}
|
|
147
|
+
const properties = result.schema.properties;
|
|
148
|
+
const isEventDef = ((_a = properties.type) === null || _a === void 0 ? void 0 : _a.const) === "event" ||
|
|
149
|
+
(((_b = properties.__is_ATMYAPP_Object) === null || _b === void 0 ? void 0 : _b.const) === true &&
|
|
150
|
+
properties.id &&
|
|
151
|
+
properties.columns);
|
|
152
|
+
if (isEventDef) {
|
|
153
|
+
// Handle event definitions
|
|
154
|
+
const eventContent = processEventDefinition(properties, result.definitionType);
|
|
155
|
+
if (eventContent) {
|
|
156
|
+
contents.push(eventContent);
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
else {
|
|
160
|
+
// Handle regular content definitions
|
|
161
|
+
const contentDefinition = processContentDefinition(properties, result.definitionType);
|
|
162
|
+
if (contentDefinition) {
|
|
163
|
+
contents.push(contentDefinition);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
return contents;
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
function extractDefinitionTypes(atmyappType) {
|
|
171
|
+
const typeNode = atmyappType.getTypeNode();
|
|
172
|
+
if (!ts_morph_1.Node.isTupleTypeNode(typeNode) && !ts_morph_1.Node.isArrayTypeNode(typeNode)) {
|
|
173
|
+
return [];
|
|
174
|
+
}
|
|
175
|
+
const elementTypes = [];
|
|
176
|
+
if (ts_morph_1.Node.isTupleTypeNode(typeNode)) {
|
|
177
|
+
typeNode.getElements().forEach((element) => {
|
|
178
|
+
elementTypes.push(element.getText());
|
|
179
|
+
});
|
|
180
|
+
}
|
|
181
|
+
else if (ts_morph_1.Node.isArrayTypeNode(typeNode)) {
|
|
182
|
+
elementTypes.push(typeNode.getElementTypeNode().getText());
|
|
183
|
+
}
|
|
184
|
+
return elementTypes;
|
|
185
|
+
}
|
|
186
|
+
function processEventDefinition(properties, definitionType) {
|
|
187
|
+
var _a, _b, _c, _d, _e;
|
|
188
|
+
let eventId = null;
|
|
189
|
+
let columns = [];
|
|
190
|
+
// Extract event ID
|
|
191
|
+
if ((_a = properties.id) === null || _a === void 0 ? void 0 : _a.const) {
|
|
192
|
+
eventId = properties.id.const;
|
|
193
|
+
}
|
|
194
|
+
// Extract columns
|
|
195
|
+
if ((_b = properties.columns) === null || _b === void 0 ? void 0 : _b.const) {
|
|
196
|
+
columns = properties.columns.const;
|
|
197
|
+
}
|
|
198
|
+
else if ((_d = (_c = properties.columns) === null || _c === void 0 ? void 0 : _c.items) === null || _d === void 0 ? void 0 : _d.const) {
|
|
199
|
+
columns = properties.columns.items.const;
|
|
200
|
+
}
|
|
201
|
+
else if (((_e = properties.columns) === null || _e === void 0 ? void 0 : _e.items) &&
|
|
202
|
+
Array.isArray(properties.columns.items)) {
|
|
203
|
+
columns = properties.columns.items
|
|
204
|
+
.map((item) => item.const)
|
|
205
|
+
.filter(Boolean);
|
|
206
|
+
}
|
|
207
|
+
if (!eventId || columns.length === 0) {
|
|
208
|
+
return null;
|
|
209
|
+
}
|
|
210
|
+
return {
|
|
211
|
+
path: eventId,
|
|
212
|
+
structure: {
|
|
213
|
+
type: "event",
|
|
214
|
+
properties: {
|
|
215
|
+
id: { const: eventId },
|
|
216
|
+
columns: { const: columns },
|
|
217
|
+
type: { const: "event" },
|
|
218
|
+
},
|
|
219
|
+
},
|
|
220
|
+
};
|
|
221
|
+
}
|
|
222
|
+
function processContentDefinition(properties, definitionType) {
|
|
223
|
+
var _a, _b;
|
|
224
|
+
let path = null;
|
|
225
|
+
let structure = null;
|
|
226
|
+
// Extract path
|
|
227
|
+
if ((_a = properties.path) === null || _a === void 0 ? void 0 : _a.const) {
|
|
228
|
+
path = properties.path.const;
|
|
229
|
+
}
|
|
230
|
+
else if ((_b = properties._path) === null || _b === void 0 ? void 0 : _b.const) {
|
|
231
|
+
path = properties._path.const;
|
|
232
|
+
}
|
|
233
|
+
// Extract structure
|
|
234
|
+
if (properties.structure) {
|
|
235
|
+
structure = properties.structure;
|
|
236
|
+
}
|
|
237
|
+
else if (properties.data) {
|
|
238
|
+
structure = properties.data;
|
|
239
|
+
}
|
|
240
|
+
else if (properties._data) {
|
|
241
|
+
structure = properties._data;
|
|
242
|
+
}
|
|
243
|
+
if (!path || !structure) {
|
|
244
|
+
return null;
|
|
245
|
+
}
|
|
246
|
+
return { path, structure };
|
|
247
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { Logger } from "../logger";
|
|
2
|
+
export interface WorkerTask {
|
|
3
|
+
id: string;
|
|
4
|
+
filePath: string;
|
|
5
|
+
tsconfigPath: string;
|
|
6
|
+
}
|
|
7
|
+
export interface WorkerResult {
|
|
8
|
+
id: string;
|
|
9
|
+
success: boolean;
|
|
10
|
+
contents: any[];
|
|
11
|
+
error?: string;
|
|
12
|
+
}
|
|
13
|
+
export declare class WorkerPool {
|
|
14
|
+
private workers;
|
|
15
|
+
private taskQueue;
|
|
16
|
+
private activeWorkers;
|
|
17
|
+
private results;
|
|
18
|
+
private logger;
|
|
19
|
+
private maxWorkers;
|
|
20
|
+
constructor(logger: Logger, maxWorkers?: number);
|
|
21
|
+
processFiles(tasks: WorkerTask[]): Promise<WorkerResult[]>;
|
|
22
|
+
private setupWorker;
|
|
23
|
+
private assignNextTask;
|
|
24
|
+
private cleanup;
|
|
25
|
+
}
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.WorkerPool = void 0;
|
|
13
|
+
const worker_threads_1 = require("worker_threads");
|
|
14
|
+
const os_1 = require("os");
|
|
15
|
+
class WorkerPool {
|
|
16
|
+
constructor(logger, maxWorkers) {
|
|
17
|
+
this.workers = [];
|
|
18
|
+
this.taskQueue = [];
|
|
19
|
+
this.activeWorkers = new Set();
|
|
20
|
+
this.results = new Map();
|
|
21
|
+
this.logger = logger;
|
|
22
|
+
this.maxWorkers = maxWorkers || Math.min((0, os_1.cpus)().length, 8); // Cap at 8 workers
|
|
23
|
+
}
|
|
24
|
+
processFiles(tasks) {
|
|
25
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
26
|
+
this.taskQueue = [...tasks];
|
|
27
|
+
this.results.clear();
|
|
28
|
+
this.logger.info(`🚀 Starting parallel processing with ${this.maxWorkers} workers`);
|
|
29
|
+
this.logger.verbose_log(`Processing ${tasks.length} tasks in parallel`);
|
|
30
|
+
// Create workers
|
|
31
|
+
const workerPromises = [];
|
|
32
|
+
const workersToCreate = Math.min(this.maxWorkers, tasks.length);
|
|
33
|
+
// Determine the correct worker script path
|
|
34
|
+
const workerScriptPath = __filename.endsWith(".ts")
|
|
35
|
+
? __filename.replace(".ts", ".js").replace("/src/", "/dist/")
|
|
36
|
+
: __filename;
|
|
37
|
+
for (let i = 0; i < workersToCreate; i++) {
|
|
38
|
+
const worker = new worker_threads_1.Worker(workerScriptPath);
|
|
39
|
+
this.workers.push(worker);
|
|
40
|
+
workerPromises.push(this.setupWorker(worker));
|
|
41
|
+
}
|
|
42
|
+
// Wait for all workers to complete
|
|
43
|
+
yield Promise.all(workerPromises);
|
|
44
|
+
// Cleanup workers
|
|
45
|
+
yield this.cleanup();
|
|
46
|
+
// Return results in original order
|
|
47
|
+
return tasks.map((task) => this.results.get(task.id));
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
setupWorker(worker) {
|
|
51
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
52
|
+
return new Promise((resolve, reject) => {
|
|
53
|
+
worker.on("message", (result) => {
|
|
54
|
+
this.results.set(result.id, result);
|
|
55
|
+
if (result.success) {
|
|
56
|
+
this.logger.verbose_log(`✅ Worker completed task ${result.id}`);
|
|
57
|
+
}
|
|
58
|
+
else {
|
|
59
|
+
this.logger.error(`❌ Worker failed task ${result.id}: ${result.error}`);
|
|
60
|
+
}
|
|
61
|
+
// Assign next task or finish
|
|
62
|
+
this.assignNextTask(worker, resolve);
|
|
63
|
+
});
|
|
64
|
+
worker.on("error", (error) => {
|
|
65
|
+
this.logger.error(`Worker error: ${error.message}`);
|
|
66
|
+
reject(error);
|
|
67
|
+
});
|
|
68
|
+
worker.on("exit", (code) => {
|
|
69
|
+
if (code !== 0 && this.activeWorkers.has(worker)) {
|
|
70
|
+
this.logger.error(`Worker exited with code ${code}`);
|
|
71
|
+
}
|
|
72
|
+
this.activeWorkers.delete(worker);
|
|
73
|
+
});
|
|
74
|
+
// Start with first task
|
|
75
|
+
this.assignNextTask(worker, resolve);
|
|
76
|
+
});
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
assignNextTask(worker, resolve) {
|
|
80
|
+
const task = this.taskQueue.shift();
|
|
81
|
+
if (task) {
|
|
82
|
+
this.activeWorkers.add(worker);
|
|
83
|
+
worker.postMessage(task);
|
|
84
|
+
}
|
|
85
|
+
else {
|
|
86
|
+
// No more tasks, resolve this worker
|
|
87
|
+
this.activeWorkers.delete(worker);
|
|
88
|
+
resolve();
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
cleanup() {
|
|
92
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
93
|
+
const terminationPromises = this.workers.map((worker) => worker
|
|
94
|
+
.terminate()
|
|
95
|
+
.catch((err) => this.logger.warn(`Error terminating worker: ${err.message}`)));
|
|
96
|
+
yield Promise.all(terminationPromises);
|
|
97
|
+
this.workers = [];
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
exports.WorkerPool = WorkerPool;
|
|
102
|
+
// Worker thread implementation
|
|
103
|
+
if (!worker_threads_1.isMainThread && worker_threads_1.parentPort) {
|
|
104
|
+
// Import required modules in worker context
|
|
105
|
+
const { processFileInWorker } = require("./worker-file-processor");
|
|
106
|
+
worker_threads_1.parentPort.on("message", (task) => __awaiter(void 0, void 0, void 0, function* () {
|
|
107
|
+
try {
|
|
108
|
+
const contents = yield processFileInWorker(task.filePath, task.tsconfigPath);
|
|
109
|
+
const result = {
|
|
110
|
+
id: task.id,
|
|
111
|
+
success: true,
|
|
112
|
+
contents,
|
|
113
|
+
};
|
|
114
|
+
worker_threads_1.parentPort.postMessage(result);
|
|
115
|
+
}
|
|
116
|
+
catch (error) {
|
|
117
|
+
const result = {
|
|
118
|
+
id: task.id,
|
|
119
|
+
success: false,
|
|
120
|
+
contents: [],
|
|
121
|
+
error: error instanceof Error ? error.message : "Unknown error",
|
|
122
|
+
};
|
|
123
|
+
worker_threads_1.parentPort.postMessage(result);
|
|
124
|
+
}
|
|
125
|
+
}));
|
|
126
|
+
}
|