@things-factory/integration-label-studio 9.1.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +85 -0
- package/EXTERNAL_DATA_SOURCING.md +484 -0
- package/IMPLEMENTATION_GUIDE.md +469 -0
- package/INTEGRATION.md +279 -0
- package/README.md +1014 -0
- package/SETUP_GUIDE.md +577 -0
- package/TEST_GUIDE.md +387 -0
- package/UI_CUSTOMIZATION.md +395 -0
- package/USER_SYNC_GUIDE.md +514 -0
- package/client/bootstrap.ts +1 -0
- package/client/index.ts +1 -0
- package/client/label-studio-label-page.ts +52 -0
- package/client/label-studio-project-create.ts +216 -0
- package/client/label-studio-project-list.ts +214 -0
- package/client/label-studio-wrapper.ts +294 -0
- package/client/route.ts +15 -0
- package/client/tsconfig.json +13 -0
- package/config/config.development.js +124 -0
- package/config/config.production.js +182 -0
- package/dist-client/bootstrap.d.ts +1 -0
- package/dist-client/bootstrap.js +2 -0
- package/dist-client/bootstrap.js.map +1 -0
- package/dist-client/index.d.ts +1 -0
- package/dist-client/index.js +2 -0
- package/dist-client/index.js.map +1 -0
- package/dist-client/label-studio-label-page.d.ts +8 -0
- package/dist-client/label-studio-label-page.js +54 -0
- package/dist-client/label-studio-label-page.js.map +1 -0
- package/dist-client/label-studio-project-create.d.ts +16 -0
- package/dist-client/label-studio-project-create.js +235 -0
- package/dist-client/label-studio-project-create.js.map +1 -0
- package/dist-client/label-studio-project-list.d.ts +16 -0
- package/dist-client/label-studio-project-list.js +222 -0
- package/dist-client/label-studio-project-list.js.map +1 -0
- package/dist-client/label-studio-wrapper.d.ts +57 -0
- package/dist-client/label-studio-wrapper.js +304 -0
- package/dist-client/label-studio-wrapper.js.map +1 -0
- package/dist-client/route.d.ts +1 -0
- package/dist-client/route.js +14 -0
- package/dist-client/route.js.map +1 -0
- package/dist-client/tsconfig.tsbuildinfo +1 -0
- package/dist-server/controller/label-studio-role-mapper.d.ts +35 -0
- package/dist-server/controller/label-studio-role-mapper.js +65 -0
- package/dist-server/controller/label-studio-role-mapper.js.map +1 -0
- package/dist-server/controller/user-provisioning-service.d.ts +66 -0
- package/dist-server/controller/user-provisioning-service.js +264 -0
- package/dist-server/controller/user-provisioning-service.js.map +1 -0
- package/dist-server/index.d.ts +7 -0
- package/dist-server/index.js +19 -0
- package/dist-server/index.js.map +1 -0
- package/dist-server/route/label-studio-sso.d.ts +2 -0
- package/dist-server/route/label-studio-sso.js +156 -0
- package/dist-server/route/label-studio-sso.js.map +1 -0
- package/dist-server/route/webhook.d.ts +65 -0
- package/dist-server/route/webhook.js +248 -0
- package/dist-server/route/webhook.js.map +1 -0
- package/dist-server/route.d.ts +1 -0
- package/dist-server/route.js +21 -0
- package/dist-server/route.js.map +1 -0
- package/dist-server/service/ai-prediction-service.d.ts +27 -0
- package/dist-server/service/ai-prediction-service.js +222 -0
- package/dist-server/service/ai-prediction-service.js.map +1 -0
- package/dist-server/service/dataset-labeling-integration.d.ts +44 -0
- package/dist-server/service/dataset-labeling-integration.js +512 -0
- package/dist-server/service/dataset-labeling-integration.js.map +1 -0
- package/dist-server/service/external-data-source-service.d.ts +78 -0
- package/dist-server/service/external-data-source-service.js +415 -0
- package/dist-server/service/external-data-source-service.js.map +1 -0
- package/dist-server/service/index.d.ts +12 -0
- package/dist-server/service/index.js +27 -0
- package/dist-server/service/index.js.map +1 -0
- package/dist-server/service/label-studio-sso-service.d.ts +38 -0
- package/dist-server/service/label-studio-sso-service.js +98 -0
- package/dist-server/service/label-studio-sso-service.js.map +1 -0
- package/dist-server/service/ml/ml-backend-service.d.ts +23 -0
- package/dist-server/service/ml/ml-backend-service.js +153 -0
- package/dist-server/service/ml/ml-backend-service.js.map +1 -0
- package/dist-server/service/prediction/prediction-management.d.ts +32 -0
- package/dist-server/service/prediction/prediction-management.js +299 -0
- package/dist-server/service/prediction/prediction-management.js.map +1 -0
- package/dist-server/service/project/project-management.d.ts +36 -0
- package/dist-server/service/project/project-management.js +309 -0
- package/dist-server/service/project/project-management.js.map +1 -0
- package/dist-server/service/task/task-management.d.ts +42 -0
- package/dist-server/service/task/task-management.js +372 -0
- package/dist-server/service/task/task-management.js.map +1 -0
- package/dist-server/service/user-provisioning/user-sync-mutation.d.ts +28 -0
- package/dist-server/service/user-provisioning/user-sync-mutation.js +111 -0
- package/dist-server/service/user-provisioning/user-sync-mutation.js.map +1 -0
- package/dist-server/service/webhook/webhook-management.d.ts +21 -0
- package/dist-server/service/webhook/webhook-management.js +134 -0
- package/dist-server/service/webhook/webhook-management.js.map +1 -0
- package/dist-server/tsconfig.tsbuildinfo +1 -0
- package/dist-server/types/dataset-labeling-types.d.ts +71 -0
- package/dist-server/types/dataset-labeling-types.js +259 -0
- package/dist-server/types/dataset-labeling-types.js.map +1 -0
- package/dist-server/types/label-studio-types.d.ts +128 -0
- package/dist-server/types/label-studio-types.js +494 -0
- package/dist-server/types/label-studio-types.js.map +1 -0
- package/dist-server/types/prediction-types.d.ts +39 -0
- package/dist-server/types/prediction-types.js +121 -0
- package/dist-server/types/prediction-types.js.map +1 -0
- package/dist-server/utils/annotation-exporter.d.ts +104 -0
- package/dist-server/utils/annotation-exporter.js +261 -0
- package/dist-server/utils/annotation-exporter.js.map +1 -0
- package/dist-server/utils/label-config-builder.d.ts +117 -0
- package/dist-server/utils/label-config-builder.js +286 -0
- package/dist-server/utils/label-config-builder.js.map +1 -0
- package/dist-server/utils/label-studio-api-client.d.ts +180 -0
- package/dist-server/utils/label-studio-api-client.js +401 -0
- package/dist-server/utils/label-studio-api-client.js.map +1 -0
- package/dist-server/utils/media-url-extractor.d.ts +45 -0
- package/dist-server/utils/media-url-extractor.js +152 -0
- package/dist-server/utils/media-url-extractor.js.map +1 -0
- package/dist-server/utils/task-transformer.d.ts +108 -0
- package/dist-server/utils/task-transformer.js +260 -0
- package/dist-server/utils/task-transformer.js.map +1 -0
- package/package.json +47 -0
- package/server/SERVER_STRUCTURE.md +351 -0
- package/server/controller/label-studio-role-mapper.ts +76 -0
- package/server/controller/user-provisioning-service.ts +340 -0
- package/server/index.ts +19 -0
- package/server/route/label-studio-sso.ts +194 -0
- package/server/route/webhook.ts +304 -0
- package/server/route.ts +35 -0
- package/server/service/ai-prediction-service.ts +239 -0
- package/server/service/dataset-labeling-integration.ts +590 -0
- package/server/service/external-data-source-service.ts +438 -0
- package/server/service/index.ts +24 -0
- package/server/service/label-studio-sso-service.ts +108 -0
- package/server/service/labeling-scenario-service.ts.deprecated +566 -0
- package/server/service/ml/ml-backend-service.ts +127 -0
- package/server/service/prediction/prediction-management.ts +281 -0
- package/server/service/project/project-management.ts +284 -0
- package/server/service/task/task-management.ts +363 -0
- package/server/service/user-provisioning/user-sync-mutation.ts +80 -0
- package/server/service/webhook/webhook-management.ts +109 -0
- package/server/tsconfig.json +11 -0
- package/server/types/dataset-labeling-types.ts +181 -0
- package/server/types/global.d.ts +23 -0
- package/server/types/label-studio-types.ts +346 -0
- package/server/types/prediction-types.ts +86 -0
- package/server/types/scenario-types.ts.deprecated +362 -0
- package/server/utils/annotation-exporter.ts +340 -0
- package/server/utils/label-config-builder.ts +340 -0
- package/server/utils/label-studio-api-client.ts +487 -0
- package/server/utils/media-url-extractor.ts +193 -0
- package/server/utils/task-transformer.ts +342 -0
- package/test-ai-prediction.js +268 -0
- package/test-dataset-integration.js +449 -0
- package/test-simple.js +89 -0
- package/things-factory.config.js +12 -0
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* External Data Source Service
|
|
3
|
+
*
|
|
4
|
+
* Import data from external sources into Label Studio
|
|
5
|
+
* Supports:
|
|
6
|
+
* - REST API endpoints
|
|
7
|
+
* - S3/Cloud Storage URLs
|
|
8
|
+
* - Database queries (via Things Factory connections)
|
|
9
|
+
* - CSV/JSON file URLs
|
|
10
|
+
*/
|
|
11
|
+
declare class ExternalDataSourceConfig {
|
|
12
|
+
sourceType: 'api' | 'url' | 's3' | 'database' | 'csv' | 'json';
|
|
13
|
+
sourceUrl: string;
|
|
14
|
+
authHeader?: string;
|
|
15
|
+
httpMethod?: string;
|
|
16
|
+
requestBody?: string;
|
|
17
|
+
dataPath?: string;
|
|
18
|
+
}
|
|
19
|
+
declare class ImportFromExternalSourceRequest {
|
|
20
|
+
projectId: number;
|
|
21
|
+
source: ExternalDataSourceConfig;
|
|
22
|
+
fieldMapping?: string;
|
|
23
|
+
imageField?: string;
|
|
24
|
+
autoGeneratePredictions?: boolean;
|
|
25
|
+
limit?: number;
|
|
26
|
+
}
|
|
27
|
+
declare class ExternalImportResult {
|
|
28
|
+
totalFetched: number;
|
|
29
|
+
tasksImported: number;
|
|
30
|
+
tasksFailed: number;
|
|
31
|
+
taskIds: number[];
|
|
32
|
+
error?: string;
|
|
33
|
+
}
|
|
34
|
+
declare class DataSourcePreview {
|
|
35
|
+
itemCount: number;
|
|
36
|
+
sampleData: string;
|
|
37
|
+
schema: string;
|
|
38
|
+
error?: string;
|
|
39
|
+
}
|
|
40
|
+
export declare class ExternalDataSourceService {
|
|
41
|
+
/**
|
|
42
|
+
* Preview external data source before import
|
|
43
|
+
*/
|
|
44
|
+
previewExternalDataSource(source: ExternalDataSourceConfig, context: ResolverContext): Promise<DataSourcePreview>;
|
|
45
|
+
/**
|
|
46
|
+
* Import data from external source to Label Studio
|
|
47
|
+
*/
|
|
48
|
+
importFromExternalSource(input: ImportFromExternalSourceRequest, context: ResolverContext): Promise<ExternalImportResult>;
|
|
49
|
+
/**
|
|
50
|
+
* Fetch data from external source
|
|
51
|
+
*/
|
|
52
|
+
private fetchFromSource;
|
|
53
|
+
/**
|
|
54
|
+
* Fetch from REST API
|
|
55
|
+
*/
|
|
56
|
+
private fetchFromApi;
|
|
57
|
+
/**
|
|
58
|
+
* Fetch from URL (JSON file)
|
|
59
|
+
*/
|
|
60
|
+
private fetchFromUrl;
|
|
61
|
+
/**
|
|
62
|
+
* Fetch from CSV
|
|
63
|
+
*/
|
|
64
|
+
private fetchFromCsv;
|
|
65
|
+
/**
|
|
66
|
+
* Extract data by JSON path (simple implementation)
|
|
67
|
+
*/
|
|
68
|
+
private extractDataByPath;
|
|
69
|
+
/**
|
|
70
|
+
* Transform raw data item to Label Studio task format
|
|
71
|
+
*/
|
|
72
|
+
private transformToLabelStudioTask;
|
|
73
|
+
/**
|
|
74
|
+
* Detect schema from data item
|
|
75
|
+
*/
|
|
76
|
+
private detectSchema;
|
|
77
|
+
}
|
|
78
|
+
export {};
|
|
@@ -0,0 +1,415 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.ExternalDataSourceService = void 0;
|
|
4
|
+
const tslib_1 = require("tslib");
|
|
5
|
+
const type_graphql_1 = require("type-graphql");
|
|
6
|
+
const type_graphql_2 = require("type-graphql");
|
|
7
|
+
const axios_1 = tslib_1.__importDefault(require("axios"));
|
|
8
|
+
const label_studio_api_client_js_1 = require("../utils/label-studio-api-client.js");
|
|
9
|
+
/**
|
|
10
|
+
* External Data Source Service
|
|
11
|
+
*
|
|
12
|
+
* Import data from external sources into Label Studio
|
|
13
|
+
* Supports:
|
|
14
|
+
* - REST API endpoints
|
|
15
|
+
* - S3/Cloud Storage URLs
|
|
16
|
+
* - Database queries (via Things Factory connections)
|
|
17
|
+
* - CSV/JSON file URLs
|
|
18
|
+
*/
|
|
19
|
+
// ============================================================================
|
|
20
|
+
// Type Definitions
|
|
21
|
+
// ============================================================================
|
|
22
|
+
let ExternalDataSourceConfig = class ExternalDataSourceConfig {
|
|
23
|
+
};
|
|
24
|
+
tslib_1.__decorate([
|
|
25
|
+
(0, type_graphql_2.Field)({ description: 'Data source type' }),
|
|
26
|
+
tslib_1.__metadata("design:type", String)
|
|
27
|
+
], ExternalDataSourceConfig.prototype, "sourceType", void 0);
|
|
28
|
+
tslib_1.__decorate([
|
|
29
|
+
(0, type_graphql_2.Field)({ description: 'Source URL or endpoint' }),
|
|
30
|
+
tslib_1.__metadata("design:type", String)
|
|
31
|
+
], ExternalDataSourceConfig.prototype, "sourceUrl", void 0);
|
|
32
|
+
tslib_1.__decorate([
|
|
33
|
+
(0, type_graphql_2.Field)({ nullable: true, description: 'Authentication header if required' }),
|
|
34
|
+
tslib_1.__metadata("design:type", String)
|
|
35
|
+
], ExternalDataSourceConfig.prototype, "authHeader", void 0);
|
|
36
|
+
tslib_1.__decorate([
|
|
37
|
+
(0, type_graphql_2.Field)({ nullable: true, description: 'HTTP method for API calls' }),
|
|
38
|
+
tslib_1.__metadata("design:type", String)
|
|
39
|
+
], ExternalDataSourceConfig.prototype, "httpMethod", void 0);
|
|
40
|
+
tslib_1.__decorate([
|
|
41
|
+
(0, type_graphql_2.Field)({ nullable: true, description: 'Request body for POST/PUT' }),
|
|
42
|
+
tslib_1.__metadata("design:type", String)
|
|
43
|
+
], ExternalDataSourceConfig.prototype, "requestBody", void 0);
|
|
44
|
+
tslib_1.__decorate([
|
|
45
|
+
(0, type_graphql_2.Field)({ nullable: true, description: 'JSONPath or XPath for data extraction' }),
|
|
46
|
+
tslib_1.__metadata("design:type", String)
|
|
47
|
+
], ExternalDataSourceConfig.prototype, "dataPath", void 0);
|
|
48
|
+
ExternalDataSourceConfig = tslib_1.__decorate([
|
|
49
|
+
(0, type_graphql_2.InputType)()
|
|
50
|
+
], ExternalDataSourceConfig);
|
|
51
|
+
let ImportFromExternalSourceRequest = class ImportFromExternalSourceRequest {
|
|
52
|
+
};
|
|
53
|
+
tslib_1.__decorate([
|
|
54
|
+
(0, type_graphql_2.Field)(type => type_graphql_1.Int, { description: 'Label Studio project ID' }),
|
|
55
|
+
tslib_1.__metadata("design:type", Number)
|
|
56
|
+
], ImportFromExternalSourceRequest.prototype, "projectId", void 0);
|
|
57
|
+
tslib_1.__decorate([
|
|
58
|
+
(0, type_graphql_2.Field)(type => ExternalDataSourceConfig, { description: 'External data source configuration' }),
|
|
59
|
+
tslib_1.__metadata("design:type", ExternalDataSourceConfig)
|
|
60
|
+
], ImportFromExternalSourceRequest.prototype, "source", void 0);
|
|
61
|
+
tslib_1.__decorate([
|
|
62
|
+
(0, type_graphql_2.Field)({ nullable: true, description: 'Field mapping JSON (source -> Label Studio)' }),
|
|
63
|
+
tslib_1.__metadata("design:type", String)
|
|
64
|
+
], ImportFromExternalSourceRequest.prototype, "fieldMapping", void 0);
|
|
65
|
+
tslib_1.__decorate([
|
|
66
|
+
(0, type_graphql_2.Field)({ nullable: true, description: 'Image field name in source data' }),
|
|
67
|
+
tslib_1.__metadata("design:type", String)
|
|
68
|
+
], ImportFromExternalSourceRequest.prototype, "imageField", void 0);
|
|
69
|
+
tslib_1.__decorate([
|
|
70
|
+
(0, type_graphql_2.Field)({ nullable: true, defaultValue: false, description: 'Auto-generate AI predictions' }),
|
|
71
|
+
tslib_1.__metadata("design:type", Boolean)
|
|
72
|
+
], ImportFromExternalSourceRequest.prototype, "autoGeneratePredictions", void 0);
|
|
73
|
+
tslib_1.__decorate([
|
|
74
|
+
(0, type_graphql_2.Field)({ nullable: true, description: 'Maximum number of items to import' }),
|
|
75
|
+
tslib_1.__metadata("design:type", Number)
|
|
76
|
+
], ImportFromExternalSourceRequest.prototype, "limit", void 0);
|
|
77
|
+
ImportFromExternalSourceRequest = tslib_1.__decorate([
|
|
78
|
+
(0, type_graphql_2.InputType)()
|
|
79
|
+
], ImportFromExternalSourceRequest);
|
|
80
|
+
let ExternalImportResult = class ExternalImportResult {
|
|
81
|
+
};
|
|
82
|
+
tslib_1.__decorate([
|
|
83
|
+
(0, type_graphql_2.Field)(type => type_graphql_1.Int, { description: 'Total items fetched from source' }),
|
|
84
|
+
tslib_1.__metadata("design:type", Number)
|
|
85
|
+
], ExternalImportResult.prototype, "totalFetched", void 0);
|
|
86
|
+
tslib_1.__decorate([
|
|
87
|
+
(0, type_graphql_2.Field)(type => type_graphql_1.Int, { description: 'Tasks successfully imported' }),
|
|
88
|
+
tslib_1.__metadata("design:type", Number)
|
|
89
|
+
], ExternalImportResult.prototype, "tasksImported", void 0);
|
|
90
|
+
tslib_1.__decorate([
|
|
91
|
+
(0, type_graphql_2.Field)(type => type_graphql_1.Int, { description: 'Tasks failed to import' }),
|
|
92
|
+
tslib_1.__metadata("design:type", Number)
|
|
93
|
+
], ExternalImportResult.prototype, "tasksFailed", void 0);
|
|
94
|
+
tslib_1.__decorate([
|
|
95
|
+
(0, type_graphql_2.Field)(type => [type_graphql_1.Int], { description: 'Imported task IDs' }),
|
|
96
|
+
tslib_1.__metadata("design:type", Array)
|
|
97
|
+
], ExternalImportResult.prototype, "taskIds", void 0);
|
|
98
|
+
tslib_1.__decorate([
|
|
99
|
+
(0, type_graphql_2.Field)({ nullable: true, description: 'Error message if any' }),
|
|
100
|
+
tslib_1.__metadata("design:type", String)
|
|
101
|
+
], ExternalImportResult.prototype, "error", void 0);
|
|
102
|
+
ExternalImportResult = tslib_1.__decorate([
|
|
103
|
+
(0, type_graphql_2.ObjectType)()
|
|
104
|
+
], ExternalImportResult);
|
|
105
|
+
let DataSourcePreview = class DataSourcePreview {
|
|
106
|
+
};
|
|
107
|
+
tslib_1.__decorate([
|
|
108
|
+
(0, type_graphql_2.Field)(type => type_graphql_1.Int, { description: 'Number of items available' }),
|
|
109
|
+
tslib_1.__metadata("design:type", Number)
|
|
110
|
+
], DataSourcePreview.prototype, "itemCount", void 0);
|
|
111
|
+
tslib_1.__decorate([
|
|
112
|
+
(0, type_graphql_2.Field)({ description: 'Sample data (first item)' }),
|
|
113
|
+
tslib_1.__metadata("design:type", String)
|
|
114
|
+
], DataSourcePreview.prototype, "sampleData", void 0);
|
|
115
|
+
tslib_1.__decorate([
|
|
116
|
+
(0, type_graphql_2.Field)({ description: 'Detected schema/fields' }),
|
|
117
|
+
tslib_1.__metadata("design:type", String)
|
|
118
|
+
], DataSourcePreview.prototype, "schema", void 0);
|
|
119
|
+
tslib_1.__decorate([
|
|
120
|
+
(0, type_graphql_2.Field)({ nullable: true, description: 'Preview error if any' }),
|
|
121
|
+
tslib_1.__metadata("design:type", String)
|
|
122
|
+
], DataSourcePreview.prototype, "error", void 0);
|
|
123
|
+
DataSourcePreview = tslib_1.__decorate([
|
|
124
|
+
(0, type_graphql_2.ObjectType)()
|
|
125
|
+
], DataSourcePreview);
|
|
126
|
+
// ============================================================================
|
|
127
|
+
// Resolver
|
|
128
|
+
// ============================================================================
|
|
129
|
+
let ExternalDataSourceService = class ExternalDataSourceService {
|
|
130
|
+
/**
|
|
131
|
+
* Preview external data source before import
|
|
132
|
+
*/
|
|
133
|
+
async previewExternalDataSource(source, context) {
|
|
134
|
+
console.log(`[External Data] Previewing source: ${source.sourceType} - ${source.sourceUrl}`);
|
|
135
|
+
try {
|
|
136
|
+
const data = await this.fetchFromSource(source, 1);
|
|
137
|
+
if (!data || data.length === 0) {
|
|
138
|
+
return {
|
|
139
|
+
itemCount: 0,
|
|
140
|
+
sampleData: '{}',
|
|
141
|
+
schema: '{}',
|
|
142
|
+
error: 'No data found'
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
const firstItem = data[0];
|
|
146
|
+
const schema = this.detectSchema(firstItem);
|
|
147
|
+
return {
|
|
148
|
+
itemCount: data.length,
|
|
149
|
+
sampleData: JSON.stringify(firstItem, null, 2),
|
|
150
|
+
schema: JSON.stringify(schema, null, 2)
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
catch (error) {
|
|
154
|
+
console.error('[External Data] Preview failed:', error);
|
|
155
|
+
return {
|
|
156
|
+
itemCount: 0,
|
|
157
|
+
sampleData: '{}',
|
|
158
|
+
schema: '{}',
|
|
159
|
+
error: error.message
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
/**
|
|
164
|
+
* Import data from external source to Label Studio
|
|
165
|
+
*/
|
|
166
|
+
async importFromExternalSource(input, context) {
|
|
167
|
+
console.log(`[External Data] Importing from ${input.source.sourceType}: ${input.source.sourceUrl}`);
|
|
168
|
+
try {
|
|
169
|
+
// 1. Fetch data from external source
|
|
170
|
+
const rawData = await this.fetchFromSource(input.source, input.limit);
|
|
171
|
+
console.log(`[External Data] Fetched ${rawData.length} items`);
|
|
172
|
+
if (rawData.length === 0) {
|
|
173
|
+
return {
|
|
174
|
+
totalFetched: 0,
|
|
175
|
+
tasksImported: 0,
|
|
176
|
+
tasksFailed: 0,
|
|
177
|
+
taskIds: [],
|
|
178
|
+
error: 'No data fetched from source'
|
|
179
|
+
};
|
|
180
|
+
}
|
|
181
|
+
// 2. Transform data to Label Studio format
|
|
182
|
+
const fieldMapping = input.fieldMapping ? JSON.parse(input.fieldMapping) : {};
|
|
183
|
+
const imageField = input.imageField || 'image';
|
|
184
|
+
const tasks = rawData.map(item => this.transformToLabelStudioTask(item, fieldMapping, imageField));
|
|
185
|
+
// 3. Import tasks to Label Studio
|
|
186
|
+
const result = {
|
|
187
|
+
totalFetched: rawData.length,
|
|
188
|
+
tasksImported: 0,
|
|
189
|
+
tasksFailed: 0,
|
|
190
|
+
taskIds: []
|
|
191
|
+
};
|
|
192
|
+
// Import in batches
|
|
193
|
+
const BATCH_SIZE = 50;
|
|
194
|
+
for (let i = 0; i < tasks.length; i += BATCH_SIZE) {
|
|
195
|
+
const batch = tasks.slice(i, i + BATCH_SIZE);
|
|
196
|
+
try {
|
|
197
|
+
const importResult = await label_studio_api_client_js_1.labelStudioApi.importTasks(input.projectId, batch);
|
|
198
|
+
// Label Studio import returns task IDs
|
|
199
|
+
if (importResult.task_count) {
|
|
200
|
+
result.tasksImported += importResult.task_count;
|
|
201
|
+
}
|
|
202
|
+
else if (Array.isArray(importResult)) {
|
|
203
|
+
result.tasksImported += importResult.length;
|
|
204
|
+
result.taskIds.push(...importResult.map((t) => t.id));
|
|
205
|
+
}
|
|
206
|
+
console.log(`[External Data] Imported batch ${i / BATCH_SIZE + 1}: ${batch.length} tasks`);
|
|
207
|
+
}
|
|
208
|
+
catch (batchError) {
|
|
209
|
+
console.error(`[External Data] Batch import failed:`, batchError);
|
|
210
|
+
result.tasksFailed += batch.length;
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
console.log(`[External Data] Import completed: ${result.tasksImported} imported, ${result.tasksFailed} failed`);
|
|
214
|
+
return result;
|
|
215
|
+
}
|
|
216
|
+
catch (error) {
|
|
217
|
+
console.error('[External Data] Import failed:', error);
|
|
218
|
+
throw new Error(`Failed to import from external source: ${error.message}`);
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
/**
|
|
222
|
+
* Fetch data from external source
|
|
223
|
+
*/
|
|
224
|
+
async fetchFromSource(source, limit) {
|
|
225
|
+
switch (source.sourceType) {
|
|
226
|
+
case 'api':
|
|
227
|
+
return await this.fetchFromApi(source, limit);
|
|
228
|
+
case 'url':
|
|
229
|
+
case 'json':
|
|
230
|
+
return await this.fetchFromUrl(source, limit);
|
|
231
|
+
case 'csv':
|
|
232
|
+
return await this.fetchFromCsv(source, limit);
|
|
233
|
+
case 's3':
|
|
234
|
+
throw new Error('S3 source not yet implemented. Use URL with S3 presigned URL instead.');
|
|
235
|
+
case 'database':
|
|
236
|
+
throw new Error('Database source not yet implemented. Use API endpoint instead.');
|
|
237
|
+
default:
|
|
238
|
+
throw new Error(`Unsupported source type: ${source.sourceType}`);
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
/**
|
|
242
|
+
* Fetch from REST API
|
|
243
|
+
*/
|
|
244
|
+
async fetchFromApi(source, limit) {
|
|
245
|
+
const method = (source.httpMethod || 'GET').toUpperCase();
|
|
246
|
+
const headers = {
|
|
247
|
+
'Content-Type': 'application/json'
|
|
248
|
+
};
|
|
249
|
+
if (source.authHeader) {
|
|
250
|
+
headers['Authorization'] = source.authHeader;
|
|
251
|
+
}
|
|
252
|
+
const config = {
|
|
253
|
+
method,
|
|
254
|
+
url: source.sourceUrl,
|
|
255
|
+
headers,
|
|
256
|
+
timeout: 30000
|
|
257
|
+
};
|
|
258
|
+
if (method === 'POST' || method === 'PUT') {
|
|
259
|
+
config.data = source.requestBody ? JSON.parse(source.requestBody) : {};
|
|
260
|
+
}
|
|
261
|
+
const response = await (0, axios_1.default)(config);
|
|
262
|
+
let data = response.data;
|
|
263
|
+
// Extract data using JSONPath if specified
|
|
264
|
+
if (source.dataPath) {
|
|
265
|
+
data = this.extractDataByPath(data, source.dataPath);
|
|
266
|
+
}
|
|
267
|
+
// Ensure data is an array
|
|
268
|
+
if (!Array.isArray(data)) {
|
|
269
|
+
data = [data];
|
|
270
|
+
}
|
|
271
|
+
// Apply limit
|
|
272
|
+
if (limit && data.length > limit) {
|
|
273
|
+
data = data.slice(0, limit);
|
|
274
|
+
}
|
|
275
|
+
return data;
|
|
276
|
+
}
|
|
277
|
+
/**
|
|
278
|
+
* Fetch from URL (JSON file)
|
|
279
|
+
*/
|
|
280
|
+
async fetchFromUrl(source, limit) {
|
|
281
|
+
const headers = {};
|
|
282
|
+
if (source.authHeader) {
|
|
283
|
+
headers['Authorization'] = source.authHeader;
|
|
284
|
+
}
|
|
285
|
+
const response = await axios_1.default.get(source.sourceUrl, { headers });
|
|
286
|
+
let data = response.data;
|
|
287
|
+
// Extract data using JSONPath if specified
|
|
288
|
+
if (source.dataPath) {
|
|
289
|
+
data = this.extractDataByPath(data, source.dataPath);
|
|
290
|
+
}
|
|
291
|
+
// Ensure data is an array
|
|
292
|
+
if (!Array.isArray(data)) {
|
|
293
|
+
data = [data];
|
|
294
|
+
}
|
|
295
|
+
// Apply limit
|
|
296
|
+
if (limit && data.length > limit) {
|
|
297
|
+
data = data.slice(0, limit);
|
|
298
|
+
}
|
|
299
|
+
return data;
|
|
300
|
+
}
|
|
301
|
+
/**
|
|
302
|
+
* Fetch from CSV
|
|
303
|
+
*/
|
|
304
|
+
async fetchFromCsv(source, limit) {
|
|
305
|
+
const headers = {};
|
|
306
|
+
if (source.authHeader) {
|
|
307
|
+
headers['Authorization'] = source.authHeader;
|
|
308
|
+
}
|
|
309
|
+
const response = await axios_1.default.get(source.sourceUrl, { headers });
|
|
310
|
+
const csvText = response.data;
|
|
311
|
+
// Simple CSV parsing (for production, use a proper CSV library)
|
|
312
|
+
const lines = csvText.trim().split('\n');
|
|
313
|
+
const headerLine = lines[0];
|
|
314
|
+
const headers_csv = headerLine.split(',').map((h) => h.trim());
|
|
315
|
+
const data = [];
|
|
316
|
+
const maxLines = limit ? Math.min(lines.length, limit + 1) : lines.length;
|
|
317
|
+
for (let i = 1; i < maxLines; i++) {
|
|
318
|
+
const line = lines[i];
|
|
319
|
+
const values = line.split(',').map((v) => v.trim());
|
|
320
|
+
const obj = {};
|
|
321
|
+
headers_csv.forEach((header, index) => {
|
|
322
|
+
obj[header] = values[index];
|
|
323
|
+
});
|
|
324
|
+
data.push(obj);
|
|
325
|
+
}
|
|
326
|
+
return data;
|
|
327
|
+
}
|
|
328
|
+
/**
|
|
329
|
+
* Extract data by JSON path (simple implementation)
|
|
330
|
+
*/
|
|
331
|
+
extractDataByPath(data, path) {
|
|
332
|
+
const parts = path.split('.');
|
|
333
|
+
let current = data;
|
|
334
|
+
for (const part of parts) {
|
|
335
|
+
if (current && typeof current === 'object' && part in current) {
|
|
336
|
+
current = current[part];
|
|
337
|
+
}
|
|
338
|
+
else {
|
|
339
|
+
throw new Error(`Path "${path}" not found in data`);
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
return current;
|
|
343
|
+
}
|
|
344
|
+
/**
|
|
345
|
+
* Transform raw data item to Label Studio task format
|
|
346
|
+
*/
|
|
347
|
+
transformToLabelStudioTask(item, fieldMapping, imageField) {
|
|
348
|
+
// Apply field mapping
|
|
349
|
+
const mappedData = {};
|
|
350
|
+
if (Object.keys(fieldMapping).length > 0) {
|
|
351
|
+
// Use explicit mapping
|
|
352
|
+
for (const [sourceField, targetField] of Object.entries(fieldMapping)) {
|
|
353
|
+
if (item[sourceField] !== undefined) {
|
|
354
|
+
mappedData[targetField] = item[sourceField];
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
}
|
|
358
|
+
else {
|
|
359
|
+
// No mapping, use data as-is
|
|
360
|
+
Object.assign(mappedData, item);
|
|
361
|
+
}
|
|
362
|
+
// Ensure image field exists
|
|
363
|
+
const imageUrl = mappedData[imageField] || item[imageField] || item.url || item.image_url;
|
|
364
|
+
if (!imageUrl) {
|
|
365
|
+
console.warn('[External Data] No image URL found in item:', item);
|
|
366
|
+
}
|
|
367
|
+
return {
|
|
368
|
+
data: {
|
|
369
|
+
image: imageUrl,
|
|
370
|
+
...mappedData
|
|
371
|
+
},
|
|
372
|
+
meta: {
|
|
373
|
+
source: 'external-import',
|
|
374
|
+
originalData: JSON.stringify(item)
|
|
375
|
+
}
|
|
376
|
+
};
|
|
377
|
+
}
|
|
378
|
+
/**
|
|
379
|
+
* Detect schema from data item
|
|
380
|
+
*/
|
|
381
|
+
detectSchema(item) {
|
|
382
|
+
const schema = {};
|
|
383
|
+
for (const [key, value] of Object.entries(item)) {
|
|
384
|
+
schema[key] = typeof value;
|
|
385
|
+
}
|
|
386
|
+
return schema;
|
|
387
|
+
}
|
|
388
|
+
};
|
|
389
|
+
exports.ExternalDataSourceService = ExternalDataSourceService;
|
|
390
|
+
tslib_1.__decorate([
|
|
391
|
+
(0, type_graphql_1.Query)(returns => DataSourcePreview, {
|
|
392
|
+
description: 'Preview external data source to verify connection and data structure'
|
|
393
|
+
}),
|
|
394
|
+
(0, type_graphql_1.Directive)('@privilege(category: "label-studio", privilege: "query")'),
|
|
395
|
+
tslib_1.__param(0, (0, type_graphql_1.Arg)('source', type => ExternalDataSourceConfig)),
|
|
396
|
+
tslib_1.__param(1, (0, type_graphql_1.Ctx)()),
|
|
397
|
+
tslib_1.__metadata("design:type", Function),
|
|
398
|
+
tslib_1.__metadata("design:paramtypes", [ExternalDataSourceConfig, Object]),
|
|
399
|
+
tslib_1.__metadata("design:returntype", Promise)
|
|
400
|
+
], ExternalDataSourceService.prototype, "previewExternalDataSource", null);
|
|
401
|
+
tslib_1.__decorate([
|
|
402
|
+
(0, type_graphql_1.Mutation)(returns => ExternalImportResult, {
|
|
403
|
+
description: 'Import data from external source into Label Studio project'
|
|
404
|
+
}),
|
|
405
|
+
(0, type_graphql_1.Directive)('@privilege(category: "label-studio", privilege: "mutation")'),
|
|
406
|
+
tslib_1.__param(0, (0, type_graphql_1.Arg)('input')),
|
|
407
|
+
tslib_1.__param(1, (0, type_graphql_1.Ctx)()),
|
|
408
|
+
tslib_1.__metadata("design:type", Function),
|
|
409
|
+
tslib_1.__metadata("design:paramtypes", [ImportFromExternalSourceRequest, Object]),
|
|
410
|
+
tslib_1.__metadata("design:returntype", Promise)
|
|
411
|
+
], ExternalDataSourceService.prototype, "importFromExternalSource", null);
|
|
412
|
+
exports.ExternalDataSourceService = ExternalDataSourceService = tslib_1.__decorate([
|
|
413
|
+
(0, type_graphql_1.Resolver)()
|
|
414
|
+
], ExternalDataSourceService);
|
|
415
|
+
//# sourceMappingURL=external-data-source-service.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"external-data-source-service.js","sourceRoot":"","sources":["../../server/service/external-data-source-service.ts"],"names":[],"mappings":";;;;AAAA,+CAAkF;AAClF,+CAA2D;AAC3D,0DAAyB;AACzB,oFAAoE;AAEpE;;;;;;;;;GASG;AAEH,+EAA+E;AAC/E,mBAAmB;AACnB,+EAA+E;AAG/E,IAAM,wBAAwB,GAA9B,MAAM,wBAAwB;CAkB7B,CAAA;AAhBC;IADC,IAAA,oBAAK,EAAC,EAAE,WAAW,EAAE,kBAAkB,EAAE,CAAC;;4DACmB;AAG9D;IADC,IAAA,oBAAK,EAAC,EAAE,WAAW,EAAE,wBAAwB,EAAE,CAAC;;2DAChC;AAGjB;IADC,IAAA,oBAAK,EAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,mCAAmC,EAAE,CAAC;;4DACzD;AAGnB;IADC,IAAA,oBAAK,EAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,2BAA2B,EAAE,CAAC;;4DACjD;AAGnB;IADC,IAAA,oBAAK,EAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,2BAA2B,EAAE,CAAC;;6DAChD;AAGpB;IADC,IAAA,oBAAK,EAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,uCAAuC,EAAE,CAAC;;0DAC/D;AAjBb,wBAAwB;IAD7B,IAAA,wBAAS,GAAE;GACN,wBAAwB,CAkB7B;AAGD,IAAM,+BAA+B,GAArC,MAAM,+BAA+B;CAkBpC,CAAA;AAhBC;IADC,IAAA,oBAAK,EAAC,IAAI,CAAC,EAAE,CAAC,kBAAG,EAAE,EAAE,WAAW,EAAE,yBAAyB,EAAE,CAAC;;kEAC9C;AAGjB;IADC,IAAA,oBAAK,EAAC,IAAI,CAAC,EAAE,CAAC,wBAAwB,EAAE,EAAE,WAAW,EAAE,oCAAoC,EAAE,CAAC;sCACvF,wBAAwB;+DAAA;AAGhC;IADC,IAAA,oBAAK,EAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,6CAA6C,EAAE,CAAC;;qEACjE;AAGrB;IADC,IAAA,oBAAK,EAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,iCAAiC,EAAE,CAAC;;mEACvD;AAGnB;IADC,IAAA,oBAAK,EAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,WAAW,EAAE,8BAA8B,EAAE,CAAC;;gFAC3D;AAGjC;IADC,IAAA,oBAAK,EAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,mCAAmC,EAAE,CAAC;;8DAC9D;AAjBV,+BAA+B;IADpC,IAAA,wBAAS,GAAE;GACN,+BAA+B,CAkBpC;AAGD,IAAM,oBAAoB,GAA1B,MAAM,oBAAoB;CAezB,CAAA;AAbC;IADC,IAAA,oBAAK,EAAC,IAAI,CAAC,EAAE,CAAC,kBAAG,EAAE,EAAE,WAAW,EAAE,iCAAiC,EAAE,CAAC;;0DACnD;AAGpB;IADC,IAAA,oBAAK,EAAC,IAAI,CAAC,EAAE,CAAC,kBAAG,EAAE,EAAE,WAAW,EAAE,6BAA6B,EAAE,CAAC;;2DAC9C;AAGrB;IADC,IAAA,oBAAK,EAAC,IAAI,CAAC,EAAE,CAAC,kBAAG,EAAE,EAAE,WAAW,EAAE,wBAAwB,EAAE,CAAC;;yDAC3C;AAGnB;IADC,IAAA,oBAAK,EAAC,IAAI,CAAC,EAAE,CAAC,CAAC,kBAAG,CAAC,EAAE,EAAE,WAAW,EAAE,mBAAmB,EAAE,CAAC;;qDAC1C;AAGjB;IADC,IAAA,oBAAK,EAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,sBAAsB,EAAE,CAAC;;mDACjD;AAdV,oBAAoB;IADzB,IAAA,yBAAU,GAAE;GACP,oBAAoB,CAezB;AAGD,IAAM,iBAAiB,GAAvB,MAAM,iBAAiB;CAYtB,CAAA;AAVC;IADC,IAAA,oBAAK,EAAC,IAAI,CAAC,EAAE,CAAC,kBAAG,EAAE,EAAE,WAAW,EAAE,2BAA2B,EAAE,CAAC;;oDAChD;AAGjB;IADC,IAAA,oBAAK,EAAC,EAAE,WAAW,EAAE,0BAA0B,EAAE,CAAC;;qDACjC;AAGlB;IADC,IAAA,oBAAK,EAAC,EAAE,WAAW,EAAE,wBAAwB,EAAE,CAAC;;iDACnC;AAGd;IADC,IAAA,oBAAK,EAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,sBAAsB,EAAE,CAAC;;gDACjD;AAXV,iBAAiB;IADtB,IAAA,yBAAU,GAAE;GACP,iBAAiB,CAYtB;AAED,+EAA+E;AAC/E,WAAW;AACX,+EAA+E;AAGxE,IAAM,yBAAyB,GAA/B,MAAM,yBAAyB;IACpC;;OAEG;IAKG,AAAN,KAAK,CAAC,yBAAyB,CACoB,MAAgC,EAC1E,OAAwB;QAE/B,OAAO,CAAC,GAAG,CAAC,sCAAsC,MAAM,CAAC,UAAU,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC,CAAA;QAE5F,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,CAAC,CAAC,CAAA;YAElD,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC/B,OAAO;oBACL,SAAS,EAAE,CAAC;oBACZ,UAAU,EAAE,IAAI;oBAChB,MAAM,EAAE,IAAI;oBACZ,KAAK,EAAE,eAAe;iBACvB,CAAA;YACH,CAAC;YAED,MAAM,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,CAAA;YACzB,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,CAAA;YAE3C,OAAO;gBACL,SAAS,EAAE,IAAI,CAAC,MAAM;gBACtB,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC9C,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;aACxC,CAAA;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,iCAAiC,EAAE,KAAK,CAAC,CAAA;YACvD,OAAO;gBACL,SAAS,EAAE,CAAC;gBACZ,UAAU,EAAE,IAAI;gBAChB,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,KAAK,CAAC,OAAO;aACrB,CAAA;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IAKG,AAAN,KAAK,CAAC,wBAAwB,CACd,KAAsC,EAC7C,OAAwB;QAE/B,OAAO,CAAC,GAAG,CAAC,kCAAkC,KAAK,CAAC,MAAM,CAAC,UAAU,KAAK,KAAK,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAA;QAEnG,IAAI,CAAC;YACH,qCAAqC;YACrC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,CAAA;YAErE,OAAO,CAAC,GAAG,CAAC,2BAA2B,OAAO,CAAC,MAAM,QAAQ,CAAC,CAAA;YAE9D,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACzB,OAAO;oBACL,YAAY,EAAE,CAAC;oBACf,aAAa,EAAE,CAAC;oBAChB,WAAW,EAAE,CAAC;oBACd,OAAO,EAAE,EAAE;oBACX,KAAK,EAAE,6BAA6B;iBACrC,CAAA;YACH,CAAC;YAED,2CAA2C;YAC3C,MAAM,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;YAC7E,MAAM,UAAU,GAAG,KAAK,CAAC,UAAU,IAAI,OAAO,CAAA;YAE9C,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,0BAA0B,CAAC,IAAI,EAAE,YAAY,EAAE,UAAU,CAAC,CAAC,CAAA;YAElG,kCAAkC;YAClC,MAAM,MAAM,GAAyB;gBACnC,YAAY,EAAE,OAAO,CAAC,MAAM;gBAC5B,aAAa,EAAE,CAAC;gBAChB,WAAW,EAAE,CAAC;gBACd,OAAO,EAAE,EAAE;aACZ,CAAA;YAED,oBAAoB;YACpB,MAAM,UAAU,GAAG,EAAE,CAAA;YACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC;gBAClD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,CAAA;gBAE5C,IAAI,CAAC;oBACH,MAAM,YAAY,GAAG,MAAM,2CAAc,CAAC,WAAW,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,CAAA;oBAE7E,uCAAuC;oBACvC,IAAI,YAAY,CAAC,UAAU,EAAE,CAAC;wBAC5B,MAAM,CAAC,aAAa,IAAI,YAAY,CAAC,UAAU,CAAA;oBACjD,CAAC;yBAAM,IAAI,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,CAAC;wBACvC,MAAM,CAAC,aAAa,IAAI,YAAY,CAAC,MAAM,CAAA;wBAC3C,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;oBAC5D,CAAC;oBAED,OAAO,CAAC,GAAG,CAAC,kCAAkC,CAAC,GAAG,UAAU,GAAG,CAAC,KAAK,KAAK,CAAC,MAAM,QAAQ,CAAC,CAAA;gBAC5F,CAAC;gBAAC,OAAO,UAAU,EAAE,CAAC;oBACpB,OAAO,CAAC,KAAK,CAAC,sCAAsC,EAAE,UAAU,CAAC,CAAA;oBACjE,MAAM,CAAC,WAAW,IAAI,KAAK,CAAC,MAAM,CAAA;gBACpC,CAAC;YACH,CAAC;YAED,OAAO,CAAC,GAAG,CACT,qCAAqC,MAAM,CAAC,aAAa,cAAc,MAAM,CAAC,WAAW,SAAS,CACnG,CAAA;YAED,OAAO,MAAM,CAAA;QACf,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAA;YACtD,MAAM,IAAI,KAAK,CAAC,0CAA0C,KAAK,CAAC,OAAO,EAAE,CAAC,CAAA;QAC5E,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,eAAe,CAC3B,MAAgC,EAChC,KAAc;QAEd,QAAQ,MAAM,CAAC,UAAU,EAAE,CAAC;YAC1B,KAAK,KAAK;gBACR,OAAO,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,KAAK,CAAC,CAAA;YAE/C,KAAK,KAAK,CAAC;YACX,KAAK,MAAM;gBACT,OAAO,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,KAAK,CAAC,CAAA;YAE/C,KAAK,KAAK;gBACR,OAAO,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,KAAK,CAAC,CAAA;YAE/C,KAAK,IAAI;gBACP,MAAM,IAAI,KAAK,CAAC,uEAAuE,CAAC,CAAA;YAE1F,KAAK,UAAU;gBACb,MAAM,IAAI,KAAK,CAAC,gEAAgE,CAAC,CAAA;YAEnF;gBACE,MAAM,IAAI,KAAK,CAAC,4BAA4B,MAAM,CAAC,UAAU,EAAE,CAAC,CAAA;QACpE,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,YAAY,CAAC,MAAgC,EAAE,KAAc;QACzE,MAAM,MAAM,GAAG,CAAC,MAAM,CAAC,UAAU,IAAI,KAAK,CAAC,CAAC,WAAW,EAAE,CAAA;QACzD,MAAM,OAAO,GAAQ;YACnB,cAAc,EAAE,kBAAkB;SACnC,CAAA;QAED,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YACtB,OAAO,CAAC,eAAe,CAAC,GAAG,MAAM,CAAC,UAAU,CAAA;QAC9C,CAAC;QAED,MAAM,MAAM,GAAQ;YAClB,MAAM;YACN,GAAG,EAAE,MAAM,CAAC,SAAS;YACrB,OAAO;YACP,OAAO,EAAE,KAAK;SACf,CAAA;QAED,IAAI,MAAM,KAAK,MAAM,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;YAC1C,MAAM,CAAC,IAAI,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;QACxE,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,IAAA,eAAK,EAAC,MAAM,CAAC,CAAA;QAEpC,IAAI,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAA;QAExB,2CAA2C;QAC3C,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YACpB,IAAI,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAA;QACtD,CAAC;QAED,0BAA0B;QAC1B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YACzB,IAAI,GAAG,CAAC,IAAI,CAAC,CAAA;QACf,CAAC;QAED,cAAc;QACd,IAAI,KAAK,IAAI,IAAI,CAAC,MAAM,GAAG,KAAK,EAAE,CAAC;YACjC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAA;QAC7B,CAAC;QAED,OAAO,IAAI,CAAA;IACb,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,YAAY,CAAC,MAAgC,EAAE,KAAc;QACzE,MAAM,OAAO,GAAQ,EAAE,CAAA;QAEvB,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YACtB,OAAO,CAAC,eAAe,CAAC,GAAG,MAAM,CAAC,UAAU,CAAA;QAC9C,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,eAAK,CAAC,GAAG,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,OAAO,EAAE,CAAC,CAAA;QAE/D,IAAI,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAA;QAExB,2CAA2C;QAC3C,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YACpB,IAAI,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAA;QACtD,CAAC;QAED,0BAA0B;QAC1B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YACzB,IAAI,GAAG,CAAC,IAAI,CAAC,CAAA;QACf,CAAC;QAED,cAAc;QACd,IAAI,KAAK,IAAI,IAAI,CAAC,MAAM,GAAG,KAAK,EAAE,CAAC;YACjC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAA;QAC7B,CAAC;QAED,OAAO,IAAI,CAAA;IACb,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,YAAY,CAAC,MAAgC,EAAE,KAAc;QACzE,MAAM,OAAO,GAAQ,EAAE,CAAA;QAEvB,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YACtB,OAAO,CAAC,eAAe,CAAC,GAAG,MAAM,CAAC,UAAU,CAAA;QAC9C,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,eAAK,CAAC,GAAG,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,OAAO,EAAE,CAAC,CAAA;QAC/D,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAA;QAE7B,gEAAgE;QAChE,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QACxC,MAAM,UAAU,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;QAC3B,MAAM,WAAW,GAAG,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;QAEtE,MAAM,IAAI,GAAG,EAAE,CAAA;QACf,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAA;QAEzE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;YAClC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;YACrB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;YAE3D,MAAM,GAAG,GAAQ,EAAE,CAAA;YACnB,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE;gBACpC,GAAG,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAA;YAC7B,CAAC,CAAC,CAAA;YAEF,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAChB,CAAC;QAED,OAAO,IAAI,CAAA;IACb,CAAC;IAED;;OAEG;IACK,iBAAiB,CAAC,IAAS,EAAE,IAAY;QAC/C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;QAC7B,IAAI,OAAO,GAAG,IAAI,CAAA;QAElB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,IAAI,IAAI,OAAO,EAAE,CAAC;gBAC9D,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;YACzB,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,KAAK,CAAC,SAAS,IAAI,qBAAqB,CAAC,CAAA;YACrD,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAA;IAChB,CAAC;IAED;;OAEG;IACK,0BAA0B,CAChC,IAAS,EACT,YAAoC,EACpC,UAAkB;QAElB,sBAAsB;QACtB,MAAM,UAAU,GAAQ,EAAE,CAAA;QAE1B,IAAI,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzC,uBAAuB;YACvB,KAAK,MAAM,CAAC,WAAW,EAAE,WAAW,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,CAAC;gBACtE,IAAI,IAAI,CAAC,WAAW,CAAC,KAAK,SAAS,EAAE,CAAC;oBACpC,UAAU,CAAC,WAAW,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,CAAA;gBAC7C,CAAC;YACH,CAAC;QACH,CAAC;aAAM,CAAC;YACN,6BAA6B;YAC7B,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,CAAA;QACjC,CAAC;QAED,4BAA4B;QAC5B,MAAM,QAAQ,GAAG,UAAU,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,SAAS,CAAA;QAEzF,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,OAAO,CAAC,IAAI,CAAC,6CAA6C,EAAE,IAAI,CAAC,CAAA;QACnE,CAAC;QAED,OAAO;YACL,IAAI,EAAE;gBACJ,KAAK,EAAE,QAAQ;gBACf,GAAG,UAAU;aACd;YACD,IAAI,EAAE;gBACJ,MAAM,EAAE,iBAAiB;gBACzB,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;aACnC;SACF,CAAA;IACH,CAAC;IAED;;OAEG;IACK,YAAY,CAAC,IAAS;QAC5B,MAAM,MAAM,GAA2B,EAAE,CAAA;QAEzC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YAChD,MAAM,CAAC,GAAG,CAAC,GAAG,OAAO,KAAK,CAAA;QAC5B,CAAC;QAED,OAAO,MAAM,CAAA;IACf,CAAC;CACF,CAAA;AAjVY,8DAAyB;AAQ9B;IAJL,IAAA,oBAAK,EAAC,OAAO,CAAC,EAAE,CAAC,iBAAiB,EAAE;QACnC,WAAW,EAAE,sEAAsE;KACpF,CAAC;IACD,IAAA,wBAAS,EAAC,0DAA0D,CAAC;IAEnE,mBAAA,IAAA,kBAAG,EAAC,QAAQ,EAAE,IAAI,CAAC,EAAE,CAAC,wBAAwB,CAAC,CAAA;IAC/C,mBAAA,IAAA,kBAAG,GAAE,CAAA;;6CADmD,wBAAwB;;0EAkClF;AASK;IAJL,IAAA,uBAAQ,EAAC,OAAO,CAAC,EAAE,CAAC,oBAAoB,EAAE;QACzC,WAAW,EAAE,4DAA4D;KAC1E,CAAC;IACD,IAAA,wBAAS,EAAC,6DAA6D,CAAC;IAEtE,mBAAA,IAAA,kBAAG,EAAC,OAAO,CAAC,CAAA;IACZ,mBAAA,IAAA,kBAAG,GAAE,CAAA;;6CADe,+BAA+B;;yEAmErD;oCAxHU,yBAAyB;IADrC,IAAA,uBAAQ,GAAE;GACE,yBAAyB,CAiVrC","sourcesContent":["import { Resolver, Mutation, Query, Arg, Ctx, Int, Directive } from 'type-graphql'\nimport { Field, InputType, ObjectType } from 'type-graphql'\nimport axios from 'axios'\nimport { labelStudioApi } from '../utils/label-studio-api-client.js'\n\n/**\n * External Data Source Service\n *\n * Import data from external sources into Label Studio\n * Supports:\n * - REST API endpoints\n * - S3/Cloud Storage URLs\n * - Database queries (via Things Factory connections)\n * - CSV/JSON file URLs\n */\n\n// ============================================================================\n// Type Definitions\n// ============================================================================\n\n@InputType()\nclass ExternalDataSourceConfig {\n @Field({ description: 'Data source type' })\n sourceType: 'api' | 'url' | 's3' | 'database' | 'csv' | 'json'\n\n @Field({ description: 'Source URL or endpoint' })\n sourceUrl: string\n\n @Field({ nullable: true, description: 'Authentication header if required' })\n authHeader?: string\n\n @Field({ nullable: true, description: 'HTTP method for API calls' })\n httpMethod?: string\n\n @Field({ nullable: true, description: 'Request body for POST/PUT' })\n requestBody?: string\n\n @Field({ nullable: true, description: 'JSONPath or XPath for data extraction' })\n dataPath?: string\n}\n\n@InputType()\nclass ImportFromExternalSourceRequest {\n @Field(type => Int, { description: 'Label Studio project ID' })\n projectId: number\n\n @Field(type => ExternalDataSourceConfig, { description: 'External data source configuration' })\n source: ExternalDataSourceConfig\n\n @Field({ nullable: true, description: 'Field mapping JSON (source -> Label Studio)' })\n fieldMapping?: string\n\n @Field({ nullable: true, description: 'Image field name in source data' })\n imageField?: string\n\n @Field({ nullable: true, defaultValue: false, description: 'Auto-generate AI predictions' })\n autoGeneratePredictions?: boolean\n\n @Field({ nullable: true, description: 'Maximum number of items to import' })\n limit?: number\n}\n\n@ObjectType()\nclass ExternalImportResult {\n @Field(type => Int, { description: 'Total items fetched from source' })\n totalFetched: number\n\n @Field(type => Int, { description: 'Tasks successfully imported' })\n tasksImported: number\n\n @Field(type => Int, { description: 'Tasks failed to import' })\n tasksFailed: number\n\n @Field(type => [Int], { description: 'Imported task IDs' })\n taskIds: number[]\n\n @Field({ nullable: true, description: 'Error message if any' })\n error?: string\n}\n\n@ObjectType()\nclass DataSourcePreview {\n @Field(type => Int, { description: 'Number of items available' })\n itemCount: number\n\n @Field({ description: 'Sample data (first item)' })\n sampleData: string\n\n @Field({ description: 'Detected schema/fields' })\n schema: string\n\n @Field({ nullable: true, description: 'Preview error if any' })\n error?: string\n}\n\n// ============================================================================\n// Resolver\n// ============================================================================\n\n@Resolver()\nexport class ExternalDataSourceService {\n /**\n * Preview external data source before import\n */\n @Query(returns => DataSourcePreview, {\n description: 'Preview external data source to verify connection and data structure'\n })\n @Directive('@privilege(category: \"label-studio\", privilege: \"query\")')\n async previewExternalDataSource(\n @Arg('source', type => ExternalDataSourceConfig) source: ExternalDataSourceConfig,\n @Ctx() context: ResolverContext\n ): Promise<DataSourcePreview> {\n console.log(`[External Data] Previewing source: ${source.sourceType} - ${source.sourceUrl}`)\n\n try {\n const data = await this.fetchFromSource(source, 1)\n\n if (!data || data.length === 0) {\n return {\n itemCount: 0,\n sampleData: '{}',\n schema: '{}',\n error: 'No data found'\n }\n }\n\n const firstItem = data[0]\n const schema = this.detectSchema(firstItem)\n\n return {\n itemCount: data.length,\n sampleData: JSON.stringify(firstItem, null, 2),\n schema: JSON.stringify(schema, null, 2)\n }\n } catch (error) {\n console.error('[External Data] Preview failed:', error)\n return {\n itemCount: 0,\n sampleData: '{}',\n schema: '{}',\n error: error.message\n }\n }\n }\n\n /**\n * Import data from external source to Label Studio\n */\n @Mutation(returns => ExternalImportResult, {\n description: 'Import data from external source into Label Studio project'\n })\n @Directive('@privilege(category: \"label-studio\", privilege: \"mutation\")')\n async importFromExternalSource(\n @Arg('input') input: ImportFromExternalSourceRequest,\n @Ctx() context: ResolverContext\n ): Promise<ExternalImportResult> {\n console.log(`[External Data] Importing from ${input.source.sourceType}: ${input.source.sourceUrl}`)\n\n try {\n // 1. Fetch data from external source\n const rawData = await this.fetchFromSource(input.source, input.limit)\n\n console.log(`[External Data] Fetched ${rawData.length} items`)\n\n if (rawData.length === 0) {\n return {\n totalFetched: 0,\n tasksImported: 0,\n tasksFailed: 0,\n taskIds: [],\n error: 'No data fetched from source'\n }\n }\n\n // 2. Transform data to Label Studio format\n const fieldMapping = input.fieldMapping ? JSON.parse(input.fieldMapping) : {}\n const imageField = input.imageField || 'image'\n\n const tasks = rawData.map(item => this.transformToLabelStudioTask(item, fieldMapping, imageField))\n\n // 3. Import tasks to Label Studio\n const result: ExternalImportResult = {\n totalFetched: rawData.length,\n tasksImported: 0,\n tasksFailed: 0,\n taskIds: []\n }\n\n // Import in batches\n const BATCH_SIZE = 50\n for (let i = 0; i < tasks.length; i += BATCH_SIZE) {\n const batch = tasks.slice(i, i + BATCH_SIZE)\n\n try {\n const importResult = await labelStudioApi.importTasks(input.projectId, batch)\n\n // Label Studio import returns task IDs\n if (importResult.task_count) {\n result.tasksImported += importResult.task_count\n } else if (Array.isArray(importResult)) {\n result.tasksImported += importResult.length\n result.taskIds.push(...importResult.map((t: any) => t.id))\n }\n\n console.log(`[External Data] Imported batch ${i / BATCH_SIZE + 1}: ${batch.length} tasks`)\n } catch (batchError) {\n console.error(`[External Data] Batch import failed:`, batchError)\n result.tasksFailed += batch.length\n }\n }\n\n console.log(\n `[External Data] Import completed: ${result.tasksImported} imported, ${result.tasksFailed} failed`\n )\n\n return result\n } catch (error) {\n console.error('[External Data] Import failed:', error)\n throw new Error(`Failed to import from external source: ${error.message}`)\n }\n }\n\n /**\n * Fetch data from external source\n */\n private async fetchFromSource(\n source: ExternalDataSourceConfig,\n limit?: number\n ): Promise<any[]> {\n switch (source.sourceType) {\n case 'api':\n return await this.fetchFromApi(source, limit)\n\n case 'url':\n case 'json':\n return await this.fetchFromUrl(source, limit)\n\n case 'csv':\n return await this.fetchFromCsv(source, limit)\n\n case 's3':\n throw new Error('S3 source not yet implemented. Use URL with S3 presigned URL instead.')\n\n case 'database':\n throw new Error('Database source not yet implemented. Use API endpoint instead.')\n\n default:\n throw new Error(`Unsupported source type: ${source.sourceType}`)\n }\n }\n\n /**\n * Fetch from REST API\n */\n private async fetchFromApi(source: ExternalDataSourceConfig, limit?: number): Promise<any[]> {\n const method = (source.httpMethod || 'GET').toUpperCase()\n const headers: any = {\n 'Content-Type': 'application/json'\n }\n\n if (source.authHeader) {\n headers['Authorization'] = source.authHeader\n }\n\n const config: any = {\n method,\n url: source.sourceUrl,\n headers,\n timeout: 30000\n }\n\n if (method === 'POST' || method === 'PUT') {\n config.data = source.requestBody ? JSON.parse(source.requestBody) : {}\n }\n\n const response = await axios(config)\n\n let data = response.data\n\n // Extract data using JSONPath if specified\n if (source.dataPath) {\n data = this.extractDataByPath(data, source.dataPath)\n }\n\n // Ensure data is an array\n if (!Array.isArray(data)) {\n data = [data]\n }\n\n // Apply limit\n if (limit && data.length > limit) {\n data = data.slice(0, limit)\n }\n\n return data\n }\n\n /**\n * Fetch from URL (JSON file)\n */\n private async fetchFromUrl(source: ExternalDataSourceConfig, limit?: number): Promise<any[]> {\n const headers: any = {}\n\n if (source.authHeader) {\n headers['Authorization'] = source.authHeader\n }\n\n const response = await axios.get(source.sourceUrl, { headers })\n\n let data = response.data\n\n // Extract data using JSONPath if specified\n if (source.dataPath) {\n data = this.extractDataByPath(data, source.dataPath)\n }\n\n // Ensure data is an array\n if (!Array.isArray(data)) {\n data = [data]\n }\n\n // Apply limit\n if (limit && data.length > limit) {\n data = data.slice(0, limit)\n }\n\n return data\n }\n\n /**\n * Fetch from CSV\n */\n private async fetchFromCsv(source: ExternalDataSourceConfig, limit?: number): Promise<any[]> {\n const headers: any = {}\n\n if (source.authHeader) {\n headers['Authorization'] = source.authHeader\n }\n\n const response = await axios.get(source.sourceUrl, { headers })\n const csvText = response.data\n\n // Simple CSV parsing (for production, use a proper CSV library)\n const lines = csvText.trim().split('\\n')\n const headerLine = lines[0]\n const headers_csv = headerLine.split(',').map((h: string) => h.trim())\n\n const data = []\n const maxLines = limit ? Math.min(lines.length, limit + 1) : lines.length\n\n for (let i = 1; i < maxLines; i++) {\n const line = lines[i]\n const values = line.split(',').map((v: string) => v.trim())\n\n const obj: any = {}\n headers_csv.forEach((header, index) => {\n obj[header] = values[index]\n })\n\n data.push(obj)\n }\n\n return data\n }\n\n /**\n * Extract data by JSON path (simple implementation)\n */\n private extractDataByPath(data: any, path: string): any {\n const parts = path.split('.')\n let current = data\n\n for (const part of parts) {\n if (current && typeof current === 'object' && part in current) {\n current = current[part]\n } else {\n throw new Error(`Path \"${path}\" not found in data`)\n }\n }\n\n return current\n }\n\n /**\n * Transform raw data item to Label Studio task format\n */\n private transformToLabelStudioTask(\n item: any,\n fieldMapping: Record<string, string>,\n imageField: string\n ): any {\n // Apply field mapping\n const mappedData: any = {}\n\n if (Object.keys(fieldMapping).length > 0) {\n // Use explicit mapping\n for (const [sourceField, targetField] of Object.entries(fieldMapping)) {\n if (item[sourceField] !== undefined) {\n mappedData[targetField] = item[sourceField]\n }\n }\n } else {\n // No mapping, use data as-is\n Object.assign(mappedData, item)\n }\n\n // Ensure image field exists\n const imageUrl = mappedData[imageField] || item[imageField] || item.url || item.image_url\n\n if (!imageUrl) {\n console.warn('[External Data] No image URL found in item:', item)\n }\n\n return {\n data: {\n image: imageUrl,\n ...mappedData\n },\n meta: {\n source: 'external-import',\n originalData: JSON.stringify(item)\n }\n }\n }\n\n /**\n * Detect schema from data item\n */\n private detectSchema(item: any): Record<string, string> {\n const schema: Record<string, string> = {}\n\n for (const [key, value] of Object.entries(item)) {\n schema[key] = typeof value\n }\n\n return schema\n }\n}\n"]}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { UserSyncMutation } from './user-provisioning/user-sync-mutation.js';
|
|
2
|
+
import { ProjectManagement } from './project/project-management.js';
|
|
3
|
+
import { TaskManagement } from './task/task-management.js';
|
|
4
|
+
import { WebhookManagement } from './webhook/webhook-management.js';
|
|
5
|
+
import { MLBackendService } from './ml/ml-backend-service.js';
|
|
6
|
+
import { PredictionManagement } from './prediction/prediction-management.js';
|
|
7
|
+
import { LabelStudioAIPredictionService } from './ai-prediction-service.js';
|
|
8
|
+
import { DatasetLabelingIntegration } from './dataset-labeling-integration.js';
|
|
9
|
+
import { ExternalDataSourceService } from './external-data-source-service.js';
|
|
10
|
+
export declare const schema: {
|
|
11
|
+
resolverClasses: (typeof UserSyncMutation | typeof ProjectManagement | typeof TaskManagement | typeof WebhookManagement | typeof MLBackendService | typeof PredictionManagement | typeof LabelStudioAIPredictionService | typeof DatasetLabelingIntegration | typeof ExternalDataSourceService)[];
|
|
12
|
+
};
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.schema = void 0;
|
|
4
|
+
const user_sync_mutation_js_1 = require("./user-provisioning/user-sync-mutation.js");
|
|
5
|
+
const project_management_js_1 = require("./project/project-management.js");
|
|
6
|
+
const task_management_js_1 = require("./task/task-management.js");
|
|
7
|
+
const webhook_management_js_1 = require("./webhook/webhook-management.js");
|
|
8
|
+
const ml_backend_service_js_1 = require("./ml/ml-backend-service.js");
|
|
9
|
+
const prediction_management_js_1 = require("./prediction/prediction-management.js");
|
|
10
|
+
const ai_prediction_service_js_1 = require("./ai-prediction-service.js");
|
|
11
|
+
const dataset_labeling_integration_js_1 = require("./dataset-labeling-integration.js");
|
|
12
|
+
const external_data_source_service_js_1 = require("./external-data-source-service.js");
|
|
13
|
+
exports.schema = {
|
|
14
|
+
resolverClasses: [
|
|
15
|
+
/* RESOLVER CLASSES */
|
|
16
|
+
user_sync_mutation_js_1.UserSyncMutation,
|
|
17
|
+
project_management_js_1.ProjectManagement,
|
|
18
|
+
task_management_js_1.TaskManagement,
|
|
19
|
+
webhook_management_js_1.WebhookManagement,
|
|
20
|
+
ml_backend_service_js_1.MLBackendService,
|
|
21
|
+
prediction_management_js_1.PredictionManagement,
|
|
22
|
+
ai_prediction_service_js_1.LabelStudioAIPredictionService,
|
|
23
|
+
dataset_labeling_integration_js_1.DatasetLabelingIntegration,
|
|
24
|
+
external_data_source_service_js_1.ExternalDataSourceService
|
|
25
|
+
]
|
|
26
|
+
};
|
|
27
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../server/service/index.ts"],"names":[],"mappings":";;;AAAA,qFAA4E;AAC5E,2EAAmE;AACnE,kEAA0D;AAC1D,2EAAmE;AACnE,sEAA6D;AAC7D,oFAA4E;AAC5E,yEAA2E;AAC3E,uFAA8E;AAC9E,uFAA6E;AAEhE,QAAA,MAAM,GAAG;IACpB,eAAe,EAAE;QACf,sBAAsB;QACtB,wCAAgB;QAChB,yCAAiB;QACjB,mCAAc;QACd,yCAAiB;QACjB,wCAAgB;QAChB,+CAAoB;QACpB,yDAA8B;QAC9B,4DAA0B;QAC1B,2DAAyB;KAC1B;CACF,CAAA","sourcesContent":["import { UserSyncMutation } from './user-provisioning/user-sync-mutation.js'\nimport { ProjectManagement } from './project/project-management.js'\nimport { TaskManagement } from './task/task-management.js'\nimport { WebhookManagement } from './webhook/webhook-management.js'\nimport { MLBackendService } from './ml/ml-backend-service.js'\nimport { PredictionManagement } from './prediction/prediction-management.js'\nimport { LabelStudioAIPredictionService } from './ai-prediction-service.js'\nimport { DatasetLabelingIntegration } from './dataset-labeling-integration.js'\nimport { ExternalDataSourceService } from './external-data-source-service.js'\n\nexport const schema = {\n resolverClasses: [\n /* RESOLVER CLASSES */\n UserSyncMutation,\n ProjectManagement,\n TaskManagement,\n WebhookManagement,\n MLBackendService,\n PredictionManagement,\n LabelStudioAIPredictionService,\n DatasetLabelingIntegration,\n ExternalDataSourceService\n ]\n}\n"]}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Label Studio SSO Service
|
|
3
|
+
*
|
|
4
|
+
* Handles JWT token acquisition from Label Studio for SSO authentication.
|
|
5
|
+
* The client application requests JWT tokens from Label Studio's token API,
|
|
6
|
+
* then uses cookie-based authentication to automatically log users in.
|
|
7
|
+
*/
|
|
8
|
+
export declare class LabelStudioSSOService {
|
|
9
|
+
/**
|
|
10
|
+
* Get Label Studio SSO configuration
|
|
11
|
+
*/
|
|
12
|
+
private static getConfig;
|
|
13
|
+
/**
|
|
14
|
+
* Request JWT token from Label Studio for SSO authentication
|
|
15
|
+
*
|
|
16
|
+
* @param email User email address
|
|
17
|
+
* @returns JWT token and expiry time, or null if failed
|
|
18
|
+
*
|
|
19
|
+
* @example
|
|
20
|
+
* ```typescript
|
|
21
|
+
* const tokenData = await LabelStudioSSOService.getSSOToken('user@example.com')
|
|
22
|
+
* if (tokenData) {
|
|
23
|
+
* console.log('Token:', tokenData.token)
|
|
24
|
+
* console.log('Expires in:', tokenData.expires_in, 'seconds')
|
|
25
|
+
* }
|
|
26
|
+
* ```
|
|
27
|
+
*/
|
|
28
|
+
static getSSOToken(email: string): Promise<{
|
|
29
|
+
token: string;
|
|
30
|
+
expires_in: number;
|
|
31
|
+
} | null>;
|
|
32
|
+
/**
|
|
33
|
+
* Verify SSO configuration is properly set up
|
|
34
|
+
*
|
|
35
|
+
* @returns True if configuration is valid
|
|
36
|
+
*/
|
|
37
|
+
static verifyConfig(): boolean;
|
|
38
|
+
}
|