@fluentcommerce/fc-connect-sdk 0.1.48 → 0.1.52
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +506 -379
- package/README.md +343 -0
- package/dist/cjs/clients/fluent-client.js +110 -14
- package/dist/cjs/data-sources/s3-data-source.js +1 -1
- package/dist/cjs/data-sources/sftp-data-source.js +1 -1
- package/dist/cjs/index.d.ts +1 -1
- package/dist/cjs/services/extraction/extraction-orchestrator.d.ts +4 -1
- package/dist/cjs/services/extraction/extraction-orchestrator.js +84 -11
- package/dist/cjs/types/index.d.ts +79 -10
- package/dist/cjs/versori/fluent-versori-client.d.ts +4 -1
- package/dist/cjs/versori/fluent-versori-client.js +131 -13
- package/dist/esm/clients/fluent-client.js +110 -14
- package/dist/esm/data-sources/s3-data-source.js +1 -1
- package/dist/esm/data-sources/sftp-data-source.js +1 -1
- package/dist/esm/index.d.ts +1 -1
- package/dist/esm/services/extraction/extraction-orchestrator.d.ts +4 -1
- package/dist/esm/services/extraction/extraction-orchestrator.js +84 -11
- package/dist/esm/types/index.d.ts +79 -10
- package/dist/esm/versori/fluent-versori-client.d.ts +4 -1
- package/dist/esm/versori/fluent-versori-client.js +131 -13
- package/dist/tsconfig.esm.tsbuildinfo +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/dist/tsconfig.types.tsbuildinfo +1 -1
- package/dist/types/index.d.ts +1 -1
- package/dist/types/services/extraction/extraction-orchestrator.d.ts +4 -1
- package/dist/types/types/index.d.ts +79 -10
- package/dist/types/versori/fluent-versori-client.d.ts +4 -1
- package/docs/02-CORE-GUIDES/api-reference/event-api-input-output-reference.md +478 -18
- package/docs/02-CORE-GUIDES/api-reference/modules/api-reference-01-client-api.md +83 -0
- package/docs/02-CORE-GUIDES/api-reference/modules/api-reference-08-types.md +52 -0
- package/docs/02-CORE-GUIDES/api-reference/modules/api-reference-12-partial-responses.md +212 -0
- package/docs/02-CORE-GUIDES/api-reference/readme.md +1 -1
- package/docs/02-CORE-GUIDES/extraction/modules/02-core-guides-extraction-08-extraction-orchestrator.md +68 -4
- package/docs/02-CORE-GUIDES/mapping/modules/mapping-01-foundations.md +450 -448
- package/docs/02-CORE-GUIDES/mapping/modules/mapping-02-quick-start.md +476 -474
- package/docs/02-CORE-GUIDES/mapping/modules/mapping-03-schema-validation.md +464 -462
- package/docs/02-CORE-GUIDES/mapping/modules/mapping-05-advanced-patterns.md +1366 -1364
- package/docs/readme.md +245 -245
- package/package.json +17 -6
- package/docs/versori-apis/ACTIVATIONS-AND-VARIABLES-GUIDE.md +0 -60
- package/docs/versori-apis/JWT-GENERATION-GUIDE.md +0 -94
- package/docs/versori-apis/QUICK-WORKFLOW.md +0 -293
- package/docs/versori-apis/README.md +0 -73
- package/docs/versori-apis/VERSORI-PLATFORM-ARCHITECTURE.md +0 -880
- package/docs/versori-apis/Versori-Platform-API.postman_collection.json +0 -2925
- package/docs/versori-apis/Versori-Platform-API.postman_environment.example.json +0 -62
- package/docs/versori-apis/Versori-Platform-API.postman_environment.json +0 -178
|
@@ -75,8 +75,9 @@ export class FluentClient {
|
|
|
75
75
|
paginationEnabled,
|
|
76
76
|
direction: paginationVars.direction,
|
|
77
77
|
});
|
|
78
|
+
const errorHandling = payload.errorHandling ?? 'throw';
|
|
78
79
|
if (!paginationEnabled) {
|
|
79
|
-
return this.executeSinglePage(payload);
|
|
80
|
+
return this.executeSinglePage(payload, errorHandling);
|
|
80
81
|
}
|
|
81
82
|
let direction = payload.pagination?.direction ?? paginationVars.direction;
|
|
82
83
|
if (paginationVars.direction === 'none' && !payload.pagination?.direction) {
|
|
@@ -96,16 +97,18 @@ export class FluentClient {
|
|
|
96
97
|
abortSignal: payload.pagination?.abortSignal,
|
|
97
98
|
onProgress: payload.pagination?.onProgress ?? (() => { }),
|
|
98
99
|
onWarning: payload.pagination?.onWarning ?? ((msg) => this.log('warn', msg)),
|
|
100
|
+
errorHandling,
|
|
99
101
|
};
|
|
100
102
|
this.log('info', '[FluentClient:graphql] Auto-pagination enabled', {
|
|
101
103
|
maxPages: config.maxPages,
|
|
102
104
|
maxRecords: config.maxRecords,
|
|
103
105
|
timeoutMs: config.timeoutMs,
|
|
104
106
|
direction: config.direction,
|
|
107
|
+
errorHandling: config.errorHandling,
|
|
105
108
|
});
|
|
106
109
|
return this.executeWithAutoPagination(payload, paginationVars, config);
|
|
107
110
|
}
|
|
108
|
-
async executeSinglePage(payload) {
|
|
111
|
+
async executeSinglePage(payload, errorHandling = 'throw') {
|
|
109
112
|
const operationName = payload.operationName || 'unnamed';
|
|
110
113
|
try {
|
|
111
114
|
const apiPayload = {
|
|
@@ -122,9 +125,21 @@ export class FluentClient {
|
|
|
122
125
|
body: apiPayload,
|
|
123
126
|
});
|
|
124
127
|
if (response.data.errors?.length) {
|
|
128
|
+
if (errorHandling === 'partial') {
|
|
129
|
+
this.log('warn', `[FluentClient:graphql] GraphQL operation "${operationName}" returned partial data with ${response.data.errors.length} error(s)`, {
|
|
130
|
+
operationName,
|
|
131
|
+
errorMessages: response.data.errors.map(e => e.message),
|
|
132
|
+
errorCount: response.data.errors.length,
|
|
133
|
+
hasData: !!response.data.data,
|
|
134
|
+
});
|
|
135
|
+
return {
|
|
136
|
+
...response.data,
|
|
137
|
+
hasPartialData: true,
|
|
138
|
+
};
|
|
139
|
+
}
|
|
125
140
|
this.log('error', `[FluentClient:graphql] GraphQL operation "${operationName}" returned ${response.data.errors.length} error(s)`, {
|
|
126
141
|
operationName,
|
|
127
|
-
|
|
142
|
+
errorMessages: response.data.errors.map(e => e.message),
|
|
128
143
|
errorCount: response.data.errors.length,
|
|
129
144
|
});
|
|
130
145
|
const firstError = response.data.errors[0];
|
|
@@ -149,6 +164,7 @@ export class FluentClient {
|
|
|
149
164
|
let lastCursor = null;
|
|
150
165
|
let emptyPageCount = 0;
|
|
151
166
|
const MAX_EMPTY_PAGES = FluentClient.DEFAULT_PAGINATION_CONFIG.maxEmptyPages;
|
|
167
|
+
const accumulatedErrors = [];
|
|
152
168
|
while (true) {
|
|
153
169
|
if (config.abortSignal?.aborted) {
|
|
154
170
|
truncated = true;
|
|
@@ -169,17 +185,29 @@ export class FluentClient {
|
|
|
169
185
|
const response = await this.executeSinglePage({
|
|
170
186
|
query: payload.query,
|
|
171
187
|
variables: currentVariables,
|
|
172
|
-
});
|
|
188
|
+
}, 'partial');
|
|
173
189
|
lastExtensions = response.extensions;
|
|
174
190
|
if (response.errors && response.errors.length > 0) {
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
191
|
+
if (config.errorHandling === 'partial') {
|
|
192
|
+
this.log('warn', `[FluentClient:pagination] Page ${pageNumber + 1} returned partial data with ${response.errors.length} error(s)`, {
|
|
193
|
+
errorMessages: response.errors.map(e => e.message),
|
|
194
|
+
pagesCompleted: pageNumber,
|
|
195
|
+
totalRecordsSoFar: totalRecords,
|
|
196
|
+
errorCount: response.errors.length,
|
|
197
|
+
hasData: !!response.data,
|
|
198
|
+
});
|
|
199
|
+
accumulatedErrors.push(...response.errors);
|
|
200
|
+
}
|
|
201
|
+
else {
|
|
202
|
+
this.log('error', `[FluentClient:pagination] GraphQL errors during pagination (page ${pageNumber + 1}, ${totalRecords} records fetched so far)`, {
|
|
203
|
+
errorMessages: response.errors.map(e => e.message),
|
|
204
|
+
pagesCompleted: pageNumber,
|
|
205
|
+
totalRecordsBeforeError: totalRecords,
|
|
206
|
+
errorCount: response.errors.length,
|
|
207
|
+
});
|
|
208
|
+
const firstError = response.errors[0];
|
|
209
|
+
throw new GraphQLExecutionError(`GraphQL error during pagination (page ${pageNumber}): ${firstError.message || 'Unknown error'}`, response.errors, payload.query, currentVariables);
|
|
210
|
+
}
|
|
183
211
|
}
|
|
184
212
|
pageNumber++;
|
|
185
213
|
const connection = extractConnection(response.data, config.connectionPath);
|
|
@@ -257,14 +285,17 @@ export class FluentClient {
|
|
|
257
285
|
await new Promise(resolve => setTimeout(resolve, config.delayMs));
|
|
258
286
|
}
|
|
259
287
|
}
|
|
260
|
-
|
|
288
|
+
const hasPartialErrors = accumulatedErrors.length > 0;
|
|
289
|
+
this.log('info', `[FluentClient:pagination] Pagination complete (${pageNumber} pages, ${totalRecords} records${truncated ? `, truncated: ${truncationReason}` : ''}${hasPartialErrors ? `, ${accumulatedErrors.length} errors` : ''})`, {
|
|
261
290
|
totalPages: pageNumber,
|
|
262
291
|
totalRecords,
|
|
263
292
|
truncated,
|
|
264
293
|
truncationReason,
|
|
265
294
|
duration: Date.now() - startTime,
|
|
295
|
+
hasPartialErrors,
|
|
296
|
+
errorCount: accumulatedErrors.length,
|
|
266
297
|
});
|
|
267
|
-
|
|
298
|
+
const response = {
|
|
268
299
|
data: allData,
|
|
269
300
|
extensions: {
|
|
270
301
|
...(lastExtensions || {}),
|
|
@@ -277,6 +308,11 @@ export class FluentClient {
|
|
|
277
308
|
},
|
|
278
309
|
},
|
|
279
310
|
};
|
|
311
|
+
if (hasPartialErrors) {
|
|
312
|
+
response.errors = accumulatedErrors;
|
|
313
|
+
response.hasPartialData = true;
|
|
314
|
+
}
|
|
315
|
+
return response;
|
|
280
316
|
}
|
|
281
317
|
mergeConnectionData(allData, newData, connection, direction) {
|
|
282
318
|
const allConnection = extractConnection(allData);
|
|
@@ -474,6 +510,56 @@ export class FluentClient {
|
|
|
474
510
|
throw error;
|
|
475
511
|
}
|
|
476
512
|
}
|
|
513
|
+
async getEvents(params = {}) {
|
|
514
|
+
const queryString = this.buildEventQueryString(params);
|
|
515
|
+
const endpoint = queryString ? `/api/v4.1/event?${queryString}` : '/api/v4.1/event';
|
|
516
|
+
this.log('info', '[FluentClient:event] Searching event logs', {
|
|
517
|
+
endpoint,
|
|
518
|
+
filterCount: Object.keys(params).length,
|
|
519
|
+
hasDateRange: !!(params.from || params.to),
|
|
520
|
+
entityType: params['context.entityType'],
|
|
521
|
+
rootEntityType: params['context.rootEntityType'],
|
|
522
|
+
eventStatus: params.eventStatus,
|
|
523
|
+
});
|
|
524
|
+
try {
|
|
525
|
+
const response = await this.request(endpoint, {
|
|
526
|
+
method: 'GET',
|
|
527
|
+
});
|
|
528
|
+
this.log('info', '[FluentClient:event] Event search completed', {
|
|
529
|
+
resultCount: response.data.count,
|
|
530
|
+
hasMore: response.data.hasMore,
|
|
531
|
+
start: response.data.start,
|
|
532
|
+
});
|
|
533
|
+
return response.data;
|
|
534
|
+
}
|
|
535
|
+
catch (error) {
|
|
536
|
+
this.log('error', '[FluentClient:event] Failed to search events', error);
|
|
537
|
+
throw error;
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
async getEventById(eventId) {
|
|
541
|
+
if (!eventId) {
|
|
542
|
+
throw new FluentValidationError('eventId is required');
|
|
543
|
+
}
|
|
544
|
+
this.log('info', `[FluentClient:event] Getting event by ID: ${eventId}`, { eventId });
|
|
545
|
+
try {
|
|
546
|
+
const response = await this.request(`/api/v4.1/event/${eventId}`, {
|
|
547
|
+
method: 'GET',
|
|
548
|
+
});
|
|
549
|
+
this.log('info', `[FluentClient:event] Event retrieved: ${response.data.name}`, {
|
|
550
|
+
eventId: response.data.id,
|
|
551
|
+
name: response.data.name,
|
|
552
|
+
type: response.data.type,
|
|
553
|
+
eventStatus: response.data.eventStatus,
|
|
554
|
+
entityType: response.data.context?.entityType,
|
|
555
|
+
});
|
|
556
|
+
return response.data;
|
|
557
|
+
}
|
|
558
|
+
catch (error) {
|
|
559
|
+
this.log('error', `[FluentClient:event] Failed to get event ${eventId}`, error);
|
|
560
|
+
throw error;
|
|
561
|
+
}
|
|
562
|
+
}
|
|
477
563
|
async validateWebhook(payload, signature, rawPayload) {
|
|
478
564
|
this.log('info', `[FluentClient:webhook] Validating webhook for event "${payload.name || 'unknown'}"`, {
|
|
479
565
|
hasSignature: !!signature,
|
|
@@ -553,6 +639,16 @@ export class FluentClient {
|
|
|
553
639
|
const result = await this.graphql(payload);
|
|
554
640
|
return result.data;
|
|
555
641
|
}
|
|
642
|
+
buildEventQueryString(params) {
|
|
643
|
+
const searchParams = new URLSearchParams();
|
|
644
|
+
for (const [key, value] of Object.entries(params)) {
|
|
645
|
+
if (value === undefined || value === null || value === '') {
|
|
646
|
+
continue;
|
|
647
|
+
}
|
|
648
|
+
searchParams.append(key, String(value));
|
|
649
|
+
}
|
|
650
|
+
return searchParams.toString();
|
|
651
|
+
}
|
|
556
652
|
async request(endpoint, config = {}) {
|
|
557
653
|
const isVersoriContext = this.context?.fetch && this.context?.fetch !== globalThis.fetch;
|
|
558
654
|
this.log('debug', 'Request mode detection', {
|
|
@@ -342,7 +342,7 @@ export class S3DataSource extends AbstractDataSource {
|
|
|
342
342
|
return Buffer.from('PAR1\x00\x00\x00\x00PAR1', 'ascii');
|
|
343
343
|
}
|
|
344
344
|
try {
|
|
345
|
-
const parquet = await import('parquetjs');
|
|
345
|
+
const parquet = await import('@dsnp/parquetjs');
|
|
346
346
|
const ParquetWriter = parquet.ParquetWriter;
|
|
347
347
|
const ParquetSchema = parquet.ParquetSchema;
|
|
348
348
|
const fs = await import('fs');
|
|
@@ -322,7 +322,7 @@ export class SftpDataSource extends AbstractDataSource {
|
|
|
322
322
|
return Buffer.from('PAR1\x00\x00\x00\x00PAR1', 'ascii');
|
|
323
323
|
}
|
|
324
324
|
try {
|
|
325
|
-
const parquet = await import('parquetjs');
|
|
325
|
+
const parquet = await import('@dsnp/parquetjs');
|
|
326
326
|
const ParquetWriter = parquet.ParquetWriter;
|
|
327
327
|
const ParquetSchema = parquet.ParquetSchema;
|
|
328
328
|
const fs = await import('fs');
|
package/dist/esm/index.d.ts
CHANGED
|
@@ -34,7 +34,7 @@ export { GraphQLTemplateGenerator } from './services/mapping/query/graphql-templ
|
|
|
34
34
|
export { MappingError, ResolverError } from './errors/index.js';
|
|
35
35
|
export type { NodeConfig, NodesConfig, FieldConfig, FieldsConfig, NodesContext, MappingContext, ResolverFunction, ResolverHelpers, ResolverContext, ResolversMap, MapResult, MapWithNodesResult, NodeValidationResult, MappingOptions, } from './services/mapping/types/index.js';
|
|
36
36
|
export { CsvDelimiter, FileEncoding, JobStrategy, FileType, BatchAction, EntityType, AwsRegion, ValidationMode, ProcessingStatus, } from './types/enums.js';
|
|
37
|
-
export type { FluentClientConfig, FluentBatchPayload, FluentInventoryBatchEntity, FluentInventoryBatchRequest, FluentBatchResponse, FluentBatchStatus, FluentJobPayload, FluentJobResponse, FluentJobMetadata, FluentJobStatus, FluentJobResults, FluentEvent, FluentEventMode, CreateEventOptions, GraphQLPayload, GraphQLResponse, PaginationConfig, PageInfo, PaginatedResponse, FluentWebhookPayload, WebhookRequestContext, GraphQLValidationResult, BatchConfiguration, DataSourceConfig, FileMetadata, Logger, StructuredLogger, LogContext, LoggerConfig, PerformanceMetrics, InventoryDataRecord, ParquetDataRecord, FieldMappingConfig, FieldMappingRule, ProcessingMetadata, AttributeValue, } from './types/index.js';
|
|
37
|
+
export type { FluentClientConfig, FluentBatchPayload, FluentInventoryBatchEntity, FluentInventoryBatchRequest, FluentBatchResponse, FluentBatchStatus, FluentJobPayload, FluentJobResponse, FluentJobMetadata, FluentJobStatus, FluentJobResults, FluentEvent, FluentEventMode, CreateEventOptions, FluentEventRootEntityType, FluentEventEntityType, FluentEventCategory, FluentEventType, FluentEventStatus, FluentEventQueryParamValue, FluentEventQueryParams, FluentEventLogAttribute, FluentEventLogContext, FluentEventLogItem, FluentEventLogResponse, GraphQLPayload, GraphQLResponse, GraphQLErrorMode, GraphQLError, PaginationConfig, PageInfo, PaginatedResponse, FluentWebhookPayload, WebhookRequestContext, GraphQLValidationResult, BatchConfiguration, DataSourceConfig, FileMetadata, Logger, StructuredLogger, LogContext, LoggerConfig, PerformanceMetrics, InventoryDataRecord, ParquetDataRecord, FieldMappingConfig, FieldMappingRule, ProcessingMetadata, AttributeValue, } from './types/index.js';
|
|
38
38
|
export type { VersoriContext, WebhookContext, DirectContext } from './types/index.js';
|
|
39
39
|
export { S3SDKTester, S3PresignedTester, S3ComparisonTester, FluentConnectionTester, type TestResult, type FluentTestConfig, type S3TestConfig, } from './testing/index.js';
|
|
40
40
|
export { S3Service, S3ServiceError, type S3ServiceConfig, type S3Config, type S3Object, type ListOptions, type PutOptions, type CopyOptions, type S3Location, } from './services/s3/index.js';
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { FluentClient } from '../../clients/fluent-client.js';
|
|
2
|
-
import type { Logger } from '../../types/index.js';
|
|
2
|
+
import type { Logger, GraphQLErrorMode, GraphQLError } from '../../types/index.js';
|
|
3
3
|
export interface ExtractionOptions<T = any> {
|
|
4
4
|
query: string;
|
|
5
5
|
resultPath: string;
|
|
@@ -10,6 +10,8 @@ export interface ExtractionOptions<T = any> {
|
|
|
10
10
|
direction?: 'forward' | 'backward';
|
|
11
11
|
timeout?: number;
|
|
12
12
|
validateItem?: (item: T) => boolean;
|
|
13
|
+
operationName?: string;
|
|
14
|
+
errorHandling?: GraphQLErrorMode;
|
|
13
15
|
}
|
|
14
16
|
export interface ExtractionStats {
|
|
15
17
|
totalRecords: number;
|
|
@@ -20,6 +22,7 @@ export interface ExtractionStats {
|
|
|
20
22
|
validRecords?: number;
|
|
21
23
|
invalidRecords?: number;
|
|
22
24
|
direction?: 'forward' | 'backward';
|
|
25
|
+
partialErrors?: GraphQLError[];
|
|
23
26
|
}
|
|
24
27
|
export interface ExtractionError {
|
|
25
28
|
type: 'graphql' | 'network' | 'validation' | 'parsing';
|
|
@@ -17,13 +17,17 @@ export class ExtractionOrchestrator {
|
|
|
17
17
|
const pageSize = options.pageSize || this.DEFAULT_PAGE_SIZE;
|
|
18
18
|
const timeout = options.timeout || this.DEFAULT_TIMEOUT;
|
|
19
19
|
const direction = options.direction || 'forward';
|
|
20
|
+
const errorHandling = options.errorHandling || 'throw';
|
|
21
|
+
const accumulatedGraphQLErrors = [];
|
|
20
22
|
this.logger.info('Starting data extraction', {
|
|
23
|
+
operationName: options.operationName || 'unnamed',
|
|
21
24
|
resultPath: options.resultPath,
|
|
22
25
|
pageSize,
|
|
23
26
|
maxRecords: options.maxRecords,
|
|
24
27
|
maxPages: options.maxPages,
|
|
25
28
|
timeout,
|
|
26
29
|
direction,
|
|
30
|
+
errorHandling,
|
|
27
31
|
});
|
|
28
32
|
try {
|
|
29
33
|
const variables = {
|
|
@@ -33,16 +37,39 @@ export class ExtractionOrchestrator {
|
|
|
33
37
|
const response = await this.executeWithTimeout(this.client.graphql({
|
|
34
38
|
query: options.query,
|
|
35
39
|
variables,
|
|
40
|
+
...(options.operationName && { operationName: options.operationName }),
|
|
41
|
+
errorHandling,
|
|
36
42
|
}), timeout);
|
|
37
43
|
if (response.errors && response.errors.length > 0) {
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
44
|
+
if (errorHandling === 'partial') {
|
|
45
|
+
this.logger.warn('GraphQL query returned partial data with errors (page 1)', {
|
|
46
|
+
errors: response.errors,
|
|
47
|
+
errorCount: response.errors.length,
|
|
48
|
+
hasData: !!response.data,
|
|
49
|
+
});
|
|
50
|
+
accumulatedGraphQLErrors.push(...response.errors);
|
|
51
|
+
}
|
|
52
|
+
else {
|
|
53
|
+
const graphqlError = new Error(`GraphQL errors: ${response.errors.map((e) => e.message).join(', ')}`);
|
|
54
|
+
this.logger.error('GraphQL query returned errors', graphqlError, {
|
|
55
|
+
errors: response.errors,
|
|
56
|
+
errorCount: response.errors.length,
|
|
57
|
+
});
|
|
58
|
+
throw graphqlError;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
let records = [];
|
|
62
|
+
if (response.data) {
|
|
63
|
+
records = this.extractFromPath(response.data, options.resultPath);
|
|
64
|
+
}
|
|
65
|
+
else if (errorHandling === 'partial' && response.errors && response.errors.length > 0) {
|
|
66
|
+
this.logger.warn('GraphQL response has errors but no data (page 1), returning empty records', {
|
|
41
67
|
errorCount: response.errors.length,
|
|
42
68
|
});
|
|
43
|
-
throw graphqlError;
|
|
44
69
|
}
|
|
45
|
-
|
|
70
|
+
else if (!response.data) {
|
|
71
|
+
records = this.extractFromPath(response.data, options.resultPath);
|
|
72
|
+
}
|
|
46
73
|
if (options.maxRecords && records.length > options.maxRecords) {
|
|
47
74
|
records = records.slice(0, options.maxRecords);
|
|
48
75
|
}
|
|
@@ -56,14 +83,21 @@ export class ExtractionOrchestrator {
|
|
|
56
83
|
const rootPath = edgesIdx >= 0 ? pathParts.slice(0, edgesIdx).join('.') : pathParts.slice(0, -1).join('.');
|
|
57
84
|
const pageInfoPath = rootPath ? `${rootPath}.pageInfo` : 'pageInfo';
|
|
58
85
|
const edgesPath = edgesIdx >= 0 ? `${rootPath}.edges` : options.resultPath;
|
|
59
|
-
const firstPageInfo = this.safeGet(response.data, pageInfoPath) || {};
|
|
86
|
+
const firstPageInfo = response.data ? this.safeGet(response.data, pageInfoPath) || {} : {};
|
|
60
87
|
let hasMore = direction === 'forward'
|
|
61
88
|
? Boolean(firstPageInfo?.hasNextPage)
|
|
62
89
|
: Boolean(firstPageInfo?.hasPreviousPage);
|
|
63
|
-
if (direction === 'backward' &&
|
|
90
|
+
if (direction === 'backward' &&
|
|
91
|
+
response.data &&
|
|
92
|
+
firstPageInfo?.hasPreviousPage === undefined) {
|
|
64
93
|
throw new Error('Backward pagination requested but pageInfo.hasPreviousPage is missing from query response');
|
|
65
94
|
}
|
|
66
|
-
|
|
95
|
+
if (!response.data && errorHandling === 'partial' && accumulatedGraphQLErrors.length > 0) {
|
|
96
|
+
hasMore = false;
|
|
97
|
+
}
|
|
98
|
+
let cursor = response.data
|
|
99
|
+
? this.pickCursor(response.data, edgesPath, direction)
|
|
100
|
+
: undefined;
|
|
67
101
|
while (hasMore) {
|
|
68
102
|
if (options.maxPages && totalPages >= options.maxPages) {
|
|
69
103
|
truncated = true;
|
|
@@ -87,11 +121,40 @@ export class ExtractionOrchestrator {
|
|
|
87
121
|
const nextPage = await this.executeWithTimeout(this.client.graphql({
|
|
88
122
|
query: options.query,
|
|
89
123
|
variables: nextVariables,
|
|
124
|
+
...(options.operationName && { operationName: options.operationName }),
|
|
125
|
+
errorHandling,
|
|
90
126
|
}), timeout);
|
|
91
|
-
if (
|
|
92
|
-
|
|
127
|
+
if (nextPage?.errors && nextPage.errors.length > 0) {
|
|
128
|
+
if (errorHandling === 'partial') {
|
|
129
|
+
this.logger.warn(`GraphQL query returned partial data with errors (page ${totalPages + 1})`, {
|
|
130
|
+
errors: nextPage.errors,
|
|
131
|
+
errorCount: nextPage.errors.length,
|
|
132
|
+
hasData: !!nextPage.data,
|
|
133
|
+
});
|
|
134
|
+
accumulatedGraphQLErrors.push(...nextPage.errors);
|
|
135
|
+
}
|
|
136
|
+
else {
|
|
137
|
+
throw new Error(`GraphQL extraction failed on subsequent page: ${JSON.stringify(nextPage.errors)}`);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
if (!nextPage) {
|
|
141
|
+
throw new Error('GraphQL extraction failed on subsequent page: no response');
|
|
142
|
+
}
|
|
143
|
+
let pageRecords = [];
|
|
144
|
+
if (nextPage.data) {
|
|
145
|
+
pageRecords = this.extractFromPath(nextPage.data, options.resultPath);
|
|
146
|
+
}
|
|
147
|
+
else if (errorHandling === 'partial' && nextPage.errors && nextPage.errors.length > 0) {
|
|
148
|
+
this.logger.warn(`GraphQL response has errors but no data (page ${totalPages + 1}), stopping pagination`, {
|
|
149
|
+
errorCount: nextPage.errors.length,
|
|
150
|
+
});
|
|
151
|
+
hasMore = false;
|
|
152
|
+
cursor = undefined;
|
|
153
|
+
continue;
|
|
154
|
+
}
|
|
155
|
+
else if (!nextPage.data) {
|
|
156
|
+
pageRecords = this.extractFromPath(nextPage.data, options.resultPath);
|
|
93
157
|
}
|
|
94
|
-
let pageRecords = this.extractFromPath(nextPage.data, options.resultPath);
|
|
95
158
|
this.validateRecords(pageRecords, options.validateItem);
|
|
96
159
|
if (options.maxRecords && records.length + pageRecords.length > options.maxRecords) {
|
|
97
160
|
const remaining = options.maxRecords - records.length;
|
|
@@ -125,6 +188,7 @@ export class ExtractionOrchestrator {
|
|
|
125
188
|
validRecords: finalValidation.validRecords,
|
|
126
189
|
invalidRecords: finalValidation.invalidRecords,
|
|
127
190
|
direction,
|
|
191
|
+
...(accumulatedGraphQLErrors.length > 0 && { partialErrors: accumulatedGraphQLErrors }),
|
|
128
192
|
};
|
|
129
193
|
this.logger.info('Data extraction completed', {
|
|
130
194
|
totalRecords: stats.totalRecords,
|
|
@@ -133,7 +197,16 @@ export class ExtractionOrchestrator {
|
|
|
133
197
|
truncated: stats.truncated,
|
|
134
198
|
truncationReason: stats.truncationReason,
|
|
135
199
|
direction,
|
|
200
|
+
hasPartialErrors: accumulatedGraphQLErrors.length > 0,
|
|
201
|
+
partialErrorCount: accumulatedGraphQLErrors.length,
|
|
136
202
|
});
|
|
203
|
+
if (accumulatedGraphQLErrors.length > 0) {
|
|
204
|
+
this.logger.warn('Extraction completed with partial errors', {
|
|
205
|
+
errorCount: accumulatedGraphQLErrors.length,
|
|
206
|
+
recordsExtracted: stats.totalRecords,
|
|
207
|
+
message: 'Some GraphQL errors occurred but extraction continued with available data.',
|
|
208
|
+
});
|
|
209
|
+
}
|
|
137
210
|
if (stats.truncated) {
|
|
138
211
|
this.logger.warn('Extraction was truncated', {
|
|
139
212
|
reason: stats.truncationReason,
|
|
@@ -95,23 +95,91 @@ export interface FluentEvent {
|
|
|
95
95
|
export interface CreateEventOptions {
|
|
96
96
|
mode?: FluentEventMode;
|
|
97
97
|
}
|
|
98
|
+
export type FluentEventRootEntityType = 'ORDER' | 'LOCATION' | 'FULFILMENT_OPTIONS' | 'PRODUCT_CATALOGUE' | 'INVENTORY_CATALOGUE' | 'VIRTUAL_CATALOGUE' | 'CONTROL_GROUP' | 'RETURN_ORDER' | 'BILLING_ACCOUNT' | 'JOB';
|
|
99
|
+
export type FluentEventEntityType = 'ORDER' | 'FULFILMENT' | 'ARTICLE' | 'CONSIGNMENT' | 'LOCATION' | 'WAVE' | 'FULFILMENT_OPTIONS' | 'FULFILMENT_PLAN' | 'PRODUCT_CATALOGUE' | 'CATEGORY' | 'PRODUCT' | 'INVENTORY_CATALOGUE' | 'INVENTORY_POSITION' | 'INVENTORY_QUANTITY' | 'VIRTUAL_CATALOGUE' | 'VIRTUAL_POSITION' | 'CONTROL_GROUP' | 'CONTROL' | 'RETURN_ORDER' | 'RETURN_FULFILMENT' | 'BILLING_ACCOUNT' | 'CREDIT_MEMO' | 'BATCH';
|
|
100
|
+
export type FluentEventCategory = 'snapshot' | 'ruleSet' | 'rule' | 'ACTION' | 'CUSTOM' | 'exception' | 'ORDER_WORKFLOW';
|
|
101
|
+
export type FluentEventType = 'ORCHESTRATION' | 'ORCHESTRATION_AUDIT' | 'API' | 'INTEGRATION' | 'SECURITY' | 'GENERAL';
|
|
102
|
+
export type FluentEventStatus = 'PENDING' | 'SCHEDULED' | 'NO_MATCH' | 'SUCCESS' | 'FAILED' | 'COMPLETE';
|
|
103
|
+
export type FluentEventQueryParamValue = string | number | boolean;
|
|
104
|
+
export interface FluentEventQueryParams {
|
|
105
|
+
start?: number;
|
|
106
|
+
count?: number;
|
|
107
|
+
from?: string;
|
|
108
|
+
to?: string;
|
|
109
|
+
name?: string;
|
|
110
|
+
category?: FluentEventCategory | string;
|
|
111
|
+
retailerId?: string | number;
|
|
112
|
+
eventType?: FluentEventType | string;
|
|
113
|
+
eventStatus?: FluentEventStatus | string;
|
|
114
|
+
eventId?: string;
|
|
115
|
+
'context.rootEntityType'?: FluentEventRootEntityType | string;
|
|
116
|
+
'context.rootEntityId'?: string | number;
|
|
117
|
+
'context.rootEntityRef'?: string;
|
|
118
|
+
'context.entityType'?: FluentEventEntityType | string;
|
|
119
|
+
'context.entityId'?: string | number;
|
|
120
|
+
'context.entityRef'?: string;
|
|
121
|
+
'context.sourceEvents'?: string;
|
|
122
|
+
[key: string]: FluentEventQueryParamValue | undefined;
|
|
123
|
+
}
|
|
124
|
+
export interface FluentEventLogContext {
|
|
125
|
+
sourceEvents?: string[];
|
|
126
|
+
entityType?: FluentEventEntityType | string;
|
|
127
|
+
entityId?: string;
|
|
128
|
+
entityRef?: string;
|
|
129
|
+
rootEntityType?: FluentEventRootEntityType | string;
|
|
130
|
+
rootEntityId?: string;
|
|
131
|
+
rootEntityRef?: string;
|
|
132
|
+
[key: string]: JsonValue | undefined;
|
|
133
|
+
}
|
|
134
|
+
export interface FluentEventLogAttribute {
|
|
135
|
+
name: string;
|
|
136
|
+
value: string;
|
|
137
|
+
type?: string;
|
|
138
|
+
[key: string]: unknown;
|
|
139
|
+
}
|
|
140
|
+
export interface FluentEventLogItem {
|
|
141
|
+
id: string;
|
|
142
|
+
name: string;
|
|
143
|
+
type?: FluentEventType | string;
|
|
144
|
+
accountId?: string;
|
|
145
|
+
retailerId?: string;
|
|
146
|
+
category?: FluentEventCategory | string;
|
|
147
|
+
context?: FluentEventLogContext;
|
|
148
|
+
eventStatus?: FluentEventStatus | string;
|
|
149
|
+
attributes?: FluentEventLogAttribute[] | Record<string, JsonValue> | null;
|
|
150
|
+
source?: string | null;
|
|
151
|
+
generatedBy?: string;
|
|
152
|
+
generatedOn?: string;
|
|
153
|
+
[key: string]: unknown;
|
|
154
|
+
}
|
|
155
|
+
export interface FluentEventLogResponse {
|
|
156
|
+
start: number;
|
|
157
|
+
count: number;
|
|
158
|
+
hasMore: boolean;
|
|
159
|
+
results: FluentEventLogItem[];
|
|
160
|
+
[key: string]: unknown;
|
|
161
|
+
}
|
|
162
|
+
export type GraphQLErrorMode = 'throw' | 'partial';
|
|
98
163
|
export interface GraphQLPayload<T = Record<string, JsonValue>> {
|
|
99
164
|
query: string;
|
|
100
165
|
variables?: T;
|
|
101
166
|
operationName?: string;
|
|
102
167
|
pagination?: PaginationConfig;
|
|
168
|
+
errorHandling?: GraphQLErrorMode;
|
|
169
|
+
}
|
|
170
|
+
export interface GraphQLError {
|
|
171
|
+
message: string;
|
|
172
|
+
locations?: Array<{
|
|
173
|
+
line: number;
|
|
174
|
+
column: number;
|
|
175
|
+
}>;
|
|
176
|
+
path?: (string | number)[];
|
|
177
|
+
extensions?: Record<string, JsonValue>;
|
|
178
|
+
[key: string]: unknown;
|
|
103
179
|
}
|
|
104
180
|
export interface GraphQLResponse<T = JsonValue> {
|
|
105
181
|
data?: T;
|
|
106
|
-
errors?:
|
|
107
|
-
message: string;
|
|
108
|
-
locations?: Array<{
|
|
109
|
-
line: number;
|
|
110
|
-
column: number;
|
|
111
|
-
}>;
|
|
112
|
-
path?: (string | number)[];
|
|
113
|
-
extensions?: Record<string, JsonValue>;
|
|
114
|
-
}>;
|
|
182
|
+
errors?: GraphQLError[];
|
|
115
183
|
extensions?: Record<string, unknown> & {
|
|
116
184
|
autoPagination?: {
|
|
117
185
|
totalPages: number;
|
|
@@ -121,6 +189,7 @@ export interface GraphQLResponse<T = JsonValue> {
|
|
|
121
189
|
direction?: 'forward' | 'backward';
|
|
122
190
|
};
|
|
123
191
|
};
|
|
192
|
+
hasPartialData?: boolean;
|
|
124
193
|
}
|
|
125
194
|
export interface FluentInventoryBatchEntity {
|
|
126
195
|
locationRef: string;
|
|
@@ -359,7 +428,7 @@ export interface Logger {
|
|
|
359
428
|
debug(message: string, meta?: Record<string, unknown> | undefined): void;
|
|
360
429
|
info(message: string, meta?: Record<string, unknown> | undefined): void;
|
|
361
430
|
warn(message: string, meta?: Record<string, unknown> | undefined): void;
|
|
362
|
-
error(message: string,
|
|
431
|
+
error(message: string, errorOrMeta?: unknown, meta?: Record<string, unknown>): void;
|
|
363
432
|
}
|
|
364
433
|
export interface LogContext {
|
|
365
434
|
[key: string]: MetadataValue;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { FluentEvent, GraphQLPayload, GraphQLResponse, FluentEventMode, JsonValue, FluentBatchPayload, FluentBatchResponse, FluentBatchStatus, FluentJobPayload, FluentJobResponse, FluentJobStatus, FluentJobResults } from '../types/index.js';
|
|
1
|
+
import type { FluentEvent, GraphQLPayload, GraphQLResponse, FluentEventMode, FluentEventLogResponse, FluentEventLogItem, FluentEventQueryParams, JsonValue, FluentBatchPayload, FluentBatchResponse, FluentBatchStatus, FluentJobPayload, FluentJobResponse, FluentJobStatus, FluentJobResults } from '../types/index.js';
|
|
2
2
|
import type { FluentWebhookPayload, EventResponse } from '../types/index.js';
|
|
3
3
|
export declare class FluentVersoriClient {
|
|
4
4
|
private ctx;
|
|
@@ -13,6 +13,8 @@ export declare class FluentVersoriClient {
|
|
|
13
13
|
private executeSinglePage;
|
|
14
14
|
private executeWithAutoPagination;
|
|
15
15
|
sendEvent(event: FluentEvent, mode?: FluentEventMode): Promise<EventResponse>;
|
|
16
|
+
getEvents(params?: FluentEventQueryParams): Promise<FluentEventLogResponse>;
|
|
17
|
+
getEventById(eventId: string): Promise<FluentEventLogItem>;
|
|
16
18
|
createJob(payload: FluentJobPayload): Promise<FluentJobResponse>;
|
|
17
19
|
sendBatch(jobId: string, payload: FluentBatchPayload): Promise<FluentBatchResponse>;
|
|
18
20
|
getJobStatus(jobId: string): Promise<FluentJobStatus>;
|
|
@@ -20,6 +22,7 @@ export declare class FluentVersoriClient {
|
|
|
20
22
|
getJobResults(jobId: string): Promise<FluentJobResults>;
|
|
21
23
|
query<T = JsonValue>(queryOrPayload: string | GraphQLPayload, variables?: Record<string, any>): Promise<T | undefined>;
|
|
22
24
|
mutate<T = JsonValue>(mutationOrPayload: string | GraphQLPayload, variables?: Record<string, any>): Promise<T | undefined>;
|
|
25
|
+
private buildEventQueryString;
|
|
23
26
|
private throwApiError;
|
|
24
27
|
private fetchWithRetry;
|
|
25
28
|
validateWebhook(payload: FluentWebhookPayload, signature?: string, rawPayload?: string, publicKey?: string): Promise<boolean>;
|