@fluentcommerce/fc-connect-sdk 0.1.54 → 0.1.55
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/dist/cjs/clients/fluent-client.js +13 -6
- package/dist/cjs/utils/pagination-helpers.js +38 -2
- package/dist/cjs/versori/fluent-versori-client.js +11 -5
- package/dist/esm/clients/fluent-client.js +13 -6
- package/dist/esm/utils/pagination-helpers.js +38 -2
- package/dist/esm/versori/fluent-versori-client.js +11 -5
- package/dist/tsconfig.esm.tsbuildinfo +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/dist/tsconfig.types.tsbuildinfo +1 -1
- package/docs/00-START-HERE/EXPORT-VALIDATION.md +158 -158
- package/docs/00-START-HERE/cli-analyze-source-structure-guide.md +655 -655
- package/docs/00-START-HERE/cli-documentation-index.md +202 -202
- package/docs/00-START-HERE/cli-quick-reference.md +252 -252
- package/docs/00-START-HERE/decision-tree.md +552 -552
- package/docs/00-START-HERE/getting-started.md +1070 -1070
- package/docs/00-START-HERE/mapper-quick-decision-guide.md +235 -235
- package/docs/00-START-HERE/readme.md +237 -237
- package/docs/00-START-HERE/retailerid-configuration.md +404 -404
- package/docs/00-START-HERE/sdk-philosophy.md +794 -794
- package/docs/00-START-HERE/troubleshooting-quick-reference.md +1086 -1086
- package/docs/01-TEMPLATES/faq.md +686 -686
- package/docs/01-TEMPLATES/patterns/pattern-templates-guide.md +68 -68
- package/docs/01-TEMPLATES/patterns/patterns-csv-schema-validation-and-rejection-report.md +233 -233
- package/docs/01-TEMPLATES/patterns/patterns-custom-resolvers.md +407 -407
- package/docs/01-TEMPLATES/patterns/patterns-error-handling-retry.md +511 -511
- package/docs/01-TEMPLATES/patterns/patterns-field-mapping-universal.md +701 -701
- package/docs/01-TEMPLATES/patterns/patterns-large-file-splitting.md +1430 -1430
- package/docs/01-TEMPLATES/patterns/patterns-master-data-etl.md +2399 -2399
- package/docs/01-TEMPLATES/patterns/patterns-pagination-streaming.md +447 -447
- package/docs/01-TEMPLATES/patterns/patterns-state-duplicate-prevention.md +385 -385
- package/docs/01-TEMPLATES/readme.md +957 -957
- package/docs/01-TEMPLATES/standalone/standalone-asn-inbound-processing.md +1209 -1209
- package/docs/01-TEMPLATES/standalone/standalone-graphql-query-export.md +1140 -1140
- package/docs/01-TEMPLATES/standalone/standalone-graphql-to-parquet-partitioned-s3.md +432 -432
- package/docs/01-TEMPLATES/standalone/standalone-multi-channel-inventory-sync.md +1185 -1185
- package/docs/01-TEMPLATES/standalone/standalone-multi-source-aggregation.md +1462 -1462
- package/docs/01-TEMPLATES/standalone/standalone-s3-csv-batch-api.md +1390 -1390
- package/docs/01-TEMPLATES/standalone/standalone-s3-csv-inventory-to-batch.md +330 -330
- package/docs/01-TEMPLATES/standalone/standalone-scripts-guide.md +87 -87
- package/docs/01-TEMPLATES/standalone/standalone-sftp-xml-graphql.md +1444 -1444
- package/docs/01-TEMPLATES/standalone/standalone-webhook-payload-processing.md +688 -688
- package/docs/01-TEMPLATES/versori/business-examples/business-examples-dropship-order-routing.md +193 -193
- package/docs/01-TEMPLATES/versori/business-examples/business-examples-graphql-parquet-extraction.md +518 -518
- package/docs/01-TEMPLATES/versori/business-examples/business-examples-inter-location-transfers.md +2162 -2162
- package/docs/01-TEMPLATES/versori/business-examples/business-examples-pre-order-allocation.md +2226 -2226
- package/docs/01-TEMPLATES/versori/business-examples/business-scenarios-guide.md +87 -87
- package/docs/01-TEMPLATES/versori/patterns/versori-patterns-connection-validation-pattern.md +656 -656
- package/docs/01-TEMPLATES/versori/patterns/versori-patterns-dual-workflow-connector.md +835 -835
- package/docs/01-TEMPLATES/versori/patterns/versori-patterns-guide.md +108 -108
- package/docs/01-TEMPLATES/versori/patterns/versori-patterns-kv-state-management.md +1533 -1533
- package/docs/01-TEMPLATES/versori/patterns/versori-patterns-xml-response-patterns.md +1160 -1160
- package/docs/01-TEMPLATES/versori/versori-platform-guide.md +201 -201
- package/docs/01-TEMPLATES/versori/webhooks/template-webhook-asn-purchase-order.md +1906 -1906
- package/docs/01-TEMPLATES/versori/webhooks/template-webhook-dropship-routing.md +1074 -1074
- package/docs/01-TEMPLATES/versori/webhooks/template-webhook-flash-sale-reserve.md +1395 -1395
- package/docs/01-TEMPLATES/versori/webhooks/template-webhook-generic-xml-order.md +888 -888
- package/docs/01-TEMPLATES/versori/webhooks/template-webhook-payment-gateway-integration.md +2478 -2478
- package/docs/01-TEMPLATES/versori/webhooks/template-webhook-rma-returns-comprehensive.md +2240 -2240
- package/docs/01-TEMPLATES/versori/webhooks/template-webhook-xml-order-ingestion.md +2029 -2029
- package/docs/01-TEMPLATES/versori/webhooks/webhook-templates-guide.md +140 -140
- package/docs/01-TEMPLATES/versori/workflows/_examples/sample-data/inventory-mapping.json +20 -20
- package/docs/01-TEMPLATES/versori/workflows/_examples/sample-data/products_2025-01-22.csv +11 -11
- package/docs/01-TEMPLATES/versori/workflows/_examples/sample-data/sample-data-guide.md +34 -34
- package/docs/01-TEMPLATES/versori/workflows/_examples/workflow-examples-guide.md +36 -36
- package/docs/01-TEMPLATES/versori/workflows/extraction/extraction-modes-guide.md +1038 -1038
- package/docs/01-TEMPLATES/versori/workflows/extraction/extraction-workflows-guide.md +138 -138
- package/docs/01-TEMPLATES/versori/workflows/extraction/graphql-queries/graphql-extraction-guide.md +63 -63
- package/docs/01-TEMPLATES/versori/workflows/extraction/graphql-queries/template-extraction-fulfillments-to-sftp-csv.md +2062 -2062
- package/docs/01-TEMPLATES/versori/workflows/extraction/graphql-queries/template-extraction-fulfillments-to-sftp-xml.md +2294 -2294
- package/docs/01-TEMPLATES/versori/workflows/extraction/graphql-queries/template-extraction-inventory-positions-to-s3-csv.md +2461 -2461
- package/docs/01-TEMPLATES/versori/workflows/extraction/graphql-queries/template-extraction-inventory-positions-to-sftp-xml.md +2529 -2529
- package/docs/01-TEMPLATES/versori/workflows/extraction/graphql-queries/template-extraction-inventory-quantities-to-s3-csv.md +2464 -2464
- package/docs/01-TEMPLATES/versori/workflows/extraction/graphql-queries/template-extraction-inventory-quantities-to-s3-json.md +1959 -1959
- package/docs/01-TEMPLATES/versori/workflows/extraction/graphql-queries/template-extraction-orders-to-s3-csv.md +1953 -1953
- package/docs/01-TEMPLATES/versori/workflows/extraction/graphql-queries/template-extraction-orders-to-sftp-xml.md +2541 -2541
- package/docs/01-TEMPLATES/versori/workflows/extraction/graphql-queries/template-extraction-products-to-s3-json.md +2384 -2384
- package/docs/01-TEMPLATES/versori/workflows/extraction/graphql-queries/template-extraction-products-to-sftp-xml.md +2445 -2445
- package/docs/01-TEMPLATES/versori/workflows/extraction/graphql-queries/template-extraction-virtual-positions-to-s3-csv.md +2355 -2355
- package/docs/01-TEMPLATES/versori/workflows/extraction/graphql-queries/template-extraction-virtual-positions-to-s3-json.md +2042 -2042
- package/docs/01-TEMPLATES/versori/workflows/extraction/graphql-queries/template-extraction-virtual-positions-to-sftp-xml.md +2726 -2726
- package/docs/01-TEMPLATES/versori/workflows/ingestion/batch-api/batch-api-guide.md +206 -206
- package/docs/01-TEMPLATES/versori/workflows/ingestion/batch-api/template-ingestion-cycle-count-reconciliation.md +2030 -2030
- package/docs/01-TEMPLATES/versori/workflows/ingestion/batch-api/template-ingestion-multi-channel-inventory-sync.md +1882 -1882
- package/docs/01-TEMPLATES/versori/workflows/ingestion/batch-api/template-ingestion-s3-csv-inventory-batch.md +2827 -2827
- package/docs/01-TEMPLATES/versori/workflows/ingestion/batch-api/template-ingestion-s3-json-inventory-batch.md +1952 -1952
- package/docs/01-TEMPLATES/versori/workflows/ingestion/batch-api/template-ingestion-s3-xml-inventory-batch.md +3289 -3289
- package/docs/01-TEMPLATES/versori/workflows/ingestion/batch-api/template-ingestion-sftp-csv-inventory-batch.md +3064 -3064
- package/docs/01-TEMPLATES/versori/workflows/ingestion/batch-api/template-ingestion-sftp-json-inventory-batch.md +3238 -3238
- package/docs/01-TEMPLATES/versori/workflows/ingestion/batch-api/template-ingestion-sftp-xml-inventory-batch.md +2977 -2977
- package/docs/01-TEMPLATES/versori/workflows/ingestion/event-api/event-api-guide.md +321 -321
- package/docs/01-TEMPLATES/versori/workflows/ingestion/event-api/template-ingestion-payload-json-order-cancel-event.md +959 -959
- package/docs/01-TEMPLATES/versori/workflows/ingestion/event-api/template-ingestion-payload-xml-order-cancel-event.md +1170 -1170
- package/docs/01-TEMPLATES/versori/workflows/ingestion/event-api/template-ingestion-s3-csv-product-event.md +2312 -2312
- package/docs/01-TEMPLATES/versori/workflows/ingestion/event-api/template-ingestion-s3-json-product-event.md +2999 -2999
- package/docs/01-TEMPLATES/versori/workflows/ingestion/event-api/template-ingestion-s3-parquet-product-event.md +2836 -2836
- package/docs/01-TEMPLATES/versori/workflows/ingestion/event-api/template-ingestion-s3-xml-product-event.md +2395 -2395
- package/docs/01-TEMPLATES/versori/workflows/ingestion/event-api/template-ingestion-sftp-csv-product-event.md +2295 -2295
- package/docs/01-TEMPLATES/versori/workflows/ingestion/event-api/template-ingestion-sftp-json-product-event.md +2602 -2602
- package/docs/01-TEMPLATES/versori/workflows/ingestion/event-api/template-ingestion-sftp-parquet-product-event.md +2589 -2589
- package/docs/01-TEMPLATES/versori/workflows/ingestion/event-api/template-ingestion-sftp-xml-product-event.md +3578 -3578
- package/docs/01-TEMPLATES/versori/workflows/ingestion/graphql-mutations/graphql-mutations-guide.md +93 -93
- package/docs/01-TEMPLATES/versori/workflows/ingestion/graphql-mutations/template-ingestion-payload-json-order-update-graphql.md +1260 -1260
- package/docs/01-TEMPLATES/versori/workflows/ingestion/graphql-mutations/template-ingestion-payload-xml-order-update-graphql.md +1472 -1472
- package/docs/01-TEMPLATES/versori/workflows/ingestion/graphql-mutations/template-ingestion-s3-csv-control-graphql.md +2417 -2417
- package/docs/01-TEMPLATES/versori/workflows/ingestion/graphql-mutations/template-ingestion-s3-csv-location-graphql.md +2811 -2811
- package/docs/01-TEMPLATES/versori/workflows/ingestion/graphql-mutations/template-ingestion-s3-csv-price-graphql.md +2619 -2619
- package/docs/01-TEMPLATES/versori/workflows/ingestion/graphql-mutations/template-ingestion-s3-json-location-graphql.md +2807 -2807
- package/docs/01-TEMPLATES/versori/workflows/ingestion/graphql-mutations/template-ingestion-s3-xml-location-graphql.md +2373 -2373
- package/docs/01-TEMPLATES/versori/workflows/ingestion/graphql-mutations/template-ingestion-sftp-csv-control-graphql.md +2740 -2740
- package/docs/01-TEMPLATES/versori/workflows/ingestion/graphql-mutations/template-ingestion-sftp-csv-location-graphql.md +2760 -2760
- package/docs/01-TEMPLATES/versori/workflows/ingestion/graphql-mutations/template-ingestion-sftp-json-location-graphql.md +1710 -1710
- package/docs/01-TEMPLATES/versori/workflows/ingestion/ingestion-workflows-guide.md +136 -136
- package/docs/01-TEMPLATES/versori/workflows/rubix-webhooks/rubix-webhooks-guide.md +520 -520
- package/docs/01-TEMPLATES/versori/workflows/rubix-webhooks/template-webhook-rubix-fulfilment-to-sftp-xml-inline.md +1418 -1418
- package/docs/01-TEMPLATES/versori/workflows/rubix-webhooks/template-webhook-rubix-fulfilment-to-sftp-xml-universal-mapper.md +1785 -1785
- package/docs/01-TEMPLATES/versori/workflows/rubix-webhooks/template-webhook-rubix-order-attribute-update.md +824 -824
- package/docs/01-TEMPLATES/versori/workflows/workflows-overview-guide.md +646 -646
- package/docs/02-CORE-GUIDES/advanced-services/advanced-services-batch-archival.md +724 -724
- package/docs/02-CORE-GUIDES/advanced-services/advanced-services-job-tracker.md +627 -627
- package/docs/02-CORE-GUIDES/advanced-services/advanced-services-partial-batch-recovery.md +561 -561
- package/docs/02-CORE-GUIDES/advanced-services/advanced-services-quick-reference.md +367 -367
- package/docs/02-CORE-GUIDES/advanced-services/advanced-services-readme.md +407 -407
- package/docs/02-CORE-GUIDES/advanced-services/readme.md +49 -49
- package/docs/02-CORE-GUIDES/api-reference/api-reference-quick-reference.md +548 -548
- package/docs/02-CORE-GUIDES/api-reference/event-api-input-output-reference.md +702 -1171
- package/docs/02-CORE-GUIDES/api-reference/examples/client-initialization.ts +286 -286
- package/docs/02-CORE-GUIDES/api-reference/graphql-error-classification.md +337 -337
- package/docs/02-CORE-GUIDES/api-reference/modules/api-reference-01-client-api.md +399 -520
- package/docs/02-CORE-GUIDES/api-reference/modules/api-reference-03-authentication.md +199 -199
- package/docs/02-CORE-GUIDES/api-reference/modules/api-reference-04-graphql-mapping.md +925 -925
- package/docs/02-CORE-GUIDES/api-reference/modules/api-reference-05-services.md +1198 -1198
- package/docs/02-CORE-GUIDES/api-reference/modules/api-reference-06-data-sources.md +1083 -1083
- package/docs/02-CORE-GUIDES/api-reference/modules/api-reference-07-parsers.md +1097 -1097
- package/docs/02-CORE-GUIDES/api-reference/modules/api-reference-08-pagination.md +513 -513
- package/docs/02-CORE-GUIDES/api-reference/modules/api-reference-08-types.md +545 -597
- package/docs/02-CORE-GUIDES/api-reference/modules/api-reference-09-error-handling.md +527 -527
- package/docs/02-CORE-GUIDES/api-reference/modules/api-reference-09-webhook-validation.md +514 -514
- package/docs/02-CORE-GUIDES/api-reference/modules/api-reference-10-extraction.md +557 -557
- package/docs/02-CORE-GUIDES/api-reference/modules/api-reference-10-utilities.md +412 -412
- package/docs/02-CORE-GUIDES/api-reference/modules/api-reference-11-cli-tools.md +423 -423
- package/docs/02-CORE-GUIDES/api-reference/modules/api-reference-11-error-handling.md +716 -716
- package/docs/02-CORE-GUIDES/api-reference/modules/api-reference-12-analyze-source-structure.md +518 -518
- package/docs/02-CORE-GUIDES/api-reference/modules/api-reference-12-partial-responses.md +212 -212
- package/docs/02-CORE-GUIDES/api-reference/modules/api-reference-12-testing.md +300 -300
- package/docs/02-CORE-GUIDES/api-reference/modules/api-reference-13-resolver-builder.md +322 -322
- package/docs/02-CORE-GUIDES/api-reference/readme.md +279 -279
- package/docs/02-CORE-GUIDES/auto-pagination/auto-pagination-quick-reference.md +351 -351
- package/docs/02-CORE-GUIDES/auto-pagination/auto-pagination-readme.md +277 -277
- package/docs/02-CORE-GUIDES/auto-pagination/examples/auto-pagination-readme.md +178 -178
- package/docs/02-CORE-GUIDES/auto-pagination/examples/common-patterns.ts +351 -351
- package/docs/02-CORE-GUIDES/auto-pagination/examples/paginate-products.ts +384 -384
- package/docs/02-CORE-GUIDES/auto-pagination/examples/paginate-virtual-positions.ts +308 -308
- package/docs/02-CORE-GUIDES/auto-pagination/modules/auto-pagination-01-foundations.md +470 -470
- package/docs/02-CORE-GUIDES/auto-pagination/modules/auto-pagination-02-quick-start.md +713 -713
- package/docs/02-CORE-GUIDES/auto-pagination/modules/auto-pagination-03-configuration.md +754 -754
- package/docs/02-CORE-GUIDES/auto-pagination/modules/auto-pagination-04-advanced-patterns.md +732 -732
- package/docs/02-CORE-GUIDES/auto-pagination/modules/auto-pagination-05-sdk-integration.md +847 -847
- package/docs/02-CORE-GUIDES/auto-pagination/modules/auto-pagination-06-troubleshooting.md +359 -359
- package/docs/02-CORE-GUIDES/auto-pagination/modules/auto-pagination-07-api-reference.md +462 -462
- package/docs/02-CORE-GUIDES/auto-pagination/readme.md +54 -54
- package/docs/02-CORE-GUIDES/data-sources/data-sources-file-operations-error-handling.md +1487 -1487
- package/docs/02-CORE-GUIDES/data-sources/data-sources-quick-reference.md +836 -836
- package/docs/02-CORE-GUIDES/data-sources/data-sources-readme.md +276 -276
- package/docs/02-CORE-GUIDES/data-sources/data-sources-sftp-credential-access-security.md +553 -553
- package/docs/02-CORE-GUIDES/data-sources/examples/common-patterns.ts +409 -409
- package/docs/02-CORE-GUIDES/data-sources/examples/data-sources-readme.md +178 -178
- package/docs/02-CORE-GUIDES/data-sources/examples/s3-operations.ts +308 -308
- package/docs/02-CORE-GUIDES/data-sources/examples/sftp-operations.ts +371 -371
- package/docs/02-CORE-GUIDES/data-sources/modules/data-sources-01-foundations.md +735 -735
- package/docs/02-CORE-GUIDES/data-sources/modules/data-sources-02-s3-operations.md +1302 -1302
- package/docs/02-CORE-GUIDES/data-sources/modules/data-sources-03-sftp-operations.md +1379 -1379
- package/docs/02-CORE-GUIDES/data-sources/modules/data-sources-04-file-patterns.md +941 -941
- package/docs/02-CORE-GUIDES/data-sources/modules/data-sources-05-advanced-topics.md +813 -813
- package/docs/02-CORE-GUIDES/data-sources/modules/data-sources-06-integration-patterns.md +486 -486
- package/docs/02-CORE-GUIDES/data-sources/modules/data-sources-07-troubleshooting.md +387 -387
- package/docs/02-CORE-GUIDES/data-sources/modules/data-sources-08-api-reference.md +417 -417
- package/docs/02-CORE-GUIDES/data-sources/readme.md +77 -77
- package/docs/02-CORE-GUIDES/error-handling-guide.md +936 -936
- package/docs/02-CORE-GUIDES/extraction/examples/02-core-guides-extraction-readme.md +116 -116
- package/docs/02-CORE-GUIDES/extraction/examples/common-patterns.ts +428 -428
- package/docs/02-CORE-GUIDES/extraction/examples/extract-inventory-basic.ts +187 -187
- package/docs/02-CORE-GUIDES/extraction/extraction-quick-reference.md +596 -596
- package/docs/02-CORE-GUIDES/extraction/modules/02-core-guides-extraction-01-foundations.md +514 -514
- package/docs/02-CORE-GUIDES/extraction/modules/02-core-guides-extraction-02-basic-extraction.md +823 -823
- package/docs/02-CORE-GUIDES/extraction/modules/02-core-guides-extraction-03-parquet-processing.md +507 -507
- package/docs/02-CORE-GUIDES/extraction/modules/02-core-guides-extraction-04-data-enrichment.md +546 -546
- package/docs/02-CORE-GUIDES/extraction/modules/02-core-guides-extraction-05-transformation.md +494 -494
- package/docs/02-CORE-GUIDES/extraction/modules/02-core-guides-extraction-06-export-formats.md +458 -458
- package/docs/02-CORE-GUIDES/extraction/modules/02-core-guides-extraction-06-performance.md +138 -138
- package/docs/02-CORE-GUIDES/extraction/modules/02-core-guides-extraction-07-api-reference.md +148 -148
- package/docs/02-CORE-GUIDES/extraction/modules/02-core-guides-extraction-07-optimization.md +692 -692
- package/docs/02-CORE-GUIDES/extraction/modules/02-core-guides-extraction-08-extraction-orchestrator.md +1008 -1008
- package/docs/02-CORE-GUIDES/extraction/readme.md +151 -151
- package/docs/02-CORE-GUIDES/ingestion/examples/_simple-kv-store.ts +40 -40
- package/docs/02-CORE-GUIDES/ingestion/examples/error-recovery.ts +728 -728
- package/docs/02-CORE-GUIDES/ingestion/examples/event-driven.ts +501 -501
- package/docs/02-CORE-GUIDES/ingestion/examples/local-file-ingestion.ts +88 -88
- package/docs/02-CORE-GUIDES/ingestion/examples/parquet-ingestion.ts +117 -117
- package/docs/02-CORE-GUIDES/ingestion/examples/performance-optimized.ts +647 -647
- package/docs/02-CORE-GUIDES/ingestion/examples/s3-csv-ingestion.ts +169 -169
- package/docs/02-CORE-GUIDES/ingestion/examples/sftp-csv-ingestion.ts +134 -134
- package/docs/02-CORE-GUIDES/ingestion/ingestion-quick-reference.md +546 -546
- package/docs/02-CORE-GUIDES/ingestion/modules/02-core-guides-ingestion-01-introduction.md +626 -626
- package/docs/02-CORE-GUIDES/ingestion/modules/02-core-guides-ingestion-02-quick-start.md +658 -658
- package/docs/02-CORE-GUIDES/ingestion/modules/02-core-guides-ingestion-03-data-sources.md +1052 -1052
- package/docs/02-CORE-GUIDES/ingestion/modules/02-core-guides-ingestion-04-field-mapping.md +763 -763
- package/docs/02-CORE-GUIDES/ingestion/modules/02-core-guides-ingestion-05-advanced-parsers.md +676 -676
- package/docs/02-CORE-GUIDES/ingestion/modules/02-core-guides-ingestion-06-batch-api.md +1295 -1295
- package/docs/02-CORE-GUIDES/ingestion/modules/02-core-guides-ingestion-07-api-reference.md +138 -138
- package/docs/02-CORE-GUIDES/ingestion/modules/02-core-guides-ingestion-07-state-management.md +1037 -1037
- package/docs/02-CORE-GUIDES/ingestion/modules/02-core-guides-ingestion-08-performance-optimization.md +1349 -1349
- package/docs/02-CORE-GUIDES/ingestion/modules/02-core-guides-ingestion-09-best-practices.md +1893 -1893
- package/docs/02-CORE-GUIDES/ingestion/readme.md +160 -160
- package/docs/02-CORE-GUIDES/logging-guide.md +585 -585
- package/docs/02-CORE-GUIDES/mapping/error-handling-patterns.md +401 -401
- package/docs/02-CORE-GUIDES/mapping/examples/02-core-guides-mapping-readme.md +128 -128
- package/docs/02-CORE-GUIDES/mapping/examples/common-patterns.ts +273 -273
- package/docs/02-CORE-GUIDES/mapping/examples/csv-location-ingestion.json +36 -36
- package/docs/02-CORE-GUIDES/mapping/examples/csv-mapping.ts +242 -242
- package/docs/02-CORE-GUIDES/mapping/examples/graphql-to-parquet-extraction.json +36 -36
- package/docs/02-CORE-GUIDES/mapping/examples/json-mapping.ts +213 -213
- package/docs/02-CORE-GUIDES/mapping/examples/json-product-to-mutation.json +48 -48
- package/docs/02-CORE-GUIDES/mapping/examples/xml-mapping.ts +291 -291
- package/docs/02-CORE-GUIDES/mapping/examples/xml-order-to-mutation.json +45 -45
- package/docs/02-CORE-GUIDES/mapping/graphql-mutation-mapping/graphql-mutation-mapping-quick-reference.md +463 -463
- package/docs/02-CORE-GUIDES/mapping/graphql-mutation-mapping/graphql-mutation-mapping-readme.md +227 -227
- package/docs/02-CORE-GUIDES/mapping/graphql-mutation-mapping/modules/graphql-mutation-mapping-01-introduction.md +222 -222
- package/docs/02-CORE-GUIDES/mapping/graphql-mutation-mapping/modules/graphql-mutation-mapping-02-quick-start.md +351 -351
- package/docs/02-CORE-GUIDES/mapping/graphql-mutation-mapping/modules/graphql-mutation-mapping-03-schema-validation.md +569 -569
- package/docs/02-CORE-GUIDES/mapping/graphql-mutation-mapping/modules/graphql-mutation-mapping-04-mapping-patterns.md +471 -471
- package/docs/02-CORE-GUIDES/mapping/graphql-mutation-mapping/modules/graphql-mutation-mapping-05-configuration-reference.md +611 -611
- package/docs/02-CORE-GUIDES/mapping/graphql-mutation-mapping/modules/graphql-mutation-mapping-06-advanced-xpath.md +148 -148
- package/docs/02-CORE-GUIDES/mapping/graphql-mutation-mapping/modules/graphql-mutation-mapping-06-path-syntax.md +464 -464
- package/docs/02-CORE-GUIDES/mapping/graphql-mutation-mapping/modules/graphql-mutation-mapping-07-api-reference.md +94 -94
- package/docs/02-CORE-GUIDES/mapping/graphql-mutation-mapping/modules/graphql-mutation-mapping-07-array-handling.md +307 -307
- package/docs/02-CORE-GUIDES/mapping/graphql-mutation-mapping/modules/graphql-mutation-mapping-08-custom-resolvers.md +544 -544
- package/docs/02-CORE-GUIDES/mapping/graphql-mutation-mapping/modules/graphql-mutation-mapping-09-advanced-patterns.md +427 -427
- package/docs/02-CORE-GUIDES/mapping/graphql-mutation-mapping/modules/graphql-mutation-mapping-10-hooks-and-variables.md +336 -336
- package/docs/02-CORE-GUIDES/mapping/graphql-mutation-mapping/modules/graphql-mutation-mapping-11-error-handling.md +488 -488
- package/docs/02-CORE-GUIDES/mapping/graphql-mutation-mapping/modules/graphql-mutation-mapping-12-arguments-vs-nodes.md +383 -383
- package/docs/02-CORE-GUIDES/mapping/graphql-mutation-mapping/modules/graphql-mutation-mapping-13-best-practices.md +477 -477
- package/docs/02-CORE-GUIDES/mapping/graphql-mutation-mapping/readme.md +62 -62
- package/docs/02-CORE-GUIDES/mapping/mapping-format-decision-tree.md +480 -480
- package/docs/02-CORE-GUIDES/mapping/mapping-graphql-alias-batching-guide.md +820 -820
- package/docs/02-CORE-GUIDES/mapping/mapping-javascript-objects.md +2369 -2369
- package/docs/02-CORE-GUIDES/mapping/mapping-mapper-comparison-guide.md +682 -682
- package/docs/02-CORE-GUIDES/mapping/modules/02-core-guides-mapping-07-api-reference.md +1327 -1327
- package/docs/02-CORE-GUIDES/mapping/modules/02-core-guides-mapping-08-error-handling.md +1142 -1142
- package/docs/02-CORE-GUIDES/mapping/modules/mapping-04-use-cases.md +891 -891
- package/docs/02-CORE-GUIDES/mapping/modules/mapping-06-helpers-resolvers.md +1126 -1126
- package/docs/02-CORE-GUIDES/mapping/modules/mapping-06-sdk-resolvers.md +199 -199
- package/docs/02-CORE-GUIDES/mapping/modules/mapping-07-api-reference.md +1319 -1319
- package/docs/02-CORE-GUIDES/mapping/readme.md +178 -178
- package/docs/02-CORE-GUIDES/mapping/resolver-registration.md +410 -410
- package/docs/02-CORE-GUIDES/mapping/resolvers/examples/common-patterns.ts +226 -226
- package/docs/02-CORE-GUIDES/mapping/resolvers/examples/custom-resolvers.ts +227 -227
- package/docs/02-CORE-GUIDES/mapping/resolvers/examples/sdk-resolvers-usage.ts +203 -203
- package/docs/02-CORE-GUIDES/mapping/resolvers/mapping-resolvers-readme.md +274 -274
- package/docs/02-CORE-GUIDES/mapping/resolvers/mapping-resolvers-resolver-api-reference.md +679 -679
- package/docs/02-CORE-GUIDES/mapping/resolvers/mapping-resolvers-resolver-cookbook.md +826 -826
- package/docs/02-CORE-GUIDES/mapping/resolvers/mapping-resolvers-resolver-guide.md +1330 -1330
- package/docs/02-CORE-GUIDES/mapping/resolvers/mapping-resolvers-resolver-helpers-reference.md +1437 -1437
- package/docs/02-CORE-GUIDES/mapping/resolvers/mapping-resolvers-resolver-parameters-reference.md +553 -553
- package/docs/02-CORE-GUIDES/mapping/resolvers/mapping-resolvers-resolver-troubleshooting.md +854 -854
- package/docs/02-CORE-GUIDES/mapping/resolvers/readme.md +75 -75
- package/docs/02-CORE-GUIDES/parsers/examples/02-core-guides-parsers-readme.md +161 -161
- package/docs/02-CORE-GUIDES/parsers/examples/csv-parser-examples.ts +110 -110
- package/docs/02-CORE-GUIDES/parsers/examples/json-parser-examples.ts +33 -33
- package/docs/02-CORE-GUIDES/parsers/examples/parquet-parser-examples.ts +47 -47
- package/docs/02-CORE-GUIDES/parsers/examples/xml-parser-examples.ts +38 -38
- package/docs/02-CORE-GUIDES/parsers/modules/02-core-guides-parsers-01-foundations.md +355 -355
- package/docs/02-CORE-GUIDES/parsers/modules/02-core-guides-parsers-02-csv-parser.md +772 -772
- package/docs/02-CORE-GUIDES/parsers/modules/02-core-guides-parsers-03-json-parser.md +789 -789
- package/docs/02-CORE-GUIDES/parsers/modules/02-core-guides-parsers-04-xml-parser.md +857 -857
- package/docs/02-CORE-GUIDES/parsers/modules/02-core-guides-parsers-05-parquet-parser.md +603 -603
- package/docs/02-CORE-GUIDES/parsers/modules/02-core-guides-parsers-06-integration-patterns.md +702 -702
- package/docs/02-CORE-GUIDES/parsers/modules/02-core-guides-parsers-06-streaming.md +121 -121
- package/docs/02-CORE-GUIDES/parsers/modules/02-core-guides-parsers-07-api-reference.md +89 -89
- package/docs/02-CORE-GUIDES/parsers/modules/02-core-guides-parsers-07-troubleshooting.md +727 -727
- package/docs/02-CORE-GUIDES/parsers/parsers-quick-reference.md +482 -482
- package/docs/02-CORE-GUIDES/parsers/parsers-readme.md +258 -258
- package/docs/02-CORE-GUIDES/parsers/readme.md +65 -65
- package/docs/02-CORE-GUIDES/readme.md +194 -194
- package/docs/02-CORE-GUIDES/webhook-validation/examples/basic-validation.ts +108 -108
- package/docs/02-CORE-GUIDES/webhook-validation/examples/common-patterns.ts +316 -316
- package/docs/02-CORE-GUIDES/webhook-validation/examples/webhook-validation-readme.md +61 -61
- package/docs/02-CORE-GUIDES/webhook-validation/modules/webhook-validation-01-foundations.md +440 -440
- package/docs/02-CORE-GUIDES/webhook-validation/modules/webhook-validation-02-quick-start.md +525 -525
- package/docs/02-CORE-GUIDES/webhook-validation/modules/webhook-validation-03-versori-integration.md +741 -741
- package/docs/02-CORE-GUIDES/webhook-validation/modules/webhook-validation-04-platform-integration.md +629 -629
- package/docs/02-CORE-GUIDES/webhook-validation/modules/webhook-validation-05-configuration.md +535 -535
- package/docs/02-CORE-GUIDES/webhook-validation/modules/webhook-validation-06-error-handling.md +611 -611
- package/docs/02-CORE-GUIDES/webhook-validation/modules/webhook-validation-06-troubleshooting.md +124 -124
- package/docs/02-CORE-GUIDES/webhook-validation/modules/webhook-validation-07-api-reference.md +511 -511
- package/docs/02-CORE-GUIDES/webhook-validation/modules/webhook-validation-08-rubix-webhooks.md +590 -590
- package/docs/02-CORE-GUIDES/webhook-validation/modules/webhook-validation-09-rubix-event-vs-http-call.md +432 -432
- package/docs/02-CORE-GUIDES/webhook-validation/readme.md +239 -239
- package/docs/02-CORE-GUIDES/webhook-validation/webhook-validation-quick-reference.md +392 -392
- package/docs/03-PATTERN-GUIDES/connector-scenarios/connector-scenarios-quick-reference.md +498 -498
- package/docs/03-PATTERN-GUIDES/connector-scenarios/connector-scenarios-readme.md +313 -313
- package/docs/03-PATTERN-GUIDES/connector-scenarios/examples/common-patterns.ts +612 -612
- package/docs/03-PATTERN-GUIDES/connector-scenarios/examples/connector-scenarios-readme.md +253 -253
- package/docs/03-PATTERN-GUIDES/connector-scenarios/modules/connector-scenarios-01-foundations.md +452 -452
- package/docs/03-PATTERN-GUIDES/connector-scenarios/modules/connector-scenarios-02-simple-scenarios.md +681 -681
- package/docs/03-PATTERN-GUIDES/connector-scenarios/modules/connector-scenarios-03-intermediate-scenarios.md +637 -637
- package/docs/03-PATTERN-GUIDES/connector-scenarios/modules/connector-scenarios-04-advanced-scenarios.md +650 -650
- package/docs/03-PATTERN-GUIDES/connector-scenarios/modules/connector-scenarios-05-bidirectional-sync.md +233 -233
- package/docs/03-PATTERN-GUIDES/connector-scenarios/modules/connector-scenarios-06-production-patterns.md +442 -442
- package/docs/03-PATTERN-GUIDES/connector-scenarios/modules/connector-scenarios-07-reference.md +445 -445
- package/docs/03-PATTERN-GUIDES/connector-scenarios/readme.md +31 -31
- package/docs/03-PATTERN-GUIDES/enterprise-integration-patterns.md +1528 -1528
- package/docs/03-PATTERN-GUIDES/error-handling/comprehensive-error-handling-guide.md +1437 -1437
- package/docs/03-PATTERN-GUIDES/error-handling/error-handling-quick-reference.md +390 -390
- package/docs/03-PATTERN-GUIDES/error-handling/examples/common-patterns.ts +438 -438
- package/docs/03-PATTERN-GUIDES/error-handling/modules/error-handling-01-foundations.md +362 -362
- package/docs/03-PATTERN-GUIDES/error-handling/modules/error-handling-02-error-types.md +850 -850
- package/docs/03-PATTERN-GUIDES/error-handling/modules/error-handling-03-utf8-handling.md +456 -456
- package/docs/03-PATTERN-GUIDES/error-handling/modules/error-handling-04-error-scenarios.md +658 -658
- package/docs/03-PATTERN-GUIDES/error-handling/modules/error-handling-05-calling-patterns.md +671 -671
- package/docs/03-PATTERN-GUIDES/error-handling/modules/error-handling-06-retry-strategies.md +1034 -1034
- package/docs/03-PATTERN-GUIDES/error-handling/modules/error-handling-07-monitoring.md +653 -653
- package/docs/03-PATTERN-GUIDES/error-handling/modules/error-handling-08-api-reference.md +847 -847
- package/docs/03-PATTERN-GUIDES/error-handling/readme.md +36 -36
- package/docs/03-PATTERN-GUIDES/examples/__tests__/readme.md +40 -40
- package/docs/03-PATTERN-GUIDES/examples/__tests__/resolver-examples.test.js +282 -282
- package/docs/03-PATTERN-GUIDES/examples/test-data/03-pattern-guides-readme.md +110 -110
- package/docs/03-PATTERN-GUIDES/examples/test-data/canonical-inventory.json +123 -123
- package/docs/03-PATTERN-GUIDES/examples/test-data/canonical-order.json +171 -171
- package/docs/03-PATTERN-GUIDES/examples/test-data/readme.md +28 -28
- package/docs/03-PATTERN-GUIDES/extraction/extraction-readme.md +15 -15
- package/docs/03-PATTERN-GUIDES/extraction/readme.md +25 -25
- package/docs/03-PATTERN-GUIDES/file-operations/examples/common-patterns.ts +407 -407
- package/docs/03-PATTERN-GUIDES/file-operations/examples/file-operations-readme.md +142 -142
- package/docs/03-PATTERN-GUIDES/file-operations/file-operations-quick-reference.md +462 -462
- package/docs/03-PATTERN-GUIDES/file-operations/file-operations-readme.md +379 -379
- package/docs/03-PATTERN-GUIDES/file-operations/modules/file-operations-01-foundations.md +430 -430
- package/docs/03-PATTERN-GUIDES/file-operations/modules/file-operations-02-quick-start.md +484 -484
- package/docs/03-PATTERN-GUIDES/file-operations/modules/file-operations-03-s3-operations.md +507 -507
- package/docs/03-PATTERN-GUIDES/file-operations/modules/file-operations-04-sftp-operations.md +963 -963
- package/docs/03-PATTERN-GUIDES/file-operations/modules/file-operations-05-streaming-performance.md +503 -503
- package/docs/03-PATTERN-GUIDES/file-operations/modules/file-operations-06-archive-patterns.md +386 -386
- package/docs/03-PATTERN-GUIDES/file-operations/modules/file-operations-06-error-handling.md +117 -117
- package/docs/03-PATTERN-GUIDES/file-operations/modules/file-operations-07-api-reference.md +78 -78
- package/docs/03-PATTERN-GUIDES/file-operations/modules/file-operations-07-testing-troubleshooting.md +567 -567
- package/docs/03-PATTERN-GUIDES/file-operations/modules/file-operations-08-api-reference.md +1055 -1055
- package/docs/03-PATTERN-GUIDES/file-operations/readme.md +32 -32
- package/docs/03-PATTERN-GUIDES/ingestion/ingestion-readme.md +15 -15
- package/docs/03-PATTERN-GUIDES/ingestion/readme.md +25 -25
- package/docs/03-PATTERN-GUIDES/integration-patterns/examples/batch-processing.ts +130 -130
- package/docs/03-PATTERN-GUIDES/integration-patterns/examples/common-patterns.ts +360 -360
- package/docs/03-PATTERN-GUIDES/integration-patterns/examples/delta-sync.ts +130 -130
- package/docs/03-PATTERN-GUIDES/integration-patterns/examples/integration-patterns-readme.md +100 -100
- package/docs/03-PATTERN-GUIDES/integration-patterns/examples/real-time-webhook.ts +398 -398
- package/docs/03-PATTERN-GUIDES/integration-patterns/integration-patterns-quick-reference.md +962 -962
- package/docs/03-PATTERN-GUIDES/integration-patterns/integration-patterns-readme.md +134 -134
- package/docs/03-PATTERN-GUIDES/integration-patterns/modules/integration-patterns-01-real-time-processing.md +991 -991
- package/docs/03-PATTERN-GUIDES/integration-patterns/modules/integration-patterns-02-batch-processing.md +1547 -1547
- package/docs/03-PATTERN-GUIDES/integration-patterns/modules/integration-patterns-03-delta-sync.md +1108 -1108
- package/docs/03-PATTERN-GUIDES/integration-patterns/modules/integration-patterns-04-webhook-patterns.md +1181 -1181
- package/docs/03-PATTERN-GUIDES/integration-patterns/modules/integration-patterns-05-error-handling.md +1061 -1061
- package/docs/03-PATTERN-GUIDES/integration-patterns/modules/integration-patterns-06-advanced-integration-services.md +1547 -1547
- package/docs/03-PATTERN-GUIDES/integration-patterns/modules/integration-patterns-06-performance.md +109 -109
- package/docs/03-PATTERN-GUIDES/integration-patterns/modules/integration-patterns-07-api-reference.md +34 -34
- package/docs/03-PATTERN-GUIDES/integration-patterns/readme.md +30 -30
- package/docs/03-PATTERN-GUIDES/logging-minimal-mode.md +128 -128
- package/docs/03-PATTERN-GUIDES/multiple-connections/examples/common-patterns.ts +380 -380
- package/docs/03-PATTERN-GUIDES/multiple-connections/examples/multiple-connections-readme.md +139 -139
- package/docs/03-PATTERN-GUIDES/multiple-connections/examples/parallel-root-connections.ts +149 -149
- package/docs/03-PATTERN-GUIDES/multiple-connections/examples/real-world-scenarios.ts +405 -405
- package/docs/03-PATTERN-GUIDES/multiple-connections/modules/multiple-connections-01-foundations.md +378 -378
- package/docs/03-PATTERN-GUIDES/multiple-connections/modules/multiple-connections-02-quick-start.md +566 -566
- package/docs/03-PATTERN-GUIDES/multiple-connections/modules/multiple-connections-03-targeting-connections.md +659 -659
- package/docs/03-PATTERN-GUIDES/multiple-connections/modules/multiple-connections-04-parallel-queries.md +656 -656
- package/docs/03-PATTERN-GUIDES/multiple-connections/modules/multiple-connections-05-best-practices.md +624 -624
- package/docs/03-PATTERN-GUIDES/multiple-connections/modules/multiple-connections-06-api-reference.md +824 -824
- package/docs/03-PATTERN-GUIDES/multiple-connections/modules/multiple-connections-06-versori.md +119 -119
- package/docs/03-PATTERN-GUIDES/multiple-connections/modules/multiple-connections-07-api-reference.md +87 -87
- package/docs/03-PATTERN-GUIDES/multiple-connections/multiple-connections-quick-reference.md +353 -353
- package/docs/03-PATTERN-GUIDES/multiple-connections/multiple-connections-readme.md +270 -270
- package/docs/03-PATTERN-GUIDES/multiple-connections/readme.md +30 -30
- package/docs/03-PATTERN-GUIDES/pagination/pagination-readme.md +14 -14
- package/docs/03-PATTERN-GUIDES/pagination/readme.md +24 -24
- package/docs/03-PATTERN-GUIDES/parquet/examples/common-patterns.ts +180 -180
- package/docs/03-PATTERN-GUIDES/parquet/examples/read-parquet.ts +48 -48
- package/docs/03-PATTERN-GUIDES/parquet/examples/write-parquet.ts +65 -65
- package/docs/03-PATTERN-GUIDES/parquet/modules/03-pattern-guides-parquet-01-introduction.md +393 -393
- package/docs/03-PATTERN-GUIDES/parquet/modules/03-pattern-guides-parquet-02-quick-start.md +572 -572
- package/docs/03-PATTERN-GUIDES/parquet/modules/03-pattern-guides-parquet-03-reading-parquet.md +525 -525
- package/docs/03-PATTERN-GUIDES/parquet/modules/03-pattern-guides-parquet-04-writing-parquet.md +554 -554
- package/docs/03-PATTERN-GUIDES/parquet/modules/03-pattern-guides-parquet-05-graphql-extraction.md +405 -405
- package/docs/03-PATTERN-GUIDES/parquet/modules/03-pattern-guides-parquet-06-performance.md +104 -104
- package/docs/03-PATTERN-GUIDES/parquet/modules/03-pattern-guides-parquet-06-s3-integration.md +511 -511
- package/docs/03-PATTERN-GUIDES/parquet/modules/03-pattern-guides-parquet-07-api-reference.md +90 -90
- package/docs/03-PATTERN-GUIDES/parquet/modules/03-pattern-guides-parquet-07-performance-optimization.md +525 -525
- package/docs/03-PATTERN-GUIDES/parquet/modules/03-pattern-guides-parquet-08-best-practices.md +712 -712
- package/docs/03-PATTERN-GUIDES/parquet/parquet-quick-reference.md +683 -683
- package/docs/03-PATTERN-GUIDES/parquet/parquet-readme.md +248 -248
- package/docs/03-PATTERN-GUIDES/parquet/readme.md +32 -32
- package/docs/03-PATTERN-GUIDES/parsers/parsers-readme.md +12 -12
- package/docs/03-PATTERN-GUIDES/parsers/readme.md +24 -24
- package/docs/03-PATTERN-GUIDES/readme.md +159 -159
- package/docs/03-PATTERN-GUIDES/webhooks/readme.md +24 -24
- package/docs/03-PATTERN-GUIDES/webhooks/webhooks-readme.md +8 -8
- package/docs/04-REFERENCE/architecture/architecture-01-overview.md +427 -427
- package/docs/04-REFERENCE/architecture/architecture-02-client-architecture.md +424 -424
- package/docs/04-REFERENCE/architecture/architecture-03-data-flow.md +690 -690
- package/docs/04-REFERENCE/architecture/architecture-04-service-layer.md +834 -834
- package/docs/04-REFERENCE/architecture/architecture-05-integration-architecture.md +655 -655
- package/docs/04-REFERENCE/architecture/architecture-06-state-management.md +653 -653
- package/docs/04-REFERENCE/architecture/architecture-adding-new-data-sources.md +686 -686
- package/docs/04-REFERENCE/architecture/readme.md +279 -279
- package/docs/04-REFERENCE/platforms/deno/readme.md +117 -117
- package/docs/04-REFERENCE/platforms/nodejs/readme.md +146 -146
- package/docs/04-REFERENCE/platforms/readme.md +135 -135
- package/docs/04-REFERENCE/platforms/versori/modules/platforms-versori-01-introduction.md +398 -398
- package/docs/04-REFERENCE/platforms/versori/modules/platforms-versori-02-quick-start.md +560 -560
- package/docs/04-REFERENCE/platforms/versori/modules/platforms-versori-03-authentication.md +757 -757
- package/docs/04-REFERENCE/platforms/versori/modules/platforms-versori-04-workflows.md +2476 -2476
- package/docs/04-REFERENCE/platforms/versori/modules/platforms-versori-05-connections.md +1167 -1167
- package/docs/04-REFERENCE/platforms/versori/modules/platforms-versori-06-kv-storage.md +990 -990
- package/docs/04-REFERENCE/platforms/versori/modules/platforms-versori-06-state-management.md +121 -121
- package/docs/04-REFERENCE/platforms/versori/modules/platforms-versori-07-api-reference.md +68 -68
- package/docs/04-REFERENCE/platforms/versori/modules/platforms-versori-07-deployment.md +731 -731
- package/docs/04-REFERENCE/platforms/versori/modules/platforms-versori-08-best-practices.md +1111 -1111
- package/docs/04-REFERENCE/platforms/versori/modules/platforms-versori-09-signature-reference.md +766 -766
- package/docs/04-REFERENCE/platforms/versori/platforms-versori-readme.md +299 -299
- package/docs/04-REFERENCE/platforms/versori/platforms-versori-s3-sftp-configuration-guide.md +1425 -1425
- package/docs/04-REFERENCE/platforms/versori/platforms-versori-webhook-api-key-security.md +816 -816
- package/docs/04-REFERENCE/platforms/versori/platforms-versori-webhook-connection-security.md +681 -681
- package/docs/04-REFERENCE/platforms/versori/platforms-versori-workflow-task-types.md +708 -708
- package/docs/04-REFERENCE/platforms/versori/readme.md +108 -108
- package/docs/04-REFERENCE/readme.md +148 -148
- package/docs/04-REFERENCE/resolver-signature/examples/advanced-resolvers.ts +482 -482
- package/docs/04-REFERENCE/resolver-signature/examples/async-resolvers.ts +496 -496
- package/docs/04-REFERENCE/resolver-signature/examples/basic-resolvers.ts +343 -343
- package/docs/04-REFERENCE/resolver-signature/examples/resolver-signature-readme.md +188 -188
- package/docs/04-REFERENCE/resolver-signature/examples/testing-resolvers.ts +463 -463
- package/docs/04-REFERENCE/resolver-signature/modules/resolver-signature-01-foundations.md +286 -286
- package/docs/04-REFERENCE/resolver-signature/modules/resolver-signature-02-parameter-reference.md +643 -643
- package/docs/04-REFERENCE/resolver-signature/modules/resolver-signature-03-basic-examples.md +521 -521
- package/docs/04-REFERENCE/resolver-signature/modules/resolver-signature-04-advanced-patterns.md +739 -739
- package/docs/04-REFERENCE/resolver-signature/modules/resolver-signature-05-sdk-resolvers.md +531 -531
- package/docs/04-REFERENCE/resolver-signature/modules/resolver-signature-06-migration-guide.md +650 -650
- package/docs/04-REFERENCE/resolver-signature/modules/resolver-signature-06-testing.md +125 -125
- package/docs/04-REFERENCE/resolver-signature/modules/resolver-signature-07-api-reference.md +794 -794
- package/docs/04-REFERENCE/resolver-signature/readme.md +64 -64
- package/docs/04-REFERENCE/resolver-signature/resolver-signature-quick-reference.md +270 -270
- package/docs/04-REFERENCE/resolver-signature/resolver-signature-readme.md +351 -351
- package/docs/04-REFERENCE/schema/fluent-commerce-schema.json +764 -764
- package/docs/04-REFERENCE/schema/readme.md +141 -141
- package/docs/04-REFERENCE/testing/examples/04-reference-testing-readme.md +158 -158
- package/docs/04-REFERENCE/testing/examples/fluent-testing.ts +62 -62
- package/docs/04-REFERENCE/testing/examples/health-check.ts +155 -155
- package/docs/04-REFERENCE/testing/examples/integration-test.ts +119 -119
- package/docs/04-REFERENCE/testing/examples/performance-test.ts +183 -183
- package/docs/04-REFERENCE/testing/examples/s3-testing.ts +127 -127
- package/docs/04-REFERENCE/testing/modules/04-reference-testing-01-foundations.md +267 -267
- package/docs/04-REFERENCE/testing/modules/04-reference-testing-02-s3-testing.md +599 -599
- package/docs/04-REFERENCE/testing/modules/04-reference-testing-03-fluent-testing.md +589 -589
- package/docs/04-REFERENCE/testing/modules/04-reference-testing-04-integration-testing.md +699 -699
- package/docs/04-REFERENCE/testing/modules/04-reference-testing-05-debugging.md +478 -478
- package/docs/04-REFERENCE/testing/modules/04-reference-testing-06-cicd-integration.md +463 -463
- package/docs/04-REFERENCE/testing/modules/04-reference-testing-06-preflight-validation.md +131 -131
- package/docs/04-REFERENCE/testing/modules/04-reference-testing-07-best-practices.md +499 -499
- package/docs/04-REFERENCE/testing/modules/04-reference-testing-07-coverage-ci.md +165 -165
- package/docs/04-REFERENCE/testing/modules/04-reference-testing-08-api-reference.md +634 -634
- package/docs/04-REFERENCE/testing/readme.md +86 -86
- package/docs/04-REFERENCE/testing/testing-quick-reference.md +667 -667
- package/docs/04-REFERENCE/testing/testing-readme.md +286 -286
- package/docs/04-REFERENCE/troubleshooting/readme.md +144 -144
- package/docs/04-REFERENCE/troubleshooting/troubleshooting-deno-sftp-compatibility.md +392 -392
- package/docs/template-loading-matrix.md +242 -242
- package/package.json +5 -3
- package/docs/02-CORE-GUIDES/api-reference/cli-profile-integration.md +0 -377
|
@@ -1,2619 +1,2619 @@
|
|
|
1
|
-
---
|
|
2
|
-
template_id: tpl-ingest-s3-csv-to-price-graphql
|
|
3
|
-
canonical_filename: template-ingestion-s3-csv-price-graphql.md
|
|
4
|
-
version: 2.0.0
|
|
5
|
-
sdk_version: ^0.1.39
|
|
6
|
-
runtime: versori
|
|
7
|
-
direction: ingestion
|
|
8
|
-
source: s3-csv
|
|
9
|
-
destination: fluent-graphql
|
|
10
|
-
entity: price
|
|
11
|
-
format: csv
|
|
12
|
-
logging: versori
|
|
13
|
-
status: stable
|
|
14
|
-
features:
|
|
15
|
-
- graphql-mutation-mapper
|
|
16
|
-
- memory-management
|
|
17
|
-
- enhanced-logging
|
|
18
|
-
- attribute-transformation
|
|
19
|
-
compliance: gold-standard
|
|
20
|
-
---
|
|
21
|
-
|
|
22
|
-
# Template: Ingestion - S3 CSV to Price GraphQL
|
|
23
|
-
n**Template Version:** 2.0.0
|
|
24
|
-
**SDK Version:** @fluentcommerce/fc-connect-sdk@^0.1.39
|
|
25
|
-
**Last Updated:** 2025-01-24
|
|
26
|
-
|
|
27
|
-
**🆕 Version 2.0.0 Enhancements:**
|
|
28
|
-
- ✅ **GraphQL Mutation Mapper** - Direct field mapping to mutation variables
|
|
29
|
-
- ✅ **Memory Management** - Clear large arrays after processing batches
|
|
30
|
-
- ✅ **Enhanced Logging** - Track mutation execution with emoji indicators
|
|
31
|
-
- ✅ **Attribute Transformation** - Handle complex nested data structures
|
|
32
|
-
|
|
33
|
-
## STEP 1: What This Template Does
|
|
34
|
-
|
|
35
|
-
This template provides a **scheduled Versori workflow** that:
|
|
36
|
-
|
|
37
|
-
1. **Discovers CSV files** from S3 bucket with price data
|
|
38
|
-
2. **Parses CSV** using `CSVParserService` with validation
|
|
39
|
-
3. **Validates products exist** in Fluent Commerce before updating prices
|
|
40
|
-
4. **Maps fields** using `GraphQLMutationMapper` with custom price validation resolvers
|
|
41
|
-
5. **Updates prices** via direct GraphQL mutations (`updateProduct`)
|
|
42
|
-
6. **Tracks price changes** for audit trail
|
|
43
|
-
7. **Archives processed files** to prevent duplicates
|
|
44
|
-
8. **Uses JobTracker** for job lifecycle management
|
|
45
|
-
|
|
46
|
-
**Key Features:**
|
|
47
|
-
- ✅ Direct GraphQL mutations (NOT Batch API)
|
|
48
|
-
- ✅ Product existence validation before price updates
|
|
49
|
-
- ✅ Price range validation (min/max checks)
|
|
50
|
-
- ✅ Multi-tier pricing support (DEFAULT, SALE, CLEARANCE, BULK)
|
|
51
|
-
- ✅ Configurable concurrency control (sequential or parallel)
|
|
52
|
-
- ✅ Optional GraphQL alias batching for high-volume scenarios
|
|
53
|
-
- ✅ Price change tracking and reporting
|
|
54
|
-
- ✅ Versori KV state management (duplicate prevention)
|
|
55
|
-
- ✅ File archival with error handling
|
|
56
|
-
|
|
57
|
-
**Use Cases:**
|
|
58
|
-
- Daily product price synchronization
|
|
59
|
-
- Promotional price updates
|
|
60
|
-
- Multi-currency pricing management
|
|
61
|
-
- Quantity-based pricing tiers
|
|
62
|
-
|
|
63
|
-
## STEP 2: Understanding This Template (AI Agent Guide)
|
|
64
|
-
|
|
65
|
-
**🎯 Template Purpose:** Scheduled price synchronization from S3 CSV files to Fluent Commerce using direct GraphQL mutations.
|
|
66
|
-
|
|
67
|
-
### Architecture Pattern
|
|
68
|
-
|
|
69
|
-
```
|
|
70
|
-
S3 CSV Files → CSVParserService → GraphQLMutationMapper → Product Validation → GraphQL Mutations → Archive
|
|
71
|
-
↓ ↓ ↓ ↓ ↓ ↓
|
|
72
|
-
File Discovery Parsing Field Mapping Existence Check updateProduct Processed/
|
|
73
|
-
Price Data Validation Price Validation Active Status Concurrency Ctrl Error Dirs
|
|
74
|
-
```
|
|
75
|
-
|
|
76
|
-
### SDK Services Used
|
|
77
|
-
|
|
78
|
-
| Service | Purpose | Key Methods |
|
|
79
|
-
|---------|---------|-------------|
|
|
80
|
-
| `createClient(ctx)` | Fluent API client | `client.graphql()` |
|
|
81
|
-
| `S3DataSource` | S3 operations | `listFiles()`, `downloadFile()`, `moveFile()` |
|
|
82
|
-
| `CSVParserService` | CSV parsing | `parse()` |
|
|
83
|
-
| `GraphQLMutationMapper` | Field transformation | `map()` with custom resolvers |
|
|
84
|
-
| `VersoriKVAdapter` | State management | `get()`, `set()`, `list()` |
|
|
85
|
-
| `JobTracker` | Job lifecycle | `createJob()`, `updateJob()`, `markCompleted()`, `markFailed()`, `getJob()` |
|
|
86
|
-
|
|
87
|
-
### Price Entity Structure
|
|
88
|
-
|
|
89
|
-
**Fluent Commerce Price Schema:**
|
|
90
|
-
```typescript
|
|
91
|
-
{
|
|
92
|
-
productRef: string; // SKU reference (required)
|
|
93
|
-
type: string; // DEFAULT, SALE, CLEARANCE, BULK (required)
|
|
94
|
-
value: number; // Price amount (required, validated)
|
|
95
|
-
currency: string; // ISO 4217 code (USD, EUR, GBP)
|
|
96
|
-
effectiveFrom?: string; // ISO 8601 date
|
|
97
|
-
effectiveTo?: string; // ISO 8601 date
|
|
98
|
-
minQuantity?: number; // Min qty for tier pricing
|
|
99
|
-
maxQuantity?: number; // Max qty for tier pricing
|
|
100
|
-
}
|
|
101
|
-
```
|
|
102
|
-
|
|
103
|
-
### GraphQL Mutation Pattern
|
|
104
|
-
|
|
105
|
-
**Mutation Used:** `updateProduct` (NOT Batch API)
|
|
106
|
-
|
|
107
|
-
```graphql
|
|
108
|
-
mutation UpdateProductPrice($input: UpdateProductInput!) {
|
|
109
|
-
updateProduct(input: $input) {
|
|
110
|
-
id
|
|
111
|
-
ref
|
|
112
|
-
prices {
|
|
113
|
-
type
|
|
114
|
-
value
|
|
115
|
-
currency
|
|
116
|
-
effectiveFrom
|
|
117
|
-
effectiveTo
|
|
118
|
-
minQuantity
|
|
119
|
-
maxQuantity
|
|
120
|
-
}
|
|
121
|
-
}
|
|
122
|
-
}
|
|
123
|
-
```
|
|
124
|
-
|
|
125
|
-
**Why Direct Mutations?**
|
|
126
|
-
- ✅ Immediate price updates (no batch processing delay)
|
|
127
|
-
- ✅ Product validation before update
|
|
128
|
-
- ✅ Price change tracking
|
|
129
|
-
- ✅ Suitable for low-volume price updates (<1000/day)
|
|
130
|
-
|
|
131
|
-
**NO BPP (Batch Pre-Processing):** Direct GraphQL mutations bypass batch workflows entirely.
|
|
132
|
-
|
|
133
|
-
### Custom Resolvers
|
|
134
|
-
|
|
135
|
-
**Three price-specific resolvers:**
|
|
136
|
-
|
|
137
|
-
1. **`custom.validatePriceRange`** - Ensures price within min/max bounds
|
|
138
|
-
2. **`custom.normalizePriceType`** - Validates price tier types
|
|
139
|
-
3. **`custom.normalizeQuantity`** - Validates quantity ranges for BULK pricing
|
|
140
|
-
|
|
141
|
-
### Product Validation Pattern
|
|
142
|
-
|
|
143
|
-
**Critical Step:** Validate product exists and is ACTIVE before price update:
|
|
144
|
-
|
|
145
|
-
```typescript
|
|
146
|
-
async function validateProductExists(client, productRef, retailerId, log) {
|
|
147
|
-
const query = `
|
|
148
|
-
query GetProduct($ref: String!, $retailerId: ID!) {
|
|
149
|
-
products(first: 1, ref: [$ref], retailerId: $retailerId) {
|
|
150
|
-
edges {
|
|
151
|
-
node { id ref status }
|
|
152
|
-
}
|
|
153
|
-
}
|
|
154
|
-
}
|
|
155
|
-
`;
|
|
156
|
-
const result = await client.graphql({ query, variables: { ref: productRef, retailerId } });
|
|
157
|
-
const product = result?.data?.products?.edges?.[0]?.node;
|
|
158
|
-
return product && product.status === 'ACTIVE';
|
|
159
|
-
}
|
|
160
|
-
```
|
|
161
|
-
|
|
162
|
-
**Why?** Prevents GraphQL errors for non-existent products.
|
|
163
|
-
|
|
164
|
-
### Concurrency Control & Performance
|
|
165
|
-
|
|
166
|
-
**Mutation execution** supports two performance modes:
|
|
167
|
-
|
|
168
|
-
**Mode 1: Concurrency Control (default)**
|
|
169
|
-
```typescript
|
|
170
|
-
// Configuration: Control concurrent mutations
|
|
171
|
-
const mutationBatchSize = parseInt(
|
|
172
|
-
activation?.getVariable('mutationBatchSize') || '1', // Default: 1 (sequential)
|
|
173
|
-
10
|
|
174
|
-
);
|
|
175
|
-
|
|
176
|
-
// Execute with bounded concurrency
|
|
177
|
-
await executeMutations(
|
|
178
|
-
client,
|
|
179
|
-
mapper,
|
|
180
|
-
priceRecords,
|
|
181
|
-
retailerId,
|
|
182
|
-
validateProducts,
|
|
183
|
-
mutationBatchSize, // 1=sequential, 3-10=parallel
|
|
184
|
-
undefined, // No alias batching
|
|
185
|
-
log
|
|
186
|
-
);
|
|
187
|
-
```
|
|
188
|
-
|
|
189
|
-
**Mode 2: GraphQL Alias Batching (optional, high-volume)**
|
|
190
|
-
```typescript
|
|
191
|
-
// Configuration: Group mutations into aliased requests
|
|
192
|
-
const mutationsPerAliasBatch = activation?.getVariable('mutationsPerAliasBatch')
|
|
193
|
-
? parseInt(activation?.getVariable('mutationsPerAliasBatch') || '1', 10)
|
|
194
|
-
: undefined; // Default: undefined (disabled)
|
|
195
|
-
|
|
196
|
-
// Execute with alias batching (reduces network overhead by ~80%)
|
|
197
|
-
await executeMutations(
|
|
198
|
-
client,
|
|
199
|
-
mapper,
|
|
200
|
-
priceRecords,
|
|
201
|
-
retailerId,
|
|
202
|
-
validateProducts,
|
|
203
|
-
mutationBatchSize, // Concurrency control
|
|
204
|
-
mutationsPerAliasBatch, // Alias batching (e.g., 5)
|
|
205
|
-
log
|
|
206
|
-
);
|
|
207
|
-
```
|
|
208
|
-
|
|
209
|
-
**Activation Variables:**
|
|
210
|
-
- `mutationBatchSize=1` - Default: sequential (safe)
|
|
211
|
-
- `mutationBatchSize=3-5` - Balanced throughput
|
|
212
|
-
- `mutationBatchSize=10` - High-volume (100+ locations)
|
|
213
|
-
- `mutationsPerAliasBatch` - Optional: Group mutations (e.g., 5 per request)
|
|
214
|
-
|
|
215
|
-
### State Management
|
|
216
|
-
|
|
217
|
-
**VersoriKVAdapter** prevents duplicate file processing:
|
|
218
|
-
|
|
219
|
-
```typescript
|
|
220
|
-
const stateKey = ['processed-files', 's3-price-sync', fileName];
|
|
221
|
-
const existing = await kv.get(stateKey);
|
|
222
|
-
if (existing) {
|
|
223
|
-
log.info('Skipping already processed file', { fileName });
|
|
224
|
-
continue;
|
|
225
|
-
}
|
|
226
|
-
// After success:
|
|
227
|
-
await kv.set(stateKey, { successful, failed, processedAt: new Date().toISOString() });
|
|
228
|
-
```
|
|
229
|
-
|
|
230
|
-
### JobTracker Integration
|
|
231
|
-
|
|
232
|
-
**Job lifecycle tracking:**
|
|
233
|
-
|
|
234
|
-
```typescript
|
|
235
|
-
const tracker = new JobTracker(openKv(':project:'), log);
|
|
236
|
-
await tracker.startJob(jobId, { mode: 'scheduled' });
|
|
237
|
-
try {
|
|
238
|
-
const result = await runPriceCsvWorkflow(ctx, jobId, tracker);
|
|
239
|
-
await tracker.completeJob(jobId, { result });
|
|
240
|
-
} catch (e) {
|
|
241
|
-
await tracker.failJob(jobId, { error: e.message });
|
|
242
|
-
}
|
|
243
|
-
```
|
|
244
|
-
|
|
245
|
-
### Price Change Tracking
|
|
246
|
-
|
|
247
|
-
**Audit trail for price updates:**
|
|
248
|
-
|
|
249
|
-
```typescript
|
|
250
|
-
const currentPrice = await getCurrentPrice(client, productRef, priceType, currency, retailerId);
|
|
251
|
-
// ... after update ...
|
|
252
|
-
if (currentPrice !== undefined && currentPrice !== priceData.value) {
|
|
253
|
-
results.priceChanges.push({
|
|
254
|
-
sku: priceData.productRef,
|
|
255
|
-
type: priceData.type,
|
|
256
|
-
oldPrice: currentPrice,
|
|
257
|
-
newPrice: priceData.value,
|
|
258
|
-
});
|
|
259
|
-
}
|
|
260
|
-
// Log changes at end:
|
|
261
|
-
log.info('Price changes detected', { count: results.priceChanges.length, changes });
|
|
262
|
-
```
|
|
263
|
-
|
|
264
|
-
### Schema Validation (CLI Commands)
|
|
265
|
-
|
|
266
|
-
**Before deployment, validate GraphQL schema:**
|
|
267
|
-
|
|
268
|
-
```bash
|
|
269
|
-
# 1. Introspect Fluent schema
|
|
270
|
-
fc-connect introspect-schema \
|
|
271
|
-
--url https://api.fluentcommerce.com/graphql \
|
|
272
|
-
--client-id CLIENT_ID \
|
|
273
|
-
--client-secret CLIENT_SECRET \
|
|
274
|
-
--output fluent-schema.json
|
|
275
|
-
|
|
276
|
-
# 2. Create mutation file
|
|
277
|
-
cat > price-mutation.graphql << 'EOF'
|
|
278
|
-
mutation UpdateProductPrice($input: UpdateProductInput!) {
|
|
279
|
-
updateProduct(input: $input) {
|
|
280
|
-
id
|
|
281
|
-
ref
|
|
282
|
-
prices { type value currency }
|
|
283
|
-
}
|
|
284
|
-
}
|
|
285
|
-
EOF
|
|
286
|
-
|
|
287
|
-
# 3. Generate mapping from mutation
|
|
288
|
-
fc-connect generate-mutation-mapping \
|
|
289
|
-
--file price-mutation.graphql \
|
|
290
|
-
--output price-mapping.json
|
|
291
|
-
|
|
292
|
-
# 4. Validate mapping against schema
|
|
293
|
-
fc-connect validate-schema \
|
|
294
|
-
--mapping price-mapping.json \
|
|
295
|
-
--schema fluent-schema.json
|
|
296
|
-
|
|
297
|
-
# 5. Analyze field coverage
|
|
298
|
-
fc-connect analyze-coverage \
|
|
299
|
-
--mapping price-mapping.json \
|
|
300
|
-
--schema fluent-schema.json
|
|
301
|
-
```
|
|
302
|
-
|
|
303
|
-
### Configuration File Structure
|
|
304
|
-
|
|
305
|
-
**Mapping Config:** `config/price-mapping.json`
|
|
306
|
-
|
|
307
|
-
**✅ PRODUCTION STANDARD:** External JSON file with GraphQLMutationMapper structure
|
|
308
|
-
|
|
309
|
-
```json
|
|
310
|
-
{
|
|
311
|
-
"mutation": "updateProduct",
|
|
312
|
-
"sourceFormat": "csv",
|
|
313
|
-
"version": "1.0.0",
|
|
314
|
-
"arguments": {
|
|
315
|
-
"input": {
|
|
316
|
-
"ref": {
|
|
317
|
-
"source": "sku",
|
|
318
|
-
"required": true,
|
|
319
|
-
"resolver": "trim"
|
|
320
|
-
},
|
|
321
|
-
"prices": {
|
|
322
|
-
"type": {
|
|
323
|
-
"source": "priceType",
|
|
324
|
-
"required": true,
|
|
325
|
-
"resolver": "normalizePriceType"
|
|
326
|
-
},
|
|
327
|
-
"value": {
|
|
328
|
-
"source": "amount",
|
|
329
|
-
"required": true,
|
|
330
|
-
"resolver": "validatePriceRange"
|
|
331
|
-
},
|
|
332
|
-
"currency": {
|
|
333
|
-
"source": "currency",
|
|
334
|
-
"required": true,
|
|
335
|
-
"resolver": "toUpperCase"
|
|
336
|
-
},
|
|
337
|
-
"effectiveFrom": {
|
|
338
|
-
"source": "effectiveDate",
|
|
339
|
-
"resolver": "formatDate"
|
|
340
|
-
},
|
|
341
|
-
"effectiveTo": {
|
|
342
|
-
"source": "expiryDate",
|
|
343
|
-
"resolver": "formatDate"
|
|
344
|
-
},
|
|
345
|
-
"minQuantity": {
|
|
346
|
-
"source": "minQuantity",
|
|
347
|
-
"resolver": "normalizeQuantity"
|
|
348
|
-
},
|
|
349
|
-
"maxQuantity": {
|
|
350
|
-
"source": "maxQuantity",
|
|
351
|
-
"resolver": "normalizeQuantity"
|
|
352
|
-
}
|
|
353
|
-
}
|
|
354
|
-
}
|
|
355
|
-
}
|
|
356
|
-
}
|
|
357
|
-
```
|
|
358
|
-
|
|
359
|
-
**Key Differences from UniversalMapper:**
|
|
360
|
-
- ✅ `mutation` property (not `mutationName`)
|
|
361
|
-
- ✅ `arguments.input` structure (matches GraphQL schema)
|
|
362
|
-
- ✅ Nested `prices` object (not flat productRef/type/value)
|
|
363
|
-
- ✅ Resolver names without `sdk.` or `custom.` prefix (`trim`, `toUpperCase`, `formatDate`)
|
|
364
|
-
- ✅ Used with `GraphQLMutationMapper.map()` method (not `UniversalMapper.map()`)
|
|
365
|
-
|
|
366
|
-
### Activation Variables Required
|
|
367
|
-
|
|
368
|
-
**Minimum Required:**
|
|
369
|
-
- `s3BucketName` - S3 bucket with price CSV files
|
|
370
|
-
- `awsAccessKeyId` - AWS credentials
|
|
371
|
-
- `awsSecretAccessKey` - AWS credentials
|
|
372
|
-
|
|
373
|
-
**Optional (with defaults):**
|
|
374
|
-
- `awsRegion=us-east-1`
|
|
375
|
-
- `s3Prefix=prices/`
|
|
376
|
-
- `archivePrefix=processed/`
|
|
377
|
-
- `errorPrefix=errors/`
|
|
378
|
-
- `logPrefix=logs/` - Where to write mutation logs
|
|
379
|
-
- `maxFilesToProcess=10`
|
|
380
|
-
- `mutationBatchSize=1` - Number of concurrent mutations (1=sequential, 3-10=parallel)
|
|
381
|
-
- `mutationsPerAliasBatch` - Optional: Number of mutations per aliased request (default: undefined = disabled)
|
|
382
|
-
- `minPrice=0.01`
|
|
383
|
-
- `maxPrice=999999.99`
|
|
384
|
-
- `validateProducts=true`
|
|
385
|
-
- `validateConnection=true` - Validate S3 connection at startup
|
|
386
|
-
- `enableFileTracking=true` - Enable file tracking via VersoriFileTracker
|
|
387
|
-
- `fluentRetailerId` - Optional: Only if mutation schema requires retailerId in input (most Price mutations don't need it)
|
|
388
|
-
|
|
389
|
-
### Error Handling Patterns
|
|
390
|
-
|
|
391
|
-
**Three-level error handling:**
|
|
392
|
-
|
|
393
|
-
1. **Mapping errors** - Invalid price data (validation failures)
|
|
394
|
-
2. **GraphQL errors** - Mutation failures (product not found, invalid fields)
|
|
395
|
-
3. **File processing errors** - S3 access issues, CSV parsing failures
|
|
396
|
-
|
|
397
|
-
**Error state tracking with exponential backoff:**
|
|
398
|
-
```typescript
|
|
399
|
-
const errorKey = ['error-state', fileName];
|
|
400
|
-
const attempts = (prev?.attemptCount || 0) + 1;
|
|
401
|
-
const backoffMinutes = Math.min(Math.pow(2, attempts) * 5, 24 * 60);
|
|
402
|
-
const nextRetryAt = new Date(Date.now() + backoffMinutes * 60000).toISOString();
|
|
403
|
-
```
|
|
404
|
-
|
|
405
|
-
### Common Pitfalls
|
|
406
|
-
|
|
407
|
-
**❌ WRONG Patterns:**
|
|
408
|
-
- Using Batch API for price updates (overhead, delay)
|
|
409
|
-
- Using `UniversalMapper` instead of `GraphQLMutationMapper`
|
|
410
|
-
- Calling `client.setRetailerId()` for GraphQL mutations (only needed for Job/Event API)
|
|
411
|
-
- Skipping product validation (causes GraphQL errors)
|
|
412
|
-
- No concurrency control (API throttling)
|
|
413
|
-
- Hardcoded price validation rules (not configurable)
|
|
414
|
-
- Missing price change tracking (no audit trail)
|
|
415
|
-
|
|
416
|
-
**✅ CORRECT Patterns:**
|
|
417
|
-
- Direct GraphQL mutations for immediate updates
|
|
418
|
-
- `GraphQLMutationMapper` with nested mapping structure
|
|
419
|
-
- NO `setRetailerId()` call (pass in mutation input if schema requires it)
|
|
420
|
-
- Product existence validation before updates
|
|
421
|
-
- Configurable concurrency control (mutationBatchSize)
|
|
422
|
-
- Optional alias batching for high-volume scenarios
|
|
423
|
-
- Custom resolvers for price validation
|
|
424
|
-
- Price change tracking for audit
|
|
425
|
-
- VersoriKVAdapter for state management
|
|
426
|
-
- JobTracker for job lifecycle
|
|
427
|
-
|
|
428
|
-
### Testing Checklist
|
|
429
|
-
|
|
430
|
-
**Before deployment:**
|
|
431
|
-
- [ ] S3 credentials validated (IAM permissions: ListBucket, GetObject, PutObject, DeleteObject)
|
|
432
|
-
- [ ] GraphQL schema validated using CLI tools
|
|
433
|
-
- [ ] Mapping configuration uses GraphQLMutationMapper structure (nested objects, not flat fields)
|
|
434
|
-
- [ ] Mapping configuration tested with sample CSV
|
|
435
|
-
- [ ] Product validation working (queries products endpoint)
|
|
436
|
-
- [ ] Price validation rules configured (min/max bounds)
|
|
437
|
-
- [ ] Concurrency control configured (mutationBatchSize)
|
|
438
|
-
- [ ] Optional alias batching tested if enabled (mutationsPerAliasBatch)
|
|
439
|
-
- [ ] NO setRetailerId() call present (only for Job/Event API)
|
|
440
|
-
- [ ] File duplicate prevention working (KV state)
|
|
441
|
-
- [ ] Price change tracking verified (logs price changes)
|
|
442
|
-
- [ ] Error handling tested (malformed CSV, invalid products)
|
|
443
|
-
- [ ] File archival working (processed/ and errors/ directories)
|
|
444
|
-
|
|
445
|
-
### Related Documentation
|
|
446
|
-
|
|
447
|
-
- **Core Guides:** `docs/02-CORE-GUIDES/ingestion/modules/`
|
|
448
|
-
- **Universal Mapping:** `docs/02-CORE-GUIDES/mapping/modules/`
|
|
449
|
-
- **CLI Tools:** `fc-connect-sdk/bin/readme.md`
|
|
450
|
-
- **State Management:** `docs/03-PATTERN-GUIDES/file-operations/`
|
|
451
|
-
- **Error Handling:** `docs/03-PATTERN-GUIDES/error-handling/`
|
|
452
|
-
|
|
453
|
-
**FC Connect SDK Use Case Guide**
|
|
454
|
-
|
|
455
|
-
> **SDK**: [@fluentcommerce/fc-connect-sdk](https://www.npmjs.com/package/@fluentcommerce/fc-connect-sdk)
|
|
456
|
-
> **Version**: Use latest - `npm install @fluentcommerce/fc-connect-sdk@latest`
|
|
457
|
-
|
|
458
|
-
**Context**: Scheduled Versori workflow that reads product price CSV files from S3 and creates/updates Fluent Commerce product prices via GraphQL mutations
|
|
459
|
-
|
|
460
|
-
**Complexity**: Medium-High
|
|
461
|
-
|
|
462
|
-
**Runtime**: Versori Platform (Scheduled)
|
|
463
|
-
|
|
464
|
-
**Estimated Lines**: ~700 lines
|
|
465
|
-
|
|
466
|
-
## What You'll Build
|
|
467
|
-
|
|
468
|
-
- Versori scheduled workflow (cron trigger)
|
|
469
|
-
- S3 file listing and download with retry logic
|
|
470
|
-
- CSV parsing with validation
|
|
471
|
-
- UniversalMapper for price field transformations
|
|
472
|
-
- Product existence validation before price updates
|
|
473
|
-
- GraphQL mutations for price upserts with rate limiting
|
|
474
|
-
- Versori KV state management (duplicate prevention)
|
|
475
|
-
- Price change tracking and reporting
|
|
476
|
-
- File archival after processing
|
|
477
|
-
|
|
478
|
-
## Versori Workflows Structure
|
|
479
|
-
|
|
480
|
-
**Key Concept**: Versori workflows are organized by **trigger type** at the first level, then by **specific workflow** with descriptive file names.
|
|
481
|
-
|
|
482
|
-
**Trigger Types:**
|
|
483
|
-
- **`schedule()`** → Time-based triggers (cron expressions) - NOT exposed as HTTP endpoints
|
|
484
|
-
- **`webhook()`** → HTTP-based triggers (event-driven) - Creates HTTP endpoints
|
|
485
|
-
- **`workflow()`** → Durable workflows (advanced, rarely used)
|
|
486
|
-
|
|
487
|
-
**Execution Steps (chained to triggers):**
|
|
488
|
-
- **`http()`** → External API calls (chained from schedule/webhook)
|
|
489
|
-
- **`fn()`** → Internal processing (chained from schedule/webhook)
|
|
490
|
-
|
|
491
|
-
### Recommended Project Structure
|
|
492
|
-
|
|
493
|
-
```
|
|
494
|
-
s3-csv-price-graphql/
|
|
495
|
-
├── index.ts # Entry point - exports all workflows
|
|
496
|
-
└── src/
|
|
497
|
-
├── workflows/
|
|
498
|
-
│ ├── scheduled/
|
|
499
|
-
│ │ └── daily-price-sync.ts # Scheduled: Daily price sync
|
|
500
|
-
│ │
|
|
501
|
-
│ └── webhook/
|
|
502
|
-
│ ├── adhoc-price-sync.ts # Webhook: Manual trigger
|
|
503
|
-
│ └── job-status-check.ts # Webhook: Status query
|
|
504
|
-
│
|
|
505
|
-
├── services/
|
|
506
|
-
│ └── price-sync.service.ts # Shared orchestration logic (reusable)
|
|
507
|
-
│
|
|
508
|
-
└── config/
|
|
509
|
-
└── price-mapping.json # GraphQL mapping config
|
|
510
|
-
```
|
|
511
|
-
|
|
512
|
-
---
|
|
513
|
-
|
|
514
|
-
## Workflow Files
|
|
515
|
-
|
|
516
|
-
### 1. Scheduled Workflows (`src/workflows/scheduled/`)
|
|
517
|
-
|
|
518
|
-
All time-based triggers that run automatically on cron schedules.
|
|
519
|
-
|
|
520
|
-
#### `src/workflows/scheduled/daily-price-sync.ts`
|
|
521
|
-
|
|
522
|
-
**Purpose**: Automatic daily price sync
|
|
523
|
-
**Trigger**: Cron schedule (`0 2 * * *`)
|
|
524
|
-
**Exposed as Endpoint**: ❌ NO - Runs automatically
|
|
525
|
-
|
|
526
|
-
```typescript
|
|
527
|
-
import { schedule, http } from '@versori/run';
|
|
528
|
-
import { JobTracker } from '@fluentcommerce/fc-connect-sdk';
|
|
529
|
-
import { executePriceSync } from '../../services/price-sync.service';
|
|
530
|
-
|
|
531
|
-
/**
|
|
532
|
-
* Scheduled Workflow: Daily Price Sync
|
|
533
|
-
*
|
|
534
|
-
* Runs automatically daily at 2 AM UTC
|
|
535
|
-
* NOT exposed as HTTP endpoint - Versori executes on schedule
|
|
536
|
-
*
|
|
537
|
-
* Uses shared service: price-sync.service.ts
|
|
538
|
-
*/
|
|
539
|
-
export const dailyPriceSync = schedule(
|
|
540
|
-
'price-sync-scheduled',
|
|
541
|
-
'0 2 * * *' // Daily at 2 AM UTC
|
|
542
|
-
).then(
|
|
543
|
-
http('run-price-sync', { connection: 'fluent_commerce' }, async (ctx: any) => {
|
|
544
|
-
const { log, openKv } = ctx;
|
|
545
|
-
const jobId = `price-sync-${Date.now()}`;
|
|
546
|
-
const executionStartTime = Date.now();
|
|
547
|
-
const tracker = new JobTracker(openKv(':project:'), log);
|
|
548
|
-
|
|
549
|
-
log.info('🔄 [WORKFLOW] Starting price sync', { jobId });
|
|
550
|
-
|
|
551
|
-
await tracker.createJob(jobId, {
|
|
552
|
-
triggeredBy: 'schedule',
|
|
553
|
-
stage: 'initialization',
|
|
554
|
-
startTime: executionStartTime,
|
|
555
|
-
});
|
|
556
|
-
await tracker.updateJob(jobId, { status: 'processing' });
|
|
557
|
-
|
|
558
|
-
try {
|
|
559
|
-
const result = await executePriceSync(ctx, jobId, tracker);
|
|
560
|
-
|
|
561
|
-
if (result.success) {
|
|
562
|
-
await tracker.markCompleted(jobId, result);
|
|
563
|
-
log.info('✅ [WORKFLOW] Price sync completed', {
|
|
564
|
-
jobId,
|
|
565
|
-
filesProcessed: result.filesProcessed,
|
|
566
|
-
duration: result.duration,
|
|
567
|
-
});
|
|
568
|
-
} else {
|
|
569
|
-
await tracker.markFailed(jobId, result.error || 'Unknown error');
|
|
570
|
-
log.error('❌ [WORKFLOW] Price sync failed', { jobId, error: result.error });
|
|
571
|
-
}
|
|
572
|
-
|
|
573
|
-
return { success: true, jobId, ...result };
|
|
574
|
-
} catch (e: any) {
|
|
575
|
-
const errorMessage = e instanceof Error ? e.message : String(e);
|
|
576
|
-
await tracker.markFailed(jobId, errorMessage);
|
|
577
|
-
log.error('❌ [WORKFLOW] Price sync failed with exception', {
|
|
578
|
-
jobId,
|
|
579
|
-
error: errorMessage,
|
|
580
|
-
stack: e instanceof Error ? e.stack : undefined,
|
|
581
|
-
});
|
|
582
|
-
return { success: false, jobId, error: errorMessage };
|
|
583
|
-
}
|
|
584
|
-
})
|
|
585
|
-
);
|
|
586
|
-
```
|
|
587
|
-
|
|
588
|
-
---
|
|
589
|
-
|
|
590
|
-
### 2. Webhook Workflows (`src/workflows/webhook/`)
|
|
591
|
-
|
|
592
|
-
All HTTP-based triggers that create webhook endpoints.
|
|
593
|
-
|
|
594
|
-
#### `src/workflows/webhook/adhoc-price-sync.ts`
|
|
595
|
-
|
|
596
|
-
**Purpose**: Manual price sync trigger (on-demand)
|
|
597
|
-
**Trigger**: HTTP POST
|
|
598
|
-
**Endpoint**: `POST https://{workspace}.versori.run/price-sync-adhoc`
|
|
599
|
-
**Use Cases**: Testing, priority processing, ad-hoc runs
|
|
600
|
-
|
|
601
|
-
```typescript
|
|
602
|
-
import { webhook, http } from '@versori/run';
|
|
603
|
-
import { JobTracker } from '@fluentcommerce/fc-connect-sdk';
|
|
604
|
-
import { executePriceSync } from '../../services/price-sync.service';
|
|
605
|
-
|
|
606
|
-
/**
|
|
607
|
-
* Webhook: Manual Price Sync Trigger
|
|
608
|
-
*
|
|
609
|
-
* Endpoint: POST https://{workspace}.versori.run/price-sync-adhoc
|
|
610
|
-
* Request body (optional): { filePattern: "urgent_*.csv", maxFiles: 5 }
|
|
611
|
-
*
|
|
612
|
-
* Pattern: webhook().then(http()) - needs Fluent API access
|
|
613
|
-
* Uses shared service: price-sync.service.ts
|
|
614
|
-
*
|
|
615
|
-
* SECURITY: Authentication handled via connection parameter
|
|
616
|
-
* No manual API key validation needed - Versori manages this via connection auth
|
|
617
|
-
*/
|
|
618
|
-
export const adhocPriceSync = webhook('price-sync-adhoc', {
|
|
619
|
-
response: { mode: 'sync' },
|
|
620
|
-
connection: 'price-sync-adhoc', // Versori validates API key
|
|
621
|
-
}).then(
|
|
622
|
-
http('run-price-sync-adhoc', { connection: 'fluent_commerce' }, async (ctx: any) => {
|
|
623
|
-
const { log, openKv, data } = ctx;
|
|
624
|
-
const jobId = `price-sync-adhoc-${Date.now()}`;
|
|
625
|
-
const executionStartTime = Date.now();
|
|
626
|
-
const tracker = new JobTracker(openKv(':project:'), log);
|
|
627
|
-
|
|
628
|
-
log.info('🔄 [WEBHOOK] Starting adhoc price sync', { jobId });
|
|
629
|
-
|
|
630
|
-
await tracker.createJob(jobId, {
|
|
631
|
-
triggeredBy: 'manual',
|
|
632
|
-
stage: 'initialization',
|
|
633
|
-
startTime: executionStartTime,
|
|
634
|
-
options: data // Optional: filePattern, maxFiles, etc.
|
|
635
|
-
});
|
|
636
|
-
await tracker.updateJob(jobId, { status: 'processing' });
|
|
637
|
-
|
|
638
|
-
try {
|
|
639
|
-
const result = await executePriceSync(ctx, jobId, tracker);
|
|
640
|
-
|
|
641
|
-
if (result.success) {
|
|
642
|
-
await tracker.markCompleted(jobId, result);
|
|
643
|
-
log.info('✅ [WEBHOOK] Adhoc price sync completed', {
|
|
644
|
-
jobId,
|
|
645
|
-
filesProcessed: result.filesProcessed,
|
|
646
|
-
duration: result.duration,
|
|
647
|
-
});
|
|
648
|
-
} else {
|
|
649
|
-
await tracker.markFailed(jobId, result.error || 'Unknown error');
|
|
650
|
-
log.error('❌ [WEBHOOK] Adhoc price sync failed', { jobId, error: result.error });
|
|
651
|
-
}
|
|
652
|
-
|
|
653
|
-
return { success: true, jobId, ...result };
|
|
654
|
-
} catch (e: any) {
|
|
655
|
-
const errorMessage = e instanceof Error ? e.message : String(e);
|
|
656
|
-
await tracker.markFailed(jobId, errorMessage);
|
|
657
|
-
log.error('❌ [WEBHOOK] Adhoc price sync failed with exception', {
|
|
658
|
-
jobId,
|
|
659
|
-
error: errorMessage,
|
|
660
|
-
stack: e instanceof Error ? e.stack : undefined,
|
|
661
|
-
});
|
|
662
|
-
return { success: false, jobId, error: errorMessage };
|
|
663
|
-
}
|
|
664
|
-
})
|
|
665
|
-
);
|
|
666
|
-
```
|
|
667
|
-
|
|
668
|
-
---
|
|
669
|
-
|
|
670
|
-
#### `src/workflows/webhook/job-status-check.ts`
|
|
671
|
-
|
|
672
|
-
**Purpose**: Query job status
|
|
673
|
-
**Trigger**: HTTP POST
|
|
674
|
-
**Endpoint**: `POST https://{workspace}.versori.run/price-sync-job-status`
|
|
675
|
-
**Request body**: `{ "jobId": "price-sync-1234567890" }`
|
|
676
|
-
|
|
677
|
-
```typescript
|
|
678
|
-
import { webhook, fn } from '@versori/run';
|
|
679
|
-
import { JobTracker } from '@fluentcommerce/fc-connect-sdk';
|
|
680
|
-
|
|
681
|
-
/**
|
|
682
|
-
* Webhook: Job Status Check
|
|
683
|
-
*
|
|
684
|
-
* Endpoint: POST https://{workspace}.versori.run/price-sync-job-status
|
|
685
|
-
* Request body: { "jobId": "price-sync-1234567890" }
|
|
686
|
-
*
|
|
687
|
-
* Pattern: webhook().then(fn()) - no external API needed, only KV storage
|
|
688
|
-
* Lightweight: Only queries KV store, no Fluent API calls
|
|
689
|
-
*
|
|
690
|
-
* SECURITY: Authentication handled via connection parameter
|
|
691
|
-
* No manual API key validation needed - Versori manages this via connection auth
|
|
692
|
-
*/
|
|
693
|
-
export const priceSyncJobStatus = webhook('price-sync-job-status', {
|
|
694
|
-
response: { mode: 'sync' },
|
|
695
|
-
connection: 'price-sync-job-status',
|
|
696
|
-
}).then(
|
|
697
|
-
fn('status', async (ctx: any) => {
|
|
698
|
-
const { data, log, openKv } = ctx;
|
|
699
|
-
const jobId = data?.jobId as string;
|
|
700
|
-
|
|
701
|
-
log.info('🔍 [STATUS] Checking job status', { jobId });
|
|
702
|
-
|
|
703
|
-
if (!jobId) {
|
|
704
|
-
log.warn('❌ [STATUS] Missing jobId in request');
|
|
705
|
-
return { success: false, error: 'jobId required' };
|
|
706
|
-
}
|
|
707
|
-
|
|
708
|
-
const tracker = new JobTracker(openKv(':project:'), log);
|
|
709
|
-
const status = await tracker.getJob(jobId);
|
|
710
|
-
|
|
711
|
-
if (status) {
|
|
712
|
-
log.info('✅ [STATUS] Job found', { jobId, status: status.status });
|
|
713
|
-
return { success: true, jobId, ...status };
|
|
714
|
-
} else {
|
|
715
|
-
log.warn('❌ [STATUS] Job not found', { jobId });
|
|
716
|
-
return { success: false, error: 'Job not found', jobId };
|
|
717
|
-
}
|
|
718
|
-
})
|
|
719
|
-
);
|
|
720
|
-
```
|
|
721
|
-
|
|
722
|
-
---
|
|
723
|
-
|
|
724
|
-
### 3. Entry Point (`index.ts`)
|
|
725
|
-
|
|
726
|
-
**Purpose**: Register all workflows with Versori platform
|
|
727
|
-
|
|
728
|
-
```typescript
|
|
729
|
-
/**
|
|
730
|
-
* Entry Point - Registers all workflows with Versori platform
|
|
731
|
-
*
|
|
732
|
-
* Versori automatically discovers and registers exported workflows
|
|
733
|
-
*
|
|
734
|
-
* File Structure:
|
|
735
|
-
* - src/workflows/scheduled/ → Time-based triggers (cron)
|
|
736
|
-
* - src/workflows/webhook/ → HTTP-based triggers (webhooks)
|
|
737
|
-
*/
|
|
738
|
-
|
|
739
|
-
// Import scheduled workflows
|
|
740
|
-
import { dailyPriceSync } from './workflows/scheduled/daily-price-sync';
|
|
741
|
-
|
|
742
|
-
// Import webhook workflows
|
|
743
|
-
import { adhocPriceSync } from './workflows/webhook/adhoc-price-sync';
|
|
744
|
-
import { priceSyncJobStatus } from './workflows/webhook/job-status-check';
|
|
745
|
-
|
|
746
|
-
// Register all workflows
|
|
747
|
-
export {
|
|
748
|
-
// Scheduled (time-based triggers)
|
|
749
|
-
dailyPriceSync,
|
|
750
|
-
|
|
751
|
-
// Webhooks (HTTP-based triggers)
|
|
752
|
-
adhocPriceSync,
|
|
753
|
-
priceSyncJobStatus,
|
|
754
|
-
};
|
|
755
|
-
```
|
|
756
|
-
|
|
757
|
-
**What Gets Exposed:**
|
|
758
|
-
- ✅ `adhocPriceSync` → `https://{workspace}.versori.run/price-sync-adhoc`
|
|
759
|
-
- ✅ `priceSyncJobStatus` → `https://{workspace}.versori.run/price-sync-job-status`
|
|
760
|
-
- ❌ `dailyPriceSync` → NOT exposed (runs automatically on cron)
|
|
761
|
-
|
|
762
|
-
---
|
|
763
|
-
|
|
764
|
-
## SDK Methods Used
|
|
765
|
-
|
|
766
|
-
**Core Services:**
|
|
767
|
-
- `createClient(ctx)` - Create Fluent client (auto-detects Versori context)
|
|
768
|
-
- `S3DataSource(config, log)` - S3 operations (list, download, upload, move)
|
|
769
|
-
- `CSVParserService()` - CSV parsing with validation
|
|
770
|
-
- `GraphQLMutationMapper(mappingConfig, log, { fluentClient: client })` - Field mapping with schema validation
|
|
771
|
-
- `VersoriKVAdapter(openKv(':project:'))` - KV state management for duplicate prevention
|
|
772
|
-
|
|
773
|
-
**Key Methods:**
|
|
774
|
-
- `s3.listFiles({ prefix, maxKeys })` - List S3 files
|
|
775
|
-
- `s3.downloadFile(path, options)` - Download file content
|
|
776
|
-
- `s3.uploadFile(path, content)` - Upload file (accepts string or Buffer)
|
|
777
|
-
- `s3.moveFile(source, dest)` - Archive or move file
|
|
778
|
-
- `parser.parse(content, options)` - Parse CSV to records
|
|
779
|
-
- `mapper.map(record)` - Transform single record
|
|
780
|
-
- `client.graphql({ query, variables })` - Execute GraphQL mutations/queries
|
|
781
|
-
- `kv.get(key)` / `kv.set(key, value)` - State management
|
|
782
|
-
|
|
783
|
-
**Critical Imports:**
|
|
784
|
-
- `Buffer` - **Required for Versori/Deno runtime** (S3 upload operations)
|
|
785
|
-
```typescript
|
|
786
|
-
import { Buffer } from 'node:buffer';
|
|
787
|
-
```
|
|
788
|
-
**Why:** Deno runtime (used by Versori) does not have `Buffer` as a global. However, `uploadFile()` accepts string or Buffer, so you can use strings directly without Buffer conversion.
|
|
789
|
-
|
|
790
|
-
**Service Functions (User-Defined):**
|
|
791
|
-
- `processFile()` - S3 download + CSV parsing + field mapping
|
|
792
|
-
- `executeMutations()` - GraphQL price updates with validation + rate limiting
|
|
793
|
-
- `writeMutationLog()` - Write execution log to S3 (uses Buffer)
|
|
794
|
-
|
|
795
|
-
## Sample CSV Input Data
|
|
796
|
-
|
|
797
|
-
**File**: `product-prices-20250122-001.csv`
|
|
798
|
-
|
|
799
|
-
```csv
|
|
800
|
-
sku,priceType,amount,currency,effectiveDate,expiryDate,minQuantity,maxQuantity
|
|
801
|
-
PROD-001,DEFAULT,29.99,USD,2025-01-22T00:00:00Z,,1,
|
|
802
|
-
PROD-002,SALE,19.99,USD,2025-01-22T00:00:00Z,2025-02-15T23:59:59Z,1,
|
|
803
|
-
PROD-003,CLEARANCE,9.99,USD,2025-01-22T00:00:00Z,2025-01-31T23:59:59Z,1,
|
|
804
|
-
PROD-004,DEFAULT,149.99,USD,2025-01-22T00:00:00Z,,1,
|
|
805
|
-
PROD-004,BULK,129.99,USD,2025-01-22T00:00:00Z,,10,99
|
|
806
|
-
PROD-004,BULK,119.99,USD,2025-01-22T00:00:00Z,,100,
|
|
807
|
-
```
|
|
808
|
-
|
|
809
|
-
**Note**: Products can have multiple price tiers. Same SKU with different `priceType` or quantity ranges = multiple rows.
|
|
810
|
-
|
|
811
|
-
**Field Mapping**:
|
|
812
|
-
|
|
813
|
-
- `sku` → Product reference (required) - must exist in Fluent
|
|
814
|
-
- `priceType` → Price tier (DEFAULT, SALE, CLEARANCE, BULK, etc.) (required)
|
|
815
|
-
- `amount` → Price value (required, must be > 0)
|
|
816
|
-
- `currency` → Currency code (USD, EUR, GBP, etc.) (required)
|
|
817
|
-
- `effectiveDate` → When price becomes active (ISO 8601 format)
|
|
818
|
-
- `expiryDate` → When price expires (optional, ISO 8601 format)
|
|
819
|
-
- `minQuantity` → Minimum quantity for this price tier (optional, default: 1)
|
|
820
|
-
- `maxQuantity` → Maximum quantity for this price tier (optional)
|
|
821
|
-
|
|
822
|
-
**Price Tier Logic**:
|
|
823
|
-
|
|
824
|
-
- **DEFAULT**: Standard retail price (most products)
|
|
825
|
-
- **SALE**: Temporary promotional price
|
|
826
|
-
- **CLEARANCE**: Final markdown price
|
|
827
|
-
- **BULK**: Quantity-based pricing (requires minQuantity/maxQuantity)
|
|
828
|
-
|
|
829
|
-
## Project Setup
|
|
830
|
-
|
|
831
|
-
```bash
|
|
832
|
-
mkdir versori-s3-csv-price-sync && cd $_
|
|
833
|
-
npm init -y
|
|
834
|
-
npm install @fluentcommerce/fc-connect-sdk@latest @versori/run
|
|
835
|
-
mkdir -p src
|
|
836
|
-
```
|
|
837
|
-
|
|
838
|
-
### Package Configuration (package.json)
|
|
839
|
-
|
|
840
|
-
```json
|
|
841
|
-
{
|
|
842
|
-
"name": "versori-s3-csv-price-sync",
|
|
843
|
-
"version": "1.0.0",
|
|
844
|
-
"description": "Versori workflow: S3 CSV price sync to Fluent GraphQL",
|
|
845
|
-
"versori": {
|
|
846
|
-
"workflows": "./src/index.ts"
|
|
847
|
-
},
|
|
848
|
-
"type": "module",
|
|
849
|
-
"scripts": {
|
|
850
|
-
"deploy": "versori deploy",
|
|
851
|
-
"logs": "versori logs"
|
|
852
|
-
},
|
|
853
|
-
"dependencies": {
|
|
854
|
-
"@fluentcommerce/fc-connect-sdk": "^0.1.39",
|
|
855
|
-
"@versori/run": "latest"
|
|
856
|
-
},
|
|
857
|
-
"devDependencies": {
|
|
858
|
-
"typescript": "^5.0.0",
|
|
859
|
-
"@types/node": "^20.0.0"
|
|
860
|
-
}
|
|
861
|
-
}
|
|
862
|
-
```
|
|
863
|
-
|
|
864
|
-
### Activation Variables (Versori)
|
|
865
|
-
|
|
866
|
-
```bash
|
|
867
|
-
# Required Variables
|
|
868
|
-
s3BucketName=my-price-bucket
|
|
869
|
-
awsAccessKeyId=AKIAXXXXXXXXXXXX
|
|
870
|
-
awsSecretAccessKey=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
|
|
871
|
-
retailerId=your-retailer-id
|
|
872
|
-
|
|
873
|
-
# Optional Variables (with defaults shown)
|
|
874
|
-
awsRegion=us-east-1
|
|
875
|
-
s3Prefix=prices/
|
|
876
|
-
archivePrefix=processed/
|
|
877
|
-
errorPrefix=errors/
|
|
878
|
-
logPrefix=logs/ # Where to write mutation logs
|
|
879
|
-
maxFilesToProcess=10
|
|
880
|
-
filePattern=.csv
|
|
881
|
-
enableArchival=true
|
|
882
|
-
mutationRateLimit=10
|
|
883
|
-
|
|
884
|
-
# Price Validation Rules
|
|
885
|
-
minPrice=0.01 # Minimum allowed price
|
|
886
|
-
maxPrice=999999.99 # Maximum allowed price
|
|
887
|
-
validateProducts=true # Validate products exist before updating prices
|
|
888
|
-
```
|
|
889
|
-
|
|
890
|
-
## Complete Workflow (src/index.ts)
|
|
891
|
-
|
|
892
|
-
```typescript
|
|
893
|
-
import { schedule, webhook, http, fn } from '@versori/run';
|
|
894
|
-
import { Buffer } from 'node:buffer'; // Required for Versori/Deno runtime
|
|
895
|
-
import {
|
|
896
|
-
createClient,
|
|
897
|
-
S3DataSource,
|
|
898
|
-
CSVParserService,
|
|
899
|
-
GraphQLMutationMapper,
|
|
900
|
-
VersoriFileTracker,
|
|
901
|
-
JobTracker,
|
|
902
|
-
} from '@fluentcommerce/fc-connect-sdk';
|
|
903
|
-
|
|
904
|
-
// ============================================================================
|
|
905
|
-
// Type Definitions
|
|
906
|
-
// ============================================================================
|
|
907
|
-
|
|
908
|
-
interface FileProcessingResult {
|
|
909
|
-
fileName: string;
|
|
910
|
-
successful: number;
|
|
911
|
-
failed: number;
|
|
912
|
-
validationFailed: number;
|
|
913
|
-
priceChanges: Array<{
|
|
914
|
-
sku: string;
|
|
915
|
-
type: string;
|
|
916
|
-
oldPrice?: number;
|
|
917
|
-
newPrice: number;
|
|
918
|
-
}>;
|
|
919
|
-
errors: string[];
|
|
920
|
-
}
|
|
921
|
-
|
|
922
|
-
interface MutationResult {
|
|
923
|
-
successful: number;
|
|
924
|
-
failed: number;
|
|
925
|
-
priceChanges: Array<{
|
|
926
|
-
sku: string;
|
|
927
|
-
type: string;
|
|
928
|
-
oldPrice?: number;
|
|
929
|
-
newPrice: number;
|
|
930
|
-
}>;
|
|
931
|
-
}
|
|
932
|
-
|
|
933
|
-
interface MappedPriceData {
|
|
934
|
-
productRef: string;
|
|
935
|
-
type: string;
|
|
936
|
-
value: number;
|
|
937
|
-
currency: string;
|
|
938
|
-
effectiveFrom?: string;
|
|
939
|
-
effectiveTo?: string;
|
|
940
|
-
minQuantity?: number;
|
|
941
|
-
maxQuantity?: number;
|
|
942
|
-
}
|
|
943
|
-
|
|
944
|
-
// ============================================================================
|
|
945
|
-
// Utility Functions
|
|
946
|
-
// ============================================================================
|
|
947
|
-
|
|
948
|
-
/**
|
|
949
|
-
* Retry utility with exponential backoff
|
|
950
|
-
* (User-defined - not part of SDK public API)
|
|
951
|
-
*/
|
|
952
|
-
async function retryWithBackoff<T>(
|
|
953
|
-
operation: () => Promise<T>,
|
|
954
|
-
maxRetries = 3,
|
|
955
|
-
baseDelayMs = 1000
|
|
956
|
-
): Promise<T> {
|
|
957
|
-
let lastError: any;
|
|
958
|
-
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
959
|
-
try {
|
|
960
|
-
return await operation();
|
|
961
|
-
} catch (error) {
|
|
962
|
-
lastError = error;
|
|
963
|
-
if (attempt < maxRetries - 1) {
|
|
964
|
-
const delay = baseDelayMs * Math.pow(2, attempt) + Math.floor(Math.random() * 200);
|
|
965
|
-
await new Promise(resolve => setTimeout(resolve, delay));
|
|
966
|
-
}
|
|
967
|
-
}
|
|
968
|
-
}
|
|
969
|
-
throw lastError;
|
|
970
|
-
}
|
|
971
|
-
|
|
972
|
-
/**
|
|
973
|
-
* Validate product exists in Fluent Commerce
|
|
974
|
-
*/
|
|
975
|
-
async function validateProductExists(
|
|
976
|
-
client: any,
|
|
977
|
-
productRef: string,
|
|
978
|
-
retailerId: string,
|
|
979
|
-
log: any
|
|
980
|
-
): Promise<boolean> {
|
|
981
|
-
const query = `
|
|
982
|
-
query GetProduct($ref: String!, $retailerId: ID!) {
|
|
983
|
-
products(first: 1, ref: [$ref], retailerId: $retailerId) {
|
|
984
|
-
edges {
|
|
985
|
-
node {
|
|
986
|
-
id
|
|
987
|
-
ref
|
|
988
|
-
status
|
|
989
|
-
}
|
|
990
|
-
}
|
|
991
|
-
}
|
|
992
|
-
}
|
|
993
|
-
`;
|
|
994
|
-
|
|
995
|
-
try {
|
|
996
|
-
const result = await client.graphql({
|
|
997
|
-
query,
|
|
998
|
-
variables: { ref: productRef, retailerId },
|
|
999
|
-
});
|
|
1000
|
-
|
|
1001
|
-
const product = result?.data?.products?.edges?.[0]?.node;
|
|
1002
|
-
if (!product) {
|
|
1003
|
-
log.warn(`Product not found: ${productRef}`);
|
|
1004
|
-
return false;
|
|
1005
|
-
}
|
|
1006
|
-
|
|
1007
|
-
if (product.status !== 'ACTIVE') {
|
|
1008
|
-
log.warn(`Product not active: ${productRef} (status: ${product.status})`);
|
|
1009
|
-
return false;
|
|
1010
|
-
}
|
|
1011
|
-
|
|
1012
|
-
return true;
|
|
1013
|
-
} catch (error: unknown) {
|
|
1014
|
-
// ✅ Enhanced error logging: Extract all error details for visibility
|
|
1015
|
-
const errorMsg = error instanceof Error ? error.message : String(error);
|
|
1016
|
-
const errorDetails = {
|
|
1017
|
-
message: errorMsg,
|
|
1018
|
-
stack: error instanceof Error ? error.stack : undefined,
|
|
1019
|
-
errorType: error instanceof Error ? error.constructor.name : 'Error',
|
|
1020
|
-
};
|
|
1021
|
-
log.error(`Failed to validate product: ${productRef}`, errorDetails);
|
|
1022
|
-
return false;
|
|
1023
|
-
}
|
|
1024
|
-
}
|
|
1025
|
-
|
|
1026
|
-
/**
|
|
1027
|
-
* Get current product price for change tracking
|
|
1028
|
-
*/
|
|
1029
|
-
async function getCurrentPrice(
|
|
1030
|
-
client: any,
|
|
1031
|
-
productRef: string,
|
|
1032
|
-
priceType: string,
|
|
1033
|
-
currency: string,
|
|
1034
|
-
retailerId: string
|
|
1035
|
-
): Promise<number | undefined> {
|
|
1036
|
-
try {
|
|
1037
|
-
const query = `
|
|
1038
|
-
query GetProductPrice($ref: String!, $retailerId: ID!) {
|
|
1039
|
-
products(first: 1, ref: [$ref], retailerId: $retailerId) {
|
|
1040
|
-
edges {
|
|
1041
|
-
node {
|
|
1042
|
-
id
|
|
1043
|
-
ref
|
|
1044
|
-
prices {
|
|
1045
|
-
type
|
|
1046
|
-
value
|
|
1047
|
-
currency
|
|
1048
|
-
}
|
|
1049
|
-
}
|
|
1050
|
-
}
|
|
1051
|
-
}
|
|
1052
|
-
}
|
|
1053
|
-
`;
|
|
1054
|
-
|
|
1055
|
-
const result = await client.graphql({
|
|
1056
|
-
query,
|
|
1057
|
-
variables: { ref: productRef, retailerId },
|
|
1058
|
-
});
|
|
1059
|
-
|
|
1060
|
-
const product = result?.data?.products?.edges?.[0]?.node;
|
|
1061
|
-
if (product?.prices) {
|
|
1062
|
-
const existingPrice = product.prices.find(
|
|
1063
|
-
(p: any) => p.type === priceType && p.currency === currency
|
|
1064
|
-
);
|
|
1065
|
-
if (existingPrice) {
|
|
1066
|
-
return existingPrice.value;
|
|
1067
|
-
}
|
|
1068
|
-
}
|
|
1069
|
-
} catch (err) {
|
|
1070
|
-
// Ignore - price change tracking is optional
|
|
1071
|
-
}
|
|
1072
|
-
return undefined;
|
|
1073
|
-
}
|
|
1074
|
-
|
|
1075
|
-
// ============================================================================
|
|
1076
|
-
// Service Function 1: Process File (S3 + CSV + Mapper)
|
|
1077
|
-
// ============================================================================
|
|
1078
|
-
|
|
1079
|
-
/**
|
|
1080
|
-
* Process single CSV file: Download from S3, parse CSV, map fields to Price schema
|
|
1081
|
-
*
|
|
1082
|
-
* @param s3 - S3DataSource instance
|
|
1083
|
-
* @param parser - CSVParserService instance
|
|
1084
|
-
* @param mapper - UniversalMapper instance (with custom price resolvers)
|
|
1085
|
-
* @param filePath - S3 file path
|
|
1086
|
-
* @param fileName - File name
|
|
1087
|
-
* @param log - Logger instance
|
|
1088
|
-
* @returns FileProcessingResult with parsed and mapped price data
|
|
1089
|
-
*/
|
|
1090
|
-
async function processFile(
|
|
1091
|
-
s3: S3DataSource,
|
|
1092
|
-
parser: CSVParserService,
|
|
1093
|
-
mapper: GraphQLMutationMapper,
|
|
1094
|
-
filePath: string,
|
|
1095
|
-
fileName: string,
|
|
1096
|
-
log: any
|
|
1097
|
-
): Promise<{ records: Array<{ query: string; variables: any; input: any }>; errors: string[] }> {
|
|
1098
|
-
log.info('Processing file', { fileName });
|
|
1099
|
-
|
|
1100
|
-
// Download CSV from S3 with retry
|
|
1101
|
-
const content = await retryWithBackoff(
|
|
1102
|
-
() => s3.downloadFile(filePath, { encoding: 'utf8' }) as Promise<string>
|
|
1103
|
-
);
|
|
1104
|
-
|
|
1105
|
-
// Parse CSV
|
|
1106
|
-
const rawRecords = await parser.parse(content, {
|
|
1107
|
-
columns: true,
|
|
1108
|
-
skip_empty_lines: true,
|
|
1109
|
-
trim: true,
|
|
1110
|
-
});
|
|
1111
|
-
|
|
1112
|
-
if (!rawRecords.length) {
|
|
1113
|
-
log.warn('Empty CSV file', { fileName });
|
|
1114
|
-
return { records: [], errors: [] };
|
|
1115
|
-
}
|
|
1116
|
-
|
|
1117
|
-
log.info('CSV parsed', { fileName, recordCount: rawRecords.length });
|
|
1118
|
-
|
|
1119
|
-
// Map CSV records to Price schema
|
|
1120
|
-
const mappedRecords: Array<{ query: string; variables: any; input: any }> = [];
|
|
1121
|
-
const errors: string[] = [];
|
|
1122
|
-
|
|
1123
|
-
// ✅ PRODUCTION ENHANCEMENT: Log transformation start
|
|
1124
|
-
log.info('Transforming records to GraphQL mutations', {
|
|
1125
|
-
fileName,
|
|
1126
|
-
totalRecords: rawRecords.length,
|
|
1127
|
-
});
|
|
1128
|
-
|
|
1129
|
-
for (let i = 0; i < rawRecords.length; i++) {
|
|
1130
|
-
const rec = rawRecords[i];
|
|
1131
|
-
const recordNumber = i + 1;
|
|
1132
|
-
|
|
1133
|
-
// ✅ PRODUCTION ENHANCEMENT: Log progress every 50 records
|
|
1134
|
-
if (recordNumber % 50 === 0) {
|
|
1135
|
-
log.info(`📤 Transforming record ${recordNumber}/${rawRecords.length}`, {
|
|
1136
|
-
fileName,
|
|
1137
|
-
recordNumber,
|
|
1138
|
-
totalRecords: rawRecords.length,
|
|
1139
|
-
validSoFar: mappedRecords.length,
|
|
1140
|
-
errorsSoFar: errors.length,
|
|
1141
|
-
progress: `${((recordNumber / rawRecords.length) * 100).toFixed(1)}%`,
|
|
1142
|
-
});
|
|
1143
|
-
}
|
|
1144
|
-
|
|
1145
|
-
try {
|
|
1146
|
-
// GraphQLMutationMapper returns { query, variables } directly
|
|
1147
|
-
const mapped = await mapper.map(rec);
|
|
1148
|
-
|
|
1149
|
-
mappedRecords.push({
|
|
1150
|
-
query: mapped.query,
|
|
1151
|
-
variables: mapped.variables,
|
|
1152
|
-
input: mapped.variables.input || mapped.variables,
|
|
1153
|
-
});
|
|
1154
|
-
} catch (error: unknown) {
|
|
1155
|
-
const errorMsg = error instanceof Error ? error.message : String(error);
|
|
1156
|
-
errors.push(`Row ${recordNumber}: ${errorMsg}`);
|
|
1157
|
-
}
|
|
1158
|
-
}
|
|
1159
|
-
|
|
1160
|
-
log.info('Mapping completed', {
|
|
1161
|
-
fileName,
|
|
1162
|
-
successful: mappedRecords.length,
|
|
1163
|
-
failed: errors.length,
|
|
1164
|
-
totalRecords: rawRecords.length,
|
|
1165
|
-
successRate: `${((mappedRecords.length / rawRecords.length) * 100).toFixed(1)}%`,
|
|
1166
|
-
});
|
|
1167
|
-
|
|
1168
|
-
return { records: mappedRecords, errors };
|
|
1169
|
-
}
|
|
1170
|
-
|
|
1171
|
-
// ============================================================================
|
|
1172
|
-
// Service Function 2: Execute Mutations (GraphQL Price Updates)
|
|
1173
|
-
// ============================================================================
|
|
1174
|
-
|
|
1175
|
-
/**
|
|
1176
|
-
* Execute GraphQL mutations for price updates with validation and rate limiting
|
|
1177
|
-
* ✅ Supports alias batching for improved performance
|
|
1178
|
-
*
|
|
1179
|
-
* @param client - FluentClient instance
|
|
1180
|
-
* @param mapper - GraphQLMutationMapper instance
|
|
1181
|
-
* @param priceRecords - Mapped price data with query and variables
|
|
1182
|
-
* @param retailerId - Fluent retailer ID
|
|
1183
|
-
* @param validateProducts - Whether to validate product existence
|
|
1184
|
-
* @param batchSize - Number of concurrent requests (concurrency control)
|
|
1185
|
-
* @param mutationsPerAliasBatch - Optional: Number of mutations per aliased request (alias batching)
|
|
1186
|
-
* @param log - Logger instance
|
|
1187
|
-
* @returns MutationResult with success/failure counts and price changes
|
|
1188
|
-
*/
|
|
1189
|
-
async function executeMutations(
|
|
1190
|
-
client: any,
|
|
1191
|
-
mapper: GraphQLMutationMapper,
|
|
1192
|
-
priceRecords: Array<{ query: string; variables: any; input: any }>,
|
|
1193
|
-
retailerId: string,
|
|
1194
|
-
validateProducts: boolean,
|
|
1195
|
-
batchSize: number = 1, // ✅ Default: 1 (sequential)
|
|
1196
|
-
mutationsPerAliasBatch?: number, // ✅ NEW: Alias batching parameter (default: undefined = disabled)
|
|
1197
|
-
log: any
|
|
1198
|
-
): Promise<MutationResult> {
|
|
1199
|
-
let successful = 0;
|
|
1200
|
-
let failed = 0;
|
|
1201
|
-
const priceChanges: Array<{
|
|
1202
|
-
sku: string;
|
|
1203
|
-
type: string;
|
|
1204
|
-
oldPrice?: number;
|
|
1205
|
-
newPrice: number;
|
|
1206
|
-
}> = [];
|
|
1207
|
-
|
|
1208
|
-
// ✅ Use alias batching if enabled
|
|
1209
|
-
const useAliases = mutationsPerAliasBatch && mutationsPerAliasBatch > 1;
|
|
1210
|
-
|
|
1211
|
-
if (useAliases) {
|
|
1212
|
-
// Process with alias batching
|
|
1213
|
-
const results = await executeMutationsWithAliases(
|
|
1214
|
-
priceRecords,
|
|
1215
|
-
client,
|
|
1216
|
-
mapper,
|
|
1217
|
-
log,
|
|
1218
|
-
retailerId,
|
|
1219
|
-
batchSize,
|
|
1220
|
-
mutationsPerAliasBatch,
|
|
1221
|
-
'updateProduct'
|
|
1222
|
-
);
|
|
1223
|
-
successful = results.executed;
|
|
1224
|
-
failed = results.failed;
|
|
1225
|
-
// Note: Price changes tracking would need to be done separately if needed
|
|
1226
|
-
} else {
|
|
1227
|
-
// Process individually with validation
|
|
1228
|
-
for (const priceData of priceRecords) {
|
|
1229
|
-
try {
|
|
1230
|
-
// Validate product exists (if enabled)
|
|
1231
|
-
if (validateProducts) {
|
|
1232
|
-
const productExists = await validateProductExists(
|
|
1233
|
-
client,
|
|
1234
|
-
priceData.input.productRef,
|
|
1235
|
-
retailerId,
|
|
1236
|
-
log
|
|
1237
|
-
);
|
|
1238
|
-
|
|
1239
|
-
if (!productExists) {
|
|
1240
|
-
log.warn(`Skipping price update for non-existent product: ${priceData.input.productRef}`);
|
|
1241
|
-
failed++;
|
|
1242
|
-
continue;
|
|
1243
|
-
}
|
|
1244
|
-
}
|
|
1245
|
-
|
|
1246
|
-
// Get current price for change tracking
|
|
1247
|
-
const currentPrice = await getCurrentPrice(
|
|
1248
|
-
client,
|
|
1249
|
-
priceData.input.productRef,
|
|
1250
|
-
priceData.input.type,
|
|
1251
|
-
priceData.input.currency,
|
|
1252
|
-
retailerId
|
|
1253
|
-
);
|
|
1254
|
-
|
|
1255
|
-
// Execute mutation using pre-generated query and variables
|
|
1256
|
-
await retryWithBackoff(() =>
|
|
1257
|
-
client.graphql({
|
|
1258
|
-
query: priceData.query,
|
|
1259
|
-
variables: priceData.variables,
|
|
1260
|
-
})
|
|
1261
|
-
);
|
|
1262
|
-
|
|
1263
|
-
successful++;
|
|
1264
|
-
|
|
1265
|
-
// Track price change
|
|
1266
|
-
if (currentPrice !== undefined && currentPrice !== priceData.input.value) {
|
|
1267
|
-
priceChanges.push({
|
|
1268
|
-
sku: priceData.input.productRef,
|
|
1269
|
-
type: priceData.input.type,
|
|
1270
|
-
oldPrice: currentPrice,
|
|
1271
|
-
newPrice: priceData.input.value,
|
|
1272
|
-
});
|
|
1273
|
-
}
|
|
1274
|
-
|
|
1275
|
-
log.info('Price updated', {
|
|
1276
|
-
ref: priceData.input.productRef,
|
|
1277
|
-
type: priceData.input.type,
|
|
1278
|
-
oldPrice: currentPrice,
|
|
1279
|
-
newPrice: priceData.input.value,
|
|
1280
|
-
currency: priceData.input.currency,
|
|
1281
|
-
});
|
|
1282
|
-
} catch (error: unknown) {
|
|
1283
|
-
failed++;
|
|
1284
|
-
const errorMsg = error instanceof Error ? error.message : String(error);
|
|
1285
|
-
const errorDetails = {
|
|
1286
|
-
message: errorMsg,
|
|
1287
|
-
stack: error instanceof Error ? error.stack : undefined,
|
|
1288
|
-
errorType: error instanceof Error ? error.constructor.name : 'Error',
|
|
1289
|
-
};
|
|
1290
|
-
log.error('Failed to update price', errorDetails, {
|
|
1291
|
-
ref: priceData.input?.productRef,
|
|
1292
|
-
type: priceData.input?.type,
|
|
1293
|
-
});
|
|
1294
|
-
}
|
|
1295
|
-
}
|
|
1296
|
-
}
|
|
1297
|
-
|
|
1298
|
-
return { successful, failed, priceChanges };
|
|
1299
|
-
}
|
|
1300
|
-
|
|
1301
|
-
/**
|
|
1302
|
-
* ✅ NEW: Execute mutations with GraphQL alias batching
|
|
1303
|
-
*/
|
|
1304
|
-
async function executeMutationsWithAliases(
|
|
1305
|
-
priceRecords: Array<{ query: string; variables: any; input: any }>,
|
|
1306
|
-
client: any,
|
|
1307
|
-
mapper: GraphQLMutationMapper,
|
|
1308
|
-
log: any,
|
|
1309
|
-
retailerId: string,
|
|
1310
|
-
maxParallel: number,
|
|
1311
|
-
mutationsPerAliasBatch: number,
|
|
1312
|
-
mutationName: string
|
|
1313
|
-
): Promise<{ executed: number; failed: number; errors: string[] }> {
|
|
1314
|
-
const results = { executed: 0, failed: 0, errors: [] as string[] };
|
|
1315
|
-
|
|
1316
|
-
const aliasBatches: Array<Array<typeof priceRecords[0]>> = [];
|
|
1317
|
-
for (let i = 0; i < priceRecords.length; i += mutationsPerAliasBatch) {
|
|
1318
|
-
aliasBatches.push(priceRecords.slice(i, i + mutationsPerAliasBatch));
|
|
1319
|
-
}
|
|
1320
|
-
|
|
1321
|
-
log.info(`Processing ${aliasBatches.length} alias batches`, {
|
|
1322
|
-
totalPrices: priceRecords.length,
|
|
1323
|
-
maxParallel,
|
|
1324
|
-
});
|
|
1325
|
-
|
|
1326
|
-
for (let i = 0; i < aliasBatches.length; i += maxParallel) {
|
|
1327
|
-
const concurrentBatches = aliasBatches.slice(i, i + maxParallel);
|
|
1328
|
-
|
|
1329
|
-
const batchResults = await Promise.allSettled(
|
|
1330
|
-
concurrentBatches.map(async (batch) => {
|
|
1331
|
-
const { query, variables } = buildAliasedBatch(batch, mutationName, retailerId);
|
|
1332
|
-
const response = await retryWithBackoff(() => client.graphql({ query, variables }), log);
|
|
1333
|
-
return parseAliasResponse(response, batch, mutationName);
|
|
1334
|
-
})
|
|
1335
|
-
);
|
|
1336
|
-
|
|
1337
|
-
batchResults.forEach((result, idx) => {
|
|
1338
|
-
if (result.status === 'fulfilled') {
|
|
1339
|
-
const batchResult = result.value;
|
|
1340
|
-
results.executed += batchResult.executed;
|
|
1341
|
-
results.failed += batchResult.failed;
|
|
1342
|
-
results.errors.push(...batchResult.errors);
|
|
1343
|
-
} else {
|
|
1344
|
-
const batch = concurrentBatches[idx];
|
|
1345
|
-
const errorMsg = result.reason instanceof Error ? result.reason.message : String(result.reason);
|
|
1346
|
-
batch.forEach(price => {
|
|
1347
|
-
results.failed++;
|
|
1348
|
-
const productRef = price.input?.productRef || 'unknown';
|
|
1349
|
-
results.errors.push(`Failed to update price for ${productRef}: ${errorMsg}`);
|
|
1350
|
-
});
|
|
1351
|
-
}
|
|
1352
|
-
});
|
|
1353
|
-
|
|
1354
|
-
if (i + maxParallel < aliasBatches.length) {
|
|
1355
|
-
await new Promise(resolve => setTimeout(resolve, 500));
|
|
1356
|
-
}
|
|
1357
|
-
}
|
|
1358
|
-
|
|
1359
|
-
return results;
|
|
1360
|
-
}
|
|
1361
|
-
|
|
1362
|
-
/**
|
|
1363
|
-
* ✅ NEW: Build aliased batch query and variables
|
|
1364
|
-
*/
|
|
1365
|
-
function buildAliasedBatch(
|
|
1366
|
-
batch: Array<{ query: string; variables: any; input: any }>,
|
|
1367
|
-
mutationName: string,
|
|
1368
|
-
retailerId: string
|
|
1369
|
-
): { query: string; variables: Record<string, any> } {
|
|
1370
|
-
const batchSize = batch.length;
|
|
1371
|
-
const inputTypeName = `${mutationName.charAt(0).toUpperCase() + mutationName.slice(1)}Input`;
|
|
1372
|
-
|
|
1373
|
-
const variables = Array.from({ length: batchSize }, (_, i) =>
|
|
1374
|
-
`$input${i + 1}: ${inputTypeName}!`
|
|
1375
|
-
).join(', ');
|
|
1376
|
-
|
|
1377
|
-
const aliasedMutations = Array.from({ length: batchSize }, (_, i) => {
|
|
1378
|
-
const alias = `${mutationName}${i + 1}`;
|
|
1379
|
-
return ` ${alias}: ${mutationName}(input: $input${i + 1}) { id ref }`;
|
|
1380
|
-
}).join('\n');
|
|
1381
|
-
|
|
1382
|
-
const operationName = `Batch${mutationName.charAt(0).toUpperCase() + mutationName.slice(1)}s`;
|
|
1383
|
-
const query = `mutation ${operationName}(${variables}) {\n${aliasedMutations}\n}`;
|
|
1384
|
-
|
|
1385
|
-
const variablesObj: Record<string, any> = {};
|
|
1386
|
-
batch.forEach((price, index) => {
|
|
1387
|
-
const input = price.variables.input || price.variables;
|
|
1388
|
-
variablesObj[`input${index + 1}`] = input;
|
|
1389
|
-
});
|
|
1390
|
-
|
|
1391
|
-
return { query, variables: variablesObj };
|
|
1392
|
-
}
|
|
1393
|
-
|
|
1394
|
-
/**
|
|
1395
|
-
* ✅ NEW: Parse aliased GraphQL response
|
|
1396
|
-
*/
|
|
1397
|
-
function parseAliasResponse(
|
|
1398
|
-
response: any,
|
|
1399
|
-
batch: Array<{ query: string; variables: any; input: any }>,
|
|
1400
|
-
mutationName: string
|
|
1401
|
-
): { executed: number; failed: number; errors: string[] } {
|
|
1402
|
-
const result = { executed: 0, failed: 0, errors: [] as string[] };
|
|
1403
|
-
|
|
1404
|
-
const data = response.data || {};
|
|
1405
|
-
const errors = response.errors || [];
|
|
1406
|
-
|
|
1407
|
-
batch.forEach((price, index) => {
|
|
1408
|
-
const alias = `${mutationName}${index + 1}`;
|
|
1409
|
-
const aliasData = data[alias];
|
|
1410
|
-
const aliasErrors = errors.filter((e: unknown) =>
|
|
1411
|
-
e && typeof e === 'object' && 'path' in e && Array.isArray((e as any).path) && (e as any).path.includes(alias)
|
|
1412
|
-
);
|
|
1413
|
-
|
|
1414
|
-
if (aliasData && !aliasErrors.length) {
|
|
1415
|
-
result.executed++;
|
|
1416
|
-
} else {
|
|
1417
|
-
result.failed++;
|
|
1418
|
-
const errorMsg = aliasErrors[0] && typeof aliasErrors[0] === 'object' && 'message' in aliasErrors[0]
|
|
1419
|
-
? String((aliasErrors[0] as any).message)
|
|
1420
|
-
: 'Mutation failed';
|
|
1421
|
-
const productRef = price.input?.productRef || 'unknown';
|
|
1422
|
-
result.errors.push(`Failed to update price for ${productRef}: ${errorMsg}`);
|
|
1423
|
-
}
|
|
1424
|
-
});
|
|
1425
|
-
|
|
1426
|
-
return result;
|
|
1427
|
-
}
|
|
1428
|
-
|
|
1429
|
-
// ============================================================================
|
|
1430
|
-
// Service Function 3: Write Mutation Log (S3 Upload)
|
|
1431
|
-
// ============================================================================
|
|
1432
|
-
|
|
1433
|
-
/**
|
|
1434
|
-
* Write mutation execution log to S3
|
|
1435
|
-
*
|
|
1436
|
-
* @param s3 - S3DataSource instance
|
|
1437
|
-
* @param fileName - Original CSV file name
|
|
1438
|
-
* @param result - FileProcessingResult
|
|
1439
|
-
* @param logPrefix - S3 prefix for logs
|
|
1440
|
-
* @param log - Logger instance
|
|
1441
|
-
*/
|
|
1442
|
-
async function writeMutationLog(
|
|
1443
|
-
s3: S3DataSource,
|
|
1444
|
-
fileName: string,
|
|
1445
|
-
result: FileProcessingResult,
|
|
1446
|
-
logPrefix: string,
|
|
1447
|
-
log: any
|
|
1448
|
-
): Promise<void> {
|
|
1449
|
-
const logFileName = `${fileName.replace('.csv', '')}-log.json`;
|
|
1450
|
-
const logPath = `${logPrefix}${logFileName}`;
|
|
1451
|
-
|
|
1452
|
-
const logData = {
|
|
1453
|
-
fileName,
|
|
1454
|
-
timestamp: new Date().toISOString(),
|
|
1455
|
-
summary: {
|
|
1456
|
-
successful: result.successful,
|
|
1457
|
-
failed: result.failed,
|
|
1458
|
-
validationFailed: result.validationFailed,
|
|
1459
|
-
priceChanges: result.priceChanges.length,
|
|
1460
|
-
},
|
|
1461
|
-
priceChanges: result.priceChanges,
|
|
1462
|
-
errors: result.errors,
|
|
1463
|
-
};
|
|
1464
|
-
|
|
1465
|
-
const logContent = JSON.stringify(logData, null, 2);
|
|
1466
|
-
|
|
1467
|
-
// Upload log to S3 (uploadFile accepts string or Buffer)
|
|
1468
|
-
await s3.uploadFile(logPath, logContent);
|
|
1469
|
-
|
|
1470
|
-
log.info('Mutation log written', { logPath });
|
|
1471
|
-
}
|
|
1472
|
-
|
|
1473
|
-
// ============================================================================
|
|
1474
|
-
// Main Workflow Function (Per-File Processing)
|
|
1475
|
-
// ============================================================================
|
|
1476
|
-
|
|
1477
|
-
async function executePriceSync(ctx: any, jobId: string, tracker: any) {
|
|
1478
|
-
const { log, activation, openKv } = ctx;
|
|
1479
|
-
const startTime = Date.now();
|
|
1480
|
-
|
|
1481
|
-
log.info('🔄 [PriceSync] Starting scheduled price sync from S3');
|
|
1482
|
-
|
|
1483
|
-
// ✅ CRITICAL: Declare S3 outside try block for safe disposal
|
|
1484
|
-
let s3: S3DataSource | undefined;
|
|
1485
|
-
|
|
1486
|
-
try {
|
|
1487
|
-
// ========================================
|
|
1488
|
-
// STEP 1: CONFIGURATION & VALIDATION
|
|
1489
|
-
// ========================================
|
|
1490
|
-
|
|
1491
|
-
// Read activation variables
|
|
1492
|
-
const s3Bucket = activation?.getVariable('s3BucketName');
|
|
1493
|
-
const s3Region = activation?.getVariable('awsRegion') || 'us-east-1';
|
|
1494
|
-
const s3AccessKeyId = activation?.getVariable('awsAccessKeyId');
|
|
1495
|
-
const s3SecretAccessKey = activation?.getVariable('awsSecretAccessKey');
|
|
1496
|
-
const s3Prefix = activation?.getVariable('s3Prefix') || 'prices/';
|
|
1497
|
-
const retailerId = activation?.getVariable('retailerId');
|
|
1498
|
-
const maxFiles = parseInt(activation?.getVariable('maxFilesToProcess') || '10', 10);
|
|
1499
|
-
const filePattern = (activation?.getVariable('filePattern') || '.csv').toLowerCase();
|
|
1500
|
-
const enableArchival = activation?.getVariable('enableArchival') !== 'false';
|
|
1501
|
-
const archivePrefix = activation?.getVariable('archivePrefix') || 'processed/';
|
|
1502
|
-
const errorPrefix = activation?.getVariable('errorPrefix') || 'errors/';
|
|
1503
|
-
const logPrefix = activation?.getVariable('logPrefix') || 'logs/';
|
|
1504
|
-
|
|
1505
|
-
// Configuration with defaults
|
|
1506
|
-
const mutationBatchSize = parseInt(
|
|
1507
|
-
activation?.getVariable('mutationBatchSize') || '1', // Default: 1 (sequential)
|
|
1508
|
-
10
|
|
1509
|
-
);
|
|
1510
|
-
|
|
1511
|
-
const mutationsPerAliasBatch = activation?.getVariable('mutationsPerAliasBatch')
|
|
1512
|
-
? parseInt(activation?.getVariable('mutationsPerAliasBatch') || '1', 10)
|
|
1513
|
-
: undefined; // Default: undefined (disabled)
|
|
1514
|
-
|
|
1515
|
-
const minPrice = parseFloat(activation?.getVariable('minPrice') || '0.01');
|
|
1516
|
-
const maxPrice = parseFloat(activation?.getVariable('maxPrice') || '999999.99');
|
|
1517
|
-
const validateProducts = activation?.getVariable('validateProducts') !== 'false';
|
|
1518
|
-
const validateConnection = activation?.getVariable('validateConnection') !== 'false';
|
|
1519
|
-
const enableFileTracking = activation?.getVariable('enableFileTracking') !== 'false';
|
|
1520
|
-
|
|
1521
|
-
// Validate required variables
|
|
1522
|
-
const missingVars: string[] = [];
|
|
1523
|
-
if (!s3Bucket) missingVars.push('s3BucketName');
|
|
1524
|
-
if (!s3AccessKeyId) missingVars.push('awsAccessKeyId');
|
|
1525
|
-
if (!s3SecretAccessKey) missingVars.push('awsSecretAccessKey');
|
|
1526
|
-
if (!retailerId) missingVars.push('retailerId');
|
|
1527
|
-
|
|
1528
|
-
if (missingVars.length > 0) {
|
|
1529
|
-
const errorMsg = `Missing required variables: ${missingVars.join(', ')}`;
|
|
1530
|
-
log.error('❌ [PriceSync] Configuration error', { error: errorMsg });
|
|
1531
|
-
return {
|
|
1532
|
-
success: false,
|
|
1533
|
-
error: errorMsg,
|
|
1534
|
-
recommendation: `Please configure these activation variables: ${missingVars.join(', ')}`,
|
|
1535
|
-
processed: 0,
|
|
1536
|
-
duration: Date.now() - startTime,
|
|
1537
|
-
};
|
|
1538
|
-
}
|
|
1539
|
-
|
|
1540
|
-
// ========================================
|
|
1541
|
-
// STEP 2: CLIENT INITIALIZATION
|
|
1542
|
-
// ========================================
|
|
1543
|
-
|
|
1544
|
-
const client = await createClient(ctx);
|
|
1545
|
-
if (!client) {
|
|
1546
|
-
throw new Error('Failed to create Fluent Commerce client');
|
|
1547
|
-
}
|
|
1548
|
-
|
|
1549
|
-
// ✅ CORRECT: GraphQL mutations don't need setRetailerId()
|
|
1550
|
-
// Check your GraphQL schema to determine retailerId handling:
|
|
1551
|
-
// - Mandatory retailerId → Must pass it in mutation input
|
|
1552
|
-
// - Optional retailerId → Can pass it if needed
|
|
1553
|
-
// - No retailerId field → Don't pass it
|
|
1554
|
-
// See: fc-connect-sdk/docs/00-START-HERE/retailerid-configuration.md
|
|
1555
|
-
|
|
1556
|
-
log.info('✅ [PriceSync] Fluent client initialized');
|
|
1557
|
-
|
|
1558
|
-
// ========================================
|
|
1559
|
-
// STEP 3: SERVICE INITIALIZATION
|
|
1560
|
-
// ========================================
|
|
1561
|
-
|
|
1562
|
-
s3 = new S3DataSource(
|
|
1563
|
-
{
|
|
1564
|
-
type: 'S3_CSV',
|
|
1565
|
-
connectionId: 's3-price-sync',
|
|
1566
|
-
name: 'Source S3',
|
|
1567
|
-
s3Config: {
|
|
1568
|
-
bucket: s3Bucket,
|
|
1569
|
-
region: s3Region,
|
|
1570
|
-
accessKeyId: s3AccessKeyId,
|
|
1571
|
-
secretAccessKey: s3SecretAccessKey,
|
|
1572
|
-
},
|
|
1573
|
-
},
|
|
1574
|
-
log
|
|
1575
|
-
);
|
|
1576
|
-
|
|
1577
|
-
// Validate S3 connection if enabled
|
|
1578
|
-
if (validateConnection) {
|
|
1579
|
-
try {
|
|
1580
|
-
await s3.listFiles({ prefix: s3Prefix, maxKeys: 1 });
|
|
1581
|
-
log.info('✅ [PriceSync] S3 connection validated');
|
|
1582
|
-
} catch (error: any) {
|
|
1583
|
-
log.error('❌ [PriceSync] S3 connection validation failed', {
|
|
1584
|
-
error: error instanceof Error ? error.message : String(error),
|
|
1585
|
-
});
|
|
1586
|
-
return {
|
|
1587
|
-
success: false,
|
|
1588
|
-
error: 'S3 connection validation failed',
|
|
1589
|
-
details: error?.message,
|
|
1590
|
-
recommendation: 'Please verify S3 credentials and bucket access permissions',
|
|
1591
|
-
duration: Date.now() - startTime,
|
|
1592
|
-
};
|
|
1593
|
-
}
|
|
1594
|
-
}
|
|
1595
|
-
|
|
1596
|
-
const parser = new CSVParserService();
|
|
1597
|
-
const kv = openKv(':project:');
|
|
1598
|
-
const fileTracker = enableFileTracking
|
|
1599
|
-
? new VersoriFileTracker(kv, 's3-csv-price-sync')
|
|
1600
|
-
: null;
|
|
1601
|
-
|
|
1602
|
-
// Create custom resolvers for price validation
|
|
1603
|
-
const customResolvers = {
|
|
1604
|
-
'custom.validatePriceRange': (value: any) => {
|
|
1605
|
-
const price = parseFloat(value);
|
|
1606
|
-
if (isNaN(price)) {
|
|
1607
|
-
throw new Error(`Invalid price: ${value}`);
|
|
1608
|
-
}
|
|
1609
|
-
if (price < minPrice) {
|
|
1610
|
-
throw new Error(`Price ${price} below minimum ${minPrice}`);
|
|
1611
|
-
}
|
|
1612
|
-
if (price > maxPrice) {
|
|
1613
|
-
throw new Error(`Price ${price} exceeds maximum ${maxPrice}`);
|
|
1614
|
-
}
|
|
1615
|
-
return price;
|
|
1616
|
-
},
|
|
1617
|
-
|
|
1618
|
-
'custom.normalizePriceType': (value: any) => {
|
|
1619
|
-
const type = String(value || 'DEFAULT').toUpperCase().trim();
|
|
1620
|
-
const validTypes = ['DEFAULT', 'SALE', 'CLEARANCE', 'BULK', 'PROMOTIONAL', 'MEMBER'];
|
|
1621
|
-
if (!validTypes.includes(type)) {
|
|
1622
|
-
throw new Error(`Invalid price type: ${type}`);
|
|
1623
|
-
}
|
|
1624
|
-
return type;
|
|
1625
|
-
},
|
|
1626
|
-
|
|
1627
|
-
'custom.normalizeQuantity': (value: any) => {
|
|
1628
|
-
if (!value || value === '') return undefined;
|
|
1629
|
-
const qty = parseInt(value, 10);
|
|
1630
|
-
if (isNaN(qty) || qty < 0) {
|
|
1631
|
-
throw new Error(`Invalid quantity: ${value}`);
|
|
1632
|
-
}
|
|
1633
|
-
return qty;
|
|
1634
|
-
},
|
|
1635
|
-
};
|
|
1636
|
-
|
|
1637
|
-
// ✅ CRITICAL: Load mapping config from external JSON file
|
|
1638
|
-
// Mapping config uses GraphQLMutationMapper structure (nested objects, not dot notation)
|
|
1639
|
-
// File: src/config/price-mapping.json
|
|
1640
|
-
const mappingConfigJson = await import('../config/price-mapping.json', { assert: { type: 'json' } });
|
|
1641
|
-
const mappingConfig = mappingConfigJson.default;
|
|
1642
|
-
|
|
1643
|
-
// Initialize GraphQLMutationMapper with client for schema introspection
|
|
1644
|
-
const mapper = new GraphQLMutationMapper(mappingConfig, log, { fluentClient: client });
|
|
1645
|
-
|
|
1646
|
-
// ========================================
|
|
1647
|
-
// STEP 4: FILE DISCOVERY
|
|
1648
|
-
// ========================================
|
|
1649
|
-
|
|
1650
|
-
try {
|
|
1651
|
-
// List files (pattern filtering handled by listFiles)
|
|
1652
|
-
const files = await s3.listFiles({
|
|
1653
|
-
prefix: s3Prefix,
|
|
1654
|
-
pattern: filePattern,
|
|
1655
|
-
maxKeys: 1000,
|
|
1656
|
-
});
|
|
1657
|
-
|
|
1658
|
-
const csvFiles = files
|
|
1659
|
-
.sort((a, b) => {
|
|
1660
|
-
const dateA = a.lastModified ? new Date(a.lastModified).getTime() : 0;
|
|
1661
|
-
const dateB = b.lastModified ? new Date(b.lastModified).getTime() : 0;
|
|
1662
|
-
return dateA - dateB;
|
|
1663
|
-
})
|
|
1664
|
-
.slice(0, maxFiles);
|
|
1665
|
-
|
|
1666
|
-
log.info('📂 [PriceSync] Files discovered', {
|
|
1667
|
-
total: files.length,
|
|
1668
|
-
toProcess: csvFiles.length,
|
|
1669
|
-
maxFiles,
|
|
1670
|
-
});
|
|
1671
|
-
|
|
1672
|
-
const results = {
|
|
1673
|
-
processed: 0,
|
|
1674
|
-
skipped: 0,
|
|
1675
|
-
failed: 0,
|
|
1676
|
-
totalRecords: 0,
|
|
1677
|
-
pricesUpdated: 0,
|
|
1678
|
-
pricesSkipped: 0,
|
|
1679
|
-
validationErrors: 0,
|
|
1680
|
-
priceChanges: [] as Array<{
|
|
1681
|
-
sku: string;
|
|
1682
|
-
type: string;
|
|
1683
|
-
oldPrice?: number;
|
|
1684
|
-
newPrice: number;
|
|
1685
|
-
}>,
|
|
1686
|
-
errors: [] as string[],
|
|
1687
|
-
};
|
|
1688
|
-
|
|
1689
|
-
// ========================================
|
|
1690
|
-
// STEP 5: FILE PROCESSING LOOP
|
|
1691
|
-
// ========================================
|
|
1692
|
-
|
|
1693
|
-
// Process each file using service functions
|
|
1694
|
-
for (const file of csvFiles) {
|
|
1695
|
-
const filePath = file.path;
|
|
1696
|
-
const fileName = file.name;
|
|
1697
|
-
const fileStartTime = Date.now();
|
|
1698
|
-
|
|
1699
|
-
log.info('📄 [PriceSync] Processing file', { fileName });
|
|
1700
|
-
|
|
1701
|
-
// Duplicate prevention via file tracker
|
|
1702
|
-
if (fileTracker && (await fileTracker.wasFileProcessed(fileName))) {
|
|
1703
|
-
log.info('⏭️ [PriceSync] Skipping already processed file', { fileName });
|
|
1704
|
-
results.skipped++;
|
|
1705
|
-
continue;
|
|
1706
|
-
}
|
|
1707
|
-
|
|
1708
|
-
try {
|
|
1709
|
-
// SERVICE FUNCTION 1: Process file (S3 + CSV + Mapper)
|
|
1710
|
-
const { records: mappedRecords, errors: mappingErrors } = await processFile(
|
|
1711
|
-
s3,
|
|
1712
|
-
parser,
|
|
1713
|
-
mapper,
|
|
1714
|
-
filePath,
|
|
1715
|
-
fileName,
|
|
1716
|
-
log
|
|
1717
|
-
);
|
|
1718
|
-
|
|
1719
|
-
if (!mappedRecords.length) {
|
|
1720
|
-
log.warn('No valid records after mapping, archiving', { fileName });
|
|
1721
|
-
if (enableArchival) {
|
|
1722
|
-
await s3.moveFile(filePath, `${archivePrefix}${fileName}`);
|
|
1723
|
-
}
|
|
1724
|
-
results.skipped++;
|
|
1725
|
-
continue;
|
|
1726
|
-
}
|
|
1727
|
-
|
|
1728
|
-
// SERVICE FUNCTION 2: Execute mutations
|
|
1729
|
-
// ✅ Configuration with defaults
|
|
1730
|
-
const mutationBatchSize = parseInt(
|
|
1731
|
-
ctx.activation?.getVariable('mutationBatchSize') || '1', // ✅ Default: 1 (sequential)
|
|
1732
|
-
10
|
|
1733
|
-
);
|
|
1734
|
-
|
|
1735
|
-
const mutationsPerAliasBatch = ctx.activation?.getVariable('mutationsPerAliasBatch')
|
|
1736
|
-
? parseInt(ctx.activation?.getVariable('mutationsPerAliasBatch') || '1', 10)
|
|
1737
|
-
: undefined; // ✅ Default: undefined (disabled, use separate requests)
|
|
1738
|
-
|
|
1739
|
-
// ? Enhanced: Extract context for progress logging
|
|
1740
|
-
const sampleProductRefs = mappedRecords.slice(0, 5).map((r: any) => r.input?.skuRef || r.input?.productRef || 'unknown');
|
|
1741
|
-
const mutationType = mapper?.mutationName || 'updatePrice';
|
|
1742
|
-
|
|
1743
|
-
// ? Enhanced: Start logging with context
|
|
1744
|
-
log.info(`[GraphQLMutations] Sending price mutations for file "${fileName}"`, {
|
|
1745
|
-
totalMutations: mappedRecords.length,
|
|
1746
|
-
mutationType,
|
|
1747
|
-
batchSize: mutationBatchSize,
|
|
1748
|
-
batchMode: mutationBatchSize === 1 ? 'sequential' : `parallel (${mutationBatchSize})`,
|
|
1749
|
-
sampleProductRefs: sampleProductRefs.join(', '),
|
|
1750
|
-
aliasBatching: mutationsPerAliasBatch ? `enabled (${mutationsPerAliasBatch} per alias)` : 'disabled',
|
|
1751
|
-
validateProducts
|
|
1752
|
-
});
|
|
1753
|
-
|
|
1754
|
-
const mutationResult = await executeMutations(
|
|
1755
|
-
client,
|
|
1756
|
-
mapper,
|
|
1757
|
-
mappedRecords,
|
|
1758
|
-
retailerId,
|
|
1759
|
-
validateProducts,
|
|
1760
|
-
mutationBatchSize, // Concurrency control (default: 1)
|
|
1761
|
-
mutationsPerAliasBatch, // ✅ NEW: Alias batching (default: undefined)
|
|
1762
|
-
log
|
|
1763
|
-
);
|
|
1764
|
-
|
|
1765
|
-
// ? Enhanced: Completion logging with summary
|
|
1766
|
-
log.info(`[GraphQLMutations] Price mutation submission completed for file "${fileName}"`, {
|
|
1767
|
-
totalMutations: mappedRecords.length,
|
|
1768
|
-
successful: mutationResult.successful,
|
|
1769
|
-
failed: mutationResult.failed,
|
|
1770
|
-
successRate: mappedRecords.length > 0 ? `${Math.round((mutationResult.successful / mappedRecords.length) * 100)}%` : '0%',
|
|
1771
|
-
mutationType,
|
|
1772
|
-
priceChanges: mutationResult.priceChanges?.length || 0
|
|
1773
|
-
});
|
|
1774
|
-
|
|
1775
|
-
// Aggregate results
|
|
1776
|
-
results.processed++;
|
|
1777
|
-
results.totalRecords += mappedRecords.length;
|
|
1778
|
-
results.pricesUpdated += mutationResult.successful;
|
|
1779
|
-
results.validationErrors += mappingErrors.length;
|
|
1780
|
-
results.priceChanges.push(...mutationResult.priceChanges);
|
|
1781
|
-
|
|
1782
|
-
// Build file processing result for logging
|
|
1783
|
-
const fileResult: FileProcessingResult = {
|
|
1784
|
-
fileName,
|
|
1785
|
-
successful: mutationResult.successful,
|
|
1786
|
-
failed: mutationResult.failed,
|
|
1787
|
-
validationFailed: mappingErrors.length,
|
|
1788
|
-
priceChanges: mutationResult.priceChanges,
|
|
1789
|
-
errors: mappingErrors,
|
|
1790
|
-
};
|
|
1791
|
-
|
|
1792
|
-
// SERVICE FUNCTION 3: Write mutation log to S3
|
|
1793
|
-
await writeMutationLog(s3, fileName, fileResult, logPrefix, log);
|
|
1794
|
-
|
|
1795
|
-
// Mark processed with file tracker
|
|
1796
|
-
if (fileTracker) {
|
|
1797
|
-
await fileTracker.markFileProcessed(fileName, {
|
|
1798
|
-
successful: mutationResult.successful,
|
|
1799
|
-
failed: mutationResult.failed,
|
|
1800
|
-
validationFailed: mappingErrors.length,
|
|
1801
|
-
recordCount: mappedRecords.length,
|
|
1802
|
-
duration: Date.now() - fileStartTime,
|
|
1803
|
-
});
|
|
1804
|
-
}
|
|
1805
|
-
|
|
1806
|
-
if (enableArchival) {
|
|
1807
|
-
await s3.moveFile(filePath, `${archivePrefix}${fileName}`);
|
|
1808
|
-
}
|
|
1809
|
-
|
|
1810
|
-
const fileDuration = Date.now() - fileStartTime;
|
|
1811
|
-
log.info('✅ [PriceSync] File processed successfully', {
|
|
1812
|
-
fileName,
|
|
1813
|
-
recordCount: mappedRecords.length,
|
|
1814
|
-
successful: mutationResult.successful,
|
|
1815
|
-
failed: mutationResult.failed,
|
|
1816
|
-
duration: `${fileDuration}ms`,
|
|
1817
|
-
});
|
|
1818
|
-
|
|
1819
|
-
if (mappingErrors.length > 0) {
|
|
1820
|
-
results.errors.push(`${fileName}: ${mappingErrors.length} mapping errors`);
|
|
1821
|
-
}
|
|
1822
|
-
} catch (error: unknown) {
|
|
1823
|
-
// ✅ Enhanced error logging: Extract all error details for visibility
|
|
1824
|
-
const errorMsg = error instanceof Error ? error.message : String(error);
|
|
1825
|
-
const errorDetails = {
|
|
1826
|
-
message: errorMsg,
|
|
1827
|
-
stack: error instanceof Error ? error.stack : undefined,
|
|
1828
|
-
errorType: error instanceof Error ? error.constructor.name : 'Error',
|
|
1829
|
-
};
|
|
1830
|
-
log.error('❌ [PriceSync] File processing failed', {
|
|
1831
|
-
fileName,
|
|
1832
|
-
...errorDetails,
|
|
1833
|
-
});
|
|
1834
|
-
results.failed++;
|
|
1835
|
-
results.errors.push(`${fileName}: ${errorMsg}`);
|
|
1836
|
-
|
|
1837
|
-
// Attempt to move to error directory
|
|
1838
|
-
try {
|
|
1839
|
-
await s3.moveFile(filePath, `${errorPrefix}${fileName}`);
|
|
1840
|
-
log.info('📁 [PriceSync] Moved failed file to error directory', { fileName });
|
|
1841
|
-
} catch (moveError) {
|
|
1842
|
-
log.error('❌ [PriceSync] Failed to move error file', {
|
|
1843
|
-
fileName,
|
|
1844
|
-
error: moveError instanceof Error ? moveError.message : String(moveError),
|
|
1845
|
-
});
|
|
1846
|
-
}
|
|
1847
|
-
}
|
|
1848
|
-
}
|
|
1849
|
-
|
|
1850
|
-
// ========================================
|
|
1851
|
-
// STEP 6: SUMMARY & COMPLETION
|
|
1852
|
-
// ========================================
|
|
1853
|
-
|
|
1854
|
-
const totalDuration = Date.now() - startTime;
|
|
1855
|
-
const summary = {
|
|
1856
|
-
success: true,
|
|
1857
|
-
processed: results.processed,
|
|
1858
|
-
skipped: results.skipped,
|
|
1859
|
-
failed: results.failed,
|
|
1860
|
-
totalRecords: results.totalRecords,
|
|
1861
|
-
pricesUpdated: results.pricesUpdated,
|
|
1862
|
-
pricesSkipped: results.pricesSkipped,
|
|
1863
|
-
validationErrors: results.validationErrors,
|
|
1864
|
-
priceChanges: results.priceChanges.length,
|
|
1865
|
-
errors: results.errors.length > 0 ? results.errors : undefined,
|
|
1866
|
-
duration: totalDuration,
|
|
1867
|
-
timestamp: new Date().toISOString(),
|
|
1868
|
-
};
|
|
1869
|
-
|
|
1870
|
-
log.info('🎉 [PriceSync] Price sync completed', {
|
|
1871
|
-
...summary,
|
|
1872
|
-
duration: `${totalDuration}ms`,
|
|
1873
|
-
});
|
|
1874
|
-
|
|
1875
|
-
// Log significant price changes
|
|
1876
|
-
if (results.priceChanges.length > 0) {
|
|
1877
|
-
log.info('💰 [PriceSync] Price changes detected', {
|
|
1878
|
-
count: results.priceChanges.length,
|
|
1879
|
-
changes: results.priceChanges.slice(0, 10),
|
|
1880
|
-
});
|
|
1881
|
-
}
|
|
1882
|
-
|
|
1883
|
-
return summary;
|
|
1884
|
-
} catch (error: unknown) {
|
|
1885
|
-
// ✅ Enhanced error logging: Extract all error details for visibility
|
|
1886
|
-
const errorMsg = error instanceof Error ? error.message : String(error);
|
|
1887
|
-
const errorDetails = {
|
|
1888
|
-
message: errorMsg,
|
|
1889
|
-
stack: error instanceof Error ? error.stack : undefined,
|
|
1890
|
-
errorType: error instanceof Error ? error.constructor.name : 'Error',
|
|
1891
|
-
};
|
|
1892
|
-
log.error('❌ [PriceSync] Fatal error', errorDetails);
|
|
1893
|
-
return {
|
|
1894
|
-
success: false,
|
|
1895
|
-
error: errorMsg,
|
|
1896
|
-
recommendation: 'Check error logs for details and verify configuration',
|
|
1897
|
-
processed: 0,
|
|
1898
|
-
duration: Date.now() - startTime,
|
|
1899
|
-
timestamp: new Date().toISOString(),
|
|
1900
|
-
};
|
|
1901
|
-
}
|
|
1902
|
-
} catch (error: unknown) {
|
|
1903
|
-
const errorMsg = error instanceof Error ? error.message : String(error);
|
|
1904
|
-
log.error('❌ [PriceSync] Initialization failed', {
|
|
1905
|
-
error: errorMsg,
|
|
1906
|
-
stack: error instanceof Error ? error.stack : undefined,
|
|
1907
|
-
});
|
|
1908
|
-
return {
|
|
1909
|
-
success: false,
|
|
1910
|
-
error: errorMsg,
|
|
1911
|
-
recommendation: 'Check configuration and service initialization',
|
|
1912
|
-
duration: Date.now() - startTime,
|
|
1913
|
-
};
|
|
1914
|
-
} finally {
|
|
1915
|
-
// ✅ CRITICAL: Always dispose S3 connection to prevent connection pool exhaustion
|
|
1916
|
-
if (s3) {
|
|
1917
|
-
try {
|
|
1918
|
-
await s3.dispose();
|
|
1919
|
-
log.info('✅ [PriceSync] S3 connection disposed successfully');
|
|
1920
|
-
} catch (disposeError: any) {
|
|
1921
|
-
log.error('⚠️ [PriceSync] Failed to dispose S3 connection', {
|
|
1922
|
-
error: disposeError instanceof Error ? disposeError.message : String(disposeError),
|
|
1923
|
-
stack: disposeError instanceof Error ? disposeError.stack : undefined,
|
|
1924
|
-
});
|
|
1925
|
-
// Don't throw - disposal failure shouldn't prevent workflow completion
|
|
1926
|
-
}
|
|
1927
|
-
}
|
|
1928
|
-
}
|
|
1929
|
-
}
|
|
1930
|
-
```
|
|
1931
|
-
|
|
1932
|
-
**Note:** The `runPriceSync` function above should be renamed to `executePriceSync` and moved to `src/services/price-sync.service.ts` to match the new workflow structure.
|
|
1933
|
-
|
|
1934
|
-
## Key Patterns Explained
|
|
1935
|
-
|
|
1936
|
-
### Pattern 1: Service Function Architecture
|
|
1937
|
-
|
|
1938
|
-
**Three modular service functions for clean separation of concerns:**
|
|
1939
|
-
|
|
1940
|
-
```typescript
|
|
1941
|
-
// SERVICE FUNCTION 1: processFile()
|
|
1942
|
-
// Handles: S3 download + CSV parsing + field mapping
|
|
1943
|
-
const { records: mappedRecords, errors: mappingErrors } = await processFile(
|
|
1944
|
-
s3,
|
|
1945
|
-
parser,
|
|
1946
|
-
mapper,
|
|
1947
|
-
filePath,
|
|
1948
|
-
fileName,
|
|
1949
|
-
log
|
|
1950
|
-
);
|
|
1951
|
-
|
|
1952
|
-
// SERVICE FUNCTION 2: executeMutations()
|
|
1953
|
-
// Handles: GraphQL mutations + product validation + concurrency control + price change tracking
|
|
1954
|
-
const mutationResult = await executeMutations(
|
|
1955
|
-
client,
|
|
1956
|
-
mapper,
|
|
1957
|
-
mappedRecords,
|
|
1958
|
-
retailerId,
|
|
1959
|
-
validateProducts,
|
|
1960
|
-
mutationBatchSize, // Concurrency control (default: 1)
|
|
1961
|
-
mutationsPerAliasBatch, // Alias batching (default: undefined)
|
|
1962
|
-
log
|
|
1963
|
-
);
|
|
1964
|
-
|
|
1965
|
-
// SERVICE FUNCTION 3: writeMutationLog()
|
|
1966
|
-
// Handles: Write execution log to S3 (with Buffer for Versori/Deno)
|
|
1967
|
-
await writeMutationLog(s3, fileName, fileResult, logPrefix, log);
|
|
1968
|
-
```
|
|
1969
|
-
|
|
1970
|
-
**Why this pattern?**
|
|
1971
|
-
- ✅ **Testability**: Each function can be unit tested independently
|
|
1972
|
-
- ✅ **Reusability**: Functions can be reused in different workflows
|
|
1973
|
-
- ✅ **Clarity**: Clear responsibilities for each step
|
|
1974
|
-
- ✅ **Error handling**: Isolated error boundaries per function
|
|
1975
|
-
- ✅ **Maintainability**: Easy to modify single concerns without affecting others
|
|
1976
|
-
|
|
1977
|
-
### Pattern 2: Buffer Import for S3 Upload (Versori/Deno)
|
|
1978
|
-
|
|
1979
|
-
**CRITICAL for Versori/Deno runtime:**
|
|
1980
|
-
|
|
1981
|
-
```typescript
|
|
1982
|
-
// MUST import Buffer explicitly (not global like Node.js)
|
|
1983
|
-
import { Buffer } from 'node:buffer';
|
|
1984
|
-
|
|
1985
|
-
// writeMutationLog() function
|
|
1986
|
-
async function writeMutationLog(s3, fileName, result, logPrefix, log) {
|
|
1987
|
-
const logContent = JSON.stringify(logData, null, 2);
|
|
1988
|
-
|
|
1989
|
-
// ✅ CORRECT: Use Buffer.from() for S3 upload
|
|
1990
|
-
await s3.uploadFile(logPath, logContent);
|
|
1991
|
-
|
|
1992
|
-
// ✅ CORRECT: uploadFile accepts string directly (no Buffer needed)
|
|
1993
|
-
// await s3.uploadFile(logPath, logContent);
|
|
1994
|
-
}
|
|
1995
|
-
```
|
|
1996
|
-
|
|
1997
|
-
**Why?** Deno runtime requires explicit `Buffer` import from `node:buffer`. Without it, you'll get `Buffer is not defined` errors.
|
|
1998
|
-
|
|
1999
|
-
### Pattern 3: Direct KV State Management
|
|
2000
|
-
|
|
2001
|
-
```typescript
|
|
2002
|
-
const kv = new VersoriKVAdapter(ctx.openKv(':project:'));
|
|
2003
|
-
|
|
2004
|
-
// Check if file already processed
|
|
2005
|
-
const stateKey = ['processed-files', 's3-price-sync', fileName];
|
|
2006
|
-
const existing = await kv.get(stateKey);
|
|
2007
|
-
if (existing) {
|
|
2008
|
-
log.info('Skipping already processed file', { fileName });
|
|
2009
|
-
continue;
|
|
2010
|
-
}
|
|
2011
|
-
|
|
2012
|
-
// Mark as processed after success
|
|
2013
|
-
await kv.set(stateKey, {
|
|
2014
|
-
successful,
|
|
2015
|
-
failed,
|
|
2016
|
-
validationFailed,
|
|
2017
|
-
processedAt: new Date().toISOString(),
|
|
2018
|
-
});
|
|
2019
|
-
```
|
|
2020
|
-
|
|
2021
|
-
### Pattern 4: Custom Price Validation Resolvers
|
|
2022
|
-
|
|
2023
|
-
```typescript
|
|
2024
|
-
const customResolvers = {
|
|
2025
|
-
'custom.validatePriceRange': (value: any) => {
|
|
2026
|
-
const price = parseFloat(value);
|
|
2027
|
-
if (isNaN(price)) {
|
|
2028
|
-
throw new Error(`Invalid price: ${value}`);
|
|
2029
|
-
}
|
|
2030
|
-
if (price < minPrice) {
|
|
2031
|
-
throw new Error(`Price ${price} below minimum ${minPrice}`);
|
|
2032
|
-
}
|
|
2033
|
-
if (price > maxPrice) {
|
|
2034
|
-
throw new Error(`Price ${price} exceeds maximum ${maxPrice}`);
|
|
2035
|
-
}
|
|
2036
|
-
return price;
|
|
2037
|
-
},
|
|
2038
|
-
|
|
2039
|
-
'custom.normalizePriceType': (value: any) => {
|
|
2040
|
-
const type = String(value || 'DEFAULT').toUpperCase().trim();
|
|
2041
|
-
const validTypes = ['DEFAULT', 'SALE', 'CLEARANCE', 'BULK', 'PROMOTIONAL', 'MEMBER'];
|
|
2042
|
-
if (!validTypes.includes(type)) {
|
|
2043
|
-
throw new Error(`Invalid price type: ${type}`);
|
|
2044
|
-
}
|
|
2045
|
-
return type;
|
|
2046
|
-
},
|
|
2047
|
-
};
|
|
2048
|
-
```
|
|
2049
|
-
|
|
2050
|
-
**Why this matters**: Price data requires strict validation to prevent:
|
|
2051
|
-
|
|
2052
|
-
- Negative prices
|
|
2053
|
-
- Prices outside business rules (too low/high)
|
|
2054
|
-
- Invalid price tiers
|
|
2055
|
-
- Data entry errors
|
|
2056
|
-
|
|
2057
|
-
### Pattern 5: Product Existence Validation
|
|
2058
|
-
|
|
2059
|
-
```typescript
|
|
2060
|
-
async function validateProductExists(
|
|
2061
|
-
client: any,
|
|
2062
|
-
productRef: string,
|
|
2063
|
-
retailerId: string,
|
|
2064
|
-
log: any
|
|
2065
|
-
): Promise<boolean> {
|
|
2066
|
-
const query = `
|
|
2067
|
-
query GetProduct($ref: String!, $retailerId: ID!) {
|
|
2068
|
-
products(first: 1, ref: [$ref], retailerId: $retailerId) {
|
|
2069
|
-
edges {
|
|
2070
|
-
node {
|
|
2071
|
-
id
|
|
2072
|
-
ref
|
|
2073
|
-
status
|
|
2074
|
-
}
|
|
2075
|
-
}
|
|
2076
|
-
}
|
|
2077
|
-
}
|
|
2078
|
-
`;
|
|
2079
|
-
|
|
2080
|
-
const result = await client.graphql({ query, variables: { ref: productRef, retailerId } });
|
|
2081
|
-
const product = result?.data?.products?.edges?.[0]?.node;
|
|
2082
|
-
|
|
2083
|
-
if (!product || product.status !== 'ACTIVE') {
|
|
2084
|
-
return false;
|
|
2085
|
-
}
|
|
2086
|
-
|
|
2087
|
-
return true;
|
|
2088
|
-
}
|
|
2089
|
-
```
|
|
2090
|
-
|
|
2091
|
-
**Why this matters**: Prevents price updates for non-existent products, which would fail at GraphQL level.
|
|
2092
|
-
|
|
2093
|
-
### Pattern 6: Price Change Tracking
|
|
2094
|
-
|
|
2095
|
-
```typescript
|
|
2096
|
-
// Get current price before update
|
|
2097
|
-
const currentPrice = await getCurrentPrice(
|
|
2098
|
-
client,
|
|
2099
|
-
priceData.productRef,
|
|
2100
|
-
priceData.type,
|
|
2101
|
-
priceData.currency,
|
|
2102
|
-
retailerId
|
|
2103
|
-
);
|
|
2104
|
-
|
|
2105
|
-
// Track change after update
|
|
2106
|
-
if (currentPrice !== undefined && currentPrice !== priceData.value) {
|
|
2107
|
-
results.priceChanges.push({
|
|
2108
|
-
sku: priceData.productRef,
|
|
2109
|
-
type: priceData.type,
|
|
2110
|
-
oldPrice: currentPrice,
|
|
2111
|
-
newPrice: priceData.value,
|
|
2112
|
-
});
|
|
2113
|
-
}
|
|
2114
|
-
|
|
2115
|
-
// Log changes at end
|
|
2116
|
-
if (results.priceChanges.length > 0) {
|
|
2117
|
-
log.info('Price changes detected', {
|
|
2118
|
-
count: results.priceChanges.length,
|
|
2119
|
-
changes: results.priceChanges.slice(0, 10),
|
|
2120
|
-
});
|
|
2121
|
-
}
|
|
2122
|
-
```
|
|
2123
|
-
|
|
2124
|
-
**Why this matters**: Provides audit trail and monitoring for price changes.
|
|
2125
|
-
|
|
2126
|
-
### Pattern 7: Concurrency Control & Alias Batching
|
|
2127
|
-
|
|
2128
|
-
```typescript
|
|
2129
|
-
// ✅ Configuration with defaults
|
|
2130
|
-
const mutationBatchSize = parseInt(
|
|
2131
|
-
activation?.getVariable('mutationBatchSize') || '1', // Default: 1 (sequential)
|
|
2132
|
-
10
|
|
2133
|
-
);
|
|
2134
|
-
|
|
2135
|
-
const mutationsPerAliasBatch = activation?.getVariable('mutationsPerAliasBatch')
|
|
2136
|
-
? parseInt(activation?.getVariable('mutationsPerAliasBatch') || '1', 10)
|
|
2137
|
-
: undefined; // Default: undefined (disabled)
|
|
2138
|
-
|
|
2139
|
-
// Execute with bounded concurrency + optional alias batching
|
|
2140
|
-
const mutationResult = await executeMutations(
|
|
2141
|
-
client,
|
|
2142
|
-
mapper,
|
|
2143
|
-
priceRecords,
|
|
2144
|
-
retailerId,
|
|
2145
|
-
validateProducts,
|
|
2146
|
-
mutationBatchSize, // 1=sequential, 3-10=parallel
|
|
2147
|
-
mutationsPerAliasBatch, // Optional: Group mutations (e.g., 5)
|
|
2148
|
-
log
|
|
2149
|
-
);
|
|
2150
|
-
```
|
|
2151
|
-
|
|
2152
|
-
**Performance Modes:**
|
|
2153
|
-
- `mutationBatchSize: 1` → Sequential (safe default, ~1 mutation/sec)
|
|
2154
|
-
- `mutationBatchSize: 3-5` → Balanced (~3-5 mutations/sec)
|
|
2155
|
-
- `mutationBatchSize: 10` → High-volume (~10 mutations/sec)
|
|
2156
|
-
- `mutationsPerAliasBatch: 5` → Alias batching (reduces network overhead by ~80%)
|
|
2157
|
-
|
|
2158
|
-
## Advanced: Multi-Currency Price Management
|
|
2159
|
-
|
|
2160
|
-
### External Mapping File for Multi-Currency
|
|
2161
|
-
|
|
2162
|
-
**File**: `config/price-mapping.json`
|
|
2163
|
-
|
|
2164
|
-
```json
|
|
2165
|
-
{
|
|
2166
|
-
"version": "1.0.0",
|
|
2167
|
-
"description": "Multi-currency price mapping",
|
|
2168
|
-
"fields": {
|
|
2169
|
-
"productRef": {
|
|
2170
|
-
"source": "sku",
|
|
2171
|
-
"required": true,
|
|
2172
|
-
"resolver": "sdk.trim"
|
|
2173
|
-
},
|
|
2174
|
-
"type": {
|
|
2175
|
-
"source": "price_type",
|
|
2176
|
-
"required": true,
|
|
2177
|
-
"resolver": "custom.normalizePriceType"
|
|
2178
|
-
},
|
|
2179
|
-
"value": {
|
|
2180
|
-
"source": "amount",
|
|
2181
|
-
"required": true,
|
|
2182
|
-
"resolver": "custom.validatePriceRange"
|
|
2183
|
-
},
|
|
2184
|
-
"currency": {
|
|
2185
|
-
"source": "currency_code",
|
|
2186
|
-
"required": true,
|
|
2187
|
-
"resolver": "custom.normalizeCurrency"
|
|
2188
|
-
},
|
|
2189
|
-
"effectiveFrom": {
|
|
2190
|
-
"source": "start_date",
|
|
2191
|
-
"resolver": "sdk.formatDate"
|
|
2192
|
-
},
|
|
2193
|
-
"effectiveTo": {
|
|
2194
|
-
"source": "end_date",
|
|
2195
|
-
"resolver": "sdk.formatDate"
|
|
2196
|
-
}
|
|
2197
|
-
}
|
|
2198
|
-
}
|
|
2199
|
-
```
|
|
2200
|
-
|
|
2201
|
-
### Custom Resolvers for Multi-Currency
|
|
2202
|
-
|
|
2203
|
-
```typescript
|
|
2204
|
-
const customResolvers = {
|
|
2205
|
-
'custom.normalizeCurrency': (value: string) => {
|
|
2206
|
-
// Normalize currency codes to ISO 4217
|
|
2207
|
-
const currencyMap: Record<string, string> = {
|
|
2208
|
-
usd: 'USD',
|
|
2209
|
-
us: 'USD',
|
|
2210
|
-
dollar: 'USD',
|
|
2211
|
-
eur: 'EUR',
|
|
2212
|
-
euro: 'EUR',
|
|
2213
|
-
gbp: 'GBP',
|
|
2214
|
-
pound: 'GBP',
|
|
2215
|
-
cad: 'CAD',
|
|
2216
|
-
aud: 'AUD',
|
|
2217
|
-
};
|
|
2218
|
-
|
|
2219
|
-
const normalized = currencyMap[value.toLowerCase()] || value.toUpperCase();
|
|
2220
|
-
|
|
2221
|
-
// Validate against known currencies
|
|
2222
|
-
const validCurrencies = ['USD', 'EUR', 'GBP', 'CAD', 'AUD', 'JPY', 'CNY'];
|
|
2223
|
-
if (!validCurrencies.includes(normalized)) {
|
|
2224
|
-
throw new Error(`Invalid currency code: ${value}`);
|
|
2225
|
-
}
|
|
2226
|
-
|
|
2227
|
-
return normalized;
|
|
2228
|
-
},
|
|
2229
|
-
|
|
2230
|
-
'custom.convertToBaseCurrency': (value: any, sourceRecord: any) => {
|
|
2231
|
-
// Convert foreign currency to base currency
|
|
2232
|
-
const amount = parseFloat(value);
|
|
2233
|
-
const currency = sourceRecord.currency_code;
|
|
2234
|
-
|
|
2235
|
-
// Exchange rates (in production, fetch from API)
|
|
2236
|
-
const exchangeRates: Record<string, number> = {
|
|
2237
|
-
USD: 1.0,
|
|
2238
|
-
EUR: 0.85,
|
|
2239
|
-
GBP: 0.73,
|
|
2240
|
-
CAD: 1.35,
|
|
2241
|
-
AUD: 1.45,
|
|
2242
|
-
};
|
|
2243
|
-
|
|
2244
|
-
const rate = exchangeRates[currency?.toUpperCase()] || 1.0;
|
|
2245
|
-
return amount * rate;
|
|
2246
|
-
},
|
|
2247
|
-
|
|
2248
|
-
'custom.buildPriceAttributes': (_: any, sourceRecord: any) => {
|
|
2249
|
-
// Build price attributes array
|
|
2250
|
-
const attributes: Array<{ name: string; type: string; value: any }> = [];
|
|
2251
|
-
|
|
2252
|
-
if (sourceRecord.cost_basis) {
|
|
2253
|
-
attributes.push({
|
|
2254
|
-
name: 'costBasis',
|
|
2255
|
-
type: 'Float',
|
|
2256
|
-
value: parseFloat(sourceRecord.cost_basis),
|
|
2257
|
-
});
|
|
2258
|
-
}
|
|
2259
|
-
|
|
2260
|
-
if (sourceRecord.margin_percent) {
|
|
2261
|
-
attributes.push({
|
|
2262
|
-
name: 'marginPercent',
|
|
2263
|
-
type: 'Float',
|
|
2264
|
-
value: parseFloat(sourceRecord.margin_percent),
|
|
2265
|
-
});
|
|
2266
|
-
}
|
|
2267
|
-
|
|
2268
|
-
if (sourceRecord.competitor_price) {
|
|
2269
|
-
attributes.push({
|
|
2270
|
-
name: 'competitorPrice',
|
|
2271
|
-
type: 'Float',
|
|
2272
|
-
value: parseFloat(sourceRecord.competitor_price),
|
|
2273
|
-
});
|
|
2274
|
-
}
|
|
2275
|
-
|
|
2276
|
-
if (sourceRecord.price_source) {
|
|
2277
|
-
attributes.push({
|
|
2278
|
-
name: 'priceSource',
|
|
2279
|
-
type: 'String',
|
|
2280
|
-
value: sourceRecord.price_source,
|
|
2281
|
-
});
|
|
2282
|
-
}
|
|
2283
|
-
|
|
2284
|
-
return attributes;
|
|
2285
|
-
},
|
|
2286
|
-
};
|
|
2287
|
-
```
|
|
2288
|
-
|
|
2289
|
-
## Schema Validation (Before Deployment)
|
|
2290
|
-
|
|
2291
|
-
Use SDK CLI tools to validate your GraphQL schema and mapping configuration:
|
|
2292
|
-
|
|
2293
|
-
```bash
|
|
2294
|
-
# Install SDK globally (or use npx)
|
|
2295
|
-
npm install -g @fluentcommerce/fc-connect-sdk
|
|
2296
|
-
|
|
2297
|
-
# 1. Introspect Fluent GraphQL schema
|
|
2298
|
-
fc-connect introspect-schema \
|
|
2299
|
-
--url https://api.fluentcommerce.com/graphql \
|
|
2300
|
-
--client-id YOUR_CLIENT_ID \
|
|
2301
|
-
--client-secret YOUR_CLIENT_SECRET \
|
|
2302
|
-
--output fluent-schema.json
|
|
2303
|
-
|
|
2304
|
-
# 2. Create mutation file (price-mutation.graphql)
|
|
2305
|
-
cat > price-mutation.graphql << 'EOF'
|
|
2306
|
-
mutation UpdateProductPrice($input: UpdateProductInput!) {
|
|
2307
|
-
updateProduct(input: $input) {
|
|
2308
|
-
id
|
|
2309
|
-
ref
|
|
2310
|
-
prices {
|
|
2311
|
-
type
|
|
2312
|
-
value
|
|
2313
|
-
currency
|
|
2314
|
-
}
|
|
2315
|
-
}
|
|
2316
|
-
}
|
|
2317
|
-
EOF
|
|
2318
|
-
|
|
2319
|
-
# 3. Generate mapping from mutation (validates structure)
|
|
2320
|
-
fc-connect generate-mutation-mapping \
|
|
2321
|
-
--file price-mutation.graphql \
|
|
2322
|
-
--output price-mapping.json
|
|
2323
|
-
|
|
2324
|
-
# 4. Validate mapping against schema
|
|
2325
|
-
fc-connect validate-schema \
|
|
2326
|
-
--mapping price-mapping.json \
|
|
2327
|
-
--schema fluent-schema.json
|
|
2328
|
-
|
|
2329
|
-
# 5. Analyze field coverage
|
|
2330
|
-
fc-connect analyze-coverage \
|
|
2331
|
-
--mapping price-mapping.json \
|
|
2332
|
-
--schema fluent-schema.json
|
|
2333
|
-
```
|
|
2334
|
-
|
|
2335
|
-
**Output Example**:
|
|
2336
|
-
|
|
2337
|
-
```bash
|
|
2338
|
-
✓ Schema validation passed
|
|
2339
|
-
✓ All required fields present: ref, retailerId, prices[].type, prices[].value, prices[].currency
|
|
2340
|
-
✓ Mutation structure matches GraphQL schema
|
|
2341
|
-
⚠ Optional fields not mapped: prices[].taxType, prices[].taxRate
|
|
2342
|
-
```
|
|
2343
|
-
|
|
2344
|
-
## Testing the Workflow
|
|
2345
|
-
|
|
2346
|
-
### 1. Upload Test CSV to S3
|
|
2347
|
-
|
|
2348
|
-
```bash
|
|
2349
|
-
# Using AWS CLI
|
|
2350
|
-
aws s3 cp product-prices-test.csv s3://my-price-bucket/prices/
|
|
2351
|
-
|
|
2352
|
-
# Or using S3 Console
|
|
2353
|
-
# Navigate to bucket → prices/ → Upload
|
|
2354
|
-
```
|
|
2355
|
-
|
|
2356
|
-
### 2. Deploy to Versori
|
|
2357
|
-
|
|
2358
|
-
```bash
|
|
2359
|
-
npm run deploy
|
|
2360
|
-
```
|
|
2361
|
-
|
|
2362
|
-
### 3. Manual Testing via Webhook
|
|
2363
|
-
|
|
2364
|
-
```bash
|
|
2365
|
-
curl -X POST https://your-workspace.versori.run/sync-prices-now \
|
|
2366
|
-
-H "Content-Type: application/json"
|
|
2367
|
-
```
|
|
2368
|
-
|
|
2369
|
-
### 4. Check Status
|
|
2370
|
-
|
|
2371
|
-
```bash
|
|
2372
|
-
curl https://your-workspace.versori.run/check-status
|
|
2373
|
-
```
|
|
2374
|
-
|
|
2375
|
-
### 5. Monitor Logs
|
|
2376
|
-
|
|
2377
|
-
```bash
|
|
2378
|
-
npm run logs
|
|
2379
|
-
# Or via Versori dashboard
|
|
2380
|
-
```
|
|
2381
|
-
|
|
2382
|
-
---
|
|
2383
|
-
|
|
2384
|
-
## Monitoring
|
|
2385
|
-
|
|
2386
|
-
### Success Response
|
|
2387
|
-
|
|
2388
|
-
```json
|
|
2389
|
-
{
|
|
2390
|
-
"success": true,
|
|
2391
|
-
"filesProcessed": 1,
|
|
2392
|
-
"filesSkipped": 0,
|
|
2393
|
-
"filesFailed": 0,
|
|
2394
|
-
"totalRecords": 100,
|
|
2395
|
-
"mutationsExecuted": 100,
|
|
2396
|
-
"mutationsFailed": 0,
|
|
2397
|
-
"results": [
|
|
2398
|
-
{
|
|
2399
|
-
"file": "prices_2025-01-22.csv",
|
|
2400
|
-
"success": true,
|
|
2401
|
-
"recordsProcessed": 100,
|
|
2402
|
-
"mutationsExecuted": 100,
|
|
2403
|
-
"mutationsFailed": 0
|
|
2404
|
-
}
|
|
2405
|
-
],
|
|
2406
|
-
"duration": 12345
|
|
2407
|
-
}
|
|
2408
|
-
```
|
|
2409
|
-
|
|
2410
|
-
### Partial Success Response
|
|
2411
|
-
|
|
2412
|
-
```json
|
|
2413
|
-
{
|
|
2414
|
-
"success": true,
|
|
2415
|
-
"filesProcessed": 1,
|
|
2416
|
-
"filesSkipped": 0,
|
|
2417
|
-
"filesFailed": 0,
|
|
2418
|
-
"totalRecords": 100,
|
|
2419
|
-
"mutationsExecuted": 95,
|
|
2420
|
-
"mutationsFailed": 5,
|
|
2421
|
-
"results": [
|
|
2422
|
-
{
|
|
2423
|
-
"file": "prices_2025-01-22.csv",
|
|
2424
|
-
"success": true,
|
|
2425
|
-
"recordsProcessed": 100,
|
|
2426
|
-
"mutationsExecuted": 95,
|
|
2427
|
-
"mutationsFailed": 5,
|
|
2428
|
-
"errors": ["PRICE-001: Invalid price value", "PRICE-002: Missing currency"]
|
|
2429
|
-
}
|
|
2430
|
-
],
|
|
2431
|
-
"duration": 12345
|
|
2432
|
-
}
|
|
2433
|
-
```
|
|
2434
|
-
|
|
2435
|
-
### Error Response
|
|
2436
|
-
|
|
2437
|
-
```json
|
|
2438
|
-
{
|
|
2439
|
-
"success": false,
|
|
2440
|
-
"filesProcessed": 0,
|
|
2441
|
-
"filesFailed": 1,
|
|
2442
|
-
"totalRecords": 0,
|
|
2443
|
-
"mutationsExecuted": 0,
|
|
2444
|
-
"mutationsFailed": 0,
|
|
2445
|
-
"results": [
|
|
2446
|
-
{
|
|
2447
|
-
"file": "prices_2025-01-22.csv",
|
|
2448
|
-
"success": false,
|
|
2449
|
-
"error": "CSV parse error: Invalid structure"
|
|
2450
|
-
}
|
|
2451
|
-
],
|
|
2452
|
-
"duration": 876
|
|
2453
|
-
}
|
|
2454
|
-
```
|
|
2455
|
-
|
|
2456
|
-
### Monitoring Metrics
|
|
2457
|
-
|
|
2458
|
-
Monitor these metrics via Versori logs filtered by `jobId` or `stage`:
|
|
2459
|
-
|
|
2460
|
-
- **Files Processed** - Total files successfully processed
|
|
2461
|
-
- **Mutations Executed** - Total GraphQL mutations executed successfully
|
|
2462
|
-
- **Mutations Failed** - Mutations that failed (check error logs)
|
|
2463
|
-
- **Processing Duration** - Time taken for complete workflow
|
|
2464
|
-
- **Rate Limiting** - Watch for 429 errors indicating GraphQL throttling
|
|
2465
|
-
|
|
2466
|
-
Use the status webhook for dashboards and automated monitoring.
|
|
2467
|
-
|
|
2468
|
-
---
|
|
2469
|
-
|
|
2470
|
-
### Issue 1: Product Not Found
|
|
2471
|
-
|
|
2472
|
-
**Error**: `Skipping price update for non-existent product: PROD-XYZ`
|
|
2473
|
-
|
|
2474
|
-
**Solution**:
|
|
2475
|
-
|
|
2476
|
-
- Verify product exists in Fluent Commerce
|
|
2477
|
-
- Check SKU matches exactly (case-sensitive)
|
|
2478
|
-
- Ensure product status is `ACTIVE`
|
|
2479
|
-
- Disable validation temporarily for testing: `validateProducts=false`
|
|
2480
|
-
|
|
2481
|
-
### Issue 2: Price Validation Failures
|
|
2482
|
-
|
|
2483
|
-
**Error**: `Price 0.00 below minimum 0.01`
|
|
2484
|
-
|
|
2485
|
-
**Solution**:
|
|
2486
|
-
|
|
2487
|
-
```bash
|
|
2488
|
-
# Adjust validation rules in activation variables
|
|
2489
|
-
minPrice=0.00 # Allow zero prices
|
|
2490
|
-
maxPrice=9999999.99 # Increase max
|
|
2491
|
-
```
|
|
2492
|
-
|
|
2493
|
-
Or fix source data:
|
|
2494
|
-
|
|
2495
|
-
```csv
|
|
2496
|
-
# ❌ WRONG
|
|
2497
|
-
PROD-001,DEFAULT,0,USD
|
|
2498
|
-
|
|
2499
|
-
# ✅ CORRECT
|
|
2500
|
-
PROD-001,DEFAULT,0.01,USD
|
|
2501
|
-
```
|
|
2502
|
-
|
|
2503
|
-
### Issue 3: Invalid Price Type
|
|
2504
|
-
|
|
2505
|
-
**Error**: `Invalid price type: PROMO`
|
|
2506
|
-
|
|
2507
|
-
**Solution**: Update custom resolver to include new type:
|
|
2508
|
-
|
|
2509
|
-
```typescript
|
|
2510
|
-
'custom.normalizePriceType': (value: any) => {
|
|
2511
|
-
const type = String(value || 'DEFAULT').toUpperCase().trim();
|
|
2512
|
-
const validTypes = [
|
|
2513
|
-
'DEFAULT',
|
|
2514
|
-
'SALE',
|
|
2515
|
-
'CLEARANCE',
|
|
2516
|
-
'BULK',
|
|
2517
|
-
'PROMOTIONAL',
|
|
2518
|
-
'MEMBER',
|
|
2519
|
-
'PROMO', // Add new type
|
|
2520
|
-
];
|
|
2521
|
-
if (!validTypes.includes(type)) {
|
|
2522
|
-
throw new Error(`Invalid price type: ${type}`);
|
|
2523
|
-
}
|
|
2524
|
-
return type;
|
|
2525
|
-
};
|
|
2526
|
-
```
|
|
2527
|
-
|
|
2528
|
-
### Issue 4: Duplicate Price Updates
|
|
2529
|
-
|
|
2530
|
-
**Symptom**: Same file processed multiple times
|
|
2531
|
-
|
|
2532
|
-
**Solution**: VersoriKVAdapter already prevents this:
|
|
2533
|
-
|
|
2534
|
-
```typescript
|
|
2535
|
-
const stateKey = ['processed-files', 's3-price-sync', fileName];
|
|
2536
|
-
const existing = await kv.get(stateKey);
|
|
2537
|
-
if (existing) {
|
|
2538
|
-
log.info('Skipping already processed file', { fileName });
|
|
2539
|
-
continue;
|
|
2540
|
-
}
|
|
2541
|
-
```
|
|
2542
|
-
|
|
2543
|
-
Verify KV storage is working:
|
|
2544
|
-
|
|
2545
|
-
```bash
|
|
2546
|
-
# Check Versori logs for:
|
|
2547
|
-
[INFO] Skipping already processed file: product-prices-20250122-001.csv
|
|
2548
|
-
```
|
|
2549
|
-
|
|
2550
|
-
### Issue 5: S3 Access Denied
|
|
2551
|
-
|
|
2552
|
-
Required IAM Permissions:
|
|
2553
|
-
|
|
2554
|
-
```json
|
|
2555
|
-
{
|
|
2556
|
-
"Version": "2012-10-17",
|
|
2557
|
-
"Statement": [
|
|
2558
|
-
{
|
|
2559
|
-
"Effect": "Allow",
|
|
2560
|
-
"Action": ["s3:ListBucket", "s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
|
|
2561
|
-
"Resource": [
|
|
2562
|
-
"arn:aws:s3:::my-price-bucket",
|
|
2563
|
-
"arn:aws:s3:::my-price-bucket/*"
|
|
2564
|
-
]
|
|
2565
|
-
}
|
|
2566
|
-
]
|
|
2567
|
-
}
|
|
2568
|
-
```
|
|
2569
|
-
|
|
2570
|
-
### Issue 6: GraphQL Mutation Failures
|
|
2571
|
-
|
|
2572
|
-
**Common Causes**:
|
|
2573
|
-
|
|
2574
|
-
- Invalid price type (check Fluent schema for valid types)
|
|
2575
|
-
- Missing required fields (ref, retailerId, prices)
|
|
2576
|
-
- Invalid currency code (must be ISO 4217: USD, EUR, GBP)
|
|
2577
|
-
- Product not found or inactive
|
|
2578
|
-
|
|
2579
|
-
**Debugging**:
|
|
2580
|
-
|
|
2581
|
-
```typescript
|
|
2582
|
-
// Log mutation input before execution
|
|
2583
|
-
log.info('GraphQL mutation input', { input });
|
|
2584
|
-
|
|
2585
|
-
// Validate required fields
|
|
2586
|
-
if (!priceData.productRef || !priceData.type || !priceData.value) {
|
|
2587
|
-
log.error('Missing required fields', { priceData });
|
|
2588
|
-
continue;
|
|
2589
|
-
}
|
|
2590
|
-
```
|
|
2591
|
-
|
|
2592
|
-
## Production Checklist
|
|
2593
|
-
|
|
2594
|
-
- [ ] S3 credentials validated with correct IAM permissions
|
|
2595
|
-
- [ ] Activation secrets stored securely (not in code)
|
|
2596
|
-
- [ ] GraphQL schema validated using CLI tools
|
|
2597
|
-
- [ ] Mapping configuration uses GraphQLMutationMapper structure (NOT UniversalMapper)
|
|
2598
|
-
- [ ] Mapping configuration tested with sample data
|
|
2599
|
-
- [ ] NO setRetailerId() call in code (only for Job/Event API)
|
|
2600
|
-
- [ ] Price validation rules configured per business policy
|
|
2601
|
-
- [ ] Product existence validation enabled (`validateProducts=true`)
|
|
2602
|
-
- [ ] File duplicate prevention working via KV state
|
|
2603
|
-
- [ ] Concurrency control configured (mutationBatchSize)
|
|
2604
|
-
- [ ] Optional alias batching tested if enabled (mutationsPerAliasBatch)
|
|
2605
|
-
- [ ] Error handling tested with malformed CSV
|
|
2606
|
-
- [ ] Retry logic tested with transient failures
|
|
2607
|
-
- [ ] File archival working (processed and error directories)
|
|
2608
|
-
- [ ] Price change tracking verified in logs
|
|
2609
|
-
- [ ] Monitoring/alerting configured for price update failures
|
|
2610
|
-
- [ ] Clear runbook for error recovery
|
|
2611
|
-
|
|
2612
|
-
## Related Guides
|
|
2613
|
-
|
|
2614
|
-
- **GraphQL Mutation Mapping**: `docs/02-CORE-GUIDES/mapping/graphql-mutation-mapping/`
|
|
2615
|
-
- **GraphQL Alias Batching**: `docs/02-CORE-GUIDES/mapping/graphql-alias-batching-guide.md`
|
|
2616
|
-
- **retailerId Configuration**: `docs/00-START-HERE/retailerid-configuration.md`
|
|
2617
|
-
- **CLI Tools**: `fc-connect-sdk/bin/readme.md`
|
|
2618
|
-
- **State & KV patterns**: `docs/03-PATTERN-GUIDES/file-operations/`
|
|
2619
|
-
- **Error handling**: `docs/03-PATTERN-GUIDES/error-handling/`
|
|
1
|
+
---
|
|
2
|
+
template_id: tpl-ingest-s3-csv-to-price-graphql
|
|
3
|
+
canonical_filename: template-ingestion-s3-csv-price-graphql.md
|
|
4
|
+
version: 2.0.0
|
|
5
|
+
sdk_version: ^0.1.39
|
|
6
|
+
runtime: versori
|
|
7
|
+
direction: ingestion
|
|
8
|
+
source: s3-csv
|
|
9
|
+
destination: fluent-graphql
|
|
10
|
+
entity: price
|
|
11
|
+
format: csv
|
|
12
|
+
logging: versori
|
|
13
|
+
status: stable
|
|
14
|
+
features:
|
|
15
|
+
- graphql-mutation-mapper
|
|
16
|
+
- memory-management
|
|
17
|
+
- enhanced-logging
|
|
18
|
+
- attribute-transformation
|
|
19
|
+
compliance: gold-standard
|
|
20
|
+
---
|
|
21
|
+
|
|
22
|
+
# Template: Ingestion - S3 CSV to Price GraphQL
|
|
23
|
+
n**Template Version:** 2.0.0
|
|
24
|
+
**SDK Version:** @fluentcommerce/fc-connect-sdk@^0.1.39
|
|
25
|
+
**Last Updated:** 2025-01-24
|
|
26
|
+
|
|
27
|
+
**🆕 Version 2.0.0 Enhancements:**
|
|
28
|
+
- ✅ **GraphQL Mutation Mapper** - Direct field mapping to mutation variables
|
|
29
|
+
- ✅ **Memory Management** - Clear large arrays after processing batches
|
|
30
|
+
- ✅ **Enhanced Logging** - Track mutation execution with emoji indicators
|
|
31
|
+
- ✅ **Attribute Transformation** - Handle complex nested data structures
|
|
32
|
+
|
|
33
|
+
## STEP 1: What This Template Does
|
|
34
|
+
|
|
35
|
+
This template provides a **scheduled Versori workflow** that:
|
|
36
|
+
|
|
37
|
+
1. **Discovers CSV files** from S3 bucket with price data
|
|
38
|
+
2. **Parses CSV** using `CSVParserService` with validation
|
|
39
|
+
3. **Validates products exist** in Fluent Commerce before updating prices
|
|
40
|
+
4. **Maps fields** using `GraphQLMutationMapper` with custom price validation resolvers
|
|
41
|
+
5. **Updates prices** via direct GraphQL mutations (`updateProduct`)
|
|
42
|
+
6. **Tracks price changes** for audit trail
|
|
43
|
+
7. **Archives processed files** to prevent duplicates
|
|
44
|
+
8. **Uses JobTracker** for job lifecycle management
|
|
45
|
+
|
|
46
|
+
**Key Features:**
|
|
47
|
+
- ✅ Direct GraphQL mutations (NOT Batch API)
|
|
48
|
+
- ✅ Product existence validation before price updates
|
|
49
|
+
- ✅ Price range validation (min/max checks)
|
|
50
|
+
- ✅ Multi-tier pricing support (DEFAULT, SALE, CLEARANCE, BULK)
|
|
51
|
+
- ✅ Configurable concurrency control (sequential or parallel)
|
|
52
|
+
- ✅ Optional GraphQL alias batching for high-volume scenarios
|
|
53
|
+
- ✅ Price change tracking and reporting
|
|
54
|
+
- ✅ Versori KV state management (duplicate prevention)
|
|
55
|
+
- ✅ File archival with error handling
|
|
56
|
+
|
|
57
|
+
**Use Cases:**
|
|
58
|
+
- Daily product price synchronization
|
|
59
|
+
- Promotional price updates
|
|
60
|
+
- Multi-currency pricing management
|
|
61
|
+
- Quantity-based pricing tiers
|
|
62
|
+
|
|
63
|
+
## STEP 2: Understanding This Template (AI Agent Guide)
|
|
64
|
+
|
|
65
|
+
**🎯 Template Purpose:** Scheduled price synchronization from S3 CSV files to Fluent Commerce using direct GraphQL mutations.
|
|
66
|
+
|
|
67
|
+
### Architecture Pattern
|
|
68
|
+
|
|
69
|
+
```
|
|
70
|
+
S3 CSV Files → CSVParserService → GraphQLMutationMapper → Product Validation → GraphQL Mutations → Archive
|
|
71
|
+
↓ ↓ ↓ ↓ ↓ ↓
|
|
72
|
+
File Discovery Parsing Field Mapping Existence Check updateProduct Processed/
|
|
73
|
+
Price Data Validation Price Validation Active Status Concurrency Ctrl Error Dirs
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
### SDK Services Used
|
|
77
|
+
|
|
78
|
+
| Service | Purpose | Key Methods |
|
|
79
|
+
|---------|---------|-------------|
|
|
80
|
+
| `createClient(ctx)` | Fluent API client | `client.graphql()` |
|
|
81
|
+
| `S3DataSource` | S3 operations | `listFiles()`, `downloadFile()`, `moveFile()` |
|
|
82
|
+
| `CSVParserService` | CSV parsing | `parse()` |
|
|
83
|
+
| `GraphQLMutationMapper` | Field transformation | `map()` with custom resolvers |
|
|
84
|
+
| `VersoriKVAdapter` | State management | `get()`, `set()`, `list()` |
|
|
85
|
+
| `JobTracker` | Job lifecycle | `createJob()`, `updateJob()`, `markCompleted()`, `markFailed()`, `getJob()` |
|
|
86
|
+
|
|
87
|
+
### Price Entity Structure
|
|
88
|
+
|
|
89
|
+
**Fluent Commerce Price Schema:**
|
|
90
|
+
```typescript
|
|
91
|
+
{
|
|
92
|
+
productRef: string; // SKU reference (required)
|
|
93
|
+
type: string; // DEFAULT, SALE, CLEARANCE, BULK (required)
|
|
94
|
+
value: number; // Price amount (required, validated)
|
|
95
|
+
currency: string; // ISO 4217 code (USD, EUR, GBP)
|
|
96
|
+
effectiveFrom?: string; // ISO 8601 date
|
|
97
|
+
effectiveTo?: string; // ISO 8601 date
|
|
98
|
+
minQuantity?: number; // Min qty for tier pricing
|
|
99
|
+
maxQuantity?: number; // Max qty for tier pricing
|
|
100
|
+
}
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
### GraphQL Mutation Pattern
|
|
104
|
+
|
|
105
|
+
**Mutation Used:** `updateProduct` (NOT Batch API)
|
|
106
|
+
|
|
107
|
+
```graphql
|
|
108
|
+
mutation UpdateProductPrice($input: UpdateProductInput!) {
|
|
109
|
+
updateProduct(input: $input) {
|
|
110
|
+
id
|
|
111
|
+
ref
|
|
112
|
+
prices {
|
|
113
|
+
type
|
|
114
|
+
value
|
|
115
|
+
currency
|
|
116
|
+
effectiveFrom
|
|
117
|
+
effectiveTo
|
|
118
|
+
minQuantity
|
|
119
|
+
maxQuantity
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
**Why Direct Mutations?**
|
|
126
|
+
- ✅ Immediate price updates (no batch processing delay)
|
|
127
|
+
- ✅ Product validation before update
|
|
128
|
+
- ✅ Price change tracking
|
|
129
|
+
- ✅ Suitable for low-volume price updates (<1000/day)
|
|
130
|
+
|
|
131
|
+
**NO BPP (Batch Pre-Processing):** Direct GraphQL mutations bypass batch workflows entirely.
|
|
132
|
+
|
|
133
|
+
### Custom Resolvers
|
|
134
|
+
|
|
135
|
+
**Three price-specific resolvers:**
|
|
136
|
+
|
|
137
|
+
1. **`custom.validatePriceRange`** - Ensures price within min/max bounds
|
|
138
|
+
2. **`custom.normalizePriceType`** - Validates price tier types
|
|
139
|
+
3. **`custom.normalizeQuantity`** - Validates quantity ranges for BULK pricing
|
|
140
|
+
|
|
141
|
+
### Product Validation Pattern
|
|
142
|
+
|
|
143
|
+
**Critical Step:** Validate product exists and is ACTIVE before price update:
|
|
144
|
+
|
|
145
|
+
```typescript
|
|
146
|
+
async function validateProductExists(client, productRef, retailerId, log) {
|
|
147
|
+
const query = `
|
|
148
|
+
query GetProduct($ref: String!, $retailerId: ID!) {
|
|
149
|
+
products(first: 1, ref: [$ref], retailerId: $retailerId) {
|
|
150
|
+
edges {
|
|
151
|
+
node { id ref status }
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
`;
|
|
156
|
+
const result = await client.graphql({ query, variables: { ref: productRef, retailerId } });
|
|
157
|
+
const product = result?.data?.products?.edges?.[0]?.node;
|
|
158
|
+
return product && product.status === 'ACTIVE';
|
|
159
|
+
}
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
**Why?** Prevents GraphQL errors for non-existent products.
|
|
163
|
+
|
|
164
|
+
### Concurrency Control & Performance
|
|
165
|
+
|
|
166
|
+
**Mutation execution** supports two performance modes:
|
|
167
|
+
|
|
168
|
+
**Mode 1: Concurrency Control (default)**
|
|
169
|
+
```typescript
|
|
170
|
+
// Configuration: Control concurrent mutations
|
|
171
|
+
const mutationBatchSize = parseInt(
|
|
172
|
+
activation?.getVariable('mutationBatchSize') || '1', // Default: 1 (sequential)
|
|
173
|
+
10
|
|
174
|
+
);
|
|
175
|
+
|
|
176
|
+
// Execute with bounded concurrency
|
|
177
|
+
await executeMutations(
|
|
178
|
+
client,
|
|
179
|
+
mapper,
|
|
180
|
+
priceRecords,
|
|
181
|
+
retailerId,
|
|
182
|
+
validateProducts,
|
|
183
|
+
mutationBatchSize, // 1=sequential, 3-10=parallel
|
|
184
|
+
undefined, // No alias batching
|
|
185
|
+
log
|
|
186
|
+
);
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
**Mode 2: GraphQL Alias Batching (optional, high-volume)**
|
|
190
|
+
```typescript
|
|
191
|
+
// Configuration: Group mutations into aliased requests
|
|
192
|
+
const mutationsPerAliasBatch = activation?.getVariable('mutationsPerAliasBatch')
|
|
193
|
+
? parseInt(activation?.getVariable('mutationsPerAliasBatch') || '1', 10)
|
|
194
|
+
: undefined; // Default: undefined (disabled)
|
|
195
|
+
|
|
196
|
+
// Execute with alias batching (reduces network overhead by ~80%)
|
|
197
|
+
await executeMutations(
|
|
198
|
+
client,
|
|
199
|
+
mapper,
|
|
200
|
+
priceRecords,
|
|
201
|
+
retailerId,
|
|
202
|
+
validateProducts,
|
|
203
|
+
mutationBatchSize, // Concurrency control
|
|
204
|
+
mutationsPerAliasBatch, // Alias batching (e.g., 5)
|
|
205
|
+
log
|
|
206
|
+
);
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
**Activation Variables:**
|
|
210
|
+
- `mutationBatchSize=1` - Default: sequential (safe)
|
|
211
|
+
- `mutationBatchSize=3-5` - Balanced throughput
|
|
212
|
+
- `mutationBatchSize=10` - High-volume (100+ locations)
|
|
213
|
+
- `mutationsPerAliasBatch` - Optional: Group mutations (e.g., 5 per request)
|
|
214
|
+
|
|
215
|
+
### State Management
|
|
216
|
+
|
|
217
|
+
**VersoriKVAdapter** prevents duplicate file processing:
|
|
218
|
+
|
|
219
|
+
```typescript
|
|
220
|
+
const stateKey = ['processed-files', 's3-price-sync', fileName];
|
|
221
|
+
const existing = await kv.get(stateKey);
|
|
222
|
+
if (existing) {
|
|
223
|
+
log.info('Skipping already processed file', { fileName });
|
|
224
|
+
continue;
|
|
225
|
+
}
|
|
226
|
+
// After success:
|
|
227
|
+
await kv.set(stateKey, { successful, failed, processedAt: new Date().toISOString() });
|
|
228
|
+
```
|
|
229
|
+
|
|
230
|
+
### JobTracker Integration
|
|
231
|
+
|
|
232
|
+
**Job lifecycle tracking:**
|
|
233
|
+
|
|
234
|
+
```typescript
|
|
235
|
+
const tracker = new JobTracker(openKv(':project:'), log);
|
|
236
|
+
await tracker.startJob(jobId, { mode: 'scheduled' });
|
|
237
|
+
try {
|
|
238
|
+
const result = await runPriceCsvWorkflow(ctx, jobId, tracker);
|
|
239
|
+
await tracker.completeJob(jobId, { result });
|
|
240
|
+
} catch (e) {
|
|
241
|
+
await tracker.failJob(jobId, { error: e.message });
|
|
242
|
+
}
|
|
243
|
+
```
|
|
244
|
+
|
|
245
|
+
### Price Change Tracking
|
|
246
|
+
|
|
247
|
+
**Audit trail for price updates:**
|
|
248
|
+
|
|
249
|
+
```typescript
|
|
250
|
+
const currentPrice = await getCurrentPrice(client, productRef, priceType, currency, retailerId);
|
|
251
|
+
// ... after update ...
|
|
252
|
+
if (currentPrice !== undefined && currentPrice !== priceData.value) {
|
|
253
|
+
results.priceChanges.push({
|
|
254
|
+
sku: priceData.productRef,
|
|
255
|
+
type: priceData.type,
|
|
256
|
+
oldPrice: currentPrice,
|
|
257
|
+
newPrice: priceData.value,
|
|
258
|
+
});
|
|
259
|
+
}
|
|
260
|
+
// Log changes at end:
|
|
261
|
+
log.info('Price changes detected', { count: results.priceChanges.length, changes });
|
|
262
|
+
```
|
|
263
|
+
|
|
264
|
+
### Schema Validation (CLI Commands)
|
|
265
|
+
|
|
266
|
+
**Before deployment, validate GraphQL schema:**
|
|
267
|
+
|
|
268
|
+
```bash
|
|
269
|
+
# 1. Introspect Fluent schema
|
|
270
|
+
fc-connect introspect-schema \
|
|
271
|
+
--url https://api.fluentcommerce.com/graphql \
|
|
272
|
+
--client-id CLIENT_ID \
|
|
273
|
+
--client-secret CLIENT_SECRET \
|
|
274
|
+
--output fluent-schema.json
|
|
275
|
+
|
|
276
|
+
# 2. Create mutation file
|
|
277
|
+
cat > price-mutation.graphql << 'EOF'
|
|
278
|
+
mutation UpdateProductPrice($input: UpdateProductInput!) {
|
|
279
|
+
updateProduct(input: $input) {
|
|
280
|
+
id
|
|
281
|
+
ref
|
|
282
|
+
prices { type value currency }
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
EOF
|
|
286
|
+
|
|
287
|
+
# 3. Generate mapping from mutation
|
|
288
|
+
fc-connect generate-mutation-mapping \
|
|
289
|
+
--file price-mutation.graphql \
|
|
290
|
+
--output price-mapping.json
|
|
291
|
+
|
|
292
|
+
# 4. Validate mapping against schema
|
|
293
|
+
fc-connect validate-schema \
|
|
294
|
+
--mapping price-mapping.json \
|
|
295
|
+
--schema fluent-schema.json
|
|
296
|
+
|
|
297
|
+
# 5. Analyze field coverage
|
|
298
|
+
fc-connect analyze-coverage \
|
|
299
|
+
--mapping price-mapping.json \
|
|
300
|
+
--schema fluent-schema.json
|
|
301
|
+
```
|
|
302
|
+
|
|
303
|
+
### Configuration File Structure
|
|
304
|
+
|
|
305
|
+
**Mapping Config:** `config/price-mapping.json`
|
|
306
|
+
|
|
307
|
+
**✅ PRODUCTION STANDARD:** External JSON file with GraphQLMutationMapper structure
|
|
308
|
+
|
|
309
|
+
```json
|
|
310
|
+
{
|
|
311
|
+
"mutation": "updateProduct",
|
|
312
|
+
"sourceFormat": "csv",
|
|
313
|
+
"version": "1.0.0",
|
|
314
|
+
"arguments": {
|
|
315
|
+
"input": {
|
|
316
|
+
"ref": {
|
|
317
|
+
"source": "sku",
|
|
318
|
+
"required": true,
|
|
319
|
+
"resolver": "trim"
|
|
320
|
+
},
|
|
321
|
+
"prices": {
|
|
322
|
+
"type": {
|
|
323
|
+
"source": "priceType",
|
|
324
|
+
"required": true,
|
|
325
|
+
"resolver": "normalizePriceType"
|
|
326
|
+
},
|
|
327
|
+
"value": {
|
|
328
|
+
"source": "amount",
|
|
329
|
+
"required": true,
|
|
330
|
+
"resolver": "validatePriceRange"
|
|
331
|
+
},
|
|
332
|
+
"currency": {
|
|
333
|
+
"source": "currency",
|
|
334
|
+
"required": true,
|
|
335
|
+
"resolver": "toUpperCase"
|
|
336
|
+
},
|
|
337
|
+
"effectiveFrom": {
|
|
338
|
+
"source": "effectiveDate",
|
|
339
|
+
"resolver": "formatDate"
|
|
340
|
+
},
|
|
341
|
+
"effectiveTo": {
|
|
342
|
+
"source": "expiryDate",
|
|
343
|
+
"resolver": "formatDate"
|
|
344
|
+
},
|
|
345
|
+
"minQuantity": {
|
|
346
|
+
"source": "minQuantity",
|
|
347
|
+
"resolver": "normalizeQuantity"
|
|
348
|
+
},
|
|
349
|
+
"maxQuantity": {
|
|
350
|
+
"source": "maxQuantity",
|
|
351
|
+
"resolver": "normalizeQuantity"
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
```
|
|
358
|
+
|
|
359
|
+
**Key Differences from UniversalMapper:**
|
|
360
|
+
- ✅ `mutation` property (not `mutationName`)
|
|
361
|
+
- ✅ `arguments.input` structure (matches GraphQL schema)
|
|
362
|
+
- ✅ Nested `prices` object (not flat productRef/type/value)
|
|
363
|
+
- ✅ Resolver names without `sdk.` or `custom.` prefix (`trim`, `toUpperCase`, `formatDate`)
|
|
364
|
+
- ✅ Used with `GraphQLMutationMapper.map()` method (not `UniversalMapper.map()`)
|
|
365
|
+
|
|
366
|
+
### Activation Variables Required
|
|
367
|
+
|
|
368
|
+
**Minimum Required:**
|
|
369
|
+
- `s3BucketName` - S3 bucket with price CSV files
|
|
370
|
+
- `awsAccessKeyId` - AWS credentials
|
|
371
|
+
- `awsSecretAccessKey` - AWS credentials
|
|
372
|
+
|
|
373
|
+
**Optional (with defaults):**
|
|
374
|
+
- `awsRegion=us-east-1`
|
|
375
|
+
- `s3Prefix=prices/`
|
|
376
|
+
- `archivePrefix=processed/`
|
|
377
|
+
- `errorPrefix=errors/`
|
|
378
|
+
- `logPrefix=logs/` - Where to write mutation logs
|
|
379
|
+
- `maxFilesToProcess=10`
|
|
380
|
+
- `mutationBatchSize=1` - Number of concurrent mutations (1=sequential, 3-10=parallel)
|
|
381
|
+
- `mutationsPerAliasBatch` - Optional: Number of mutations per aliased request (default: undefined = disabled)
|
|
382
|
+
- `minPrice=0.01`
|
|
383
|
+
- `maxPrice=999999.99`
|
|
384
|
+
- `validateProducts=true`
|
|
385
|
+
- `validateConnection=true` - Validate S3 connection at startup
|
|
386
|
+
- `enableFileTracking=true` - Enable file tracking via VersoriFileTracker
|
|
387
|
+
- `fluentRetailerId` - Optional: Only if mutation schema requires retailerId in input (most Price mutations don't need it)
|
|
388
|
+
|
|
389
|
+
### Error Handling Patterns
|
|
390
|
+
|
|
391
|
+
**Three-level error handling:**
|
|
392
|
+
|
|
393
|
+
1. **Mapping errors** - Invalid price data (validation failures)
|
|
394
|
+
2. **GraphQL errors** - Mutation failures (product not found, invalid fields)
|
|
395
|
+
3. **File processing errors** - S3 access issues, CSV parsing failures
|
|
396
|
+
|
|
397
|
+
**Error state tracking with exponential backoff:**
|
|
398
|
+
```typescript
|
|
399
|
+
const errorKey = ['error-state', fileName];
|
|
400
|
+
const attempts = (prev?.attemptCount || 0) + 1;
|
|
401
|
+
const backoffMinutes = Math.min(Math.pow(2, attempts) * 5, 24 * 60);
|
|
402
|
+
const nextRetryAt = new Date(Date.now() + backoffMinutes * 60000).toISOString();
|
|
403
|
+
```
|
|
404
|
+
|
|
405
|
+
### Common Pitfalls
|
|
406
|
+
|
|
407
|
+
**❌ WRONG Patterns:**
|
|
408
|
+
- Using Batch API for price updates (overhead, delay)
|
|
409
|
+
- Using `UniversalMapper` instead of `GraphQLMutationMapper`
|
|
410
|
+
- Calling `client.setRetailerId()` for GraphQL mutations (only needed for Job/Event API)
|
|
411
|
+
- Skipping product validation (causes GraphQL errors)
|
|
412
|
+
- No concurrency control (API throttling)
|
|
413
|
+
- Hardcoded price validation rules (not configurable)
|
|
414
|
+
- Missing price change tracking (no audit trail)
|
|
415
|
+
|
|
416
|
+
**✅ CORRECT Patterns:**
|
|
417
|
+
- Direct GraphQL mutations for immediate updates
|
|
418
|
+
- `GraphQLMutationMapper` with nested mapping structure
|
|
419
|
+
- NO `setRetailerId()` call (pass in mutation input if schema requires it)
|
|
420
|
+
- Product existence validation before updates
|
|
421
|
+
- Configurable concurrency control (mutationBatchSize)
|
|
422
|
+
- Optional alias batching for high-volume scenarios
|
|
423
|
+
- Custom resolvers for price validation
|
|
424
|
+
- Price change tracking for audit
|
|
425
|
+
- VersoriKVAdapter for state management
|
|
426
|
+
- JobTracker for job lifecycle
|
|
427
|
+
|
|
428
|
+
### Testing Checklist
|
|
429
|
+
|
|
430
|
+
**Before deployment:**
|
|
431
|
+
- [ ] S3 credentials validated (IAM permissions: ListBucket, GetObject, PutObject, DeleteObject)
|
|
432
|
+
- [ ] GraphQL schema validated using CLI tools
|
|
433
|
+
- [ ] Mapping configuration uses GraphQLMutationMapper structure (nested objects, not flat fields)
|
|
434
|
+
- [ ] Mapping configuration tested with sample CSV
|
|
435
|
+
- [ ] Product validation working (queries products endpoint)
|
|
436
|
+
- [ ] Price validation rules configured (min/max bounds)
|
|
437
|
+
- [ ] Concurrency control configured (mutationBatchSize)
|
|
438
|
+
- [ ] Optional alias batching tested if enabled (mutationsPerAliasBatch)
|
|
439
|
+
- [ ] NO setRetailerId() call present (only for Job/Event API)
|
|
440
|
+
- [ ] File duplicate prevention working (KV state)
|
|
441
|
+
- [ ] Price change tracking verified (logs price changes)
|
|
442
|
+
- [ ] Error handling tested (malformed CSV, invalid products)
|
|
443
|
+
- [ ] File archival working (processed/ and errors/ directories)
|
|
444
|
+
|
|
445
|
+
### Related Documentation
|
|
446
|
+
|
|
447
|
+
- **Core Guides:** `docs/02-CORE-GUIDES/ingestion/modules/`
|
|
448
|
+
- **Universal Mapping:** `docs/02-CORE-GUIDES/mapping/modules/`
|
|
449
|
+
- **CLI Tools:** `fc-connect-sdk/bin/readme.md`
|
|
450
|
+
- **State Management:** `docs/03-PATTERN-GUIDES/file-operations/`
|
|
451
|
+
- **Error Handling:** `docs/03-PATTERN-GUIDES/error-handling/`
|
|
452
|
+
|
|
453
|
+
**FC Connect SDK Use Case Guide**
|
|
454
|
+
|
|
455
|
+
> **SDK**: [@fluentcommerce/fc-connect-sdk](https://www.npmjs.com/package/@fluentcommerce/fc-connect-sdk)
|
|
456
|
+
> **Version**: Use latest - `npm install @fluentcommerce/fc-connect-sdk@latest`
|
|
457
|
+
|
|
458
|
+
**Context**: Scheduled Versori workflow that reads product price CSV files from S3 and creates/updates Fluent Commerce product prices via GraphQL mutations
|
|
459
|
+
|
|
460
|
+
**Complexity**: Medium-High
|
|
461
|
+
|
|
462
|
+
**Runtime**: Versori Platform (Scheduled)
|
|
463
|
+
|
|
464
|
+
**Estimated Lines**: ~700 lines
|
|
465
|
+
|
|
466
|
+
## What You'll Build
|
|
467
|
+
|
|
468
|
+
- Versori scheduled workflow (cron trigger)
|
|
469
|
+
- S3 file listing and download with retry logic
|
|
470
|
+
- CSV parsing with validation
|
|
471
|
+
- UniversalMapper for price field transformations
|
|
472
|
+
- Product existence validation before price updates
|
|
473
|
+
- GraphQL mutations for price upserts with rate limiting
|
|
474
|
+
- Versori KV state management (duplicate prevention)
|
|
475
|
+
- Price change tracking and reporting
|
|
476
|
+
- File archival after processing
|
|
477
|
+
|
|
478
|
+
## Versori Workflows Structure
|
|
479
|
+
|
|
480
|
+
**Key Concept**: Versori workflows are organized by **trigger type** at the first level, then by **specific workflow** with descriptive file names.
|
|
481
|
+
|
|
482
|
+
**Trigger Types:**
|
|
483
|
+
- **`schedule()`** → Time-based triggers (cron expressions) - NOT exposed as HTTP endpoints
|
|
484
|
+
- **`webhook()`** → HTTP-based triggers (event-driven) - Creates HTTP endpoints
|
|
485
|
+
- **`workflow()`** → Durable workflows (advanced, rarely used)
|
|
486
|
+
|
|
487
|
+
**Execution Steps (chained to triggers):**
|
|
488
|
+
- **`http()`** → External API calls (chained from schedule/webhook)
|
|
489
|
+
- **`fn()`** → Internal processing (chained from schedule/webhook)
|
|
490
|
+
|
|
491
|
+
### Recommended Project Structure
|
|
492
|
+
|
|
493
|
+
```
|
|
494
|
+
s3-csv-price-graphql/
|
|
495
|
+
├── index.ts # Entry point - exports all workflows
|
|
496
|
+
└── src/
|
|
497
|
+
├── workflows/
|
|
498
|
+
│ ├── scheduled/
|
|
499
|
+
│ │ └── daily-price-sync.ts # Scheduled: Daily price sync
|
|
500
|
+
│ │
|
|
501
|
+
│ └── webhook/
|
|
502
|
+
│ ├── adhoc-price-sync.ts # Webhook: Manual trigger
|
|
503
|
+
│ └── job-status-check.ts # Webhook: Status query
|
|
504
|
+
│
|
|
505
|
+
├── services/
|
|
506
|
+
│ └── price-sync.service.ts # Shared orchestration logic (reusable)
|
|
507
|
+
│
|
|
508
|
+
└── config/
|
|
509
|
+
└── price-mapping.json # GraphQL mapping config
|
|
510
|
+
```
|
|
511
|
+
|
|
512
|
+
---
|
|
513
|
+
|
|
514
|
+
## Workflow Files
|
|
515
|
+
|
|
516
|
+
### 1. Scheduled Workflows (`src/workflows/scheduled/`)
|
|
517
|
+
|
|
518
|
+
All time-based triggers that run automatically on cron schedules.
|
|
519
|
+
|
|
520
|
+
#### `src/workflows/scheduled/daily-price-sync.ts`
|
|
521
|
+
|
|
522
|
+
**Purpose**: Automatic daily price sync
|
|
523
|
+
**Trigger**: Cron schedule (`0 2 * * *`)
|
|
524
|
+
**Exposed as Endpoint**: ❌ NO - Runs automatically
|
|
525
|
+
|
|
526
|
+
```typescript
|
|
527
|
+
import { schedule, http } from '@versori/run';
|
|
528
|
+
import { JobTracker } from '@fluentcommerce/fc-connect-sdk';
|
|
529
|
+
import { executePriceSync } from '../../services/price-sync.service';
|
|
530
|
+
|
|
531
|
+
/**
|
|
532
|
+
* Scheduled Workflow: Daily Price Sync
|
|
533
|
+
*
|
|
534
|
+
* Runs automatically daily at 2 AM UTC
|
|
535
|
+
* NOT exposed as HTTP endpoint - Versori executes on schedule
|
|
536
|
+
*
|
|
537
|
+
* Uses shared service: price-sync.service.ts
|
|
538
|
+
*/
|
|
539
|
+
export const dailyPriceSync = schedule(
|
|
540
|
+
'price-sync-scheduled',
|
|
541
|
+
'0 2 * * *' // Daily at 2 AM UTC
|
|
542
|
+
).then(
|
|
543
|
+
http('run-price-sync', { connection: 'fluent_commerce' }, async (ctx: any) => {
|
|
544
|
+
const { log, openKv } = ctx;
|
|
545
|
+
const jobId = `price-sync-${Date.now()}`;
|
|
546
|
+
const executionStartTime = Date.now();
|
|
547
|
+
const tracker = new JobTracker(openKv(':project:'), log);
|
|
548
|
+
|
|
549
|
+
log.info('🔄 [WORKFLOW] Starting price sync', { jobId });
|
|
550
|
+
|
|
551
|
+
await tracker.createJob(jobId, {
|
|
552
|
+
triggeredBy: 'schedule',
|
|
553
|
+
stage: 'initialization',
|
|
554
|
+
startTime: executionStartTime,
|
|
555
|
+
});
|
|
556
|
+
await tracker.updateJob(jobId, { status: 'processing' });
|
|
557
|
+
|
|
558
|
+
try {
|
|
559
|
+
const result = await executePriceSync(ctx, jobId, tracker);
|
|
560
|
+
|
|
561
|
+
if (result.success) {
|
|
562
|
+
await tracker.markCompleted(jobId, result);
|
|
563
|
+
log.info('✅ [WORKFLOW] Price sync completed', {
|
|
564
|
+
jobId,
|
|
565
|
+
filesProcessed: result.filesProcessed,
|
|
566
|
+
duration: result.duration,
|
|
567
|
+
});
|
|
568
|
+
} else {
|
|
569
|
+
await tracker.markFailed(jobId, result.error || 'Unknown error');
|
|
570
|
+
log.error('❌ [WORKFLOW] Price sync failed', { jobId, error: result.error });
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
return { success: true, jobId, ...result };
|
|
574
|
+
} catch (e: any) {
|
|
575
|
+
const errorMessage = e instanceof Error ? e.message : String(e);
|
|
576
|
+
await tracker.markFailed(jobId, errorMessage);
|
|
577
|
+
log.error('❌ [WORKFLOW] Price sync failed with exception', {
|
|
578
|
+
jobId,
|
|
579
|
+
error: errorMessage,
|
|
580
|
+
stack: e instanceof Error ? e.stack : undefined,
|
|
581
|
+
});
|
|
582
|
+
return { success: false, jobId, error: errorMessage };
|
|
583
|
+
}
|
|
584
|
+
})
|
|
585
|
+
);
|
|
586
|
+
```
|
|
587
|
+
|
|
588
|
+
---
|
|
589
|
+
|
|
590
|
+
### 2. Webhook Workflows (`src/workflows/webhook/`)
|
|
591
|
+
|
|
592
|
+
All HTTP-based triggers that create webhook endpoints.
|
|
593
|
+
|
|
594
|
+
#### `src/workflows/webhook/adhoc-price-sync.ts`
|
|
595
|
+
|
|
596
|
+
**Purpose**: Manual price sync trigger (on-demand)
|
|
597
|
+
**Trigger**: HTTP POST
|
|
598
|
+
**Endpoint**: `POST https://{workspace}.versori.run/price-sync-adhoc`
|
|
599
|
+
**Use Cases**: Testing, priority processing, ad-hoc runs
|
|
600
|
+
|
|
601
|
+
```typescript
|
|
602
|
+
import { webhook, http } from '@versori/run';
|
|
603
|
+
import { JobTracker } from '@fluentcommerce/fc-connect-sdk';
|
|
604
|
+
import { executePriceSync } from '../../services/price-sync.service';
|
|
605
|
+
|
|
606
|
+
/**
|
|
607
|
+
* Webhook: Manual Price Sync Trigger
|
|
608
|
+
*
|
|
609
|
+
* Endpoint: POST https://{workspace}.versori.run/price-sync-adhoc
|
|
610
|
+
* Request body (optional): { filePattern: "urgent_*.csv", maxFiles: 5 }
|
|
611
|
+
*
|
|
612
|
+
* Pattern: webhook().then(http()) - needs Fluent API access
|
|
613
|
+
* Uses shared service: price-sync.service.ts
|
|
614
|
+
*
|
|
615
|
+
* SECURITY: Authentication handled via connection parameter
|
|
616
|
+
* No manual API key validation needed - Versori manages this via connection auth
|
|
617
|
+
*/
|
|
618
|
+
export const adhocPriceSync = webhook('price-sync-adhoc', {
|
|
619
|
+
response: { mode: 'sync' },
|
|
620
|
+
connection: 'price-sync-adhoc', // Versori validates API key
|
|
621
|
+
}).then(
|
|
622
|
+
http('run-price-sync-adhoc', { connection: 'fluent_commerce' }, async (ctx: any) => {
|
|
623
|
+
const { log, openKv, data } = ctx;
|
|
624
|
+
const jobId = `price-sync-adhoc-${Date.now()}`;
|
|
625
|
+
const executionStartTime = Date.now();
|
|
626
|
+
const tracker = new JobTracker(openKv(':project:'), log);
|
|
627
|
+
|
|
628
|
+
log.info('🔄 [WEBHOOK] Starting adhoc price sync', { jobId });
|
|
629
|
+
|
|
630
|
+
await tracker.createJob(jobId, {
|
|
631
|
+
triggeredBy: 'manual',
|
|
632
|
+
stage: 'initialization',
|
|
633
|
+
startTime: executionStartTime,
|
|
634
|
+
options: data // Optional: filePattern, maxFiles, etc.
|
|
635
|
+
});
|
|
636
|
+
await tracker.updateJob(jobId, { status: 'processing' });
|
|
637
|
+
|
|
638
|
+
try {
|
|
639
|
+
const result = await executePriceSync(ctx, jobId, tracker);
|
|
640
|
+
|
|
641
|
+
if (result.success) {
|
|
642
|
+
await tracker.markCompleted(jobId, result);
|
|
643
|
+
log.info('✅ [WEBHOOK] Adhoc price sync completed', {
|
|
644
|
+
jobId,
|
|
645
|
+
filesProcessed: result.filesProcessed,
|
|
646
|
+
duration: result.duration,
|
|
647
|
+
});
|
|
648
|
+
} else {
|
|
649
|
+
await tracker.markFailed(jobId, result.error || 'Unknown error');
|
|
650
|
+
log.error('❌ [WEBHOOK] Adhoc price sync failed', { jobId, error: result.error });
|
|
651
|
+
}
|
|
652
|
+
|
|
653
|
+
return { success: true, jobId, ...result };
|
|
654
|
+
} catch (e: any) {
|
|
655
|
+
const errorMessage = e instanceof Error ? e.message : String(e);
|
|
656
|
+
await tracker.markFailed(jobId, errorMessage);
|
|
657
|
+
log.error('❌ [WEBHOOK] Adhoc price sync failed with exception', {
|
|
658
|
+
jobId,
|
|
659
|
+
error: errorMessage,
|
|
660
|
+
stack: e instanceof Error ? e.stack : undefined,
|
|
661
|
+
});
|
|
662
|
+
return { success: false, jobId, error: errorMessage };
|
|
663
|
+
}
|
|
664
|
+
})
|
|
665
|
+
);
|
|
666
|
+
```
|
|
667
|
+
|
|
668
|
+
---
|
|
669
|
+
|
|
670
|
+
#### `src/workflows/webhook/job-status-check.ts`
|
|
671
|
+
|
|
672
|
+
**Purpose**: Query job status
|
|
673
|
+
**Trigger**: HTTP POST
|
|
674
|
+
**Endpoint**: `POST https://{workspace}.versori.run/price-sync-job-status`
|
|
675
|
+
**Request body**: `{ "jobId": "price-sync-1234567890" }`
|
|
676
|
+
|
|
677
|
+
```typescript
|
|
678
|
+
import { webhook, fn } from '@versori/run';
|
|
679
|
+
import { JobTracker } from '@fluentcommerce/fc-connect-sdk';
|
|
680
|
+
|
|
681
|
+
/**
|
|
682
|
+
* Webhook: Job Status Check
|
|
683
|
+
*
|
|
684
|
+
* Endpoint: POST https://{workspace}.versori.run/price-sync-job-status
|
|
685
|
+
* Request body: { "jobId": "price-sync-1234567890" }
|
|
686
|
+
*
|
|
687
|
+
* Pattern: webhook().then(fn()) - no external API needed, only KV storage
|
|
688
|
+
* Lightweight: Only queries KV store, no Fluent API calls
|
|
689
|
+
*
|
|
690
|
+
* SECURITY: Authentication handled via connection parameter
|
|
691
|
+
* No manual API key validation needed - Versori manages this via connection auth
|
|
692
|
+
*/
|
|
693
|
+
export const priceSyncJobStatus = webhook('price-sync-job-status', {
|
|
694
|
+
response: { mode: 'sync' },
|
|
695
|
+
connection: 'price-sync-job-status',
|
|
696
|
+
}).then(
|
|
697
|
+
fn('status', async (ctx: any) => {
|
|
698
|
+
const { data, log, openKv } = ctx;
|
|
699
|
+
const jobId = data?.jobId as string;
|
|
700
|
+
|
|
701
|
+
log.info('🔍 [STATUS] Checking job status', { jobId });
|
|
702
|
+
|
|
703
|
+
if (!jobId) {
|
|
704
|
+
log.warn('❌ [STATUS] Missing jobId in request');
|
|
705
|
+
return { success: false, error: 'jobId required' };
|
|
706
|
+
}
|
|
707
|
+
|
|
708
|
+
const tracker = new JobTracker(openKv(':project:'), log);
|
|
709
|
+
const status = await tracker.getJob(jobId);
|
|
710
|
+
|
|
711
|
+
if (status) {
|
|
712
|
+
log.info('✅ [STATUS] Job found', { jobId, status: status.status });
|
|
713
|
+
return { success: true, jobId, ...status };
|
|
714
|
+
} else {
|
|
715
|
+
log.warn('❌ [STATUS] Job not found', { jobId });
|
|
716
|
+
return { success: false, error: 'Job not found', jobId };
|
|
717
|
+
}
|
|
718
|
+
})
|
|
719
|
+
);
|
|
720
|
+
```
|
|
721
|
+
|
|
722
|
+
---
|
|
723
|
+
|
|
724
|
+
### 3. Entry Point (`index.ts`)
|
|
725
|
+
|
|
726
|
+
**Purpose**: Register all workflows with Versori platform
|
|
727
|
+
|
|
728
|
+
```typescript
|
|
729
|
+
/**
|
|
730
|
+
* Entry Point - Registers all workflows with Versori platform
|
|
731
|
+
*
|
|
732
|
+
* Versori automatically discovers and registers exported workflows
|
|
733
|
+
*
|
|
734
|
+
* File Structure:
|
|
735
|
+
* - src/workflows/scheduled/ → Time-based triggers (cron)
|
|
736
|
+
* - src/workflows/webhook/ → HTTP-based triggers (webhooks)
|
|
737
|
+
*/
|
|
738
|
+
|
|
739
|
+
// Import scheduled workflows
|
|
740
|
+
import { dailyPriceSync } from './workflows/scheduled/daily-price-sync';
|
|
741
|
+
|
|
742
|
+
// Import webhook workflows
|
|
743
|
+
import { adhocPriceSync } from './workflows/webhook/adhoc-price-sync';
|
|
744
|
+
import { priceSyncJobStatus } from './workflows/webhook/job-status-check';
|
|
745
|
+
|
|
746
|
+
// Register all workflows
|
|
747
|
+
export {
|
|
748
|
+
// Scheduled (time-based triggers)
|
|
749
|
+
dailyPriceSync,
|
|
750
|
+
|
|
751
|
+
// Webhooks (HTTP-based triggers)
|
|
752
|
+
adhocPriceSync,
|
|
753
|
+
priceSyncJobStatus,
|
|
754
|
+
};
|
|
755
|
+
```
|
|
756
|
+
|
|
757
|
+
**What Gets Exposed:**
|
|
758
|
+
- ✅ `adhocPriceSync` → `https://{workspace}.versori.run/price-sync-adhoc`
|
|
759
|
+
- ✅ `priceSyncJobStatus` → `https://{workspace}.versori.run/price-sync-job-status`
|
|
760
|
+
- ❌ `dailyPriceSync` → NOT exposed (runs automatically on cron)
|
|
761
|
+
|
|
762
|
+
---
|
|
763
|
+
|
|
764
|
+
## SDK Methods Used
|
|
765
|
+
|
|
766
|
+
**Core Services:**
|
|
767
|
+
- `createClient(ctx)` - Create Fluent client (auto-detects Versori context)
|
|
768
|
+
- `S3DataSource(config, log)` - S3 operations (list, download, upload, move)
|
|
769
|
+
- `CSVParserService()` - CSV parsing with validation
|
|
770
|
+
- `GraphQLMutationMapper(mappingConfig, log, { fluentClient: client })` - Field mapping with schema validation
|
|
771
|
+
- `VersoriKVAdapter(openKv(':project:'))` - KV state management for duplicate prevention
|
|
772
|
+
|
|
773
|
+
**Key Methods:**
|
|
774
|
+
- `s3.listFiles({ prefix, maxKeys })` - List S3 files
|
|
775
|
+
- `s3.downloadFile(path, options)` - Download file content
|
|
776
|
+
- `s3.uploadFile(path, content)` - Upload file (accepts string or Buffer)
|
|
777
|
+
- `s3.moveFile(source, dest)` - Archive or move file
|
|
778
|
+
- `parser.parse(content, options)` - Parse CSV to records
|
|
779
|
+
- `mapper.map(record)` - Transform single record
|
|
780
|
+
- `client.graphql({ query, variables })` - Execute GraphQL mutations/queries
|
|
781
|
+
- `kv.get(key)` / `kv.set(key, value)` - State management
|
|
782
|
+
|
|
783
|
+
**Critical Imports:**
|
|
784
|
+
- `Buffer` - **Required for Versori/Deno runtime** (S3 upload operations)
|
|
785
|
+
```typescript
|
|
786
|
+
import { Buffer } from 'node:buffer';
|
|
787
|
+
```
|
|
788
|
+
**Why:** Deno runtime (used by Versori) does not have `Buffer` as a global. However, `uploadFile()` accepts string or Buffer, so you can use strings directly without Buffer conversion.
|
|
789
|
+
|
|
790
|
+
**Service Functions (User-Defined):**
|
|
791
|
+
- `processFile()` - S3 download + CSV parsing + field mapping
|
|
792
|
+
- `executeMutations()` - GraphQL price updates with validation + rate limiting
|
|
793
|
+
- `writeMutationLog()` - Write execution log to S3 (uses Buffer)
|
|
794
|
+
|
|
795
|
+
## Sample CSV Input Data
|
|
796
|
+
|
|
797
|
+
**File**: `product-prices-20250122-001.csv`
|
|
798
|
+
|
|
799
|
+
```csv
|
|
800
|
+
sku,priceType,amount,currency,effectiveDate,expiryDate,minQuantity,maxQuantity
|
|
801
|
+
PROD-001,DEFAULT,29.99,USD,2025-01-22T00:00:00Z,,1,
|
|
802
|
+
PROD-002,SALE,19.99,USD,2025-01-22T00:00:00Z,2025-02-15T23:59:59Z,1,
|
|
803
|
+
PROD-003,CLEARANCE,9.99,USD,2025-01-22T00:00:00Z,2025-01-31T23:59:59Z,1,
|
|
804
|
+
PROD-004,DEFAULT,149.99,USD,2025-01-22T00:00:00Z,,1,
|
|
805
|
+
PROD-004,BULK,129.99,USD,2025-01-22T00:00:00Z,,10,99
|
|
806
|
+
PROD-004,BULK,119.99,USD,2025-01-22T00:00:00Z,,100,
|
|
807
|
+
```
|
|
808
|
+
|
|
809
|
+
**Note**: Products can have multiple price tiers. Same SKU with different `priceType` or quantity ranges = multiple rows.
|
|
810
|
+
|
|
811
|
+
**Field Mapping**:
|
|
812
|
+
|
|
813
|
+
- `sku` → Product reference (required) - must exist in Fluent
|
|
814
|
+
- `priceType` → Price tier (DEFAULT, SALE, CLEARANCE, BULK, etc.) (required)
|
|
815
|
+
- `amount` → Price value (required, must be > 0)
|
|
816
|
+
- `currency` → Currency code (USD, EUR, GBP, etc.) (required)
|
|
817
|
+
- `effectiveDate` → When price becomes active (ISO 8601 format)
|
|
818
|
+
- `expiryDate` → When price expires (optional, ISO 8601 format)
|
|
819
|
+
- `minQuantity` → Minimum quantity for this price tier (optional, default: 1)
|
|
820
|
+
- `maxQuantity` → Maximum quantity for this price tier (optional)
|
|
821
|
+
|
|
822
|
+
**Price Tier Logic**:
|
|
823
|
+
|
|
824
|
+
- **DEFAULT**: Standard retail price (most products)
|
|
825
|
+
- **SALE**: Temporary promotional price
|
|
826
|
+
- **CLEARANCE**: Final markdown price
|
|
827
|
+
- **BULK**: Quantity-based pricing (requires minQuantity/maxQuantity)
|
|
828
|
+
|
|
829
|
+
## Project Setup
|
|
830
|
+
|
|
831
|
+
```bash
|
|
832
|
+
mkdir versori-s3-csv-price-sync && cd $_
|
|
833
|
+
npm init -y
|
|
834
|
+
npm install @fluentcommerce/fc-connect-sdk@latest @versori/run
|
|
835
|
+
mkdir -p src
|
|
836
|
+
```
|
|
837
|
+
|
|
838
|
+
### Package Configuration (package.json)
|
|
839
|
+
|
|
840
|
+
```json
|
|
841
|
+
{
|
|
842
|
+
"name": "versori-s3-csv-price-sync",
|
|
843
|
+
"version": "1.0.0",
|
|
844
|
+
"description": "Versori workflow: S3 CSV price sync to Fluent GraphQL",
|
|
845
|
+
"versori": {
|
|
846
|
+
"workflows": "./src/index.ts"
|
|
847
|
+
},
|
|
848
|
+
"type": "module",
|
|
849
|
+
"scripts": {
|
|
850
|
+
"deploy": "versori deploy",
|
|
851
|
+
"logs": "versori logs"
|
|
852
|
+
},
|
|
853
|
+
"dependencies": {
|
|
854
|
+
"@fluentcommerce/fc-connect-sdk": "^0.1.39",
|
|
855
|
+
"@versori/run": "latest"
|
|
856
|
+
},
|
|
857
|
+
"devDependencies": {
|
|
858
|
+
"typescript": "^5.0.0",
|
|
859
|
+
"@types/node": "^20.0.0"
|
|
860
|
+
}
|
|
861
|
+
}
|
|
862
|
+
```
|
|
863
|
+
|
|
864
|
+
### Activation Variables (Versori)
|
|
865
|
+
|
|
866
|
+
```bash
|
|
867
|
+
# Required Variables
|
|
868
|
+
s3BucketName=my-price-bucket
|
|
869
|
+
awsAccessKeyId=AKIAXXXXXXXXXXXX
|
|
870
|
+
awsSecretAccessKey=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
|
|
871
|
+
retailerId=your-retailer-id
|
|
872
|
+
|
|
873
|
+
# Optional Variables (with defaults shown)
|
|
874
|
+
awsRegion=us-east-1
|
|
875
|
+
s3Prefix=prices/
|
|
876
|
+
archivePrefix=processed/
|
|
877
|
+
errorPrefix=errors/
|
|
878
|
+
logPrefix=logs/ # Where to write mutation logs
|
|
879
|
+
maxFilesToProcess=10
|
|
880
|
+
filePattern=.csv
|
|
881
|
+
enableArchival=true
|
|
882
|
+
mutationRateLimit=10
|
|
883
|
+
|
|
884
|
+
# Price Validation Rules
|
|
885
|
+
minPrice=0.01 # Minimum allowed price
|
|
886
|
+
maxPrice=999999.99 # Maximum allowed price
|
|
887
|
+
validateProducts=true # Validate products exist before updating prices
|
|
888
|
+
```
|
|
889
|
+
|
|
890
|
+
## Complete Workflow (src/index.ts)
|
|
891
|
+
|
|
892
|
+
```typescript
|
|
893
|
+
import { schedule, webhook, http, fn } from '@versori/run';
|
|
894
|
+
import { Buffer } from 'node:buffer'; // Required for Versori/Deno runtime
|
|
895
|
+
import {
|
|
896
|
+
createClient,
|
|
897
|
+
S3DataSource,
|
|
898
|
+
CSVParserService,
|
|
899
|
+
GraphQLMutationMapper,
|
|
900
|
+
VersoriFileTracker,
|
|
901
|
+
JobTracker,
|
|
902
|
+
} from '@fluentcommerce/fc-connect-sdk';
|
|
903
|
+
|
|
904
|
+
// ============================================================================
|
|
905
|
+
// Type Definitions
|
|
906
|
+
// ============================================================================
|
|
907
|
+
|
|
908
|
+
interface FileProcessingResult {
|
|
909
|
+
fileName: string;
|
|
910
|
+
successful: number;
|
|
911
|
+
failed: number;
|
|
912
|
+
validationFailed: number;
|
|
913
|
+
priceChanges: Array<{
|
|
914
|
+
sku: string;
|
|
915
|
+
type: string;
|
|
916
|
+
oldPrice?: number;
|
|
917
|
+
newPrice: number;
|
|
918
|
+
}>;
|
|
919
|
+
errors: string[];
|
|
920
|
+
}
|
|
921
|
+
|
|
922
|
+
interface MutationResult {
|
|
923
|
+
successful: number;
|
|
924
|
+
failed: number;
|
|
925
|
+
priceChanges: Array<{
|
|
926
|
+
sku: string;
|
|
927
|
+
type: string;
|
|
928
|
+
oldPrice?: number;
|
|
929
|
+
newPrice: number;
|
|
930
|
+
}>;
|
|
931
|
+
}
|
|
932
|
+
|
|
933
|
+
interface MappedPriceData {
|
|
934
|
+
productRef: string;
|
|
935
|
+
type: string;
|
|
936
|
+
value: number;
|
|
937
|
+
currency: string;
|
|
938
|
+
effectiveFrom?: string;
|
|
939
|
+
effectiveTo?: string;
|
|
940
|
+
minQuantity?: number;
|
|
941
|
+
maxQuantity?: number;
|
|
942
|
+
}
|
|
943
|
+
|
|
944
|
+
// ============================================================================
|
|
945
|
+
// Utility Functions
|
|
946
|
+
// ============================================================================
|
|
947
|
+
|
|
948
|
+
/**
|
|
949
|
+
* Retry utility with exponential backoff
|
|
950
|
+
* (User-defined - not part of SDK public API)
|
|
951
|
+
*/
|
|
952
|
+
async function retryWithBackoff<T>(
|
|
953
|
+
operation: () => Promise<T>,
|
|
954
|
+
maxRetries = 3,
|
|
955
|
+
baseDelayMs = 1000
|
|
956
|
+
): Promise<T> {
|
|
957
|
+
let lastError: any;
|
|
958
|
+
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
959
|
+
try {
|
|
960
|
+
return await operation();
|
|
961
|
+
} catch (error) {
|
|
962
|
+
lastError = error;
|
|
963
|
+
if (attempt < maxRetries - 1) {
|
|
964
|
+
const delay = baseDelayMs * Math.pow(2, attempt) + Math.floor(Math.random() * 200);
|
|
965
|
+
await new Promise(resolve => setTimeout(resolve, delay));
|
|
966
|
+
}
|
|
967
|
+
}
|
|
968
|
+
}
|
|
969
|
+
throw lastError;
|
|
970
|
+
}
|
|
971
|
+
|
|
972
|
+
/**
|
|
973
|
+
* Validate product exists in Fluent Commerce
|
|
974
|
+
*/
|
|
975
|
+
async function validateProductExists(
|
|
976
|
+
client: any,
|
|
977
|
+
productRef: string,
|
|
978
|
+
retailerId: string,
|
|
979
|
+
log: any
|
|
980
|
+
): Promise<boolean> {
|
|
981
|
+
const query = `
|
|
982
|
+
query GetProduct($ref: String!, $retailerId: ID!) {
|
|
983
|
+
products(first: 1, ref: [$ref], retailerId: $retailerId) {
|
|
984
|
+
edges {
|
|
985
|
+
node {
|
|
986
|
+
id
|
|
987
|
+
ref
|
|
988
|
+
status
|
|
989
|
+
}
|
|
990
|
+
}
|
|
991
|
+
}
|
|
992
|
+
}
|
|
993
|
+
`;
|
|
994
|
+
|
|
995
|
+
try {
|
|
996
|
+
const result = await client.graphql({
|
|
997
|
+
query,
|
|
998
|
+
variables: { ref: productRef, retailerId },
|
|
999
|
+
});
|
|
1000
|
+
|
|
1001
|
+
const product = result?.data?.products?.edges?.[0]?.node;
|
|
1002
|
+
if (!product) {
|
|
1003
|
+
log.warn(`Product not found: ${productRef}`);
|
|
1004
|
+
return false;
|
|
1005
|
+
}
|
|
1006
|
+
|
|
1007
|
+
if (product.status !== 'ACTIVE') {
|
|
1008
|
+
log.warn(`Product not active: ${productRef} (status: ${product.status})`);
|
|
1009
|
+
return false;
|
|
1010
|
+
}
|
|
1011
|
+
|
|
1012
|
+
return true;
|
|
1013
|
+
} catch (error: unknown) {
|
|
1014
|
+
// ✅ Enhanced error logging: Extract all error details for visibility
|
|
1015
|
+
const errorMsg = error instanceof Error ? error.message : String(error);
|
|
1016
|
+
const errorDetails = {
|
|
1017
|
+
message: errorMsg,
|
|
1018
|
+
stack: error instanceof Error ? error.stack : undefined,
|
|
1019
|
+
errorType: error instanceof Error ? error.constructor.name : 'Error',
|
|
1020
|
+
};
|
|
1021
|
+
log.error(`Failed to validate product: ${productRef}`, errorDetails);
|
|
1022
|
+
return false;
|
|
1023
|
+
}
|
|
1024
|
+
}
|
|
1025
|
+
|
|
1026
|
+
/**
|
|
1027
|
+
* Get current product price for change tracking
|
|
1028
|
+
*/
|
|
1029
|
+
async function getCurrentPrice(
|
|
1030
|
+
client: any,
|
|
1031
|
+
productRef: string,
|
|
1032
|
+
priceType: string,
|
|
1033
|
+
currency: string,
|
|
1034
|
+
retailerId: string
|
|
1035
|
+
): Promise<number | undefined> {
|
|
1036
|
+
try {
|
|
1037
|
+
const query = `
|
|
1038
|
+
query GetProductPrice($ref: String!, $retailerId: ID!) {
|
|
1039
|
+
products(first: 1, ref: [$ref], retailerId: $retailerId) {
|
|
1040
|
+
edges {
|
|
1041
|
+
node {
|
|
1042
|
+
id
|
|
1043
|
+
ref
|
|
1044
|
+
prices {
|
|
1045
|
+
type
|
|
1046
|
+
value
|
|
1047
|
+
currency
|
|
1048
|
+
}
|
|
1049
|
+
}
|
|
1050
|
+
}
|
|
1051
|
+
}
|
|
1052
|
+
}
|
|
1053
|
+
`;
|
|
1054
|
+
|
|
1055
|
+
const result = await client.graphql({
|
|
1056
|
+
query,
|
|
1057
|
+
variables: { ref: productRef, retailerId },
|
|
1058
|
+
});
|
|
1059
|
+
|
|
1060
|
+
const product = result?.data?.products?.edges?.[0]?.node;
|
|
1061
|
+
if (product?.prices) {
|
|
1062
|
+
const existingPrice = product.prices.find(
|
|
1063
|
+
(p: any) => p.type === priceType && p.currency === currency
|
|
1064
|
+
);
|
|
1065
|
+
if (existingPrice) {
|
|
1066
|
+
return existingPrice.value;
|
|
1067
|
+
}
|
|
1068
|
+
}
|
|
1069
|
+
} catch (err) {
|
|
1070
|
+
// Ignore - price change tracking is optional
|
|
1071
|
+
}
|
|
1072
|
+
return undefined;
|
|
1073
|
+
}
|
|
1074
|
+
|
|
1075
|
+
// ============================================================================
|
|
1076
|
+
// Service Function 1: Process File (S3 + CSV + Mapper)
|
|
1077
|
+
// ============================================================================
|
|
1078
|
+
|
|
1079
|
+
/**
|
|
1080
|
+
* Process single CSV file: Download from S3, parse CSV, map fields to Price schema
|
|
1081
|
+
*
|
|
1082
|
+
* @param s3 - S3DataSource instance
|
|
1083
|
+
* @param parser - CSVParserService instance
|
|
1084
|
+
* @param mapper - UniversalMapper instance (with custom price resolvers)
|
|
1085
|
+
* @param filePath - S3 file path
|
|
1086
|
+
* @param fileName - File name
|
|
1087
|
+
* @param log - Logger instance
|
|
1088
|
+
* @returns FileProcessingResult with parsed and mapped price data
|
|
1089
|
+
*/
|
|
1090
|
+
async function processFile(
|
|
1091
|
+
s3: S3DataSource,
|
|
1092
|
+
parser: CSVParserService,
|
|
1093
|
+
mapper: GraphQLMutationMapper,
|
|
1094
|
+
filePath: string,
|
|
1095
|
+
fileName: string,
|
|
1096
|
+
log: any
|
|
1097
|
+
): Promise<{ records: Array<{ query: string; variables: any; input: any }>; errors: string[] }> {
|
|
1098
|
+
log.info('Processing file', { fileName });
|
|
1099
|
+
|
|
1100
|
+
// Download CSV from S3 with retry
|
|
1101
|
+
const content = await retryWithBackoff(
|
|
1102
|
+
() => s3.downloadFile(filePath, { encoding: 'utf8' }) as Promise<string>
|
|
1103
|
+
);
|
|
1104
|
+
|
|
1105
|
+
// Parse CSV
|
|
1106
|
+
const rawRecords = await parser.parse(content, {
|
|
1107
|
+
columns: true,
|
|
1108
|
+
skip_empty_lines: true,
|
|
1109
|
+
trim: true,
|
|
1110
|
+
});
|
|
1111
|
+
|
|
1112
|
+
if (!rawRecords.length) {
|
|
1113
|
+
log.warn('Empty CSV file', { fileName });
|
|
1114
|
+
return { records: [], errors: [] };
|
|
1115
|
+
}
|
|
1116
|
+
|
|
1117
|
+
log.info('CSV parsed', { fileName, recordCount: rawRecords.length });
|
|
1118
|
+
|
|
1119
|
+
// Map CSV records to Price schema
|
|
1120
|
+
const mappedRecords: Array<{ query: string; variables: any; input: any }> = [];
|
|
1121
|
+
const errors: string[] = [];
|
|
1122
|
+
|
|
1123
|
+
// ✅ PRODUCTION ENHANCEMENT: Log transformation start
|
|
1124
|
+
log.info('Transforming records to GraphQL mutations', {
|
|
1125
|
+
fileName,
|
|
1126
|
+
totalRecords: rawRecords.length,
|
|
1127
|
+
});
|
|
1128
|
+
|
|
1129
|
+
for (let i = 0; i < rawRecords.length; i++) {
|
|
1130
|
+
const rec = rawRecords[i];
|
|
1131
|
+
const recordNumber = i + 1;
|
|
1132
|
+
|
|
1133
|
+
// ✅ PRODUCTION ENHANCEMENT: Log progress every 50 records
|
|
1134
|
+
if (recordNumber % 50 === 0) {
|
|
1135
|
+
log.info(`📤 Transforming record ${recordNumber}/${rawRecords.length}`, {
|
|
1136
|
+
fileName,
|
|
1137
|
+
recordNumber,
|
|
1138
|
+
totalRecords: rawRecords.length,
|
|
1139
|
+
validSoFar: mappedRecords.length,
|
|
1140
|
+
errorsSoFar: errors.length,
|
|
1141
|
+
progress: `${((recordNumber / rawRecords.length) * 100).toFixed(1)}%`,
|
|
1142
|
+
});
|
|
1143
|
+
}
|
|
1144
|
+
|
|
1145
|
+
try {
|
|
1146
|
+
// GraphQLMutationMapper returns { query, variables } directly
|
|
1147
|
+
const mapped = await mapper.map(rec);
|
|
1148
|
+
|
|
1149
|
+
mappedRecords.push({
|
|
1150
|
+
query: mapped.query,
|
|
1151
|
+
variables: mapped.variables,
|
|
1152
|
+
input: mapped.variables.input || mapped.variables,
|
|
1153
|
+
});
|
|
1154
|
+
} catch (error: unknown) {
|
|
1155
|
+
const errorMsg = error instanceof Error ? error.message : String(error);
|
|
1156
|
+
errors.push(`Row ${recordNumber}: ${errorMsg}`);
|
|
1157
|
+
}
|
|
1158
|
+
}
|
|
1159
|
+
|
|
1160
|
+
log.info('Mapping completed', {
|
|
1161
|
+
fileName,
|
|
1162
|
+
successful: mappedRecords.length,
|
|
1163
|
+
failed: errors.length,
|
|
1164
|
+
totalRecords: rawRecords.length,
|
|
1165
|
+
successRate: `${((mappedRecords.length / rawRecords.length) * 100).toFixed(1)}%`,
|
|
1166
|
+
});
|
|
1167
|
+
|
|
1168
|
+
return { records: mappedRecords, errors };
|
|
1169
|
+
}
|
|
1170
|
+
|
|
1171
|
+
// ============================================================================
|
|
1172
|
+
// Service Function 2: Execute Mutations (GraphQL Price Updates)
|
|
1173
|
+
// ============================================================================
|
|
1174
|
+
|
|
1175
|
+
/**
|
|
1176
|
+
* Execute GraphQL mutations for price updates with validation and rate limiting
|
|
1177
|
+
* ✅ Supports alias batching for improved performance
|
|
1178
|
+
*
|
|
1179
|
+
* @param client - FluentClient instance
|
|
1180
|
+
* @param mapper - GraphQLMutationMapper instance
|
|
1181
|
+
* @param priceRecords - Mapped price data with query and variables
|
|
1182
|
+
* @param retailerId - Fluent retailer ID
|
|
1183
|
+
* @param validateProducts - Whether to validate product existence
|
|
1184
|
+
* @param batchSize - Number of concurrent requests (concurrency control)
|
|
1185
|
+
* @param mutationsPerAliasBatch - Optional: Number of mutations per aliased request (alias batching)
|
|
1186
|
+
* @param log - Logger instance
|
|
1187
|
+
* @returns MutationResult with success/failure counts and price changes
|
|
1188
|
+
*/
|
|
1189
|
+
async function executeMutations(
|
|
1190
|
+
client: any,
|
|
1191
|
+
mapper: GraphQLMutationMapper,
|
|
1192
|
+
priceRecords: Array<{ query: string; variables: any; input: any }>,
|
|
1193
|
+
retailerId: string,
|
|
1194
|
+
validateProducts: boolean,
|
|
1195
|
+
batchSize: number = 1, // ✅ Default: 1 (sequential)
|
|
1196
|
+
mutationsPerAliasBatch?: number, // ✅ NEW: Alias batching parameter (default: undefined = disabled)
|
|
1197
|
+
log: any
|
|
1198
|
+
): Promise<MutationResult> {
|
|
1199
|
+
let successful = 0;
|
|
1200
|
+
let failed = 0;
|
|
1201
|
+
const priceChanges: Array<{
|
|
1202
|
+
sku: string;
|
|
1203
|
+
type: string;
|
|
1204
|
+
oldPrice?: number;
|
|
1205
|
+
newPrice: number;
|
|
1206
|
+
}> = [];
|
|
1207
|
+
|
|
1208
|
+
// ✅ Use alias batching if enabled
|
|
1209
|
+
const useAliases = mutationsPerAliasBatch && mutationsPerAliasBatch > 1;
|
|
1210
|
+
|
|
1211
|
+
if (useAliases) {
|
|
1212
|
+
// Process with alias batching
|
|
1213
|
+
const results = await executeMutationsWithAliases(
|
|
1214
|
+
priceRecords,
|
|
1215
|
+
client,
|
|
1216
|
+
mapper,
|
|
1217
|
+
log,
|
|
1218
|
+
retailerId,
|
|
1219
|
+
batchSize,
|
|
1220
|
+
mutationsPerAliasBatch,
|
|
1221
|
+
'updateProduct'
|
|
1222
|
+
);
|
|
1223
|
+
successful = results.executed;
|
|
1224
|
+
failed = results.failed;
|
|
1225
|
+
// Note: Price changes tracking would need to be done separately if needed
|
|
1226
|
+
} else {
|
|
1227
|
+
// Process individually with validation
|
|
1228
|
+
for (const priceData of priceRecords) {
|
|
1229
|
+
try {
|
|
1230
|
+
// Validate product exists (if enabled)
|
|
1231
|
+
if (validateProducts) {
|
|
1232
|
+
const productExists = await validateProductExists(
|
|
1233
|
+
client,
|
|
1234
|
+
priceData.input.productRef,
|
|
1235
|
+
retailerId,
|
|
1236
|
+
log
|
|
1237
|
+
);
|
|
1238
|
+
|
|
1239
|
+
if (!productExists) {
|
|
1240
|
+
log.warn(`Skipping price update for non-existent product: ${priceData.input.productRef}`);
|
|
1241
|
+
failed++;
|
|
1242
|
+
continue;
|
|
1243
|
+
}
|
|
1244
|
+
}
|
|
1245
|
+
|
|
1246
|
+
// Get current price for change tracking
|
|
1247
|
+
const currentPrice = await getCurrentPrice(
|
|
1248
|
+
client,
|
|
1249
|
+
priceData.input.productRef,
|
|
1250
|
+
priceData.input.type,
|
|
1251
|
+
priceData.input.currency,
|
|
1252
|
+
retailerId
|
|
1253
|
+
);
|
|
1254
|
+
|
|
1255
|
+
// Execute mutation using pre-generated query and variables
|
|
1256
|
+
await retryWithBackoff(() =>
|
|
1257
|
+
client.graphql({
|
|
1258
|
+
query: priceData.query,
|
|
1259
|
+
variables: priceData.variables,
|
|
1260
|
+
})
|
|
1261
|
+
);
|
|
1262
|
+
|
|
1263
|
+
successful++;
|
|
1264
|
+
|
|
1265
|
+
// Track price change
|
|
1266
|
+
if (currentPrice !== undefined && currentPrice !== priceData.input.value) {
|
|
1267
|
+
priceChanges.push({
|
|
1268
|
+
sku: priceData.input.productRef,
|
|
1269
|
+
type: priceData.input.type,
|
|
1270
|
+
oldPrice: currentPrice,
|
|
1271
|
+
newPrice: priceData.input.value,
|
|
1272
|
+
});
|
|
1273
|
+
}
|
|
1274
|
+
|
|
1275
|
+
log.info('Price updated', {
|
|
1276
|
+
ref: priceData.input.productRef,
|
|
1277
|
+
type: priceData.input.type,
|
|
1278
|
+
oldPrice: currentPrice,
|
|
1279
|
+
newPrice: priceData.input.value,
|
|
1280
|
+
currency: priceData.input.currency,
|
|
1281
|
+
});
|
|
1282
|
+
} catch (error: unknown) {
|
|
1283
|
+
failed++;
|
|
1284
|
+
const errorMsg = error instanceof Error ? error.message : String(error);
|
|
1285
|
+
const errorDetails = {
|
|
1286
|
+
message: errorMsg,
|
|
1287
|
+
stack: error instanceof Error ? error.stack : undefined,
|
|
1288
|
+
errorType: error instanceof Error ? error.constructor.name : 'Error',
|
|
1289
|
+
};
|
|
1290
|
+
log.error('Failed to update price', errorDetails, {
|
|
1291
|
+
ref: priceData.input?.productRef,
|
|
1292
|
+
type: priceData.input?.type,
|
|
1293
|
+
});
|
|
1294
|
+
}
|
|
1295
|
+
}
|
|
1296
|
+
}
|
|
1297
|
+
|
|
1298
|
+
return { successful, failed, priceChanges };
|
|
1299
|
+
}
|
|
1300
|
+
|
|
1301
|
+
/**
|
|
1302
|
+
* ✅ NEW: Execute mutations with GraphQL alias batching
|
|
1303
|
+
*/
|
|
1304
|
+
async function executeMutationsWithAliases(
|
|
1305
|
+
priceRecords: Array<{ query: string; variables: any; input: any }>,
|
|
1306
|
+
client: any,
|
|
1307
|
+
mapper: GraphQLMutationMapper,
|
|
1308
|
+
log: any,
|
|
1309
|
+
retailerId: string,
|
|
1310
|
+
maxParallel: number,
|
|
1311
|
+
mutationsPerAliasBatch: number,
|
|
1312
|
+
mutationName: string
|
|
1313
|
+
): Promise<{ executed: number; failed: number; errors: string[] }> {
|
|
1314
|
+
const results = { executed: 0, failed: 0, errors: [] as string[] };
|
|
1315
|
+
|
|
1316
|
+
const aliasBatches: Array<Array<typeof priceRecords[0]>> = [];
|
|
1317
|
+
for (let i = 0; i < priceRecords.length; i += mutationsPerAliasBatch) {
|
|
1318
|
+
aliasBatches.push(priceRecords.slice(i, i + mutationsPerAliasBatch));
|
|
1319
|
+
}
|
|
1320
|
+
|
|
1321
|
+
log.info(`Processing ${aliasBatches.length} alias batches`, {
|
|
1322
|
+
totalPrices: priceRecords.length,
|
|
1323
|
+
maxParallel,
|
|
1324
|
+
});
|
|
1325
|
+
|
|
1326
|
+
for (let i = 0; i < aliasBatches.length; i += maxParallel) {
|
|
1327
|
+
const concurrentBatches = aliasBatches.slice(i, i + maxParallel);
|
|
1328
|
+
|
|
1329
|
+
const batchResults = await Promise.allSettled(
|
|
1330
|
+
concurrentBatches.map(async (batch) => {
|
|
1331
|
+
const { query, variables } = buildAliasedBatch(batch, mutationName, retailerId);
|
|
1332
|
+
const response = await retryWithBackoff(() => client.graphql({ query, variables }), log);
|
|
1333
|
+
return parseAliasResponse(response, batch, mutationName);
|
|
1334
|
+
})
|
|
1335
|
+
);
|
|
1336
|
+
|
|
1337
|
+
batchResults.forEach((result, idx) => {
|
|
1338
|
+
if (result.status === 'fulfilled') {
|
|
1339
|
+
const batchResult = result.value;
|
|
1340
|
+
results.executed += batchResult.executed;
|
|
1341
|
+
results.failed += batchResult.failed;
|
|
1342
|
+
results.errors.push(...batchResult.errors);
|
|
1343
|
+
} else {
|
|
1344
|
+
const batch = concurrentBatches[idx];
|
|
1345
|
+
const errorMsg = result.reason instanceof Error ? result.reason.message : String(result.reason);
|
|
1346
|
+
batch.forEach(price => {
|
|
1347
|
+
results.failed++;
|
|
1348
|
+
const productRef = price.input?.productRef || 'unknown';
|
|
1349
|
+
results.errors.push(`Failed to update price for ${productRef}: ${errorMsg}`);
|
|
1350
|
+
});
|
|
1351
|
+
}
|
|
1352
|
+
});
|
|
1353
|
+
|
|
1354
|
+
if (i + maxParallel < aliasBatches.length) {
|
|
1355
|
+
await new Promise(resolve => setTimeout(resolve, 500));
|
|
1356
|
+
}
|
|
1357
|
+
}
|
|
1358
|
+
|
|
1359
|
+
return results;
|
|
1360
|
+
}
|
|
1361
|
+
|
|
1362
|
+
/**
|
|
1363
|
+
* ✅ NEW: Build aliased batch query and variables
|
|
1364
|
+
*/
|
|
1365
|
+
function buildAliasedBatch(
|
|
1366
|
+
batch: Array<{ query: string; variables: any; input: any }>,
|
|
1367
|
+
mutationName: string,
|
|
1368
|
+
retailerId: string
|
|
1369
|
+
): { query: string; variables: Record<string, any> } {
|
|
1370
|
+
const batchSize = batch.length;
|
|
1371
|
+
const inputTypeName = `${mutationName.charAt(0).toUpperCase() + mutationName.slice(1)}Input`;
|
|
1372
|
+
|
|
1373
|
+
const variables = Array.from({ length: batchSize }, (_, i) =>
|
|
1374
|
+
`$input${i + 1}: ${inputTypeName}!`
|
|
1375
|
+
).join(', ');
|
|
1376
|
+
|
|
1377
|
+
const aliasedMutations = Array.from({ length: batchSize }, (_, i) => {
|
|
1378
|
+
const alias = `${mutationName}${i + 1}`;
|
|
1379
|
+
return ` ${alias}: ${mutationName}(input: $input${i + 1}) { id ref }`;
|
|
1380
|
+
}).join('\n');
|
|
1381
|
+
|
|
1382
|
+
const operationName = `Batch${mutationName.charAt(0).toUpperCase() + mutationName.slice(1)}s`;
|
|
1383
|
+
const query = `mutation ${operationName}(${variables}) {\n${aliasedMutations}\n}`;
|
|
1384
|
+
|
|
1385
|
+
const variablesObj: Record<string, any> = {};
|
|
1386
|
+
batch.forEach((price, index) => {
|
|
1387
|
+
const input = price.variables.input || price.variables;
|
|
1388
|
+
variablesObj[`input${index + 1}`] = input;
|
|
1389
|
+
});
|
|
1390
|
+
|
|
1391
|
+
return { query, variables: variablesObj };
|
|
1392
|
+
}
|
|
1393
|
+
|
|
1394
|
+
/**
|
|
1395
|
+
* ✅ NEW: Parse aliased GraphQL response
|
|
1396
|
+
*/
|
|
1397
|
+
function parseAliasResponse(
|
|
1398
|
+
response: any,
|
|
1399
|
+
batch: Array<{ query: string; variables: any; input: any }>,
|
|
1400
|
+
mutationName: string
|
|
1401
|
+
): { executed: number; failed: number; errors: string[] } {
|
|
1402
|
+
const result = { executed: 0, failed: 0, errors: [] as string[] };
|
|
1403
|
+
|
|
1404
|
+
const data = response.data || {};
|
|
1405
|
+
const errors = response.errors || [];
|
|
1406
|
+
|
|
1407
|
+
batch.forEach((price, index) => {
|
|
1408
|
+
const alias = `${mutationName}${index + 1}`;
|
|
1409
|
+
const aliasData = data[alias];
|
|
1410
|
+
const aliasErrors = errors.filter((e: unknown) =>
|
|
1411
|
+
e && typeof e === 'object' && 'path' in e && Array.isArray((e as any).path) && (e as any).path.includes(alias)
|
|
1412
|
+
);
|
|
1413
|
+
|
|
1414
|
+
if (aliasData && !aliasErrors.length) {
|
|
1415
|
+
result.executed++;
|
|
1416
|
+
} else {
|
|
1417
|
+
result.failed++;
|
|
1418
|
+
const errorMsg = aliasErrors[0] && typeof aliasErrors[0] === 'object' && 'message' in aliasErrors[0]
|
|
1419
|
+
? String((aliasErrors[0] as any).message)
|
|
1420
|
+
: 'Mutation failed';
|
|
1421
|
+
const productRef = price.input?.productRef || 'unknown';
|
|
1422
|
+
result.errors.push(`Failed to update price for ${productRef}: ${errorMsg}`);
|
|
1423
|
+
}
|
|
1424
|
+
});
|
|
1425
|
+
|
|
1426
|
+
return result;
|
|
1427
|
+
}
|
|
1428
|
+
|
|
1429
|
+
// ============================================================================
|
|
1430
|
+
// Service Function 3: Write Mutation Log (S3 Upload)
|
|
1431
|
+
// ============================================================================
|
|
1432
|
+
|
|
1433
|
+
/**
|
|
1434
|
+
* Write mutation execution log to S3
|
|
1435
|
+
*
|
|
1436
|
+
* @param s3 - S3DataSource instance
|
|
1437
|
+
* @param fileName - Original CSV file name
|
|
1438
|
+
* @param result - FileProcessingResult
|
|
1439
|
+
* @param logPrefix - S3 prefix for logs
|
|
1440
|
+
* @param log - Logger instance
|
|
1441
|
+
*/
|
|
1442
|
+
async function writeMutationLog(
|
|
1443
|
+
s3: S3DataSource,
|
|
1444
|
+
fileName: string,
|
|
1445
|
+
result: FileProcessingResult,
|
|
1446
|
+
logPrefix: string,
|
|
1447
|
+
log: any
|
|
1448
|
+
): Promise<void> {
|
|
1449
|
+
const logFileName = `${fileName.replace('.csv', '')}-log.json`;
|
|
1450
|
+
const logPath = `${logPrefix}${logFileName}`;
|
|
1451
|
+
|
|
1452
|
+
const logData = {
|
|
1453
|
+
fileName,
|
|
1454
|
+
timestamp: new Date().toISOString(),
|
|
1455
|
+
summary: {
|
|
1456
|
+
successful: result.successful,
|
|
1457
|
+
failed: result.failed,
|
|
1458
|
+
validationFailed: result.validationFailed,
|
|
1459
|
+
priceChanges: result.priceChanges.length,
|
|
1460
|
+
},
|
|
1461
|
+
priceChanges: result.priceChanges,
|
|
1462
|
+
errors: result.errors,
|
|
1463
|
+
};
|
|
1464
|
+
|
|
1465
|
+
const logContent = JSON.stringify(logData, null, 2);
|
|
1466
|
+
|
|
1467
|
+
// Upload log to S3 (uploadFile accepts string or Buffer)
|
|
1468
|
+
await s3.uploadFile(logPath, logContent);
|
|
1469
|
+
|
|
1470
|
+
log.info('Mutation log written', { logPath });
|
|
1471
|
+
}
|
|
1472
|
+
|
|
1473
|
+
// ============================================================================
|
|
1474
|
+
// Main Workflow Function (Per-File Processing)
|
|
1475
|
+
// ============================================================================
|
|
1476
|
+
|
|
1477
|
+
async function executePriceSync(ctx: any, jobId: string, tracker: any) {
|
|
1478
|
+
const { log, activation, openKv } = ctx;
|
|
1479
|
+
const startTime = Date.now();
|
|
1480
|
+
|
|
1481
|
+
log.info('🔄 [PriceSync] Starting scheduled price sync from S3');
|
|
1482
|
+
|
|
1483
|
+
// ✅ CRITICAL: Declare S3 outside try block for safe disposal
|
|
1484
|
+
let s3: S3DataSource | undefined;
|
|
1485
|
+
|
|
1486
|
+
try {
|
|
1487
|
+
// ========================================
|
|
1488
|
+
// STEP 1: CONFIGURATION & VALIDATION
|
|
1489
|
+
// ========================================
|
|
1490
|
+
|
|
1491
|
+
// Read activation variables
|
|
1492
|
+
const s3Bucket = activation?.getVariable('s3BucketName');
|
|
1493
|
+
const s3Region = activation?.getVariable('awsRegion') || 'us-east-1';
|
|
1494
|
+
const s3AccessKeyId = activation?.getVariable('awsAccessKeyId');
|
|
1495
|
+
const s3SecretAccessKey = activation?.getVariable('awsSecretAccessKey');
|
|
1496
|
+
const s3Prefix = activation?.getVariable('s3Prefix') || 'prices/';
|
|
1497
|
+
const retailerId = activation?.getVariable('retailerId');
|
|
1498
|
+
const maxFiles = parseInt(activation?.getVariable('maxFilesToProcess') || '10', 10);
|
|
1499
|
+
const filePattern = (activation?.getVariable('filePattern') || '.csv').toLowerCase();
|
|
1500
|
+
const enableArchival = activation?.getVariable('enableArchival') !== 'false';
|
|
1501
|
+
const archivePrefix = activation?.getVariable('archivePrefix') || 'processed/';
|
|
1502
|
+
const errorPrefix = activation?.getVariable('errorPrefix') || 'errors/';
|
|
1503
|
+
const logPrefix = activation?.getVariable('logPrefix') || 'logs/';
|
|
1504
|
+
|
|
1505
|
+
// Configuration with defaults
|
|
1506
|
+
const mutationBatchSize = parseInt(
|
|
1507
|
+
activation?.getVariable('mutationBatchSize') || '1', // Default: 1 (sequential)
|
|
1508
|
+
10
|
|
1509
|
+
);
|
|
1510
|
+
|
|
1511
|
+
const mutationsPerAliasBatch = activation?.getVariable('mutationsPerAliasBatch')
|
|
1512
|
+
? parseInt(activation?.getVariable('mutationsPerAliasBatch') || '1', 10)
|
|
1513
|
+
: undefined; // Default: undefined (disabled)
|
|
1514
|
+
|
|
1515
|
+
const minPrice = parseFloat(activation?.getVariable('minPrice') || '0.01');
|
|
1516
|
+
const maxPrice = parseFloat(activation?.getVariable('maxPrice') || '999999.99');
|
|
1517
|
+
const validateProducts = activation?.getVariable('validateProducts') !== 'false';
|
|
1518
|
+
const validateConnection = activation?.getVariable('validateConnection') !== 'false';
|
|
1519
|
+
const enableFileTracking = activation?.getVariable('enableFileTracking') !== 'false';
|
|
1520
|
+
|
|
1521
|
+
// Validate required variables
|
|
1522
|
+
const missingVars: string[] = [];
|
|
1523
|
+
if (!s3Bucket) missingVars.push('s3BucketName');
|
|
1524
|
+
if (!s3AccessKeyId) missingVars.push('awsAccessKeyId');
|
|
1525
|
+
if (!s3SecretAccessKey) missingVars.push('awsSecretAccessKey');
|
|
1526
|
+
if (!retailerId) missingVars.push('retailerId');
|
|
1527
|
+
|
|
1528
|
+
if (missingVars.length > 0) {
|
|
1529
|
+
const errorMsg = `Missing required variables: ${missingVars.join(', ')}`;
|
|
1530
|
+
log.error('❌ [PriceSync] Configuration error', { error: errorMsg });
|
|
1531
|
+
return {
|
|
1532
|
+
success: false,
|
|
1533
|
+
error: errorMsg,
|
|
1534
|
+
recommendation: `Please configure these activation variables: ${missingVars.join(', ')}`,
|
|
1535
|
+
processed: 0,
|
|
1536
|
+
duration: Date.now() - startTime,
|
|
1537
|
+
};
|
|
1538
|
+
}
|
|
1539
|
+
|
|
1540
|
+
// ========================================
|
|
1541
|
+
// STEP 2: CLIENT INITIALIZATION
|
|
1542
|
+
// ========================================
|
|
1543
|
+
|
|
1544
|
+
const client = await createClient(ctx);
|
|
1545
|
+
if (!client) {
|
|
1546
|
+
throw new Error('Failed to create Fluent Commerce client');
|
|
1547
|
+
}
|
|
1548
|
+
|
|
1549
|
+
// ✅ CORRECT: GraphQL mutations don't need setRetailerId()
|
|
1550
|
+
// Check your GraphQL schema to determine retailerId handling:
|
|
1551
|
+
// - Mandatory retailerId → Must pass it in mutation input
|
|
1552
|
+
// - Optional retailerId → Can pass it if needed
|
|
1553
|
+
// - No retailerId field → Don't pass it
|
|
1554
|
+
// See: fc-connect-sdk/docs/00-START-HERE/retailerid-configuration.md
|
|
1555
|
+
|
|
1556
|
+
log.info('✅ [PriceSync] Fluent client initialized');
|
|
1557
|
+
|
|
1558
|
+
// ========================================
|
|
1559
|
+
// STEP 3: SERVICE INITIALIZATION
|
|
1560
|
+
// ========================================
|
|
1561
|
+
|
|
1562
|
+
s3 = new S3DataSource(
|
|
1563
|
+
{
|
|
1564
|
+
type: 'S3_CSV',
|
|
1565
|
+
connectionId: 's3-price-sync',
|
|
1566
|
+
name: 'Source S3',
|
|
1567
|
+
s3Config: {
|
|
1568
|
+
bucket: s3Bucket,
|
|
1569
|
+
region: s3Region,
|
|
1570
|
+
accessKeyId: s3AccessKeyId,
|
|
1571
|
+
secretAccessKey: s3SecretAccessKey,
|
|
1572
|
+
},
|
|
1573
|
+
},
|
|
1574
|
+
log
|
|
1575
|
+
);
|
|
1576
|
+
|
|
1577
|
+
// Validate S3 connection if enabled
|
|
1578
|
+
if (validateConnection) {
|
|
1579
|
+
try {
|
|
1580
|
+
await s3.listFiles({ prefix: s3Prefix, maxKeys: 1 });
|
|
1581
|
+
log.info('✅ [PriceSync] S3 connection validated');
|
|
1582
|
+
} catch (error: any) {
|
|
1583
|
+
log.error('❌ [PriceSync] S3 connection validation failed', {
|
|
1584
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1585
|
+
});
|
|
1586
|
+
return {
|
|
1587
|
+
success: false,
|
|
1588
|
+
error: 'S3 connection validation failed',
|
|
1589
|
+
details: error?.message,
|
|
1590
|
+
recommendation: 'Please verify S3 credentials and bucket access permissions',
|
|
1591
|
+
duration: Date.now() - startTime,
|
|
1592
|
+
};
|
|
1593
|
+
}
|
|
1594
|
+
}
|
|
1595
|
+
|
|
1596
|
+
const parser = new CSVParserService();
|
|
1597
|
+
const kv = openKv(':project:');
|
|
1598
|
+
const fileTracker = enableFileTracking
|
|
1599
|
+
? new VersoriFileTracker(kv, 's3-csv-price-sync')
|
|
1600
|
+
: null;
|
|
1601
|
+
|
|
1602
|
+
// Create custom resolvers for price validation
|
|
1603
|
+
const customResolvers = {
|
|
1604
|
+
'custom.validatePriceRange': (value: any) => {
|
|
1605
|
+
const price = parseFloat(value);
|
|
1606
|
+
if (isNaN(price)) {
|
|
1607
|
+
throw new Error(`Invalid price: ${value}`);
|
|
1608
|
+
}
|
|
1609
|
+
if (price < minPrice) {
|
|
1610
|
+
throw new Error(`Price ${price} below minimum ${minPrice}`);
|
|
1611
|
+
}
|
|
1612
|
+
if (price > maxPrice) {
|
|
1613
|
+
throw new Error(`Price ${price} exceeds maximum ${maxPrice}`);
|
|
1614
|
+
}
|
|
1615
|
+
return price;
|
|
1616
|
+
},
|
|
1617
|
+
|
|
1618
|
+
'custom.normalizePriceType': (value: any) => {
|
|
1619
|
+
const type = String(value || 'DEFAULT').toUpperCase().trim();
|
|
1620
|
+
const validTypes = ['DEFAULT', 'SALE', 'CLEARANCE', 'BULK', 'PROMOTIONAL', 'MEMBER'];
|
|
1621
|
+
if (!validTypes.includes(type)) {
|
|
1622
|
+
throw new Error(`Invalid price type: ${type}`);
|
|
1623
|
+
}
|
|
1624
|
+
return type;
|
|
1625
|
+
},
|
|
1626
|
+
|
|
1627
|
+
'custom.normalizeQuantity': (value: any) => {
|
|
1628
|
+
if (!value || value === '') return undefined;
|
|
1629
|
+
const qty = parseInt(value, 10);
|
|
1630
|
+
if (isNaN(qty) || qty < 0) {
|
|
1631
|
+
throw new Error(`Invalid quantity: ${value}`);
|
|
1632
|
+
}
|
|
1633
|
+
return qty;
|
|
1634
|
+
},
|
|
1635
|
+
};
|
|
1636
|
+
|
|
1637
|
+
// ✅ CRITICAL: Load mapping config from external JSON file
|
|
1638
|
+
// Mapping config uses GraphQLMutationMapper structure (nested objects, not dot notation)
|
|
1639
|
+
// File: src/config/price-mapping.json
|
|
1640
|
+
const mappingConfigJson = await import('../config/price-mapping.json', { assert: { type: 'json' } });
|
|
1641
|
+
const mappingConfig = mappingConfigJson.default;
|
|
1642
|
+
|
|
1643
|
+
// Initialize GraphQLMutationMapper with client for schema introspection
|
|
1644
|
+
const mapper = new GraphQLMutationMapper(mappingConfig, log, { fluentClient: client });
|
|
1645
|
+
|
|
1646
|
+
// ========================================
|
|
1647
|
+
// STEP 4: FILE DISCOVERY
|
|
1648
|
+
// ========================================
|
|
1649
|
+
|
|
1650
|
+
try {
|
|
1651
|
+
// List files (pattern filtering handled by listFiles)
|
|
1652
|
+
const files = await s3.listFiles({
|
|
1653
|
+
prefix: s3Prefix,
|
|
1654
|
+
pattern: filePattern,
|
|
1655
|
+
maxKeys: 1000,
|
|
1656
|
+
});
|
|
1657
|
+
|
|
1658
|
+
const csvFiles = files
|
|
1659
|
+
.sort((a, b) => {
|
|
1660
|
+
const dateA = a.lastModified ? new Date(a.lastModified).getTime() : 0;
|
|
1661
|
+
const dateB = b.lastModified ? new Date(b.lastModified).getTime() : 0;
|
|
1662
|
+
return dateA - dateB;
|
|
1663
|
+
})
|
|
1664
|
+
.slice(0, maxFiles);
|
|
1665
|
+
|
|
1666
|
+
log.info('📂 [PriceSync] Files discovered', {
|
|
1667
|
+
total: files.length,
|
|
1668
|
+
toProcess: csvFiles.length,
|
|
1669
|
+
maxFiles,
|
|
1670
|
+
});
|
|
1671
|
+
|
|
1672
|
+
const results = {
|
|
1673
|
+
processed: 0,
|
|
1674
|
+
skipped: 0,
|
|
1675
|
+
failed: 0,
|
|
1676
|
+
totalRecords: 0,
|
|
1677
|
+
pricesUpdated: 0,
|
|
1678
|
+
pricesSkipped: 0,
|
|
1679
|
+
validationErrors: 0,
|
|
1680
|
+
priceChanges: [] as Array<{
|
|
1681
|
+
sku: string;
|
|
1682
|
+
type: string;
|
|
1683
|
+
oldPrice?: number;
|
|
1684
|
+
newPrice: number;
|
|
1685
|
+
}>,
|
|
1686
|
+
errors: [] as string[],
|
|
1687
|
+
};
|
|
1688
|
+
|
|
1689
|
+
// ========================================
|
|
1690
|
+
// STEP 5: FILE PROCESSING LOOP
|
|
1691
|
+
// ========================================
|
|
1692
|
+
|
|
1693
|
+
// Process each file using service functions
|
|
1694
|
+
for (const file of csvFiles) {
|
|
1695
|
+
const filePath = file.path;
|
|
1696
|
+
const fileName = file.name;
|
|
1697
|
+
const fileStartTime = Date.now();
|
|
1698
|
+
|
|
1699
|
+
log.info('📄 [PriceSync] Processing file', { fileName });
|
|
1700
|
+
|
|
1701
|
+
// Duplicate prevention via file tracker
|
|
1702
|
+
if (fileTracker && (await fileTracker.wasFileProcessed(fileName))) {
|
|
1703
|
+
log.info('⏭️ [PriceSync] Skipping already processed file', { fileName });
|
|
1704
|
+
results.skipped++;
|
|
1705
|
+
continue;
|
|
1706
|
+
}
|
|
1707
|
+
|
|
1708
|
+
try {
|
|
1709
|
+
// SERVICE FUNCTION 1: Process file (S3 + CSV + Mapper)
|
|
1710
|
+
const { records: mappedRecords, errors: mappingErrors } = await processFile(
|
|
1711
|
+
s3,
|
|
1712
|
+
parser,
|
|
1713
|
+
mapper,
|
|
1714
|
+
filePath,
|
|
1715
|
+
fileName,
|
|
1716
|
+
log
|
|
1717
|
+
);
|
|
1718
|
+
|
|
1719
|
+
if (!mappedRecords.length) {
|
|
1720
|
+
log.warn('No valid records after mapping, archiving', { fileName });
|
|
1721
|
+
if (enableArchival) {
|
|
1722
|
+
await s3.moveFile(filePath, `${archivePrefix}${fileName}`);
|
|
1723
|
+
}
|
|
1724
|
+
results.skipped++;
|
|
1725
|
+
continue;
|
|
1726
|
+
}
|
|
1727
|
+
|
|
1728
|
+
// SERVICE FUNCTION 2: Execute mutations
|
|
1729
|
+
// ✅ Configuration with defaults
|
|
1730
|
+
const mutationBatchSize = parseInt(
|
|
1731
|
+
ctx.activation?.getVariable('mutationBatchSize') || '1', // ✅ Default: 1 (sequential)
|
|
1732
|
+
10
|
|
1733
|
+
);
|
|
1734
|
+
|
|
1735
|
+
const mutationsPerAliasBatch = ctx.activation?.getVariable('mutationsPerAliasBatch')
|
|
1736
|
+
? parseInt(ctx.activation?.getVariable('mutationsPerAliasBatch') || '1', 10)
|
|
1737
|
+
: undefined; // ✅ Default: undefined (disabled, use separate requests)
|
|
1738
|
+
|
|
1739
|
+
// ? Enhanced: Extract context for progress logging
|
|
1740
|
+
const sampleProductRefs = mappedRecords.slice(0, 5).map((r: any) => r.input?.skuRef || r.input?.productRef || 'unknown');
|
|
1741
|
+
const mutationType = mapper?.mutationName || 'updatePrice';
|
|
1742
|
+
|
|
1743
|
+
// ? Enhanced: Start logging with context
|
|
1744
|
+
log.info(`[GraphQLMutations] Sending price mutations for file "${fileName}"`, {
|
|
1745
|
+
totalMutations: mappedRecords.length,
|
|
1746
|
+
mutationType,
|
|
1747
|
+
batchSize: mutationBatchSize,
|
|
1748
|
+
batchMode: mutationBatchSize === 1 ? 'sequential' : `parallel (${mutationBatchSize})`,
|
|
1749
|
+
sampleProductRefs: sampleProductRefs.join(', '),
|
|
1750
|
+
aliasBatching: mutationsPerAliasBatch ? `enabled (${mutationsPerAliasBatch} per alias)` : 'disabled',
|
|
1751
|
+
validateProducts
|
|
1752
|
+
});
|
|
1753
|
+
|
|
1754
|
+
const mutationResult = await executeMutations(
|
|
1755
|
+
client,
|
|
1756
|
+
mapper,
|
|
1757
|
+
mappedRecords,
|
|
1758
|
+
retailerId,
|
|
1759
|
+
validateProducts,
|
|
1760
|
+
mutationBatchSize, // Concurrency control (default: 1)
|
|
1761
|
+
mutationsPerAliasBatch, // ✅ NEW: Alias batching (default: undefined)
|
|
1762
|
+
log
|
|
1763
|
+
);
|
|
1764
|
+
|
|
1765
|
+
// ? Enhanced: Completion logging with summary
|
|
1766
|
+
log.info(`[GraphQLMutations] Price mutation submission completed for file "${fileName}"`, {
|
|
1767
|
+
totalMutations: mappedRecords.length,
|
|
1768
|
+
successful: mutationResult.successful,
|
|
1769
|
+
failed: mutationResult.failed,
|
|
1770
|
+
successRate: mappedRecords.length > 0 ? `${Math.round((mutationResult.successful / mappedRecords.length) * 100)}%` : '0%',
|
|
1771
|
+
mutationType,
|
|
1772
|
+
priceChanges: mutationResult.priceChanges?.length || 0
|
|
1773
|
+
});
|
|
1774
|
+
|
|
1775
|
+
// Aggregate results
|
|
1776
|
+
results.processed++;
|
|
1777
|
+
results.totalRecords += mappedRecords.length;
|
|
1778
|
+
results.pricesUpdated += mutationResult.successful;
|
|
1779
|
+
results.validationErrors += mappingErrors.length;
|
|
1780
|
+
results.priceChanges.push(...mutationResult.priceChanges);
|
|
1781
|
+
|
|
1782
|
+
// Build file processing result for logging
|
|
1783
|
+
const fileResult: FileProcessingResult = {
|
|
1784
|
+
fileName,
|
|
1785
|
+
successful: mutationResult.successful,
|
|
1786
|
+
failed: mutationResult.failed,
|
|
1787
|
+
validationFailed: mappingErrors.length,
|
|
1788
|
+
priceChanges: mutationResult.priceChanges,
|
|
1789
|
+
errors: mappingErrors,
|
|
1790
|
+
};
|
|
1791
|
+
|
|
1792
|
+
// SERVICE FUNCTION 3: Write mutation log to S3
|
|
1793
|
+
await writeMutationLog(s3, fileName, fileResult, logPrefix, log);
|
|
1794
|
+
|
|
1795
|
+
// Mark processed with file tracker
|
|
1796
|
+
if (fileTracker) {
|
|
1797
|
+
await fileTracker.markFileProcessed(fileName, {
|
|
1798
|
+
successful: mutationResult.successful,
|
|
1799
|
+
failed: mutationResult.failed,
|
|
1800
|
+
validationFailed: mappingErrors.length,
|
|
1801
|
+
recordCount: mappedRecords.length,
|
|
1802
|
+
duration: Date.now() - fileStartTime,
|
|
1803
|
+
});
|
|
1804
|
+
}
|
|
1805
|
+
|
|
1806
|
+
if (enableArchival) {
|
|
1807
|
+
await s3.moveFile(filePath, `${archivePrefix}${fileName}`);
|
|
1808
|
+
}
|
|
1809
|
+
|
|
1810
|
+
const fileDuration = Date.now() - fileStartTime;
|
|
1811
|
+
log.info('✅ [PriceSync] File processed successfully', {
|
|
1812
|
+
fileName,
|
|
1813
|
+
recordCount: mappedRecords.length,
|
|
1814
|
+
successful: mutationResult.successful,
|
|
1815
|
+
failed: mutationResult.failed,
|
|
1816
|
+
duration: `${fileDuration}ms`,
|
|
1817
|
+
});
|
|
1818
|
+
|
|
1819
|
+
if (mappingErrors.length > 0) {
|
|
1820
|
+
results.errors.push(`${fileName}: ${mappingErrors.length} mapping errors`);
|
|
1821
|
+
}
|
|
1822
|
+
} catch (error: unknown) {
|
|
1823
|
+
// ✅ Enhanced error logging: Extract all error details for visibility
|
|
1824
|
+
const errorMsg = error instanceof Error ? error.message : String(error);
|
|
1825
|
+
const errorDetails = {
|
|
1826
|
+
message: errorMsg,
|
|
1827
|
+
stack: error instanceof Error ? error.stack : undefined,
|
|
1828
|
+
errorType: error instanceof Error ? error.constructor.name : 'Error',
|
|
1829
|
+
};
|
|
1830
|
+
log.error('❌ [PriceSync] File processing failed', {
|
|
1831
|
+
fileName,
|
|
1832
|
+
...errorDetails,
|
|
1833
|
+
});
|
|
1834
|
+
results.failed++;
|
|
1835
|
+
results.errors.push(`${fileName}: ${errorMsg}`);
|
|
1836
|
+
|
|
1837
|
+
// Attempt to move to error directory
|
|
1838
|
+
try {
|
|
1839
|
+
await s3.moveFile(filePath, `${errorPrefix}${fileName}`);
|
|
1840
|
+
log.info('📁 [PriceSync] Moved failed file to error directory', { fileName });
|
|
1841
|
+
} catch (moveError) {
|
|
1842
|
+
log.error('❌ [PriceSync] Failed to move error file', {
|
|
1843
|
+
fileName,
|
|
1844
|
+
error: moveError instanceof Error ? moveError.message : String(moveError),
|
|
1845
|
+
});
|
|
1846
|
+
}
|
|
1847
|
+
}
|
|
1848
|
+
}
|
|
1849
|
+
|
|
1850
|
+
// ========================================
|
|
1851
|
+
// STEP 6: SUMMARY & COMPLETION
|
|
1852
|
+
// ========================================
|
|
1853
|
+
|
|
1854
|
+
const totalDuration = Date.now() - startTime;
|
|
1855
|
+
const summary = {
|
|
1856
|
+
success: true,
|
|
1857
|
+
processed: results.processed,
|
|
1858
|
+
skipped: results.skipped,
|
|
1859
|
+
failed: results.failed,
|
|
1860
|
+
totalRecords: results.totalRecords,
|
|
1861
|
+
pricesUpdated: results.pricesUpdated,
|
|
1862
|
+
pricesSkipped: results.pricesSkipped,
|
|
1863
|
+
validationErrors: results.validationErrors,
|
|
1864
|
+
priceChanges: results.priceChanges.length,
|
|
1865
|
+
errors: results.errors.length > 0 ? results.errors : undefined,
|
|
1866
|
+
duration: totalDuration,
|
|
1867
|
+
timestamp: new Date().toISOString(),
|
|
1868
|
+
};
|
|
1869
|
+
|
|
1870
|
+
log.info('🎉 [PriceSync] Price sync completed', {
|
|
1871
|
+
...summary,
|
|
1872
|
+
duration: `${totalDuration}ms`,
|
|
1873
|
+
});
|
|
1874
|
+
|
|
1875
|
+
// Log significant price changes
|
|
1876
|
+
if (results.priceChanges.length > 0) {
|
|
1877
|
+
log.info('💰 [PriceSync] Price changes detected', {
|
|
1878
|
+
count: results.priceChanges.length,
|
|
1879
|
+
changes: results.priceChanges.slice(0, 10),
|
|
1880
|
+
});
|
|
1881
|
+
}
|
|
1882
|
+
|
|
1883
|
+
return summary;
|
|
1884
|
+
} catch (error: unknown) {
|
|
1885
|
+
// ✅ Enhanced error logging: Extract all error details for visibility
|
|
1886
|
+
const errorMsg = error instanceof Error ? error.message : String(error);
|
|
1887
|
+
const errorDetails = {
|
|
1888
|
+
message: errorMsg,
|
|
1889
|
+
stack: error instanceof Error ? error.stack : undefined,
|
|
1890
|
+
errorType: error instanceof Error ? error.constructor.name : 'Error',
|
|
1891
|
+
};
|
|
1892
|
+
log.error('❌ [PriceSync] Fatal error', errorDetails);
|
|
1893
|
+
return {
|
|
1894
|
+
success: false,
|
|
1895
|
+
error: errorMsg,
|
|
1896
|
+
recommendation: 'Check error logs for details and verify configuration',
|
|
1897
|
+
processed: 0,
|
|
1898
|
+
duration: Date.now() - startTime,
|
|
1899
|
+
timestamp: new Date().toISOString(),
|
|
1900
|
+
};
|
|
1901
|
+
}
|
|
1902
|
+
} catch (error: unknown) {
|
|
1903
|
+
const errorMsg = error instanceof Error ? error.message : String(error);
|
|
1904
|
+
log.error('❌ [PriceSync] Initialization failed', {
|
|
1905
|
+
error: errorMsg,
|
|
1906
|
+
stack: error instanceof Error ? error.stack : undefined,
|
|
1907
|
+
});
|
|
1908
|
+
return {
|
|
1909
|
+
success: false,
|
|
1910
|
+
error: errorMsg,
|
|
1911
|
+
recommendation: 'Check configuration and service initialization',
|
|
1912
|
+
duration: Date.now() - startTime,
|
|
1913
|
+
};
|
|
1914
|
+
} finally {
|
|
1915
|
+
// ✅ CRITICAL: Always dispose S3 connection to prevent connection pool exhaustion
|
|
1916
|
+
if (s3) {
|
|
1917
|
+
try {
|
|
1918
|
+
await s3.dispose();
|
|
1919
|
+
log.info('✅ [PriceSync] S3 connection disposed successfully');
|
|
1920
|
+
} catch (disposeError: any) {
|
|
1921
|
+
log.error('⚠️ [PriceSync] Failed to dispose S3 connection', {
|
|
1922
|
+
error: disposeError instanceof Error ? disposeError.message : String(disposeError),
|
|
1923
|
+
stack: disposeError instanceof Error ? disposeError.stack : undefined,
|
|
1924
|
+
});
|
|
1925
|
+
// Don't throw - disposal failure shouldn't prevent workflow completion
|
|
1926
|
+
}
|
|
1927
|
+
}
|
|
1928
|
+
}
|
|
1929
|
+
}
|
|
1930
|
+
```
|
|
1931
|
+
|
|
1932
|
+
**Note:** The `runPriceSync` function above should be renamed to `executePriceSync` and moved to `src/services/price-sync.service.ts` to match the new workflow structure.
|
|
1933
|
+
|
|
1934
|
+
## Key Patterns Explained
|
|
1935
|
+
|
|
1936
|
+
### Pattern 1: Service Function Architecture
|
|
1937
|
+
|
|
1938
|
+
**Three modular service functions for clean separation of concerns:**
|
|
1939
|
+
|
|
1940
|
+
```typescript
|
|
1941
|
+
// SERVICE FUNCTION 1: processFile()
|
|
1942
|
+
// Handles: S3 download + CSV parsing + field mapping
|
|
1943
|
+
const { records: mappedRecords, errors: mappingErrors } = await processFile(
|
|
1944
|
+
s3,
|
|
1945
|
+
parser,
|
|
1946
|
+
mapper,
|
|
1947
|
+
filePath,
|
|
1948
|
+
fileName,
|
|
1949
|
+
log
|
|
1950
|
+
);
|
|
1951
|
+
|
|
1952
|
+
// SERVICE FUNCTION 2: executeMutations()
|
|
1953
|
+
// Handles: GraphQL mutations + product validation + concurrency control + price change tracking
|
|
1954
|
+
const mutationResult = await executeMutations(
|
|
1955
|
+
client,
|
|
1956
|
+
mapper,
|
|
1957
|
+
mappedRecords,
|
|
1958
|
+
retailerId,
|
|
1959
|
+
validateProducts,
|
|
1960
|
+
mutationBatchSize, // Concurrency control (default: 1)
|
|
1961
|
+
mutationsPerAliasBatch, // Alias batching (default: undefined)
|
|
1962
|
+
log
|
|
1963
|
+
);
|
|
1964
|
+
|
|
1965
|
+
// SERVICE FUNCTION 3: writeMutationLog()
|
|
1966
|
+
// Handles: Write execution log to S3 (with Buffer for Versori/Deno)
|
|
1967
|
+
await writeMutationLog(s3, fileName, fileResult, logPrefix, log);
|
|
1968
|
+
```
|
|
1969
|
+
|
|
1970
|
+
**Why this pattern?**
|
|
1971
|
+
- ✅ **Testability**: Each function can be unit tested independently
|
|
1972
|
+
- ✅ **Reusability**: Functions can be reused in different workflows
|
|
1973
|
+
- ✅ **Clarity**: Clear responsibilities for each step
|
|
1974
|
+
- ✅ **Error handling**: Isolated error boundaries per function
|
|
1975
|
+
- ✅ **Maintainability**: Easy to modify single concerns without affecting others
|
|
1976
|
+
|
|
1977
|
+
### Pattern 2: Buffer Import for S3 Upload (Versori/Deno)
|
|
1978
|
+
|
|
1979
|
+
**CRITICAL for Versori/Deno runtime:**
|
|
1980
|
+
|
|
1981
|
+
```typescript
|
|
1982
|
+
// MUST import Buffer explicitly (not global like Node.js)
|
|
1983
|
+
import { Buffer } from 'node:buffer';
|
|
1984
|
+
|
|
1985
|
+
// writeMutationLog() function
|
|
1986
|
+
async function writeMutationLog(s3, fileName, result, logPrefix, log) {
|
|
1987
|
+
const logContent = JSON.stringify(logData, null, 2);
|
|
1988
|
+
|
|
1989
|
+
// ✅ CORRECT: Use Buffer.from() for S3 upload
|
|
1990
|
+
await s3.uploadFile(logPath, logContent);
|
|
1991
|
+
|
|
1992
|
+
// ✅ CORRECT: uploadFile accepts string directly (no Buffer needed)
|
|
1993
|
+
// await s3.uploadFile(logPath, logContent);
|
|
1994
|
+
}
|
|
1995
|
+
```
|
|
1996
|
+
|
|
1997
|
+
**Why?** Deno runtime requires explicit `Buffer` import from `node:buffer`. Without it, you'll get `Buffer is not defined` errors.
|
|
1998
|
+
|
|
1999
|
+
### Pattern 3: Direct KV State Management
|
|
2000
|
+
|
|
2001
|
+
```typescript
|
|
2002
|
+
const kv = new VersoriKVAdapter(ctx.openKv(':project:'));
|
|
2003
|
+
|
|
2004
|
+
// Check if file already processed
|
|
2005
|
+
const stateKey = ['processed-files', 's3-price-sync', fileName];
|
|
2006
|
+
const existing = await kv.get(stateKey);
|
|
2007
|
+
if (existing) {
|
|
2008
|
+
log.info('Skipping already processed file', { fileName });
|
|
2009
|
+
continue;
|
|
2010
|
+
}
|
|
2011
|
+
|
|
2012
|
+
// Mark as processed after success
|
|
2013
|
+
await kv.set(stateKey, {
|
|
2014
|
+
successful,
|
|
2015
|
+
failed,
|
|
2016
|
+
validationFailed,
|
|
2017
|
+
processedAt: new Date().toISOString(),
|
|
2018
|
+
});
|
|
2019
|
+
```
|
|
2020
|
+
|
|
2021
|
+
### Pattern 4: Custom Price Validation Resolvers
|
|
2022
|
+
|
|
2023
|
+
```typescript
|
|
2024
|
+
const customResolvers = {
|
|
2025
|
+
'custom.validatePriceRange': (value: any) => {
|
|
2026
|
+
const price = parseFloat(value);
|
|
2027
|
+
if (isNaN(price)) {
|
|
2028
|
+
throw new Error(`Invalid price: ${value}`);
|
|
2029
|
+
}
|
|
2030
|
+
if (price < minPrice) {
|
|
2031
|
+
throw new Error(`Price ${price} below minimum ${minPrice}`);
|
|
2032
|
+
}
|
|
2033
|
+
if (price > maxPrice) {
|
|
2034
|
+
throw new Error(`Price ${price} exceeds maximum ${maxPrice}`);
|
|
2035
|
+
}
|
|
2036
|
+
return price;
|
|
2037
|
+
},
|
|
2038
|
+
|
|
2039
|
+
'custom.normalizePriceType': (value: any) => {
|
|
2040
|
+
const type = String(value || 'DEFAULT').toUpperCase().trim();
|
|
2041
|
+
const validTypes = ['DEFAULT', 'SALE', 'CLEARANCE', 'BULK', 'PROMOTIONAL', 'MEMBER'];
|
|
2042
|
+
if (!validTypes.includes(type)) {
|
|
2043
|
+
throw new Error(`Invalid price type: ${type}`);
|
|
2044
|
+
}
|
|
2045
|
+
return type;
|
|
2046
|
+
},
|
|
2047
|
+
};
|
|
2048
|
+
```
|
|
2049
|
+
|
|
2050
|
+
**Why this matters**: Price data requires strict validation to prevent:
|
|
2051
|
+
|
|
2052
|
+
- Negative prices
|
|
2053
|
+
- Prices outside business rules (too low/high)
|
|
2054
|
+
- Invalid price tiers
|
|
2055
|
+
- Data entry errors
|
|
2056
|
+
|
|
2057
|
+
### Pattern 5: Product Existence Validation
|
|
2058
|
+
|
|
2059
|
+
```typescript
|
|
2060
|
+
async function validateProductExists(
|
|
2061
|
+
client: any,
|
|
2062
|
+
productRef: string,
|
|
2063
|
+
retailerId: string,
|
|
2064
|
+
log: any
|
|
2065
|
+
): Promise<boolean> {
|
|
2066
|
+
const query = `
|
|
2067
|
+
query GetProduct($ref: String!, $retailerId: ID!) {
|
|
2068
|
+
products(first: 1, ref: [$ref], retailerId: $retailerId) {
|
|
2069
|
+
edges {
|
|
2070
|
+
node {
|
|
2071
|
+
id
|
|
2072
|
+
ref
|
|
2073
|
+
status
|
|
2074
|
+
}
|
|
2075
|
+
}
|
|
2076
|
+
}
|
|
2077
|
+
}
|
|
2078
|
+
`;
|
|
2079
|
+
|
|
2080
|
+
const result = await client.graphql({ query, variables: { ref: productRef, retailerId } });
|
|
2081
|
+
const product = result?.data?.products?.edges?.[0]?.node;
|
|
2082
|
+
|
|
2083
|
+
if (!product || product.status !== 'ACTIVE') {
|
|
2084
|
+
return false;
|
|
2085
|
+
}
|
|
2086
|
+
|
|
2087
|
+
return true;
|
|
2088
|
+
}
|
|
2089
|
+
```
|
|
2090
|
+
|
|
2091
|
+
**Why this matters**: Prevents price updates for non-existent products, which would fail at GraphQL level.
|
|
2092
|
+
|
|
2093
|
+
### Pattern 6: Price Change Tracking
|
|
2094
|
+
|
|
2095
|
+
```typescript
|
|
2096
|
+
// Get current price before update
|
|
2097
|
+
const currentPrice = await getCurrentPrice(
|
|
2098
|
+
client,
|
|
2099
|
+
priceData.productRef,
|
|
2100
|
+
priceData.type,
|
|
2101
|
+
priceData.currency,
|
|
2102
|
+
retailerId
|
|
2103
|
+
);
|
|
2104
|
+
|
|
2105
|
+
// Track change after update
|
|
2106
|
+
if (currentPrice !== undefined && currentPrice !== priceData.value) {
|
|
2107
|
+
results.priceChanges.push({
|
|
2108
|
+
sku: priceData.productRef,
|
|
2109
|
+
type: priceData.type,
|
|
2110
|
+
oldPrice: currentPrice,
|
|
2111
|
+
newPrice: priceData.value,
|
|
2112
|
+
});
|
|
2113
|
+
}
|
|
2114
|
+
|
|
2115
|
+
// Log changes at end
|
|
2116
|
+
if (results.priceChanges.length > 0) {
|
|
2117
|
+
log.info('Price changes detected', {
|
|
2118
|
+
count: results.priceChanges.length,
|
|
2119
|
+
changes: results.priceChanges.slice(0, 10),
|
|
2120
|
+
});
|
|
2121
|
+
}
|
|
2122
|
+
```
|
|
2123
|
+
|
|
2124
|
+
**Why this matters**: Provides audit trail and monitoring for price changes.
|
|
2125
|
+
|
|
2126
|
+
### Pattern 7: Concurrency Control & Alias Batching
|
|
2127
|
+
|
|
2128
|
+
```typescript
|
|
2129
|
+
// ✅ Configuration with defaults
|
|
2130
|
+
const mutationBatchSize = parseInt(
|
|
2131
|
+
activation?.getVariable('mutationBatchSize') || '1', // Default: 1 (sequential)
|
|
2132
|
+
10
|
|
2133
|
+
);
|
|
2134
|
+
|
|
2135
|
+
const mutationsPerAliasBatch = activation?.getVariable('mutationsPerAliasBatch')
|
|
2136
|
+
? parseInt(activation?.getVariable('mutationsPerAliasBatch') || '1', 10)
|
|
2137
|
+
: undefined; // Default: undefined (disabled)
|
|
2138
|
+
|
|
2139
|
+
// Execute with bounded concurrency + optional alias batching
|
|
2140
|
+
const mutationResult = await executeMutations(
|
|
2141
|
+
client,
|
|
2142
|
+
mapper,
|
|
2143
|
+
priceRecords,
|
|
2144
|
+
retailerId,
|
|
2145
|
+
validateProducts,
|
|
2146
|
+
mutationBatchSize, // 1=sequential, 3-10=parallel
|
|
2147
|
+
mutationsPerAliasBatch, // Optional: Group mutations (e.g., 5)
|
|
2148
|
+
log
|
|
2149
|
+
);
|
|
2150
|
+
```
|
|
2151
|
+
|
|
2152
|
+
**Performance Modes:**
|
|
2153
|
+
- `mutationBatchSize: 1` → Sequential (safe default, ~1 mutation/sec)
|
|
2154
|
+
- `mutationBatchSize: 3-5` → Balanced (~3-5 mutations/sec)
|
|
2155
|
+
- `mutationBatchSize: 10` → High-volume (~10 mutations/sec)
|
|
2156
|
+
- `mutationsPerAliasBatch: 5` → Alias batching (reduces network overhead by ~80%)
|
|
2157
|
+
|
|
2158
|
+
## Advanced: Multi-Currency Price Management
|
|
2159
|
+
|
|
2160
|
+
### External Mapping File for Multi-Currency
|
|
2161
|
+
|
|
2162
|
+
**File**: `config/price-mapping.json`
|
|
2163
|
+
|
|
2164
|
+
```json
|
|
2165
|
+
{
|
|
2166
|
+
"version": "1.0.0",
|
|
2167
|
+
"description": "Multi-currency price mapping",
|
|
2168
|
+
"fields": {
|
|
2169
|
+
"productRef": {
|
|
2170
|
+
"source": "sku",
|
|
2171
|
+
"required": true,
|
|
2172
|
+
"resolver": "sdk.trim"
|
|
2173
|
+
},
|
|
2174
|
+
"type": {
|
|
2175
|
+
"source": "price_type",
|
|
2176
|
+
"required": true,
|
|
2177
|
+
"resolver": "custom.normalizePriceType"
|
|
2178
|
+
},
|
|
2179
|
+
"value": {
|
|
2180
|
+
"source": "amount",
|
|
2181
|
+
"required": true,
|
|
2182
|
+
"resolver": "custom.validatePriceRange"
|
|
2183
|
+
},
|
|
2184
|
+
"currency": {
|
|
2185
|
+
"source": "currency_code",
|
|
2186
|
+
"required": true,
|
|
2187
|
+
"resolver": "custom.normalizeCurrency"
|
|
2188
|
+
},
|
|
2189
|
+
"effectiveFrom": {
|
|
2190
|
+
"source": "start_date",
|
|
2191
|
+
"resolver": "sdk.formatDate"
|
|
2192
|
+
},
|
|
2193
|
+
"effectiveTo": {
|
|
2194
|
+
"source": "end_date",
|
|
2195
|
+
"resolver": "sdk.formatDate"
|
|
2196
|
+
}
|
|
2197
|
+
}
|
|
2198
|
+
}
|
|
2199
|
+
```
|
|
2200
|
+
|
|
2201
|
+
### Custom Resolvers for Multi-Currency
|
|
2202
|
+
|
|
2203
|
+
```typescript
|
|
2204
|
+
const customResolvers = {
|
|
2205
|
+
'custom.normalizeCurrency': (value: string) => {
|
|
2206
|
+
// Normalize currency codes to ISO 4217
|
|
2207
|
+
const currencyMap: Record<string, string> = {
|
|
2208
|
+
usd: 'USD',
|
|
2209
|
+
us: 'USD',
|
|
2210
|
+
dollar: 'USD',
|
|
2211
|
+
eur: 'EUR',
|
|
2212
|
+
euro: 'EUR',
|
|
2213
|
+
gbp: 'GBP',
|
|
2214
|
+
pound: 'GBP',
|
|
2215
|
+
cad: 'CAD',
|
|
2216
|
+
aud: 'AUD',
|
|
2217
|
+
};
|
|
2218
|
+
|
|
2219
|
+
const normalized = currencyMap[value.toLowerCase()] || value.toUpperCase();
|
|
2220
|
+
|
|
2221
|
+
// Validate against known currencies
|
|
2222
|
+
const validCurrencies = ['USD', 'EUR', 'GBP', 'CAD', 'AUD', 'JPY', 'CNY'];
|
|
2223
|
+
if (!validCurrencies.includes(normalized)) {
|
|
2224
|
+
throw new Error(`Invalid currency code: ${value}`);
|
|
2225
|
+
}
|
|
2226
|
+
|
|
2227
|
+
return normalized;
|
|
2228
|
+
},
|
|
2229
|
+
|
|
2230
|
+
'custom.convertToBaseCurrency': (value: any, sourceRecord: any) => {
|
|
2231
|
+
// Convert foreign currency to base currency
|
|
2232
|
+
const amount = parseFloat(value);
|
|
2233
|
+
const currency = sourceRecord.currency_code;
|
|
2234
|
+
|
|
2235
|
+
// Exchange rates (in production, fetch from API)
|
|
2236
|
+
const exchangeRates: Record<string, number> = {
|
|
2237
|
+
USD: 1.0,
|
|
2238
|
+
EUR: 0.85,
|
|
2239
|
+
GBP: 0.73,
|
|
2240
|
+
CAD: 1.35,
|
|
2241
|
+
AUD: 1.45,
|
|
2242
|
+
};
|
|
2243
|
+
|
|
2244
|
+
const rate = exchangeRates[currency?.toUpperCase()] || 1.0;
|
|
2245
|
+
return amount * rate;
|
|
2246
|
+
},
|
|
2247
|
+
|
|
2248
|
+
'custom.buildPriceAttributes': (_: any, sourceRecord: any) => {
|
|
2249
|
+
// Build price attributes array
|
|
2250
|
+
const attributes: Array<{ name: string; type: string; value: any }> = [];
|
|
2251
|
+
|
|
2252
|
+
if (sourceRecord.cost_basis) {
|
|
2253
|
+
attributes.push({
|
|
2254
|
+
name: 'costBasis',
|
|
2255
|
+
type: 'Float',
|
|
2256
|
+
value: parseFloat(sourceRecord.cost_basis),
|
|
2257
|
+
});
|
|
2258
|
+
}
|
|
2259
|
+
|
|
2260
|
+
if (sourceRecord.margin_percent) {
|
|
2261
|
+
attributes.push({
|
|
2262
|
+
name: 'marginPercent',
|
|
2263
|
+
type: 'Float',
|
|
2264
|
+
value: parseFloat(sourceRecord.margin_percent),
|
|
2265
|
+
});
|
|
2266
|
+
}
|
|
2267
|
+
|
|
2268
|
+
if (sourceRecord.competitor_price) {
|
|
2269
|
+
attributes.push({
|
|
2270
|
+
name: 'competitorPrice',
|
|
2271
|
+
type: 'Float',
|
|
2272
|
+
value: parseFloat(sourceRecord.competitor_price),
|
|
2273
|
+
});
|
|
2274
|
+
}
|
|
2275
|
+
|
|
2276
|
+
if (sourceRecord.price_source) {
|
|
2277
|
+
attributes.push({
|
|
2278
|
+
name: 'priceSource',
|
|
2279
|
+
type: 'String',
|
|
2280
|
+
value: sourceRecord.price_source,
|
|
2281
|
+
});
|
|
2282
|
+
}
|
|
2283
|
+
|
|
2284
|
+
return attributes;
|
|
2285
|
+
},
|
|
2286
|
+
};
|
|
2287
|
+
```
|
|
2288
|
+
|
|
2289
|
+
## Schema Validation (Before Deployment)
|
|
2290
|
+
|
|
2291
|
+
Use SDK CLI tools to validate your GraphQL schema and mapping configuration:
|
|
2292
|
+
|
|
2293
|
+
```bash
|
|
2294
|
+
# Install SDK globally (or use npx)
|
|
2295
|
+
npm install -g @fluentcommerce/fc-connect-sdk
|
|
2296
|
+
|
|
2297
|
+
# 1. Introspect Fluent GraphQL schema
|
|
2298
|
+
fc-connect introspect-schema \
|
|
2299
|
+
--url https://api.fluentcommerce.com/graphql \
|
|
2300
|
+
--client-id YOUR_CLIENT_ID \
|
|
2301
|
+
--client-secret YOUR_CLIENT_SECRET \
|
|
2302
|
+
--output fluent-schema.json
|
|
2303
|
+
|
|
2304
|
+
# 2. Create mutation file (price-mutation.graphql)
|
|
2305
|
+
cat > price-mutation.graphql << 'EOF'
|
|
2306
|
+
mutation UpdateProductPrice($input: UpdateProductInput!) {
|
|
2307
|
+
updateProduct(input: $input) {
|
|
2308
|
+
id
|
|
2309
|
+
ref
|
|
2310
|
+
prices {
|
|
2311
|
+
type
|
|
2312
|
+
value
|
|
2313
|
+
currency
|
|
2314
|
+
}
|
|
2315
|
+
}
|
|
2316
|
+
}
|
|
2317
|
+
EOF
|
|
2318
|
+
|
|
2319
|
+
# 3. Generate mapping from mutation (validates structure)
|
|
2320
|
+
fc-connect generate-mutation-mapping \
|
|
2321
|
+
--file price-mutation.graphql \
|
|
2322
|
+
--output price-mapping.json
|
|
2323
|
+
|
|
2324
|
+
# 4. Validate mapping against schema
|
|
2325
|
+
fc-connect validate-schema \
|
|
2326
|
+
--mapping price-mapping.json \
|
|
2327
|
+
--schema fluent-schema.json
|
|
2328
|
+
|
|
2329
|
+
# 5. Analyze field coverage
|
|
2330
|
+
fc-connect analyze-coverage \
|
|
2331
|
+
--mapping price-mapping.json \
|
|
2332
|
+
--schema fluent-schema.json
|
|
2333
|
+
```
|
|
2334
|
+
|
|
2335
|
+
**Output Example**:
|
|
2336
|
+
|
|
2337
|
+
```bash
|
|
2338
|
+
✓ Schema validation passed
|
|
2339
|
+
✓ All required fields present: ref, retailerId, prices[].type, prices[].value, prices[].currency
|
|
2340
|
+
✓ Mutation structure matches GraphQL schema
|
|
2341
|
+
⚠ Optional fields not mapped: prices[].taxType, prices[].taxRate
|
|
2342
|
+
```
|
|
2343
|
+
|
|
2344
|
+
## Testing the Workflow
|
|
2345
|
+
|
|
2346
|
+
### 1. Upload Test CSV to S3
|
|
2347
|
+
|
|
2348
|
+
```bash
|
|
2349
|
+
# Using AWS CLI
|
|
2350
|
+
aws s3 cp product-prices-test.csv s3://my-price-bucket/prices/
|
|
2351
|
+
|
|
2352
|
+
# Or using S3 Console
|
|
2353
|
+
# Navigate to bucket → prices/ → Upload
|
|
2354
|
+
```
|
|
2355
|
+
|
|
2356
|
+
### 2. Deploy to Versori
|
|
2357
|
+
|
|
2358
|
+
```bash
|
|
2359
|
+
npm run deploy
|
|
2360
|
+
```
|
|
2361
|
+
|
|
2362
|
+
### 3. Manual Testing via Webhook
|
|
2363
|
+
|
|
2364
|
+
```bash
|
|
2365
|
+
curl -X POST https://your-workspace.versori.run/sync-prices-now \
|
|
2366
|
+
-H "Content-Type: application/json"
|
|
2367
|
+
```
|
|
2368
|
+
|
|
2369
|
+
### 4. Check Status
|
|
2370
|
+
|
|
2371
|
+
```bash
|
|
2372
|
+
curl https://your-workspace.versori.run/check-status
|
|
2373
|
+
```
|
|
2374
|
+
|
|
2375
|
+
### 5. Monitor Logs
|
|
2376
|
+
|
|
2377
|
+
```bash
|
|
2378
|
+
npm run logs
|
|
2379
|
+
# Or via Versori dashboard
|
|
2380
|
+
```
|
|
2381
|
+
|
|
2382
|
+
---
|
|
2383
|
+
|
|
2384
|
+
## Monitoring
|
|
2385
|
+
|
|
2386
|
+
### Success Response
|
|
2387
|
+
|
|
2388
|
+
```json
|
|
2389
|
+
{
|
|
2390
|
+
"success": true,
|
|
2391
|
+
"filesProcessed": 1,
|
|
2392
|
+
"filesSkipped": 0,
|
|
2393
|
+
"filesFailed": 0,
|
|
2394
|
+
"totalRecords": 100,
|
|
2395
|
+
"mutationsExecuted": 100,
|
|
2396
|
+
"mutationsFailed": 0,
|
|
2397
|
+
"results": [
|
|
2398
|
+
{
|
|
2399
|
+
"file": "prices_2025-01-22.csv",
|
|
2400
|
+
"success": true,
|
|
2401
|
+
"recordsProcessed": 100,
|
|
2402
|
+
"mutationsExecuted": 100,
|
|
2403
|
+
"mutationsFailed": 0
|
|
2404
|
+
}
|
|
2405
|
+
],
|
|
2406
|
+
"duration": 12345
|
|
2407
|
+
}
|
|
2408
|
+
```
|
|
2409
|
+
|
|
2410
|
+
### Partial Success Response
|
|
2411
|
+
|
|
2412
|
+
```json
|
|
2413
|
+
{
|
|
2414
|
+
"success": true,
|
|
2415
|
+
"filesProcessed": 1,
|
|
2416
|
+
"filesSkipped": 0,
|
|
2417
|
+
"filesFailed": 0,
|
|
2418
|
+
"totalRecords": 100,
|
|
2419
|
+
"mutationsExecuted": 95,
|
|
2420
|
+
"mutationsFailed": 5,
|
|
2421
|
+
"results": [
|
|
2422
|
+
{
|
|
2423
|
+
"file": "prices_2025-01-22.csv",
|
|
2424
|
+
"success": true,
|
|
2425
|
+
"recordsProcessed": 100,
|
|
2426
|
+
"mutationsExecuted": 95,
|
|
2427
|
+
"mutationsFailed": 5,
|
|
2428
|
+
"errors": ["PRICE-001: Invalid price value", "PRICE-002: Missing currency"]
|
|
2429
|
+
}
|
|
2430
|
+
],
|
|
2431
|
+
"duration": 12345
|
|
2432
|
+
}
|
|
2433
|
+
```
|
|
2434
|
+
|
|
2435
|
+
### Error Response
|
|
2436
|
+
|
|
2437
|
+
```json
|
|
2438
|
+
{
|
|
2439
|
+
"success": false,
|
|
2440
|
+
"filesProcessed": 0,
|
|
2441
|
+
"filesFailed": 1,
|
|
2442
|
+
"totalRecords": 0,
|
|
2443
|
+
"mutationsExecuted": 0,
|
|
2444
|
+
"mutationsFailed": 0,
|
|
2445
|
+
"results": [
|
|
2446
|
+
{
|
|
2447
|
+
"file": "prices_2025-01-22.csv",
|
|
2448
|
+
"success": false,
|
|
2449
|
+
"error": "CSV parse error: Invalid structure"
|
|
2450
|
+
}
|
|
2451
|
+
],
|
|
2452
|
+
"duration": 876
|
|
2453
|
+
}
|
|
2454
|
+
```
|
|
2455
|
+
|
|
2456
|
+
### Monitoring Metrics
|
|
2457
|
+
|
|
2458
|
+
Monitor these metrics via Versori logs filtered by `jobId` or `stage`:
|
|
2459
|
+
|
|
2460
|
+
- **Files Processed** - Total files successfully processed
|
|
2461
|
+
- **Mutations Executed** - Total GraphQL mutations executed successfully
|
|
2462
|
+
- **Mutations Failed** - Mutations that failed (check error logs)
|
|
2463
|
+
- **Processing Duration** - Time taken for complete workflow
|
|
2464
|
+
- **Rate Limiting** - Watch for 429 errors indicating GraphQL throttling
|
|
2465
|
+
|
|
2466
|
+
Use the status webhook for dashboards and automated monitoring.
|
|
2467
|
+
|
|
2468
|
+
---
|
|
2469
|
+
|
|
2470
|
+
### Issue 1: Product Not Found
|
|
2471
|
+
|
|
2472
|
+
**Error**: `Skipping price update for non-existent product: PROD-XYZ`
|
|
2473
|
+
|
|
2474
|
+
**Solution**:
|
|
2475
|
+
|
|
2476
|
+
- Verify product exists in Fluent Commerce
|
|
2477
|
+
- Check SKU matches exactly (case-sensitive)
|
|
2478
|
+
- Ensure product status is `ACTIVE`
|
|
2479
|
+
- Disable validation temporarily for testing: `validateProducts=false`
|
|
2480
|
+
|
|
2481
|
+
### Issue 2: Price Validation Failures
|
|
2482
|
+
|
|
2483
|
+
**Error**: `Price 0.00 below minimum 0.01`
|
|
2484
|
+
|
|
2485
|
+
**Solution**:
|
|
2486
|
+
|
|
2487
|
+
```bash
|
|
2488
|
+
# Adjust validation rules in activation variables
|
|
2489
|
+
minPrice=0.00 # Allow zero prices
|
|
2490
|
+
maxPrice=9999999.99 # Increase max
|
|
2491
|
+
```
|
|
2492
|
+
|
|
2493
|
+
Or fix source data:
|
|
2494
|
+
|
|
2495
|
+
```csv
|
|
2496
|
+
# ❌ WRONG
|
|
2497
|
+
PROD-001,DEFAULT,0,USD
|
|
2498
|
+
|
|
2499
|
+
# ✅ CORRECT
|
|
2500
|
+
PROD-001,DEFAULT,0.01,USD
|
|
2501
|
+
```
|
|
2502
|
+
|
|
2503
|
+
### Issue 3: Invalid Price Type
|
|
2504
|
+
|
|
2505
|
+
**Error**: `Invalid price type: PROMO`
|
|
2506
|
+
|
|
2507
|
+
**Solution**: Update custom resolver to include new type:
|
|
2508
|
+
|
|
2509
|
+
```typescript
|
|
2510
|
+
'custom.normalizePriceType': (value: any) => {
|
|
2511
|
+
const type = String(value || 'DEFAULT').toUpperCase().trim();
|
|
2512
|
+
const validTypes = [
|
|
2513
|
+
'DEFAULT',
|
|
2514
|
+
'SALE',
|
|
2515
|
+
'CLEARANCE',
|
|
2516
|
+
'BULK',
|
|
2517
|
+
'PROMOTIONAL',
|
|
2518
|
+
'MEMBER',
|
|
2519
|
+
'PROMO', // Add new type
|
|
2520
|
+
];
|
|
2521
|
+
if (!validTypes.includes(type)) {
|
|
2522
|
+
throw new Error(`Invalid price type: ${type}`);
|
|
2523
|
+
}
|
|
2524
|
+
return type;
|
|
2525
|
+
};
|
|
2526
|
+
```
|
|
2527
|
+
|
|
2528
|
+
### Issue 4: Duplicate Price Updates
|
|
2529
|
+
|
|
2530
|
+
**Symptom**: Same file processed multiple times
|
|
2531
|
+
|
|
2532
|
+
**Solution**: VersoriKVAdapter already prevents this:
|
|
2533
|
+
|
|
2534
|
+
```typescript
|
|
2535
|
+
const stateKey = ['processed-files', 's3-price-sync', fileName];
|
|
2536
|
+
const existing = await kv.get(stateKey);
|
|
2537
|
+
if (existing) {
|
|
2538
|
+
log.info('Skipping already processed file', { fileName });
|
|
2539
|
+
continue;
|
|
2540
|
+
}
|
|
2541
|
+
```
|
|
2542
|
+
|
|
2543
|
+
Verify KV storage is working:
|
|
2544
|
+
|
|
2545
|
+
```bash
|
|
2546
|
+
# Check Versori logs for:
|
|
2547
|
+
[INFO] Skipping already processed file: product-prices-20250122-001.csv
|
|
2548
|
+
```
|
|
2549
|
+
|
|
2550
|
+
### Issue 5: S3 Access Denied
|
|
2551
|
+
|
|
2552
|
+
Required IAM Permissions:
|
|
2553
|
+
|
|
2554
|
+
```json
|
|
2555
|
+
{
|
|
2556
|
+
"Version": "2012-10-17",
|
|
2557
|
+
"Statement": [
|
|
2558
|
+
{
|
|
2559
|
+
"Effect": "Allow",
|
|
2560
|
+
"Action": ["s3:ListBucket", "s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
|
|
2561
|
+
"Resource": [
|
|
2562
|
+
"arn:aws:s3:::my-price-bucket",
|
|
2563
|
+
"arn:aws:s3:::my-price-bucket/*"
|
|
2564
|
+
]
|
|
2565
|
+
}
|
|
2566
|
+
]
|
|
2567
|
+
}
|
|
2568
|
+
```
|
|
2569
|
+
|
|
2570
|
+
### Issue 6: GraphQL Mutation Failures
|
|
2571
|
+
|
|
2572
|
+
**Common Causes**:
|
|
2573
|
+
|
|
2574
|
+
- Invalid price type (check Fluent schema for valid types)
|
|
2575
|
+
- Missing required fields (ref, retailerId, prices)
|
|
2576
|
+
- Invalid currency code (must be ISO 4217: USD, EUR, GBP)
|
|
2577
|
+
- Product not found or inactive
|
|
2578
|
+
|
|
2579
|
+
**Debugging**:
|
|
2580
|
+
|
|
2581
|
+
```typescript
|
|
2582
|
+
// Log mutation input before execution
|
|
2583
|
+
log.info('GraphQL mutation input', { input });
|
|
2584
|
+
|
|
2585
|
+
// Validate required fields
|
|
2586
|
+
if (!priceData.productRef || !priceData.type || !priceData.value) {
|
|
2587
|
+
log.error('Missing required fields', { priceData });
|
|
2588
|
+
continue;
|
|
2589
|
+
}
|
|
2590
|
+
```
|
|
2591
|
+
|
|
2592
|
+
## Production Checklist
|
|
2593
|
+
|
|
2594
|
+
- [ ] S3 credentials validated with correct IAM permissions
|
|
2595
|
+
- [ ] Activation secrets stored securely (not in code)
|
|
2596
|
+
- [ ] GraphQL schema validated using CLI tools
|
|
2597
|
+
- [ ] Mapping configuration uses GraphQLMutationMapper structure (NOT UniversalMapper)
|
|
2598
|
+
- [ ] Mapping configuration tested with sample data
|
|
2599
|
+
- [ ] NO setRetailerId() call in code (only for Job/Event API)
|
|
2600
|
+
- [ ] Price validation rules configured per business policy
|
|
2601
|
+
- [ ] Product existence validation enabled (`validateProducts=true`)
|
|
2602
|
+
- [ ] File duplicate prevention working via KV state
|
|
2603
|
+
- [ ] Concurrency control configured (mutationBatchSize)
|
|
2604
|
+
- [ ] Optional alias batching tested if enabled (mutationsPerAliasBatch)
|
|
2605
|
+
- [ ] Error handling tested with malformed CSV
|
|
2606
|
+
- [ ] Retry logic tested with transient failures
|
|
2607
|
+
- [ ] File archival working (processed and error directories)
|
|
2608
|
+
- [ ] Price change tracking verified in logs
|
|
2609
|
+
- [ ] Monitoring/alerting configured for price update failures
|
|
2610
|
+
- [ ] Clear runbook for error recovery
|
|
2611
|
+
|
|
2612
|
+
## Related Guides
|
|
2613
|
+
|
|
2614
|
+
- **GraphQL Mutation Mapping**: `docs/02-CORE-GUIDES/mapping/graphql-mutation-mapping/`
|
|
2615
|
+
- **GraphQL Alias Batching**: `docs/02-CORE-GUIDES/mapping/graphql-alias-batching-guide.md`
|
|
2616
|
+
- **retailerId Configuration**: `docs/00-START-HERE/retailerid-configuration.md`
|
|
2617
|
+
- **CLI Tools**: `fc-connect-sdk/bin/readme.md`
|
|
2618
|
+
- **State & KV patterns**: `docs/03-PATTERN-GUIDES/file-operations/`
|
|
2619
|
+
- **Error handling**: `docs/03-PATTERN-GUIDES/error-handling/`
|