@fluentcommerce/fc-connect-sdk 0.1.53 → 0.1.55

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (495) hide show
  1. package/CHANGELOG.md +30 -2
  2. package/README.md +39 -0
  3. package/dist/cjs/auth/index.d.ts +3 -0
  4. package/dist/cjs/auth/index.js +13 -0
  5. package/dist/cjs/auth/profile-loader.d.ts +18 -0
  6. package/dist/cjs/auth/profile-loader.js +208 -0
  7. package/dist/cjs/client-factory.d.ts +4 -0
  8. package/dist/cjs/client-factory.js +10 -0
  9. package/dist/cjs/clients/fluent-client.js +13 -6
  10. package/dist/cjs/index.d.ts +3 -1
  11. package/dist/cjs/index.js +8 -2
  12. package/dist/cjs/utils/pagination-helpers.js +38 -2
  13. package/dist/cjs/versori/fluent-versori-client.js +11 -5
  14. package/dist/esm/auth/index.d.ts +3 -0
  15. package/dist/esm/auth/index.js +2 -0
  16. package/dist/esm/auth/profile-loader.d.ts +18 -0
  17. package/dist/esm/auth/profile-loader.js +169 -0
  18. package/dist/esm/client-factory.d.ts +4 -0
  19. package/dist/esm/client-factory.js +9 -0
  20. package/dist/esm/clients/fluent-client.js +13 -6
  21. package/dist/esm/index.d.ts +3 -1
  22. package/dist/esm/index.js +2 -1
  23. package/dist/esm/utils/pagination-helpers.js +38 -2
  24. package/dist/esm/versori/fluent-versori-client.js +11 -5
  25. package/dist/tsconfig.esm.tsbuildinfo +1 -1
  26. package/dist/tsconfig.tsbuildinfo +1 -1
  27. package/dist/tsconfig.types.tsbuildinfo +1 -1
  28. package/dist/types/auth/index.d.ts +3 -0
  29. package/dist/types/auth/profile-loader.d.ts +18 -0
  30. package/dist/types/client-factory.d.ts +4 -0
  31. package/dist/types/index.d.ts +3 -1
  32. package/docs/00-START-HERE/EXPORT-VALIDATION.md +158 -158
  33. package/docs/00-START-HERE/cli-analyze-source-structure-guide.md +655 -655
  34. package/docs/00-START-HERE/cli-documentation-index.md +202 -202
  35. package/docs/00-START-HERE/cli-quick-reference.md +252 -252
  36. package/docs/00-START-HERE/decision-tree.md +552 -552
  37. package/docs/00-START-HERE/getting-started.md +1070 -1070
  38. package/docs/00-START-HERE/mapper-quick-decision-guide.md +235 -235
  39. package/docs/00-START-HERE/readme.md +237 -237
  40. package/docs/00-START-HERE/retailerid-configuration.md +404 -404
  41. package/docs/00-START-HERE/sdk-philosophy.md +794 -794
  42. package/docs/00-START-HERE/troubleshooting-quick-reference.md +1086 -1086
  43. package/docs/01-TEMPLATES/faq.md +686 -686
  44. package/docs/01-TEMPLATES/patterns/pattern-templates-guide.md +68 -68
  45. package/docs/01-TEMPLATES/patterns/patterns-csv-schema-validation-and-rejection-report.md +233 -233
  46. package/docs/01-TEMPLATES/patterns/patterns-custom-resolvers.md +407 -407
  47. package/docs/01-TEMPLATES/patterns/patterns-error-handling-retry.md +511 -511
  48. package/docs/01-TEMPLATES/patterns/patterns-field-mapping-universal.md +701 -701
  49. package/docs/01-TEMPLATES/patterns/patterns-large-file-splitting.md +1430 -1430
  50. package/docs/01-TEMPLATES/patterns/patterns-master-data-etl.md +2399 -2399
  51. package/docs/01-TEMPLATES/patterns/patterns-pagination-streaming.md +447 -447
  52. package/docs/01-TEMPLATES/patterns/patterns-state-duplicate-prevention.md +385 -385
  53. package/docs/01-TEMPLATES/readme.md +957 -957
  54. package/docs/01-TEMPLATES/standalone/standalone-asn-inbound-processing.md +1209 -1209
  55. package/docs/01-TEMPLATES/standalone/standalone-graphql-query-export.md +1140 -1140
  56. package/docs/01-TEMPLATES/standalone/standalone-graphql-to-parquet-partitioned-s3.md +432 -432
  57. package/docs/01-TEMPLATES/standalone/standalone-multi-channel-inventory-sync.md +1185 -1185
  58. package/docs/01-TEMPLATES/standalone/standalone-multi-source-aggregation.md +1462 -1462
  59. package/docs/01-TEMPLATES/standalone/standalone-s3-csv-batch-api.md +1390 -1390
  60. package/docs/01-TEMPLATES/standalone/standalone-s3-csv-inventory-to-batch.md +330 -330
  61. package/docs/01-TEMPLATES/standalone/standalone-scripts-guide.md +87 -87
  62. package/docs/01-TEMPLATES/standalone/standalone-sftp-xml-graphql.md +1444 -1444
  63. package/docs/01-TEMPLATES/standalone/standalone-webhook-payload-processing.md +688 -688
  64. package/docs/01-TEMPLATES/versori/business-examples/business-examples-dropship-order-routing.md +193 -193
  65. package/docs/01-TEMPLATES/versori/business-examples/business-examples-graphql-parquet-extraction.md +518 -518
  66. package/docs/01-TEMPLATES/versori/business-examples/business-examples-inter-location-transfers.md +2162 -2162
  67. package/docs/01-TEMPLATES/versori/business-examples/business-examples-pre-order-allocation.md +2226 -2226
  68. package/docs/01-TEMPLATES/versori/business-examples/business-scenarios-guide.md +87 -87
  69. package/docs/01-TEMPLATES/versori/patterns/versori-patterns-connection-validation-pattern.md +656 -656
  70. package/docs/01-TEMPLATES/versori/patterns/versori-patterns-dual-workflow-connector.md +835 -835
  71. package/docs/01-TEMPLATES/versori/patterns/versori-patterns-guide.md +108 -108
  72. package/docs/01-TEMPLATES/versori/patterns/versori-patterns-kv-state-management.md +1533 -1533
  73. package/docs/01-TEMPLATES/versori/patterns/versori-patterns-xml-response-patterns.md +1160 -1160
  74. package/docs/01-TEMPLATES/versori/versori-platform-guide.md +201 -201
  75. package/docs/01-TEMPLATES/versori/webhooks/template-webhook-asn-purchase-order.md +1906 -1906
  76. package/docs/01-TEMPLATES/versori/webhooks/template-webhook-dropship-routing.md +1074 -1074
  77. package/docs/01-TEMPLATES/versori/webhooks/template-webhook-flash-sale-reserve.md +1395 -1395
  78. package/docs/01-TEMPLATES/versori/webhooks/template-webhook-generic-xml-order.md +888 -888
  79. package/docs/01-TEMPLATES/versori/webhooks/template-webhook-payment-gateway-integration.md +2478 -2478
  80. package/docs/01-TEMPLATES/versori/webhooks/template-webhook-rma-returns-comprehensive.md +2240 -2240
  81. package/docs/01-TEMPLATES/versori/webhooks/template-webhook-xml-order-ingestion.md +2029 -2029
  82. package/docs/01-TEMPLATES/versori/webhooks/webhook-templates-guide.md +140 -140
  83. package/docs/01-TEMPLATES/versori/workflows/_examples/sample-data/inventory-mapping.json +20 -20
  84. package/docs/01-TEMPLATES/versori/workflows/_examples/sample-data/products_2025-01-22.csv +11 -11
  85. package/docs/01-TEMPLATES/versori/workflows/_examples/sample-data/sample-data-guide.md +34 -34
  86. package/docs/01-TEMPLATES/versori/workflows/_examples/workflow-examples-guide.md +36 -36
  87. package/docs/01-TEMPLATES/versori/workflows/extraction/extraction-modes-guide.md +1038 -1038
  88. package/docs/01-TEMPLATES/versori/workflows/extraction/extraction-workflows-guide.md +138 -138
  89. package/docs/01-TEMPLATES/versori/workflows/extraction/graphql-queries/graphql-extraction-guide.md +63 -63
  90. package/docs/01-TEMPLATES/versori/workflows/extraction/graphql-queries/template-extraction-fulfillments-to-sftp-csv.md +2062 -2062
  91. package/docs/01-TEMPLATES/versori/workflows/extraction/graphql-queries/template-extraction-fulfillments-to-sftp-xml.md +2294 -2294
  92. package/docs/01-TEMPLATES/versori/workflows/extraction/graphql-queries/template-extraction-inventory-positions-to-s3-csv.md +2461 -2461
  93. package/docs/01-TEMPLATES/versori/workflows/extraction/graphql-queries/template-extraction-inventory-positions-to-sftp-xml.md +2529 -2529
  94. package/docs/01-TEMPLATES/versori/workflows/extraction/graphql-queries/template-extraction-inventory-quantities-to-s3-csv.md +2464 -2464
  95. package/docs/01-TEMPLATES/versori/workflows/extraction/graphql-queries/template-extraction-inventory-quantities-to-s3-json.md +1959 -1959
  96. package/docs/01-TEMPLATES/versori/workflows/extraction/graphql-queries/template-extraction-orders-to-s3-csv.md +1953 -1953
  97. package/docs/01-TEMPLATES/versori/workflows/extraction/graphql-queries/template-extraction-orders-to-sftp-xml.md +2541 -2541
  98. package/docs/01-TEMPLATES/versori/workflows/extraction/graphql-queries/template-extraction-products-to-s3-json.md +2384 -2384
  99. package/docs/01-TEMPLATES/versori/workflows/extraction/graphql-queries/template-extraction-products-to-sftp-xml.md +2445 -2445
  100. package/docs/01-TEMPLATES/versori/workflows/extraction/graphql-queries/template-extraction-virtual-positions-to-s3-csv.md +2355 -2355
  101. package/docs/01-TEMPLATES/versori/workflows/extraction/graphql-queries/template-extraction-virtual-positions-to-s3-json.md +2042 -2042
  102. package/docs/01-TEMPLATES/versori/workflows/extraction/graphql-queries/template-extraction-virtual-positions-to-sftp-xml.md +2726 -2726
  103. package/docs/01-TEMPLATES/versori/workflows/ingestion/batch-api/batch-api-guide.md +206 -206
  104. package/docs/01-TEMPLATES/versori/workflows/ingestion/batch-api/template-ingestion-cycle-count-reconciliation.md +2030 -2030
  105. package/docs/01-TEMPLATES/versori/workflows/ingestion/batch-api/template-ingestion-multi-channel-inventory-sync.md +1882 -1882
  106. package/docs/01-TEMPLATES/versori/workflows/ingestion/batch-api/template-ingestion-s3-csv-inventory-batch.md +2827 -2827
  107. package/docs/01-TEMPLATES/versori/workflows/ingestion/batch-api/template-ingestion-s3-json-inventory-batch.md +1952 -1952
  108. package/docs/01-TEMPLATES/versori/workflows/ingestion/batch-api/template-ingestion-s3-xml-inventory-batch.md +3289 -3289
  109. package/docs/01-TEMPLATES/versori/workflows/ingestion/batch-api/template-ingestion-sftp-csv-inventory-batch.md +3064 -3064
  110. package/docs/01-TEMPLATES/versori/workflows/ingestion/batch-api/template-ingestion-sftp-json-inventory-batch.md +3238 -3238
  111. package/docs/01-TEMPLATES/versori/workflows/ingestion/batch-api/template-ingestion-sftp-xml-inventory-batch.md +2977 -2977
  112. package/docs/01-TEMPLATES/versori/workflows/ingestion/event-api/event-api-guide.md +321 -321
  113. package/docs/01-TEMPLATES/versori/workflows/ingestion/event-api/template-ingestion-payload-json-order-cancel-event.md +959 -959
  114. package/docs/01-TEMPLATES/versori/workflows/ingestion/event-api/template-ingestion-payload-xml-order-cancel-event.md +1170 -1170
  115. package/docs/01-TEMPLATES/versori/workflows/ingestion/event-api/template-ingestion-s3-csv-product-event.md +2312 -2312
  116. package/docs/01-TEMPLATES/versori/workflows/ingestion/event-api/template-ingestion-s3-json-product-event.md +2999 -2999
  117. package/docs/01-TEMPLATES/versori/workflows/ingestion/event-api/template-ingestion-s3-parquet-product-event.md +2836 -2836
  118. package/docs/01-TEMPLATES/versori/workflows/ingestion/event-api/template-ingestion-s3-xml-product-event.md +2395 -2395
  119. package/docs/01-TEMPLATES/versori/workflows/ingestion/event-api/template-ingestion-sftp-csv-product-event.md +2295 -2295
  120. package/docs/01-TEMPLATES/versori/workflows/ingestion/event-api/template-ingestion-sftp-json-product-event.md +2602 -2602
  121. package/docs/01-TEMPLATES/versori/workflows/ingestion/event-api/template-ingestion-sftp-parquet-product-event.md +2589 -2589
  122. package/docs/01-TEMPLATES/versori/workflows/ingestion/event-api/template-ingestion-sftp-xml-product-event.md +3578 -3578
  123. package/docs/01-TEMPLATES/versori/workflows/ingestion/graphql-mutations/graphql-mutations-guide.md +93 -93
  124. package/docs/01-TEMPLATES/versori/workflows/ingestion/graphql-mutations/template-ingestion-payload-json-order-update-graphql.md +1260 -1260
  125. package/docs/01-TEMPLATES/versori/workflows/ingestion/graphql-mutations/template-ingestion-payload-xml-order-update-graphql.md +1472 -1472
  126. package/docs/01-TEMPLATES/versori/workflows/ingestion/graphql-mutations/template-ingestion-s3-csv-control-graphql.md +2417 -2417
  127. package/docs/01-TEMPLATES/versori/workflows/ingestion/graphql-mutations/template-ingestion-s3-csv-location-graphql.md +2811 -2811
  128. package/docs/01-TEMPLATES/versori/workflows/ingestion/graphql-mutations/template-ingestion-s3-csv-price-graphql.md +2619 -2619
  129. package/docs/01-TEMPLATES/versori/workflows/ingestion/graphql-mutations/template-ingestion-s3-json-location-graphql.md +2807 -2807
  130. package/docs/01-TEMPLATES/versori/workflows/ingestion/graphql-mutations/template-ingestion-s3-xml-location-graphql.md +2373 -2373
  131. package/docs/01-TEMPLATES/versori/workflows/ingestion/graphql-mutations/template-ingestion-sftp-csv-control-graphql.md +2740 -2740
  132. package/docs/01-TEMPLATES/versori/workflows/ingestion/graphql-mutations/template-ingestion-sftp-csv-location-graphql.md +2760 -2760
  133. package/docs/01-TEMPLATES/versori/workflows/ingestion/graphql-mutations/template-ingestion-sftp-json-location-graphql.md +1710 -1710
  134. package/docs/01-TEMPLATES/versori/workflows/ingestion/ingestion-workflows-guide.md +136 -136
  135. package/docs/01-TEMPLATES/versori/workflows/rubix-webhooks/rubix-webhooks-guide.md +520 -520
  136. package/docs/01-TEMPLATES/versori/workflows/rubix-webhooks/template-webhook-rubix-fulfilment-to-sftp-xml-inline.md +1418 -1418
  137. package/docs/01-TEMPLATES/versori/workflows/rubix-webhooks/template-webhook-rubix-fulfilment-to-sftp-xml-universal-mapper.md +1785 -1785
  138. package/docs/01-TEMPLATES/versori/workflows/rubix-webhooks/template-webhook-rubix-order-attribute-update.md +824 -824
  139. package/docs/01-TEMPLATES/versori/workflows/workflows-overview-guide.md +646 -646
  140. package/docs/02-CORE-GUIDES/advanced-services/advanced-services-batch-archival.md +724 -724
  141. package/docs/02-CORE-GUIDES/advanced-services/advanced-services-job-tracker.md +627 -627
  142. package/docs/02-CORE-GUIDES/advanced-services/advanced-services-partial-batch-recovery.md +561 -561
  143. package/docs/02-CORE-GUIDES/advanced-services/advanced-services-quick-reference.md +367 -367
  144. package/docs/02-CORE-GUIDES/advanced-services/advanced-services-readme.md +407 -407
  145. package/docs/02-CORE-GUIDES/advanced-services/readme.md +49 -49
  146. package/docs/02-CORE-GUIDES/api-reference/api-reference-quick-reference.md +548 -548
  147. package/docs/02-CORE-GUIDES/api-reference/event-api-input-output-reference.md +702 -1171
  148. package/docs/02-CORE-GUIDES/api-reference/examples/client-initialization.ts +286 -286
  149. package/docs/02-CORE-GUIDES/api-reference/graphql-error-classification.md +337 -337
  150. package/docs/02-CORE-GUIDES/api-reference/modules/api-reference-01-client-api.md +399 -482
  151. package/docs/02-CORE-GUIDES/api-reference/modules/api-reference-03-authentication.md +199 -199
  152. package/docs/02-CORE-GUIDES/api-reference/modules/api-reference-04-graphql-mapping.md +925 -925
  153. package/docs/02-CORE-GUIDES/api-reference/modules/api-reference-05-services.md +1198 -1198
  154. package/docs/02-CORE-GUIDES/api-reference/modules/api-reference-06-data-sources.md +1083 -1083
  155. package/docs/02-CORE-GUIDES/api-reference/modules/api-reference-07-parsers.md +1097 -1097
  156. package/docs/02-CORE-GUIDES/api-reference/modules/api-reference-08-pagination.md +513 -513
  157. package/docs/02-CORE-GUIDES/api-reference/modules/api-reference-08-types.md +545 -597
  158. package/docs/02-CORE-GUIDES/api-reference/modules/api-reference-09-error-handling.md +527 -527
  159. package/docs/02-CORE-GUIDES/api-reference/modules/api-reference-09-webhook-validation.md +514 -514
  160. package/docs/02-CORE-GUIDES/api-reference/modules/api-reference-10-extraction.md +557 -557
  161. package/docs/02-CORE-GUIDES/api-reference/modules/api-reference-10-utilities.md +412 -412
  162. package/docs/02-CORE-GUIDES/api-reference/modules/api-reference-11-cli-tools.md +423 -423
  163. package/docs/02-CORE-GUIDES/api-reference/modules/api-reference-11-error-handling.md +716 -716
  164. package/docs/02-CORE-GUIDES/api-reference/modules/api-reference-12-analyze-source-structure.md +518 -518
  165. package/docs/02-CORE-GUIDES/api-reference/modules/api-reference-12-partial-responses.md +212 -212
  166. package/docs/02-CORE-GUIDES/api-reference/modules/api-reference-12-testing.md +300 -300
  167. package/docs/02-CORE-GUIDES/api-reference/modules/api-reference-13-resolver-builder.md +322 -322
  168. package/docs/02-CORE-GUIDES/api-reference/readme.md +279 -279
  169. package/docs/02-CORE-GUIDES/auto-pagination/auto-pagination-quick-reference.md +351 -351
  170. package/docs/02-CORE-GUIDES/auto-pagination/auto-pagination-readme.md +277 -277
  171. package/docs/02-CORE-GUIDES/auto-pagination/examples/auto-pagination-readme.md +178 -178
  172. package/docs/02-CORE-GUIDES/auto-pagination/examples/common-patterns.ts +351 -351
  173. package/docs/02-CORE-GUIDES/auto-pagination/examples/paginate-products.ts +384 -384
  174. package/docs/02-CORE-GUIDES/auto-pagination/examples/paginate-virtual-positions.ts +308 -308
  175. package/docs/02-CORE-GUIDES/auto-pagination/modules/auto-pagination-01-foundations.md +470 -470
  176. package/docs/02-CORE-GUIDES/auto-pagination/modules/auto-pagination-02-quick-start.md +713 -713
  177. package/docs/02-CORE-GUIDES/auto-pagination/modules/auto-pagination-03-configuration.md +754 -754
  178. package/docs/02-CORE-GUIDES/auto-pagination/modules/auto-pagination-04-advanced-patterns.md +732 -732
  179. package/docs/02-CORE-GUIDES/auto-pagination/modules/auto-pagination-05-sdk-integration.md +847 -847
  180. package/docs/02-CORE-GUIDES/auto-pagination/modules/auto-pagination-06-troubleshooting.md +359 -359
  181. package/docs/02-CORE-GUIDES/auto-pagination/modules/auto-pagination-07-api-reference.md +462 -462
  182. package/docs/02-CORE-GUIDES/auto-pagination/readme.md +54 -54
  183. package/docs/02-CORE-GUIDES/data-sources/data-sources-file-operations-error-handling.md +1487 -1487
  184. package/docs/02-CORE-GUIDES/data-sources/data-sources-quick-reference.md +836 -836
  185. package/docs/02-CORE-GUIDES/data-sources/data-sources-readme.md +276 -276
  186. package/docs/02-CORE-GUIDES/data-sources/data-sources-sftp-credential-access-security.md +553 -553
  187. package/docs/02-CORE-GUIDES/data-sources/examples/common-patterns.ts +409 -409
  188. package/docs/02-CORE-GUIDES/data-sources/examples/data-sources-readme.md +178 -178
  189. package/docs/02-CORE-GUIDES/data-sources/examples/s3-operations.ts +308 -308
  190. package/docs/02-CORE-GUIDES/data-sources/examples/sftp-operations.ts +371 -371
  191. package/docs/02-CORE-GUIDES/data-sources/modules/data-sources-01-foundations.md +735 -735
  192. package/docs/02-CORE-GUIDES/data-sources/modules/data-sources-02-s3-operations.md +1302 -1302
  193. package/docs/02-CORE-GUIDES/data-sources/modules/data-sources-03-sftp-operations.md +1379 -1379
  194. package/docs/02-CORE-GUIDES/data-sources/modules/data-sources-04-file-patterns.md +941 -941
  195. package/docs/02-CORE-GUIDES/data-sources/modules/data-sources-05-advanced-topics.md +813 -813
  196. package/docs/02-CORE-GUIDES/data-sources/modules/data-sources-06-integration-patterns.md +486 -486
  197. package/docs/02-CORE-GUIDES/data-sources/modules/data-sources-07-troubleshooting.md +387 -387
  198. package/docs/02-CORE-GUIDES/data-sources/modules/data-sources-08-api-reference.md +417 -417
  199. package/docs/02-CORE-GUIDES/data-sources/readme.md +77 -77
  200. package/docs/02-CORE-GUIDES/error-handling-guide.md +936 -936
  201. package/docs/02-CORE-GUIDES/extraction/examples/02-core-guides-extraction-readme.md +116 -116
  202. package/docs/02-CORE-GUIDES/extraction/examples/common-patterns.ts +428 -428
  203. package/docs/02-CORE-GUIDES/extraction/examples/extract-inventory-basic.ts +187 -187
  204. package/docs/02-CORE-GUIDES/extraction/extraction-quick-reference.md +596 -596
  205. package/docs/02-CORE-GUIDES/extraction/modules/02-core-guides-extraction-01-foundations.md +514 -514
  206. package/docs/02-CORE-GUIDES/extraction/modules/02-core-guides-extraction-02-basic-extraction.md +823 -823
  207. package/docs/02-CORE-GUIDES/extraction/modules/02-core-guides-extraction-03-parquet-processing.md +507 -507
  208. package/docs/02-CORE-GUIDES/extraction/modules/02-core-guides-extraction-04-data-enrichment.md +546 -546
  209. package/docs/02-CORE-GUIDES/extraction/modules/02-core-guides-extraction-05-transformation.md +494 -494
  210. package/docs/02-CORE-GUIDES/extraction/modules/02-core-guides-extraction-06-export-formats.md +458 -458
  211. package/docs/02-CORE-GUIDES/extraction/modules/02-core-guides-extraction-06-performance.md +138 -138
  212. package/docs/02-CORE-GUIDES/extraction/modules/02-core-guides-extraction-07-api-reference.md +148 -148
  213. package/docs/02-CORE-GUIDES/extraction/modules/02-core-guides-extraction-07-optimization.md +692 -692
  214. package/docs/02-CORE-GUIDES/extraction/modules/02-core-guides-extraction-08-extraction-orchestrator.md +1008 -1008
  215. package/docs/02-CORE-GUIDES/extraction/readme.md +151 -151
  216. package/docs/02-CORE-GUIDES/ingestion/examples/_simple-kv-store.ts +40 -40
  217. package/docs/02-CORE-GUIDES/ingestion/examples/error-recovery.ts +728 -728
  218. package/docs/02-CORE-GUIDES/ingestion/examples/event-driven.ts +501 -501
  219. package/docs/02-CORE-GUIDES/ingestion/examples/local-file-ingestion.ts +88 -88
  220. package/docs/02-CORE-GUIDES/ingestion/examples/parquet-ingestion.ts +117 -117
  221. package/docs/02-CORE-GUIDES/ingestion/examples/performance-optimized.ts +647 -647
  222. package/docs/02-CORE-GUIDES/ingestion/examples/s3-csv-ingestion.ts +169 -169
  223. package/docs/02-CORE-GUIDES/ingestion/examples/sftp-csv-ingestion.ts +134 -134
  224. package/docs/02-CORE-GUIDES/ingestion/ingestion-quick-reference.md +546 -546
  225. package/docs/02-CORE-GUIDES/ingestion/modules/02-core-guides-ingestion-01-introduction.md +626 -626
  226. package/docs/02-CORE-GUIDES/ingestion/modules/02-core-guides-ingestion-02-quick-start.md +658 -658
  227. package/docs/02-CORE-GUIDES/ingestion/modules/02-core-guides-ingestion-03-data-sources.md +1052 -1052
  228. package/docs/02-CORE-GUIDES/ingestion/modules/02-core-guides-ingestion-04-field-mapping.md +763 -763
  229. package/docs/02-CORE-GUIDES/ingestion/modules/02-core-guides-ingestion-05-advanced-parsers.md +676 -676
  230. package/docs/02-CORE-GUIDES/ingestion/modules/02-core-guides-ingestion-06-batch-api.md +1295 -1295
  231. package/docs/02-CORE-GUIDES/ingestion/modules/02-core-guides-ingestion-07-api-reference.md +138 -138
  232. package/docs/02-CORE-GUIDES/ingestion/modules/02-core-guides-ingestion-07-state-management.md +1037 -1037
  233. package/docs/02-CORE-GUIDES/ingestion/modules/02-core-guides-ingestion-08-performance-optimization.md +1349 -1349
  234. package/docs/02-CORE-GUIDES/ingestion/modules/02-core-guides-ingestion-09-best-practices.md +1893 -1893
  235. package/docs/02-CORE-GUIDES/ingestion/readme.md +160 -160
  236. package/docs/02-CORE-GUIDES/logging-guide.md +585 -585
  237. package/docs/02-CORE-GUIDES/mapping/error-handling-patterns.md +401 -401
  238. package/docs/02-CORE-GUIDES/mapping/examples/02-core-guides-mapping-readme.md +128 -128
  239. package/docs/02-CORE-GUIDES/mapping/examples/common-patterns.ts +273 -273
  240. package/docs/02-CORE-GUIDES/mapping/examples/csv-location-ingestion.json +36 -36
  241. package/docs/02-CORE-GUIDES/mapping/examples/csv-mapping.ts +242 -242
  242. package/docs/02-CORE-GUIDES/mapping/examples/graphql-to-parquet-extraction.json +36 -36
  243. package/docs/02-CORE-GUIDES/mapping/examples/json-mapping.ts +213 -213
  244. package/docs/02-CORE-GUIDES/mapping/examples/json-product-to-mutation.json +48 -48
  245. package/docs/02-CORE-GUIDES/mapping/examples/xml-mapping.ts +291 -291
  246. package/docs/02-CORE-GUIDES/mapping/examples/xml-order-to-mutation.json +45 -45
  247. package/docs/02-CORE-GUIDES/mapping/graphql-mutation-mapping/graphql-mutation-mapping-quick-reference.md +463 -463
  248. package/docs/02-CORE-GUIDES/mapping/graphql-mutation-mapping/graphql-mutation-mapping-readme.md +227 -227
  249. package/docs/02-CORE-GUIDES/mapping/graphql-mutation-mapping/modules/graphql-mutation-mapping-01-introduction.md +222 -222
  250. package/docs/02-CORE-GUIDES/mapping/graphql-mutation-mapping/modules/graphql-mutation-mapping-02-quick-start.md +351 -351
  251. package/docs/02-CORE-GUIDES/mapping/graphql-mutation-mapping/modules/graphql-mutation-mapping-03-schema-validation.md +569 -569
  252. package/docs/02-CORE-GUIDES/mapping/graphql-mutation-mapping/modules/graphql-mutation-mapping-04-mapping-patterns.md +471 -471
  253. package/docs/02-CORE-GUIDES/mapping/graphql-mutation-mapping/modules/graphql-mutation-mapping-05-configuration-reference.md +611 -611
  254. package/docs/02-CORE-GUIDES/mapping/graphql-mutation-mapping/modules/graphql-mutation-mapping-06-advanced-xpath.md +148 -148
  255. package/docs/02-CORE-GUIDES/mapping/graphql-mutation-mapping/modules/graphql-mutation-mapping-06-path-syntax.md +464 -464
  256. package/docs/02-CORE-GUIDES/mapping/graphql-mutation-mapping/modules/graphql-mutation-mapping-07-api-reference.md +94 -94
  257. package/docs/02-CORE-GUIDES/mapping/graphql-mutation-mapping/modules/graphql-mutation-mapping-07-array-handling.md +307 -307
  258. package/docs/02-CORE-GUIDES/mapping/graphql-mutation-mapping/modules/graphql-mutation-mapping-08-custom-resolvers.md +544 -544
  259. package/docs/02-CORE-GUIDES/mapping/graphql-mutation-mapping/modules/graphql-mutation-mapping-09-advanced-patterns.md +427 -427
  260. package/docs/02-CORE-GUIDES/mapping/graphql-mutation-mapping/modules/graphql-mutation-mapping-10-hooks-and-variables.md +336 -336
  261. package/docs/02-CORE-GUIDES/mapping/graphql-mutation-mapping/modules/graphql-mutation-mapping-11-error-handling.md +488 -488
  262. package/docs/02-CORE-GUIDES/mapping/graphql-mutation-mapping/modules/graphql-mutation-mapping-12-arguments-vs-nodes.md +383 -383
  263. package/docs/02-CORE-GUIDES/mapping/graphql-mutation-mapping/modules/graphql-mutation-mapping-13-best-practices.md +477 -477
  264. package/docs/02-CORE-GUIDES/mapping/graphql-mutation-mapping/readme.md +62 -62
  265. package/docs/02-CORE-GUIDES/mapping/mapping-format-decision-tree.md +480 -480
  266. package/docs/02-CORE-GUIDES/mapping/mapping-graphql-alias-batching-guide.md +820 -820
  267. package/docs/02-CORE-GUIDES/mapping/mapping-javascript-objects.md +2369 -2369
  268. package/docs/02-CORE-GUIDES/mapping/mapping-mapper-comparison-guide.md +682 -682
  269. package/docs/02-CORE-GUIDES/mapping/modules/02-core-guides-mapping-07-api-reference.md +1327 -1327
  270. package/docs/02-CORE-GUIDES/mapping/modules/02-core-guides-mapping-08-error-handling.md +1142 -1142
  271. package/docs/02-CORE-GUIDES/mapping/modules/mapping-04-use-cases.md +891 -891
  272. package/docs/02-CORE-GUIDES/mapping/modules/mapping-06-helpers-resolvers.md +1126 -1126
  273. package/docs/02-CORE-GUIDES/mapping/modules/mapping-06-sdk-resolvers.md +199 -199
  274. package/docs/02-CORE-GUIDES/mapping/modules/mapping-07-api-reference.md +1319 -1319
  275. package/docs/02-CORE-GUIDES/mapping/readme.md +178 -178
  276. package/docs/02-CORE-GUIDES/mapping/resolver-registration.md +410 -410
  277. package/docs/02-CORE-GUIDES/mapping/resolvers/examples/common-patterns.ts +226 -226
  278. package/docs/02-CORE-GUIDES/mapping/resolvers/examples/custom-resolvers.ts +227 -227
  279. package/docs/02-CORE-GUIDES/mapping/resolvers/examples/sdk-resolvers-usage.ts +203 -203
  280. package/docs/02-CORE-GUIDES/mapping/resolvers/mapping-resolvers-readme.md +274 -274
  281. package/docs/02-CORE-GUIDES/mapping/resolvers/mapping-resolvers-resolver-api-reference.md +679 -679
  282. package/docs/02-CORE-GUIDES/mapping/resolvers/mapping-resolvers-resolver-cookbook.md +826 -826
  283. package/docs/02-CORE-GUIDES/mapping/resolvers/mapping-resolvers-resolver-guide.md +1330 -1330
  284. package/docs/02-CORE-GUIDES/mapping/resolvers/mapping-resolvers-resolver-helpers-reference.md +1437 -1437
  285. package/docs/02-CORE-GUIDES/mapping/resolvers/mapping-resolvers-resolver-parameters-reference.md +553 -553
  286. package/docs/02-CORE-GUIDES/mapping/resolvers/mapping-resolvers-resolver-troubleshooting.md +854 -854
  287. package/docs/02-CORE-GUIDES/mapping/resolvers/readme.md +75 -75
  288. package/docs/02-CORE-GUIDES/parsers/examples/02-core-guides-parsers-readme.md +161 -161
  289. package/docs/02-CORE-GUIDES/parsers/examples/csv-parser-examples.ts +110 -110
  290. package/docs/02-CORE-GUIDES/parsers/examples/json-parser-examples.ts +33 -33
  291. package/docs/02-CORE-GUIDES/parsers/examples/parquet-parser-examples.ts +47 -47
  292. package/docs/02-CORE-GUIDES/parsers/examples/xml-parser-examples.ts +38 -38
  293. package/docs/02-CORE-GUIDES/parsers/modules/02-core-guides-parsers-01-foundations.md +355 -355
  294. package/docs/02-CORE-GUIDES/parsers/modules/02-core-guides-parsers-02-csv-parser.md +772 -772
  295. package/docs/02-CORE-GUIDES/parsers/modules/02-core-guides-parsers-03-json-parser.md +789 -789
  296. package/docs/02-CORE-GUIDES/parsers/modules/02-core-guides-parsers-04-xml-parser.md +857 -857
  297. package/docs/02-CORE-GUIDES/parsers/modules/02-core-guides-parsers-05-parquet-parser.md +603 -603
  298. package/docs/02-CORE-GUIDES/parsers/modules/02-core-guides-parsers-06-integration-patterns.md +702 -702
  299. package/docs/02-CORE-GUIDES/parsers/modules/02-core-guides-parsers-06-streaming.md +121 -121
  300. package/docs/02-CORE-GUIDES/parsers/modules/02-core-guides-parsers-07-api-reference.md +89 -89
  301. package/docs/02-CORE-GUIDES/parsers/modules/02-core-guides-parsers-07-troubleshooting.md +727 -727
  302. package/docs/02-CORE-GUIDES/parsers/parsers-quick-reference.md +482 -482
  303. package/docs/02-CORE-GUIDES/parsers/parsers-readme.md +258 -258
  304. package/docs/02-CORE-GUIDES/parsers/readme.md +65 -65
  305. package/docs/02-CORE-GUIDES/readme.md +194 -194
  306. package/docs/02-CORE-GUIDES/webhook-validation/examples/basic-validation.ts +108 -108
  307. package/docs/02-CORE-GUIDES/webhook-validation/examples/common-patterns.ts +316 -316
  308. package/docs/02-CORE-GUIDES/webhook-validation/examples/webhook-validation-readme.md +61 -61
  309. package/docs/02-CORE-GUIDES/webhook-validation/modules/webhook-validation-01-foundations.md +440 -440
  310. package/docs/02-CORE-GUIDES/webhook-validation/modules/webhook-validation-02-quick-start.md +525 -525
  311. package/docs/02-CORE-GUIDES/webhook-validation/modules/webhook-validation-03-versori-integration.md +741 -741
  312. package/docs/02-CORE-GUIDES/webhook-validation/modules/webhook-validation-04-platform-integration.md +629 -629
  313. package/docs/02-CORE-GUIDES/webhook-validation/modules/webhook-validation-05-configuration.md +535 -535
  314. package/docs/02-CORE-GUIDES/webhook-validation/modules/webhook-validation-06-error-handling.md +611 -611
  315. package/docs/02-CORE-GUIDES/webhook-validation/modules/webhook-validation-06-troubleshooting.md +124 -124
  316. package/docs/02-CORE-GUIDES/webhook-validation/modules/webhook-validation-07-api-reference.md +511 -511
  317. package/docs/02-CORE-GUIDES/webhook-validation/modules/webhook-validation-08-rubix-webhooks.md +590 -590
  318. package/docs/02-CORE-GUIDES/webhook-validation/modules/webhook-validation-09-rubix-event-vs-http-call.md +432 -432
  319. package/docs/02-CORE-GUIDES/webhook-validation/readme.md +239 -239
  320. package/docs/02-CORE-GUIDES/webhook-validation/webhook-validation-quick-reference.md +392 -392
  321. package/docs/03-PATTERN-GUIDES/connector-scenarios/connector-scenarios-quick-reference.md +498 -498
  322. package/docs/03-PATTERN-GUIDES/connector-scenarios/connector-scenarios-readme.md +313 -313
  323. package/docs/03-PATTERN-GUIDES/connector-scenarios/examples/common-patterns.ts +612 -612
  324. package/docs/03-PATTERN-GUIDES/connector-scenarios/examples/connector-scenarios-readme.md +253 -253
  325. package/docs/03-PATTERN-GUIDES/connector-scenarios/modules/connector-scenarios-01-foundations.md +452 -452
  326. package/docs/03-PATTERN-GUIDES/connector-scenarios/modules/connector-scenarios-02-simple-scenarios.md +681 -681
  327. package/docs/03-PATTERN-GUIDES/connector-scenarios/modules/connector-scenarios-03-intermediate-scenarios.md +637 -637
  328. package/docs/03-PATTERN-GUIDES/connector-scenarios/modules/connector-scenarios-04-advanced-scenarios.md +650 -650
  329. package/docs/03-PATTERN-GUIDES/connector-scenarios/modules/connector-scenarios-05-bidirectional-sync.md +233 -233
  330. package/docs/03-PATTERN-GUIDES/connector-scenarios/modules/connector-scenarios-06-production-patterns.md +442 -442
  331. package/docs/03-PATTERN-GUIDES/connector-scenarios/modules/connector-scenarios-07-reference.md +445 -445
  332. package/docs/03-PATTERN-GUIDES/connector-scenarios/readme.md +31 -31
  333. package/docs/03-PATTERN-GUIDES/enterprise-integration-patterns.md +1528 -1528
  334. package/docs/03-PATTERN-GUIDES/error-handling/comprehensive-error-handling-guide.md +1437 -1437
  335. package/docs/03-PATTERN-GUIDES/error-handling/error-handling-quick-reference.md +390 -390
  336. package/docs/03-PATTERN-GUIDES/error-handling/examples/common-patterns.ts +438 -438
  337. package/docs/03-PATTERN-GUIDES/error-handling/modules/error-handling-01-foundations.md +362 -362
  338. package/docs/03-PATTERN-GUIDES/error-handling/modules/error-handling-02-error-types.md +850 -850
  339. package/docs/03-PATTERN-GUIDES/error-handling/modules/error-handling-03-utf8-handling.md +456 -456
  340. package/docs/03-PATTERN-GUIDES/error-handling/modules/error-handling-04-error-scenarios.md +658 -658
  341. package/docs/03-PATTERN-GUIDES/error-handling/modules/error-handling-05-calling-patterns.md +671 -671
  342. package/docs/03-PATTERN-GUIDES/error-handling/modules/error-handling-06-retry-strategies.md +1034 -1034
  343. package/docs/03-PATTERN-GUIDES/error-handling/modules/error-handling-07-monitoring.md +653 -653
  344. package/docs/03-PATTERN-GUIDES/error-handling/modules/error-handling-08-api-reference.md +847 -847
  345. package/docs/03-PATTERN-GUIDES/error-handling/readme.md +36 -36
  346. package/docs/03-PATTERN-GUIDES/examples/__tests__/readme.md +40 -40
  347. package/docs/03-PATTERN-GUIDES/examples/__tests__/resolver-examples.test.js +282 -282
  348. package/docs/03-PATTERN-GUIDES/examples/test-data/03-pattern-guides-readme.md +110 -110
  349. package/docs/03-PATTERN-GUIDES/examples/test-data/canonical-inventory.json +123 -123
  350. package/docs/03-PATTERN-GUIDES/examples/test-data/canonical-order.json +171 -171
  351. package/docs/03-PATTERN-GUIDES/examples/test-data/readme.md +28 -28
  352. package/docs/03-PATTERN-GUIDES/extraction/extraction-readme.md +15 -15
  353. package/docs/03-PATTERN-GUIDES/extraction/readme.md +25 -25
  354. package/docs/03-PATTERN-GUIDES/file-operations/examples/common-patterns.ts +407 -407
  355. package/docs/03-PATTERN-GUIDES/file-operations/examples/file-operations-readme.md +142 -142
  356. package/docs/03-PATTERN-GUIDES/file-operations/file-operations-quick-reference.md +462 -462
  357. package/docs/03-PATTERN-GUIDES/file-operations/file-operations-readme.md +379 -379
  358. package/docs/03-PATTERN-GUIDES/file-operations/modules/file-operations-01-foundations.md +430 -430
  359. package/docs/03-PATTERN-GUIDES/file-operations/modules/file-operations-02-quick-start.md +484 -484
  360. package/docs/03-PATTERN-GUIDES/file-operations/modules/file-operations-03-s3-operations.md +507 -507
  361. package/docs/03-PATTERN-GUIDES/file-operations/modules/file-operations-04-sftp-operations.md +963 -963
  362. package/docs/03-PATTERN-GUIDES/file-operations/modules/file-operations-05-streaming-performance.md +503 -503
  363. package/docs/03-PATTERN-GUIDES/file-operations/modules/file-operations-06-archive-patterns.md +386 -386
  364. package/docs/03-PATTERN-GUIDES/file-operations/modules/file-operations-06-error-handling.md +117 -117
  365. package/docs/03-PATTERN-GUIDES/file-operations/modules/file-operations-07-api-reference.md +78 -78
  366. package/docs/03-PATTERN-GUIDES/file-operations/modules/file-operations-07-testing-troubleshooting.md +567 -567
  367. package/docs/03-PATTERN-GUIDES/file-operations/modules/file-operations-08-api-reference.md +1055 -1055
  368. package/docs/03-PATTERN-GUIDES/file-operations/readme.md +32 -32
  369. package/docs/03-PATTERN-GUIDES/ingestion/ingestion-readme.md +15 -15
  370. package/docs/03-PATTERN-GUIDES/ingestion/readme.md +25 -25
  371. package/docs/03-PATTERN-GUIDES/integration-patterns/examples/batch-processing.ts +130 -130
  372. package/docs/03-PATTERN-GUIDES/integration-patterns/examples/common-patterns.ts +360 -360
  373. package/docs/03-PATTERN-GUIDES/integration-patterns/examples/delta-sync.ts +130 -130
  374. package/docs/03-PATTERN-GUIDES/integration-patterns/examples/integration-patterns-readme.md +100 -100
  375. package/docs/03-PATTERN-GUIDES/integration-patterns/examples/real-time-webhook.ts +398 -398
  376. package/docs/03-PATTERN-GUIDES/integration-patterns/integration-patterns-quick-reference.md +962 -962
  377. package/docs/03-PATTERN-GUIDES/integration-patterns/integration-patterns-readme.md +134 -134
  378. package/docs/03-PATTERN-GUIDES/integration-patterns/modules/integration-patterns-01-real-time-processing.md +991 -991
  379. package/docs/03-PATTERN-GUIDES/integration-patterns/modules/integration-patterns-02-batch-processing.md +1547 -1547
  380. package/docs/03-PATTERN-GUIDES/integration-patterns/modules/integration-patterns-03-delta-sync.md +1108 -1108
  381. package/docs/03-PATTERN-GUIDES/integration-patterns/modules/integration-patterns-04-webhook-patterns.md +1181 -1181
  382. package/docs/03-PATTERN-GUIDES/integration-patterns/modules/integration-patterns-05-error-handling.md +1061 -1061
  383. package/docs/03-PATTERN-GUIDES/integration-patterns/modules/integration-patterns-06-advanced-integration-services.md +1547 -1547
  384. package/docs/03-PATTERN-GUIDES/integration-patterns/modules/integration-patterns-06-performance.md +109 -109
  385. package/docs/03-PATTERN-GUIDES/integration-patterns/modules/integration-patterns-07-api-reference.md +34 -34
  386. package/docs/03-PATTERN-GUIDES/integration-patterns/readme.md +30 -30
  387. package/docs/03-PATTERN-GUIDES/logging-minimal-mode.md +128 -128
  388. package/docs/03-PATTERN-GUIDES/multiple-connections/examples/common-patterns.ts +380 -380
  389. package/docs/03-PATTERN-GUIDES/multiple-connections/examples/multiple-connections-readme.md +139 -139
  390. package/docs/03-PATTERN-GUIDES/multiple-connections/examples/parallel-root-connections.ts +149 -149
  391. package/docs/03-PATTERN-GUIDES/multiple-connections/examples/real-world-scenarios.ts +405 -405
  392. package/docs/03-PATTERN-GUIDES/multiple-connections/modules/multiple-connections-01-foundations.md +378 -378
  393. package/docs/03-PATTERN-GUIDES/multiple-connections/modules/multiple-connections-02-quick-start.md +566 -566
  394. package/docs/03-PATTERN-GUIDES/multiple-connections/modules/multiple-connections-03-targeting-connections.md +659 -659
  395. package/docs/03-PATTERN-GUIDES/multiple-connections/modules/multiple-connections-04-parallel-queries.md +656 -656
  396. package/docs/03-PATTERN-GUIDES/multiple-connections/modules/multiple-connections-05-best-practices.md +624 -624
  397. package/docs/03-PATTERN-GUIDES/multiple-connections/modules/multiple-connections-06-api-reference.md +824 -824
  398. package/docs/03-PATTERN-GUIDES/multiple-connections/modules/multiple-connections-06-versori.md +119 -119
  399. package/docs/03-PATTERN-GUIDES/multiple-connections/modules/multiple-connections-07-api-reference.md +87 -87
  400. package/docs/03-PATTERN-GUIDES/multiple-connections/multiple-connections-quick-reference.md +353 -353
  401. package/docs/03-PATTERN-GUIDES/multiple-connections/multiple-connections-readme.md +270 -270
  402. package/docs/03-PATTERN-GUIDES/multiple-connections/readme.md +30 -30
  403. package/docs/03-PATTERN-GUIDES/pagination/pagination-readme.md +14 -14
  404. package/docs/03-PATTERN-GUIDES/pagination/readme.md +24 -24
  405. package/docs/03-PATTERN-GUIDES/parquet/examples/common-patterns.ts +180 -180
  406. package/docs/03-PATTERN-GUIDES/parquet/examples/read-parquet.ts +48 -48
  407. package/docs/03-PATTERN-GUIDES/parquet/examples/write-parquet.ts +65 -65
  408. package/docs/03-PATTERN-GUIDES/parquet/modules/03-pattern-guides-parquet-01-introduction.md +393 -393
  409. package/docs/03-PATTERN-GUIDES/parquet/modules/03-pattern-guides-parquet-02-quick-start.md +572 -572
  410. package/docs/03-PATTERN-GUIDES/parquet/modules/03-pattern-guides-parquet-03-reading-parquet.md +525 -525
  411. package/docs/03-PATTERN-GUIDES/parquet/modules/03-pattern-guides-parquet-04-writing-parquet.md +554 -554
  412. package/docs/03-PATTERN-GUIDES/parquet/modules/03-pattern-guides-parquet-05-graphql-extraction.md +405 -405
  413. package/docs/03-PATTERN-GUIDES/parquet/modules/03-pattern-guides-parquet-06-performance.md +104 -104
  414. package/docs/03-PATTERN-GUIDES/parquet/modules/03-pattern-guides-parquet-06-s3-integration.md +511 -511
  415. package/docs/03-PATTERN-GUIDES/parquet/modules/03-pattern-guides-parquet-07-api-reference.md +90 -90
  416. package/docs/03-PATTERN-GUIDES/parquet/modules/03-pattern-guides-parquet-07-performance-optimization.md +525 -525
  417. package/docs/03-PATTERN-GUIDES/parquet/modules/03-pattern-guides-parquet-08-best-practices.md +712 -712
  418. package/docs/03-PATTERN-GUIDES/parquet/parquet-quick-reference.md +683 -683
  419. package/docs/03-PATTERN-GUIDES/parquet/parquet-readme.md +248 -248
  420. package/docs/03-PATTERN-GUIDES/parquet/readme.md +32 -32
  421. package/docs/03-PATTERN-GUIDES/parsers/parsers-readme.md +12 -12
  422. package/docs/03-PATTERN-GUIDES/parsers/readme.md +24 -24
  423. package/docs/03-PATTERN-GUIDES/readme.md +159 -159
  424. package/docs/03-PATTERN-GUIDES/webhooks/readme.md +24 -24
  425. package/docs/03-PATTERN-GUIDES/webhooks/webhooks-readme.md +8 -8
  426. package/docs/04-REFERENCE/architecture/architecture-01-overview.md +427 -427
  427. package/docs/04-REFERENCE/architecture/architecture-02-client-architecture.md +424 -424
  428. package/docs/04-REFERENCE/architecture/architecture-03-data-flow.md +690 -690
  429. package/docs/04-REFERENCE/architecture/architecture-04-service-layer.md +834 -834
  430. package/docs/04-REFERENCE/architecture/architecture-05-integration-architecture.md +655 -655
  431. package/docs/04-REFERENCE/architecture/architecture-06-state-management.md +653 -653
  432. package/docs/04-REFERENCE/architecture/architecture-adding-new-data-sources.md +686 -686
  433. package/docs/04-REFERENCE/architecture/readme.md +279 -279
  434. package/docs/04-REFERENCE/platforms/deno/readme.md +117 -117
  435. package/docs/04-REFERENCE/platforms/nodejs/readme.md +146 -146
  436. package/docs/04-REFERENCE/platforms/readme.md +135 -135
  437. package/docs/04-REFERENCE/platforms/versori/modules/platforms-versori-01-introduction.md +398 -398
  438. package/docs/04-REFERENCE/platforms/versori/modules/platforms-versori-02-quick-start.md +560 -560
  439. package/docs/04-REFERENCE/platforms/versori/modules/platforms-versori-03-authentication.md +757 -757
  440. package/docs/04-REFERENCE/platforms/versori/modules/platforms-versori-04-workflows.md +2476 -2476
  441. package/docs/04-REFERENCE/platforms/versori/modules/platforms-versori-05-connections.md +1167 -1167
  442. package/docs/04-REFERENCE/platforms/versori/modules/platforms-versori-06-kv-storage.md +990 -990
  443. package/docs/04-REFERENCE/platforms/versori/modules/platforms-versori-06-state-management.md +121 -121
  444. package/docs/04-REFERENCE/platforms/versori/modules/platforms-versori-07-api-reference.md +68 -68
  445. package/docs/04-REFERENCE/platforms/versori/modules/platforms-versori-07-deployment.md +731 -731
  446. package/docs/04-REFERENCE/platforms/versori/modules/platforms-versori-08-best-practices.md +1111 -1111
  447. package/docs/04-REFERENCE/platforms/versori/modules/platforms-versori-09-signature-reference.md +766 -766
  448. package/docs/04-REFERENCE/platforms/versori/platforms-versori-readme.md +299 -299
  449. package/docs/04-REFERENCE/platforms/versori/platforms-versori-s3-sftp-configuration-guide.md +1425 -1425
  450. package/docs/04-REFERENCE/platforms/versori/platforms-versori-webhook-api-key-security.md +816 -816
  451. package/docs/04-REFERENCE/platforms/versori/platforms-versori-webhook-connection-security.md +681 -681
  452. package/docs/04-REFERENCE/platforms/versori/platforms-versori-workflow-task-types.md +708 -708
  453. package/docs/04-REFERENCE/platforms/versori/readme.md +108 -108
  454. package/docs/04-REFERENCE/readme.md +148 -148
  455. package/docs/04-REFERENCE/resolver-signature/examples/advanced-resolvers.ts +482 -482
  456. package/docs/04-REFERENCE/resolver-signature/examples/async-resolvers.ts +496 -496
  457. package/docs/04-REFERENCE/resolver-signature/examples/basic-resolvers.ts +343 -343
  458. package/docs/04-REFERENCE/resolver-signature/examples/resolver-signature-readme.md +188 -188
  459. package/docs/04-REFERENCE/resolver-signature/examples/testing-resolvers.ts +463 -463
  460. package/docs/04-REFERENCE/resolver-signature/modules/resolver-signature-01-foundations.md +286 -286
  461. package/docs/04-REFERENCE/resolver-signature/modules/resolver-signature-02-parameter-reference.md +643 -643
  462. package/docs/04-REFERENCE/resolver-signature/modules/resolver-signature-03-basic-examples.md +521 -521
  463. package/docs/04-REFERENCE/resolver-signature/modules/resolver-signature-04-advanced-patterns.md +739 -739
  464. package/docs/04-REFERENCE/resolver-signature/modules/resolver-signature-05-sdk-resolvers.md +531 -531
  465. package/docs/04-REFERENCE/resolver-signature/modules/resolver-signature-06-migration-guide.md +650 -650
  466. package/docs/04-REFERENCE/resolver-signature/modules/resolver-signature-06-testing.md +125 -125
  467. package/docs/04-REFERENCE/resolver-signature/modules/resolver-signature-07-api-reference.md +794 -794
  468. package/docs/04-REFERENCE/resolver-signature/readme.md +64 -64
  469. package/docs/04-REFERENCE/resolver-signature/resolver-signature-quick-reference.md +270 -270
  470. package/docs/04-REFERENCE/resolver-signature/resolver-signature-readme.md +351 -351
  471. package/docs/04-REFERENCE/schema/fluent-commerce-schema.json +764 -764
  472. package/docs/04-REFERENCE/schema/readme.md +141 -141
  473. package/docs/04-REFERENCE/testing/examples/04-reference-testing-readme.md +158 -158
  474. package/docs/04-REFERENCE/testing/examples/fluent-testing.ts +62 -62
  475. package/docs/04-REFERENCE/testing/examples/health-check.ts +155 -155
  476. package/docs/04-REFERENCE/testing/examples/integration-test.ts +119 -119
  477. package/docs/04-REFERENCE/testing/examples/performance-test.ts +183 -183
  478. package/docs/04-REFERENCE/testing/examples/s3-testing.ts +127 -127
  479. package/docs/04-REFERENCE/testing/modules/04-reference-testing-01-foundations.md +267 -267
  480. package/docs/04-REFERENCE/testing/modules/04-reference-testing-02-s3-testing.md +599 -599
  481. package/docs/04-REFERENCE/testing/modules/04-reference-testing-03-fluent-testing.md +589 -589
  482. package/docs/04-REFERENCE/testing/modules/04-reference-testing-04-integration-testing.md +699 -699
  483. package/docs/04-REFERENCE/testing/modules/04-reference-testing-05-debugging.md +478 -478
  484. package/docs/04-REFERENCE/testing/modules/04-reference-testing-06-cicd-integration.md +463 -463
  485. package/docs/04-REFERENCE/testing/modules/04-reference-testing-06-preflight-validation.md +131 -131
  486. package/docs/04-REFERENCE/testing/modules/04-reference-testing-07-best-practices.md +499 -499
  487. package/docs/04-REFERENCE/testing/modules/04-reference-testing-07-coverage-ci.md +165 -165
  488. package/docs/04-REFERENCE/testing/modules/04-reference-testing-08-api-reference.md +634 -634
  489. package/docs/04-REFERENCE/testing/readme.md +86 -86
  490. package/docs/04-REFERENCE/testing/testing-quick-reference.md +667 -667
  491. package/docs/04-REFERENCE/testing/testing-readme.md +286 -286
  492. package/docs/04-REFERENCE/troubleshooting/readme.md +144 -144
  493. package/docs/04-REFERENCE/troubleshooting/troubleshooting-deno-sftp-compatibility.md +392 -392
  494. package/docs/template-loading-matrix.md +242 -242
  495. package/package.json +5 -3
@@ -1,2619 +1,2619 @@
1
- ---
2
- template_id: tpl-ingest-s3-csv-to-price-graphql
3
- canonical_filename: template-ingestion-s3-csv-price-graphql.md
4
- version: 2.0.0
5
- sdk_version: ^0.1.39
6
- runtime: versori
7
- direction: ingestion
8
- source: s3-csv
9
- destination: fluent-graphql
10
- entity: price
11
- format: csv
12
- logging: versori
13
- status: stable
14
- features:
15
- - graphql-mutation-mapper
16
- - memory-management
17
- - enhanced-logging
18
- - attribute-transformation
19
- compliance: gold-standard
20
- ---
21
-
22
- # Template: Ingestion - S3 CSV to Price GraphQL
23
- n**Template Version:** 2.0.0
24
- **SDK Version:** @fluentcommerce/fc-connect-sdk@^0.1.39
25
- **Last Updated:** 2025-01-24
26
-
27
- **🆕 Version 2.0.0 Enhancements:**
28
- - ✅ **GraphQL Mutation Mapper** - Direct field mapping to mutation variables
29
- - ✅ **Memory Management** - Clear large arrays after processing batches
30
- - ✅ **Enhanced Logging** - Track mutation execution with emoji indicators
31
- - ✅ **Attribute Transformation** - Handle complex nested data structures
32
-
33
- ## STEP 1: What This Template Does
34
-
35
- This template provides a **scheduled Versori workflow** that:
36
-
37
- 1. **Discovers CSV files** from S3 bucket with price data
38
- 2. **Parses CSV** using `CSVParserService` with validation
39
- 3. **Validates products exist** in Fluent Commerce before updating prices
40
- 4. **Maps fields** using `GraphQLMutationMapper` with custom price validation resolvers
41
- 5. **Updates prices** via direct GraphQL mutations (`updateProduct`)
42
- 6. **Tracks price changes** for audit trail
43
- 7. **Archives processed files** to prevent duplicates
44
- 8. **Uses JobTracker** for job lifecycle management
45
-
46
- **Key Features:**
47
- - ✅ Direct GraphQL mutations (NOT Batch API)
48
- - ✅ Product existence validation before price updates
49
- - ✅ Price range validation (min/max checks)
50
- - ✅ Multi-tier pricing support (DEFAULT, SALE, CLEARANCE, BULK)
51
- - ✅ Configurable concurrency control (sequential or parallel)
52
- - ✅ Optional GraphQL alias batching for high-volume scenarios
53
- - ✅ Price change tracking and reporting
54
- - ✅ Versori KV state management (duplicate prevention)
55
- - ✅ File archival with error handling
56
-
57
- **Use Cases:**
58
- - Daily product price synchronization
59
- - Promotional price updates
60
- - Multi-currency pricing management
61
- - Quantity-based pricing tiers
62
-
63
- ## STEP 2: Understanding This Template (AI Agent Guide)
64
-
65
- **🎯 Template Purpose:** Scheduled price synchronization from S3 CSV files to Fluent Commerce using direct GraphQL mutations.
66
-
67
- ### Architecture Pattern
68
-
69
- ```
70
- S3 CSV Files → CSVParserService → GraphQLMutationMapper → Product Validation → GraphQL Mutations → Archive
71
- ↓ ↓ ↓ ↓ ↓ ↓
72
- File Discovery Parsing Field Mapping Existence Check updateProduct Processed/
73
- Price Data Validation Price Validation Active Status Concurrency Ctrl Error Dirs
74
- ```
75
-
76
- ### SDK Services Used
77
-
78
- | Service | Purpose | Key Methods |
79
- |---------|---------|-------------|
80
- | `createClient(ctx)` | Fluent API client | `client.graphql()` |
81
- | `S3DataSource` | S3 operations | `listFiles()`, `downloadFile()`, `moveFile()` |
82
- | `CSVParserService` | CSV parsing | `parse()` |
83
- | `GraphQLMutationMapper` | Field transformation | `map()` with custom resolvers |
84
- | `VersoriKVAdapter` | State management | `get()`, `set()`, `list()` |
85
- | `JobTracker` | Job lifecycle | `createJob()`, `updateJob()`, `markCompleted()`, `markFailed()`, `getJob()` |
86
-
87
- ### Price Entity Structure
88
-
89
- **Fluent Commerce Price Schema:**
90
- ```typescript
91
- {
92
- productRef: string; // SKU reference (required)
93
- type: string; // DEFAULT, SALE, CLEARANCE, BULK (required)
94
- value: number; // Price amount (required, validated)
95
- currency: string; // ISO 4217 code (USD, EUR, GBP)
96
- effectiveFrom?: string; // ISO 8601 date
97
- effectiveTo?: string; // ISO 8601 date
98
- minQuantity?: number; // Min qty for tier pricing
99
- maxQuantity?: number; // Max qty for tier pricing
100
- }
101
- ```
102
-
103
- ### GraphQL Mutation Pattern
104
-
105
- **Mutation Used:** `updateProduct` (NOT Batch API)
106
-
107
- ```graphql
108
- mutation UpdateProductPrice($input: UpdateProductInput!) {
109
- updateProduct(input: $input) {
110
- id
111
- ref
112
- prices {
113
- type
114
- value
115
- currency
116
- effectiveFrom
117
- effectiveTo
118
- minQuantity
119
- maxQuantity
120
- }
121
- }
122
- }
123
- ```
124
-
125
- **Why Direct Mutations?**
126
- - ✅ Immediate price updates (no batch processing delay)
127
- - ✅ Product validation before update
128
- - ✅ Price change tracking
129
- - ✅ Suitable for low-volume price updates (<1000/day)
130
-
131
- **NO BPP (Batch Pre-Processing):** Direct GraphQL mutations bypass batch workflows entirely.
132
-
133
- ### Custom Resolvers
134
-
135
- **Three price-specific resolvers:**
136
-
137
- 1. **`custom.validatePriceRange`** - Ensures price within min/max bounds
138
- 2. **`custom.normalizePriceType`** - Validates price tier types
139
- 3. **`custom.normalizeQuantity`** - Validates quantity ranges for BULK pricing
140
-
141
- ### Product Validation Pattern
142
-
143
- **Critical Step:** Validate product exists and is ACTIVE before price update:
144
-
145
- ```typescript
146
- async function validateProductExists(client, productRef, retailerId, log) {
147
- const query = `
148
- query GetProduct($ref: String!, $retailerId: ID!) {
149
- products(first: 1, ref: [$ref], retailerId: $retailerId) {
150
- edges {
151
- node { id ref status }
152
- }
153
- }
154
- }
155
- `;
156
- const result = await client.graphql({ query, variables: { ref: productRef, retailerId } });
157
- const product = result?.data?.products?.edges?.[0]?.node;
158
- return product && product.status === 'ACTIVE';
159
- }
160
- ```
161
-
162
- **Why?** Prevents GraphQL errors for non-existent products.
163
-
164
- ### Concurrency Control & Performance
165
-
166
- **Mutation execution** supports two performance modes:
167
-
168
- **Mode 1: Concurrency Control (default)**
169
- ```typescript
170
- // Configuration: Control concurrent mutations
171
- const mutationBatchSize = parseInt(
172
- activation?.getVariable('mutationBatchSize') || '1', // Default: 1 (sequential)
173
- 10
174
- );
175
-
176
- // Execute with bounded concurrency
177
- await executeMutations(
178
- client,
179
- mapper,
180
- priceRecords,
181
- retailerId,
182
- validateProducts,
183
- mutationBatchSize, // 1=sequential, 3-10=parallel
184
- undefined, // No alias batching
185
- log
186
- );
187
- ```
188
-
189
- **Mode 2: GraphQL Alias Batching (optional, high-volume)**
190
- ```typescript
191
- // Configuration: Group mutations into aliased requests
192
- const mutationsPerAliasBatch = activation?.getVariable('mutationsPerAliasBatch')
193
- ? parseInt(activation?.getVariable('mutationsPerAliasBatch') || '1', 10)
194
- : undefined; // Default: undefined (disabled)
195
-
196
- // Execute with alias batching (reduces network overhead by ~80%)
197
- await executeMutations(
198
- client,
199
- mapper,
200
- priceRecords,
201
- retailerId,
202
- validateProducts,
203
- mutationBatchSize, // Concurrency control
204
- mutationsPerAliasBatch, // Alias batching (e.g., 5)
205
- log
206
- );
207
- ```
208
-
209
- **Activation Variables:**
210
- - `mutationBatchSize=1` - Default: sequential (safe)
211
- - `mutationBatchSize=3-5` - Balanced throughput
212
- - `mutationBatchSize=10` - High-volume (100+ locations)
213
- - `mutationsPerAliasBatch` - Optional: Group mutations (e.g., 5 per request)
214
-
215
- ### State Management
216
-
217
- **VersoriKVAdapter** prevents duplicate file processing:
218
-
219
- ```typescript
220
- const stateKey = ['processed-files', 's3-price-sync', fileName];
221
- const existing = await kv.get(stateKey);
222
- if (existing) {
223
- log.info('Skipping already processed file', { fileName });
224
- continue;
225
- }
226
- // After success:
227
- await kv.set(stateKey, { successful, failed, processedAt: new Date().toISOString() });
228
- ```
229
-
230
- ### JobTracker Integration
231
-
232
- **Job lifecycle tracking:**
233
-
234
- ```typescript
235
- const tracker = new JobTracker(openKv(':project:'), log);
236
- await tracker.startJob(jobId, { mode: 'scheduled' });
237
- try {
238
- const result = await runPriceCsvWorkflow(ctx, jobId, tracker);
239
- await tracker.completeJob(jobId, { result });
240
- } catch (e) {
241
- await tracker.failJob(jobId, { error: e.message });
242
- }
243
- ```
244
-
245
- ### Price Change Tracking
246
-
247
- **Audit trail for price updates:**
248
-
249
- ```typescript
250
- const currentPrice = await getCurrentPrice(client, productRef, priceType, currency, retailerId);
251
- // ... after update ...
252
- if (currentPrice !== undefined && currentPrice !== priceData.value) {
253
- results.priceChanges.push({
254
- sku: priceData.productRef,
255
- type: priceData.type,
256
- oldPrice: currentPrice,
257
- newPrice: priceData.value,
258
- });
259
- }
260
- // Log changes at end:
261
- log.info('Price changes detected', { count: results.priceChanges.length, changes });
262
- ```
263
-
264
- ### Schema Validation (CLI Commands)
265
-
266
- **Before deployment, validate GraphQL schema:**
267
-
268
- ```bash
269
- # 1. Introspect Fluent schema
270
- fc-connect introspect-schema \
271
- --url https://api.fluentcommerce.com/graphql \
272
- --client-id CLIENT_ID \
273
- --client-secret CLIENT_SECRET \
274
- --output fluent-schema.json
275
-
276
- # 2. Create mutation file
277
- cat > price-mutation.graphql << 'EOF'
278
- mutation UpdateProductPrice($input: UpdateProductInput!) {
279
- updateProduct(input: $input) {
280
- id
281
- ref
282
- prices { type value currency }
283
- }
284
- }
285
- EOF
286
-
287
- # 3. Generate mapping from mutation
288
- fc-connect generate-mutation-mapping \
289
- --file price-mutation.graphql \
290
- --output price-mapping.json
291
-
292
- # 4. Validate mapping against schema
293
- fc-connect validate-schema \
294
- --mapping price-mapping.json \
295
- --schema fluent-schema.json
296
-
297
- # 5. Analyze field coverage
298
- fc-connect analyze-coverage \
299
- --mapping price-mapping.json \
300
- --schema fluent-schema.json
301
- ```
302
-
303
- ### Configuration File Structure
304
-
305
- **Mapping Config:** `config/price-mapping.json`
306
-
307
- **✅ PRODUCTION STANDARD:** External JSON file with GraphQLMutationMapper structure
308
-
309
- ```json
310
- {
311
- "mutation": "updateProduct",
312
- "sourceFormat": "csv",
313
- "version": "1.0.0",
314
- "arguments": {
315
- "input": {
316
- "ref": {
317
- "source": "sku",
318
- "required": true,
319
- "resolver": "trim"
320
- },
321
- "prices": {
322
- "type": {
323
- "source": "priceType",
324
- "required": true,
325
- "resolver": "normalizePriceType"
326
- },
327
- "value": {
328
- "source": "amount",
329
- "required": true,
330
- "resolver": "validatePriceRange"
331
- },
332
- "currency": {
333
- "source": "currency",
334
- "required": true,
335
- "resolver": "toUpperCase"
336
- },
337
- "effectiveFrom": {
338
- "source": "effectiveDate",
339
- "resolver": "formatDate"
340
- },
341
- "effectiveTo": {
342
- "source": "expiryDate",
343
- "resolver": "formatDate"
344
- },
345
- "minQuantity": {
346
- "source": "minQuantity",
347
- "resolver": "normalizeQuantity"
348
- },
349
- "maxQuantity": {
350
- "source": "maxQuantity",
351
- "resolver": "normalizeQuantity"
352
- }
353
- }
354
- }
355
- }
356
- }
357
- ```
358
-
359
- **Key Differences from UniversalMapper:**
360
- - ✅ `mutation` property (not `mutationName`)
361
- - ✅ `arguments.input` structure (matches GraphQL schema)
362
- - ✅ Nested `prices` object (not flat productRef/type/value)
363
- - ✅ Resolver names without `sdk.` or `custom.` prefix (`trim`, `toUpperCase`, `formatDate`)
364
- - ✅ Used with `GraphQLMutationMapper.map()` method (not `UniversalMapper.map()`)
365
-
366
- ### Activation Variables Required
367
-
368
- **Minimum Required:**
369
- - `s3BucketName` - S3 bucket with price CSV files
370
- - `awsAccessKeyId` - AWS credentials
371
- - `awsSecretAccessKey` - AWS credentials
372
-
373
- **Optional (with defaults):**
374
- - `awsRegion=us-east-1`
375
- - `s3Prefix=prices/`
376
- - `archivePrefix=processed/`
377
- - `errorPrefix=errors/`
378
- - `logPrefix=logs/` - Where to write mutation logs
379
- - `maxFilesToProcess=10`
380
- - `mutationBatchSize=1` - Number of concurrent mutations (1=sequential, 3-10=parallel)
381
- - `mutationsPerAliasBatch` - Optional: Number of mutations per aliased request (default: undefined = disabled)
382
- - `minPrice=0.01`
383
- - `maxPrice=999999.99`
384
- - `validateProducts=true`
385
- - `validateConnection=true` - Validate S3 connection at startup
386
- - `enableFileTracking=true` - Enable file tracking via VersoriFileTracker
387
- - `fluentRetailerId` - Optional: Only if mutation schema requires retailerId in input (most Price mutations don't need it)
388
-
389
- ### Error Handling Patterns
390
-
391
- **Three-level error handling:**
392
-
393
- 1. **Mapping errors** - Invalid price data (validation failures)
394
- 2. **GraphQL errors** - Mutation failures (product not found, invalid fields)
395
- 3. **File processing errors** - S3 access issues, CSV parsing failures
396
-
397
- **Error state tracking with exponential backoff:**
398
- ```typescript
399
- const errorKey = ['error-state', fileName];
400
- const attempts = (prev?.attemptCount || 0) + 1;
401
- const backoffMinutes = Math.min(Math.pow(2, attempts) * 5, 24 * 60);
402
- const nextRetryAt = new Date(Date.now() + backoffMinutes * 60000).toISOString();
403
- ```
404
-
405
- ### Common Pitfalls
406
-
407
- **❌ WRONG Patterns:**
408
- - Using Batch API for price updates (overhead, delay)
409
- - Using `UniversalMapper` instead of `GraphQLMutationMapper`
410
- - Calling `client.setRetailerId()` for GraphQL mutations (only needed for Job/Event API)
411
- - Skipping product validation (causes GraphQL errors)
412
- - No concurrency control (API throttling)
413
- - Hardcoded price validation rules (not configurable)
414
- - Missing price change tracking (no audit trail)
415
-
416
- **✅ CORRECT Patterns:**
417
- - Direct GraphQL mutations for immediate updates
418
- - `GraphQLMutationMapper` with nested mapping structure
419
- - NO `setRetailerId()` call (pass in mutation input if schema requires it)
420
- - Product existence validation before updates
421
- - Configurable concurrency control (mutationBatchSize)
422
- - Optional alias batching for high-volume scenarios
423
- - Custom resolvers for price validation
424
- - Price change tracking for audit
425
- - VersoriKVAdapter for state management
426
- - JobTracker for job lifecycle
427
-
428
- ### Testing Checklist
429
-
430
- **Before deployment:**
431
- - [ ] S3 credentials validated (IAM permissions: ListBucket, GetObject, PutObject, DeleteObject)
432
- - [ ] GraphQL schema validated using CLI tools
433
- - [ ] Mapping configuration uses GraphQLMutationMapper structure (nested objects, not flat fields)
434
- - [ ] Mapping configuration tested with sample CSV
435
- - [ ] Product validation working (queries products endpoint)
436
- - [ ] Price validation rules configured (min/max bounds)
437
- - [ ] Concurrency control configured (mutationBatchSize)
438
- - [ ] Optional alias batching tested if enabled (mutationsPerAliasBatch)
439
- - [ ] NO setRetailerId() call present (only for Job/Event API)
440
- - [ ] File duplicate prevention working (KV state)
441
- - [ ] Price change tracking verified (logs price changes)
442
- - [ ] Error handling tested (malformed CSV, invalid products)
443
- - [ ] File archival working (processed/ and errors/ directories)
444
-
445
- ### Related Documentation
446
-
447
- - **Core Guides:** `docs/02-CORE-GUIDES/ingestion/modules/`
448
- - **Universal Mapping:** `docs/02-CORE-GUIDES/mapping/modules/`
449
- - **CLI Tools:** `fc-connect-sdk/bin/readme.md`
450
- - **State Management:** `docs/03-PATTERN-GUIDES/file-operations/`
451
- - **Error Handling:** `docs/03-PATTERN-GUIDES/error-handling/`
452
-
453
- **FC Connect SDK Use Case Guide**
454
-
455
- > **SDK**: [@fluentcommerce/fc-connect-sdk](https://www.npmjs.com/package/@fluentcommerce/fc-connect-sdk)
456
- > **Version**: Use latest - `npm install @fluentcommerce/fc-connect-sdk@latest`
457
-
458
- **Context**: Scheduled Versori workflow that reads product price CSV files from S3 and creates/updates Fluent Commerce product prices via GraphQL mutations
459
-
460
- **Complexity**: Medium-High
461
-
462
- **Runtime**: Versori Platform (Scheduled)
463
-
464
- **Estimated Lines**: ~700 lines
465
-
466
- ## What You'll Build
467
-
468
- - Versori scheduled workflow (cron trigger)
469
- - S3 file listing and download with retry logic
470
- - CSV parsing with validation
471
- - UniversalMapper for price field transformations
472
- - Product existence validation before price updates
473
- - GraphQL mutations for price upserts with rate limiting
474
- - Versori KV state management (duplicate prevention)
475
- - Price change tracking and reporting
476
- - File archival after processing
477
-
478
- ## Versori Workflows Structure
479
-
480
- **Key Concept**: Versori workflows are organized by **trigger type** at the first level, then by **specific workflow** with descriptive file names.
481
-
482
- **Trigger Types:**
483
- - **`schedule()`** → Time-based triggers (cron expressions) - NOT exposed as HTTP endpoints
484
- - **`webhook()`** → HTTP-based triggers (event-driven) - Creates HTTP endpoints
485
- - **`workflow()`** → Durable workflows (advanced, rarely used)
486
-
487
- **Execution Steps (chained to triggers):**
488
- - **`http()`** → External API calls (chained from schedule/webhook)
489
- - **`fn()`** → Internal processing (chained from schedule/webhook)
490
-
491
- ### Recommended Project Structure
492
-
493
- ```
494
- s3-csv-price-graphql/
495
- ├── index.ts # Entry point - exports all workflows
496
- └── src/
497
- ├── workflows/
498
- │ ├── scheduled/
499
- │ │ └── daily-price-sync.ts # Scheduled: Daily price sync
500
- │ │
501
- │ └── webhook/
502
- │ ├── adhoc-price-sync.ts # Webhook: Manual trigger
503
- │ └── job-status-check.ts # Webhook: Status query
504
-
505
- ├── services/
506
- │ └── price-sync.service.ts # Shared orchestration logic (reusable)
507
-
508
- └── config/
509
- └── price-mapping.json # GraphQL mapping config
510
- ```
511
-
512
- ---
513
-
514
- ## Workflow Files
515
-
516
- ### 1. Scheduled Workflows (`src/workflows/scheduled/`)
517
-
518
- All time-based triggers that run automatically on cron schedules.
519
-
520
- #### `src/workflows/scheduled/daily-price-sync.ts`
521
-
522
- **Purpose**: Automatic daily price sync
523
- **Trigger**: Cron schedule (`0 2 * * *`)
524
- **Exposed as Endpoint**: ❌ NO - Runs automatically
525
-
526
- ```typescript
527
- import { schedule, http } from '@versori/run';
528
- import { JobTracker } from '@fluentcommerce/fc-connect-sdk';
529
- import { executePriceSync } from '../../services/price-sync.service';
530
-
531
- /**
532
- * Scheduled Workflow: Daily Price Sync
533
- *
534
- * Runs automatically daily at 2 AM UTC
535
- * NOT exposed as HTTP endpoint - Versori executes on schedule
536
- *
537
- * Uses shared service: price-sync.service.ts
538
- */
539
- export const dailyPriceSync = schedule(
540
- 'price-sync-scheduled',
541
- '0 2 * * *' // Daily at 2 AM UTC
542
- ).then(
543
- http('run-price-sync', { connection: 'fluent_commerce' }, async (ctx: any) => {
544
- const { log, openKv } = ctx;
545
- const jobId = `price-sync-${Date.now()}`;
546
- const executionStartTime = Date.now();
547
- const tracker = new JobTracker(openKv(':project:'), log);
548
-
549
- log.info('🔄 [WORKFLOW] Starting price sync', { jobId });
550
-
551
- await tracker.createJob(jobId, {
552
- triggeredBy: 'schedule',
553
- stage: 'initialization',
554
- startTime: executionStartTime,
555
- });
556
- await tracker.updateJob(jobId, { status: 'processing' });
557
-
558
- try {
559
- const result = await executePriceSync(ctx, jobId, tracker);
560
-
561
- if (result.success) {
562
- await tracker.markCompleted(jobId, result);
563
- log.info('✅ [WORKFLOW] Price sync completed', {
564
- jobId,
565
- filesProcessed: result.filesProcessed,
566
- duration: result.duration,
567
- });
568
- } else {
569
- await tracker.markFailed(jobId, result.error || 'Unknown error');
570
- log.error('❌ [WORKFLOW] Price sync failed', { jobId, error: result.error });
571
- }
572
-
573
- return { success: true, jobId, ...result };
574
- } catch (e: any) {
575
- const errorMessage = e instanceof Error ? e.message : String(e);
576
- await tracker.markFailed(jobId, errorMessage);
577
- log.error('❌ [WORKFLOW] Price sync failed with exception', {
578
- jobId,
579
- error: errorMessage,
580
- stack: e instanceof Error ? e.stack : undefined,
581
- });
582
- return { success: false, jobId, error: errorMessage };
583
- }
584
- })
585
- );
586
- ```
587
-
588
- ---
589
-
590
- ### 2. Webhook Workflows (`src/workflows/webhook/`)
591
-
592
- All HTTP-based triggers that create webhook endpoints.
593
-
594
- #### `src/workflows/webhook/adhoc-price-sync.ts`
595
-
596
- **Purpose**: Manual price sync trigger (on-demand)
597
- **Trigger**: HTTP POST
598
- **Endpoint**: `POST https://{workspace}.versori.run/price-sync-adhoc`
599
- **Use Cases**: Testing, priority processing, ad-hoc runs
600
-
601
- ```typescript
602
- import { webhook, http } from '@versori/run';
603
- import { JobTracker } from '@fluentcommerce/fc-connect-sdk';
604
- import { executePriceSync } from '../../services/price-sync.service';
605
-
606
- /**
607
- * Webhook: Manual Price Sync Trigger
608
- *
609
- * Endpoint: POST https://{workspace}.versori.run/price-sync-adhoc
610
- * Request body (optional): { filePattern: "urgent_*.csv", maxFiles: 5 }
611
- *
612
- * Pattern: webhook().then(http()) - needs Fluent API access
613
- * Uses shared service: price-sync.service.ts
614
- *
615
- * SECURITY: Authentication handled via connection parameter
616
- * No manual API key validation needed - Versori manages this via connection auth
617
- */
618
- export const adhocPriceSync = webhook('price-sync-adhoc', {
619
- response: { mode: 'sync' },
620
- connection: 'price-sync-adhoc', // Versori validates API key
621
- }).then(
622
- http('run-price-sync-adhoc', { connection: 'fluent_commerce' }, async (ctx: any) => {
623
- const { log, openKv, data } = ctx;
624
- const jobId = `price-sync-adhoc-${Date.now()}`;
625
- const executionStartTime = Date.now();
626
- const tracker = new JobTracker(openKv(':project:'), log);
627
-
628
- log.info('🔄 [WEBHOOK] Starting adhoc price sync', { jobId });
629
-
630
- await tracker.createJob(jobId, {
631
- triggeredBy: 'manual',
632
- stage: 'initialization',
633
- startTime: executionStartTime,
634
- options: data // Optional: filePattern, maxFiles, etc.
635
- });
636
- await tracker.updateJob(jobId, { status: 'processing' });
637
-
638
- try {
639
- const result = await executePriceSync(ctx, jobId, tracker);
640
-
641
- if (result.success) {
642
- await tracker.markCompleted(jobId, result);
643
- log.info('✅ [WEBHOOK] Adhoc price sync completed', {
644
- jobId,
645
- filesProcessed: result.filesProcessed,
646
- duration: result.duration,
647
- });
648
- } else {
649
- await tracker.markFailed(jobId, result.error || 'Unknown error');
650
- log.error('❌ [WEBHOOK] Adhoc price sync failed', { jobId, error: result.error });
651
- }
652
-
653
- return { success: true, jobId, ...result };
654
- } catch (e: any) {
655
- const errorMessage = e instanceof Error ? e.message : String(e);
656
- await tracker.markFailed(jobId, errorMessage);
657
- log.error('❌ [WEBHOOK] Adhoc price sync failed with exception', {
658
- jobId,
659
- error: errorMessage,
660
- stack: e instanceof Error ? e.stack : undefined,
661
- });
662
- return { success: false, jobId, error: errorMessage };
663
- }
664
- })
665
- );
666
- ```
667
-
668
- ---
669
-
670
- #### `src/workflows/webhook/job-status-check.ts`
671
-
672
- **Purpose**: Query job status
673
- **Trigger**: HTTP POST
674
- **Endpoint**: `POST https://{workspace}.versori.run/price-sync-job-status`
675
- **Request body**: `{ "jobId": "price-sync-1234567890" }`
676
-
677
- ```typescript
678
- import { webhook, fn } from '@versori/run';
679
- import { JobTracker } from '@fluentcommerce/fc-connect-sdk';
680
-
681
- /**
682
- * Webhook: Job Status Check
683
- *
684
- * Endpoint: POST https://{workspace}.versori.run/price-sync-job-status
685
- * Request body: { "jobId": "price-sync-1234567890" }
686
- *
687
- * Pattern: webhook().then(fn()) - no external API needed, only KV storage
688
- * Lightweight: Only queries KV store, no Fluent API calls
689
- *
690
- * SECURITY: Authentication handled via connection parameter
691
- * No manual API key validation needed - Versori manages this via connection auth
692
- */
693
- export const priceSyncJobStatus = webhook('price-sync-job-status', {
694
- response: { mode: 'sync' },
695
- connection: 'price-sync-job-status',
696
- }).then(
697
- fn('status', async (ctx: any) => {
698
- const { data, log, openKv } = ctx;
699
- const jobId = data?.jobId as string;
700
-
701
- log.info('🔍 [STATUS] Checking job status', { jobId });
702
-
703
- if (!jobId) {
704
- log.warn('❌ [STATUS] Missing jobId in request');
705
- return { success: false, error: 'jobId required' };
706
- }
707
-
708
- const tracker = new JobTracker(openKv(':project:'), log);
709
- const status = await tracker.getJob(jobId);
710
-
711
- if (status) {
712
- log.info('✅ [STATUS] Job found', { jobId, status: status.status });
713
- return { success: true, jobId, ...status };
714
- } else {
715
- log.warn('❌ [STATUS] Job not found', { jobId });
716
- return { success: false, error: 'Job not found', jobId };
717
- }
718
- })
719
- );
720
- ```
721
-
722
- ---
723
-
724
- ### 3. Entry Point (`index.ts`)
725
-
726
- **Purpose**: Register all workflows with Versori platform
727
-
728
- ```typescript
729
- /**
730
- * Entry Point - Registers all workflows with Versori platform
731
- *
732
- * Versori automatically discovers and registers exported workflows
733
- *
734
- * File Structure:
735
- * - src/workflows/scheduled/ → Time-based triggers (cron)
736
- * - src/workflows/webhook/ → HTTP-based triggers (webhooks)
737
- */
738
-
739
- // Import scheduled workflows
740
- import { dailyPriceSync } from './workflows/scheduled/daily-price-sync';
741
-
742
- // Import webhook workflows
743
- import { adhocPriceSync } from './workflows/webhook/adhoc-price-sync';
744
- import { priceSyncJobStatus } from './workflows/webhook/job-status-check';
745
-
746
- // Register all workflows
747
- export {
748
- // Scheduled (time-based triggers)
749
- dailyPriceSync,
750
-
751
- // Webhooks (HTTP-based triggers)
752
- adhocPriceSync,
753
- priceSyncJobStatus,
754
- };
755
- ```
756
-
757
- **What Gets Exposed:**
758
- - ✅ `adhocPriceSync` → `https://{workspace}.versori.run/price-sync-adhoc`
759
- - ✅ `priceSyncJobStatus` → `https://{workspace}.versori.run/price-sync-job-status`
760
- - ❌ `dailyPriceSync` → NOT exposed (runs automatically on cron)
761
-
762
- ---
763
-
764
- ## SDK Methods Used
765
-
766
- **Core Services:**
767
- - `createClient(ctx)` - Create Fluent client (auto-detects Versori context)
768
- - `S3DataSource(config, log)` - S3 operations (list, download, upload, move)
769
- - `CSVParserService()` - CSV parsing with validation
770
- - `GraphQLMutationMapper(mappingConfig, log, { fluentClient: client })` - Field mapping with schema validation
771
- - `VersoriKVAdapter(openKv(':project:'))` - KV state management for duplicate prevention
772
-
773
- **Key Methods:**
774
- - `s3.listFiles({ prefix, maxKeys })` - List S3 files
775
- - `s3.downloadFile(path, options)` - Download file content
776
- - `s3.uploadFile(path, content)` - Upload file (accepts string or Buffer)
777
- - `s3.moveFile(source, dest)` - Archive or move file
778
- - `parser.parse(content, options)` - Parse CSV to records
779
- - `mapper.map(record)` - Transform single record
780
- - `client.graphql({ query, variables })` - Execute GraphQL mutations/queries
781
- - `kv.get(key)` / `kv.set(key, value)` - State management
782
-
783
- **Critical Imports:**
784
- - `Buffer` - **Required for Versori/Deno runtime** (S3 upload operations)
785
- ```typescript
786
- import { Buffer } from 'node:buffer';
787
- ```
788
- **Why:** Deno runtime (used by Versori) does not have `Buffer` as a global. However, `uploadFile()` accepts string or Buffer, so you can use strings directly without Buffer conversion.
789
-
790
- **Service Functions (User-Defined):**
791
- - `processFile()` - S3 download + CSV parsing + field mapping
792
- - `executeMutations()` - GraphQL price updates with validation + rate limiting
793
- - `writeMutationLog()` - Write execution log to S3 (uses Buffer)
794
-
795
- ## Sample CSV Input Data
796
-
797
- **File**: `product-prices-20250122-001.csv`
798
-
799
- ```csv
800
- sku,priceType,amount,currency,effectiveDate,expiryDate,minQuantity,maxQuantity
801
- PROD-001,DEFAULT,29.99,USD,2025-01-22T00:00:00Z,,1,
802
- PROD-002,SALE,19.99,USD,2025-01-22T00:00:00Z,2025-02-15T23:59:59Z,1,
803
- PROD-003,CLEARANCE,9.99,USD,2025-01-22T00:00:00Z,2025-01-31T23:59:59Z,1,
804
- PROD-004,DEFAULT,149.99,USD,2025-01-22T00:00:00Z,,1,
805
- PROD-004,BULK,129.99,USD,2025-01-22T00:00:00Z,,10,99
806
- PROD-004,BULK,119.99,USD,2025-01-22T00:00:00Z,,100,
807
- ```
808
-
809
- **Note**: Products can have multiple price tiers. Same SKU with different `priceType` or quantity ranges = multiple rows.
810
-
811
- **Field Mapping**:
812
-
813
- - `sku` → Product reference (required) - must exist in Fluent
814
- - `priceType` → Price tier (DEFAULT, SALE, CLEARANCE, BULK, etc.) (required)
815
- - `amount` → Price value (required, must be > 0)
816
- - `currency` → Currency code (USD, EUR, GBP, etc.) (required)
817
- - `effectiveDate` → When price becomes active (ISO 8601 format)
818
- - `expiryDate` → When price expires (optional, ISO 8601 format)
819
- - `minQuantity` → Minimum quantity for this price tier (optional, default: 1)
820
- - `maxQuantity` → Maximum quantity for this price tier (optional)
821
-
822
- **Price Tier Logic**:
823
-
824
- - **DEFAULT**: Standard retail price (most products)
825
- - **SALE**: Temporary promotional price
826
- - **CLEARANCE**: Final markdown price
827
- - **BULK**: Quantity-based pricing (requires minQuantity/maxQuantity)
828
-
829
- ## Project Setup
830
-
831
- ```bash
832
- mkdir versori-s3-csv-price-sync && cd $_
833
- npm init -y
834
- npm install @fluentcommerce/fc-connect-sdk@latest @versori/run
835
- mkdir -p src
836
- ```
837
-
838
- ### Package Configuration (package.json)
839
-
840
- ```json
841
- {
842
- "name": "versori-s3-csv-price-sync",
843
- "version": "1.0.0",
844
- "description": "Versori workflow: S3 CSV price sync to Fluent GraphQL",
845
- "versori": {
846
- "workflows": "./src/index.ts"
847
- },
848
- "type": "module",
849
- "scripts": {
850
- "deploy": "versori deploy",
851
- "logs": "versori logs"
852
- },
853
- "dependencies": {
854
- "@fluentcommerce/fc-connect-sdk": "^0.1.39",
855
- "@versori/run": "latest"
856
- },
857
- "devDependencies": {
858
- "typescript": "^5.0.0",
859
- "@types/node": "^20.0.0"
860
- }
861
- }
862
- ```
863
-
864
- ### Activation Variables (Versori)
865
-
866
- ```bash
867
- # Required Variables
868
- s3BucketName=my-price-bucket
869
- awsAccessKeyId=AKIAXXXXXXXXXXXX
870
- awsSecretAccessKey=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
871
- retailerId=your-retailer-id
872
-
873
- # Optional Variables (with defaults shown)
874
- awsRegion=us-east-1
875
- s3Prefix=prices/
876
- archivePrefix=processed/
877
- errorPrefix=errors/
878
- logPrefix=logs/ # Where to write mutation logs
879
- maxFilesToProcess=10
880
- filePattern=.csv
881
- enableArchival=true
882
- mutationRateLimit=10
883
-
884
- # Price Validation Rules
885
- minPrice=0.01 # Minimum allowed price
886
- maxPrice=999999.99 # Maximum allowed price
887
- validateProducts=true # Validate products exist before updating prices
888
- ```
889
-
890
- ## Complete Workflow (src/index.ts)
891
-
892
- ```typescript
893
- import { schedule, webhook, http, fn } from '@versori/run';
894
- import { Buffer } from 'node:buffer'; // Required for Versori/Deno runtime
895
- import {
896
- createClient,
897
- S3DataSource,
898
- CSVParserService,
899
- GraphQLMutationMapper,
900
- VersoriFileTracker,
901
- JobTracker,
902
- } from '@fluentcommerce/fc-connect-sdk';
903
-
904
- // ============================================================================
905
- // Type Definitions
906
- // ============================================================================
907
-
908
- interface FileProcessingResult {
909
- fileName: string;
910
- successful: number;
911
- failed: number;
912
- validationFailed: number;
913
- priceChanges: Array<{
914
- sku: string;
915
- type: string;
916
- oldPrice?: number;
917
- newPrice: number;
918
- }>;
919
- errors: string[];
920
- }
921
-
922
- interface MutationResult {
923
- successful: number;
924
- failed: number;
925
- priceChanges: Array<{
926
- sku: string;
927
- type: string;
928
- oldPrice?: number;
929
- newPrice: number;
930
- }>;
931
- }
932
-
933
- interface MappedPriceData {
934
- productRef: string;
935
- type: string;
936
- value: number;
937
- currency: string;
938
- effectiveFrom?: string;
939
- effectiveTo?: string;
940
- minQuantity?: number;
941
- maxQuantity?: number;
942
- }
943
-
944
- // ============================================================================
945
- // Utility Functions
946
- // ============================================================================
947
-
948
- /**
949
- * Retry utility with exponential backoff
950
- * (User-defined - not part of SDK public API)
951
- */
952
- async function retryWithBackoff<T>(
953
- operation: () => Promise<T>,
954
- maxRetries = 3,
955
- baseDelayMs = 1000
956
- ): Promise<T> {
957
- let lastError: any;
958
- for (let attempt = 0; attempt < maxRetries; attempt++) {
959
- try {
960
- return await operation();
961
- } catch (error) {
962
- lastError = error;
963
- if (attempt < maxRetries - 1) {
964
- const delay = baseDelayMs * Math.pow(2, attempt) + Math.floor(Math.random() * 200);
965
- await new Promise(resolve => setTimeout(resolve, delay));
966
- }
967
- }
968
- }
969
- throw lastError;
970
- }
971
-
972
- /**
973
- * Validate product exists in Fluent Commerce
974
- */
975
- async function validateProductExists(
976
- client: any,
977
- productRef: string,
978
- retailerId: string,
979
- log: any
980
- ): Promise<boolean> {
981
- const query = `
982
- query GetProduct($ref: String!, $retailerId: ID!) {
983
- products(first: 1, ref: [$ref], retailerId: $retailerId) {
984
- edges {
985
- node {
986
- id
987
- ref
988
- status
989
- }
990
- }
991
- }
992
- }
993
- `;
994
-
995
- try {
996
- const result = await client.graphql({
997
- query,
998
- variables: { ref: productRef, retailerId },
999
- });
1000
-
1001
- const product = result?.data?.products?.edges?.[0]?.node;
1002
- if (!product) {
1003
- log.warn(`Product not found: ${productRef}`);
1004
- return false;
1005
- }
1006
-
1007
- if (product.status !== 'ACTIVE') {
1008
- log.warn(`Product not active: ${productRef} (status: ${product.status})`);
1009
- return false;
1010
- }
1011
-
1012
- return true;
1013
- } catch (error: unknown) {
1014
- // ✅ Enhanced error logging: Extract all error details for visibility
1015
- const errorMsg = error instanceof Error ? error.message : String(error);
1016
- const errorDetails = {
1017
- message: errorMsg,
1018
- stack: error instanceof Error ? error.stack : undefined,
1019
- errorType: error instanceof Error ? error.constructor.name : 'Error',
1020
- };
1021
- log.error(`Failed to validate product: ${productRef}`, errorDetails);
1022
- return false;
1023
- }
1024
- }
1025
-
1026
- /**
1027
- * Get current product price for change tracking
1028
- */
1029
- async function getCurrentPrice(
1030
- client: any,
1031
- productRef: string,
1032
- priceType: string,
1033
- currency: string,
1034
- retailerId: string
1035
- ): Promise<number | undefined> {
1036
- try {
1037
- const query = `
1038
- query GetProductPrice($ref: String!, $retailerId: ID!) {
1039
- products(first: 1, ref: [$ref], retailerId: $retailerId) {
1040
- edges {
1041
- node {
1042
- id
1043
- ref
1044
- prices {
1045
- type
1046
- value
1047
- currency
1048
- }
1049
- }
1050
- }
1051
- }
1052
- }
1053
- `;
1054
-
1055
- const result = await client.graphql({
1056
- query,
1057
- variables: { ref: productRef, retailerId },
1058
- });
1059
-
1060
- const product = result?.data?.products?.edges?.[0]?.node;
1061
- if (product?.prices) {
1062
- const existingPrice = product.prices.find(
1063
- (p: any) => p.type === priceType && p.currency === currency
1064
- );
1065
- if (existingPrice) {
1066
- return existingPrice.value;
1067
- }
1068
- }
1069
- } catch (err) {
1070
- // Ignore - price change tracking is optional
1071
- }
1072
- return undefined;
1073
- }
1074
-
1075
- // ============================================================================
1076
- // Service Function 1: Process File (S3 + CSV + Mapper)
1077
- // ============================================================================
1078
-
1079
- /**
1080
- * Process single CSV file: Download from S3, parse CSV, map fields to Price schema
1081
- *
1082
- * @param s3 - S3DataSource instance
1083
- * @param parser - CSVParserService instance
1084
- * @param mapper - UniversalMapper instance (with custom price resolvers)
1085
- * @param filePath - S3 file path
1086
- * @param fileName - File name
1087
- * @param log - Logger instance
1088
- * @returns FileProcessingResult with parsed and mapped price data
1089
- */
1090
- async function processFile(
1091
- s3: S3DataSource,
1092
- parser: CSVParserService,
1093
- mapper: GraphQLMutationMapper,
1094
- filePath: string,
1095
- fileName: string,
1096
- log: any
1097
- ): Promise<{ records: Array<{ query: string; variables: any; input: any }>; errors: string[] }> {
1098
- log.info('Processing file', { fileName });
1099
-
1100
- // Download CSV from S3 with retry
1101
- const content = await retryWithBackoff(
1102
- () => s3.downloadFile(filePath, { encoding: 'utf8' }) as Promise<string>
1103
- );
1104
-
1105
- // Parse CSV
1106
- const rawRecords = await parser.parse(content, {
1107
- columns: true,
1108
- skip_empty_lines: true,
1109
- trim: true,
1110
- });
1111
-
1112
- if (!rawRecords.length) {
1113
- log.warn('Empty CSV file', { fileName });
1114
- return { records: [], errors: [] };
1115
- }
1116
-
1117
- log.info('CSV parsed', { fileName, recordCount: rawRecords.length });
1118
-
1119
- // Map CSV records to Price schema
1120
- const mappedRecords: Array<{ query: string; variables: any; input: any }> = [];
1121
- const errors: string[] = [];
1122
-
1123
- // ✅ PRODUCTION ENHANCEMENT: Log transformation start
1124
- log.info('Transforming records to GraphQL mutations', {
1125
- fileName,
1126
- totalRecords: rawRecords.length,
1127
- });
1128
-
1129
- for (let i = 0; i < rawRecords.length; i++) {
1130
- const rec = rawRecords[i];
1131
- const recordNumber = i + 1;
1132
-
1133
- // ✅ PRODUCTION ENHANCEMENT: Log progress every 50 records
1134
- if (recordNumber % 50 === 0) {
1135
- log.info(`📤 Transforming record ${recordNumber}/${rawRecords.length}`, {
1136
- fileName,
1137
- recordNumber,
1138
- totalRecords: rawRecords.length,
1139
- validSoFar: mappedRecords.length,
1140
- errorsSoFar: errors.length,
1141
- progress: `${((recordNumber / rawRecords.length) * 100).toFixed(1)}%`,
1142
- });
1143
- }
1144
-
1145
- try {
1146
- // GraphQLMutationMapper returns { query, variables } directly
1147
- const mapped = await mapper.map(rec);
1148
-
1149
- mappedRecords.push({
1150
- query: mapped.query,
1151
- variables: mapped.variables,
1152
- input: mapped.variables.input || mapped.variables,
1153
- });
1154
- } catch (error: unknown) {
1155
- const errorMsg = error instanceof Error ? error.message : String(error);
1156
- errors.push(`Row ${recordNumber}: ${errorMsg}`);
1157
- }
1158
- }
1159
-
1160
- log.info('Mapping completed', {
1161
- fileName,
1162
- successful: mappedRecords.length,
1163
- failed: errors.length,
1164
- totalRecords: rawRecords.length,
1165
- successRate: `${((mappedRecords.length / rawRecords.length) * 100).toFixed(1)}%`,
1166
- });
1167
-
1168
- return { records: mappedRecords, errors };
1169
- }
1170
-
1171
- // ============================================================================
1172
- // Service Function 2: Execute Mutations (GraphQL Price Updates)
1173
- // ============================================================================
1174
-
1175
- /**
1176
- * Execute GraphQL mutations for price updates with validation and rate limiting
1177
- * ✅ Supports alias batching for improved performance
1178
- *
1179
- * @param client - FluentClient instance
1180
- * @param mapper - GraphQLMutationMapper instance
1181
- * @param priceRecords - Mapped price data with query and variables
1182
- * @param retailerId - Fluent retailer ID
1183
- * @param validateProducts - Whether to validate product existence
1184
- * @param batchSize - Number of concurrent requests (concurrency control)
1185
- * @param mutationsPerAliasBatch - Optional: Number of mutations per aliased request (alias batching)
1186
- * @param log - Logger instance
1187
- * @returns MutationResult with success/failure counts and price changes
1188
- */
1189
- async function executeMutations(
1190
- client: any,
1191
- mapper: GraphQLMutationMapper,
1192
- priceRecords: Array<{ query: string; variables: any; input: any }>,
1193
- retailerId: string,
1194
- validateProducts: boolean,
1195
- batchSize: number = 1, // ✅ Default: 1 (sequential)
1196
- mutationsPerAliasBatch?: number, // ✅ NEW: Alias batching parameter (default: undefined = disabled)
1197
- log: any
1198
- ): Promise<MutationResult> {
1199
- let successful = 0;
1200
- let failed = 0;
1201
- const priceChanges: Array<{
1202
- sku: string;
1203
- type: string;
1204
- oldPrice?: number;
1205
- newPrice: number;
1206
- }> = [];
1207
-
1208
- // ✅ Use alias batching if enabled
1209
- const useAliases = mutationsPerAliasBatch && mutationsPerAliasBatch > 1;
1210
-
1211
- if (useAliases) {
1212
- // Process with alias batching
1213
- const results = await executeMutationsWithAliases(
1214
- priceRecords,
1215
- client,
1216
- mapper,
1217
- log,
1218
- retailerId,
1219
- batchSize,
1220
- mutationsPerAliasBatch,
1221
- 'updateProduct'
1222
- );
1223
- successful = results.executed;
1224
- failed = results.failed;
1225
- // Note: Price changes tracking would need to be done separately if needed
1226
- } else {
1227
- // Process individually with validation
1228
- for (const priceData of priceRecords) {
1229
- try {
1230
- // Validate product exists (if enabled)
1231
- if (validateProducts) {
1232
- const productExists = await validateProductExists(
1233
- client,
1234
- priceData.input.productRef,
1235
- retailerId,
1236
- log
1237
- );
1238
-
1239
- if (!productExists) {
1240
- log.warn(`Skipping price update for non-existent product: ${priceData.input.productRef}`);
1241
- failed++;
1242
- continue;
1243
- }
1244
- }
1245
-
1246
- // Get current price for change tracking
1247
- const currentPrice = await getCurrentPrice(
1248
- client,
1249
- priceData.input.productRef,
1250
- priceData.input.type,
1251
- priceData.input.currency,
1252
- retailerId
1253
- );
1254
-
1255
- // Execute mutation using pre-generated query and variables
1256
- await retryWithBackoff(() =>
1257
- client.graphql({
1258
- query: priceData.query,
1259
- variables: priceData.variables,
1260
- })
1261
- );
1262
-
1263
- successful++;
1264
-
1265
- // Track price change
1266
- if (currentPrice !== undefined && currentPrice !== priceData.input.value) {
1267
- priceChanges.push({
1268
- sku: priceData.input.productRef,
1269
- type: priceData.input.type,
1270
- oldPrice: currentPrice,
1271
- newPrice: priceData.input.value,
1272
- });
1273
- }
1274
-
1275
- log.info('Price updated', {
1276
- ref: priceData.input.productRef,
1277
- type: priceData.input.type,
1278
- oldPrice: currentPrice,
1279
- newPrice: priceData.input.value,
1280
- currency: priceData.input.currency,
1281
- });
1282
- } catch (error: unknown) {
1283
- failed++;
1284
- const errorMsg = error instanceof Error ? error.message : String(error);
1285
- const errorDetails = {
1286
- message: errorMsg,
1287
- stack: error instanceof Error ? error.stack : undefined,
1288
- errorType: error instanceof Error ? error.constructor.name : 'Error',
1289
- };
1290
- log.error('Failed to update price', errorDetails, {
1291
- ref: priceData.input?.productRef,
1292
- type: priceData.input?.type,
1293
- });
1294
- }
1295
- }
1296
- }
1297
-
1298
- return { successful, failed, priceChanges };
1299
- }
1300
-
1301
- /**
1302
- * ✅ NEW: Execute mutations with GraphQL alias batching
1303
- */
1304
- async function executeMutationsWithAliases(
1305
- priceRecords: Array<{ query: string; variables: any; input: any }>,
1306
- client: any,
1307
- mapper: GraphQLMutationMapper,
1308
- log: any,
1309
- retailerId: string,
1310
- maxParallel: number,
1311
- mutationsPerAliasBatch: number,
1312
- mutationName: string
1313
- ): Promise<{ executed: number; failed: number; errors: string[] }> {
1314
- const results = { executed: 0, failed: 0, errors: [] as string[] };
1315
-
1316
- const aliasBatches: Array<Array<typeof priceRecords[0]>> = [];
1317
- for (let i = 0; i < priceRecords.length; i += mutationsPerAliasBatch) {
1318
- aliasBatches.push(priceRecords.slice(i, i + mutationsPerAliasBatch));
1319
- }
1320
-
1321
- log.info(`Processing ${aliasBatches.length} alias batches`, {
1322
- totalPrices: priceRecords.length,
1323
- maxParallel,
1324
- });
1325
-
1326
- for (let i = 0; i < aliasBatches.length; i += maxParallel) {
1327
- const concurrentBatches = aliasBatches.slice(i, i + maxParallel);
1328
-
1329
- const batchResults = await Promise.allSettled(
1330
- concurrentBatches.map(async (batch) => {
1331
- const { query, variables } = buildAliasedBatch(batch, mutationName, retailerId);
1332
- const response = await retryWithBackoff(() => client.graphql({ query, variables }), log);
1333
- return parseAliasResponse(response, batch, mutationName);
1334
- })
1335
- );
1336
-
1337
- batchResults.forEach((result, idx) => {
1338
- if (result.status === 'fulfilled') {
1339
- const batchResult = result.value;
1340
- results.executed += batchResult.executed;
1341
- results.failed += batchResult.failed;
1342
- results.errors.push(...batchResult.errors);
1343
- } else {
1344
- const batch = concurrentBatches[idx];
1345
- const errorMsg = result.reason instanceof Error ? result.reason.message : String(result.reason);
1346
- batch.forEach(price => {
1347
- results.failed++;
1348
- const productRef = price.input?.productRef || 'unknown';
1349
- results.errors.push(`Failed to update price for ${productRef}: ${errorMsg}`);
1350
- });
1351
- }
1352
- });
1353
-
1354
- if (i + maxParallel < aliasBatches.length) {
1355
- await new Promise(resolve => setTimeout(resolve, 500));
1356
- }
1357
- }
1358
-
1359
- return results;
1360
- }
1361
-
1362
- /**
1363
- * ✅ NEW: Build aliased batch query and variables
1364
- */
1365
- function buildAliasedBatch(
1366
- batch: Array<{ query: string; variables: any; input: any }>,
1367
- mutationName: string,
1368
- retailerId: string
1369
- ): { query: string; variables: Record<string, any> } {
1370
- const batchSize = batch.length;
1371
- const inputTypeName = `${mutationName.charAt(0).toUpperCase() + mutationName.slice(1)}Input`;
1372
-
1373
- const variables = Array.from({ length: batchSize }, (_, i) =>
1374
- `$input${i + 1}: ${inputTypeName}!`
1375
- ).join(', ');
1376
-
1377
- const aliasedMutations = Array.from({ length: batchSize }, (_, i) => {
1378
- const alias = `${mutationName}${i + 1}`;
1379
- return ` ${alias}: ${mutationName}(input: $input${i + 1}) { id ref }`;
1380
- }).join('\n');
1381
-
1382
- const operationName = `Batch${mutationName.charAt(0).toUpperCase() + mutationName.slice(1)}s`;
1383
- const query = `mutation ${operationName}(${variables}) {\n${aliasedMutations}\n}`;
1384
-
1385
- const variablesObj: Record<string, any> = {};
1386
- batch.forEach((price, index) => {
1387
- const input = price.variables.input || price.variables;
1388
- variablesObj[`input${index + 1}`] = input;
1389
- });
1390
-
1391
- return { query, variables: variablesObj };
1392
- }
1393
-
1394
- /**
1395
- * ✅ NEW: Parse aliased GraphQL response
1396
- */
1397
- function parseAliasResponse(
1398
- response: any,
1399
- batch: Array<{ query: string; variables: any; input: any }>,
1400
- mutationName: string
1401
- ): { executed: number; failed: number; errors: string[] } {
1402
- const result = { executed: 0, failed: 0, errors: [] as string[] };
1403
-
1404
- const data = response.data || {};
1405
- const errors = response.errors || [];
1406
-
1407
- batch.forEach((price, index) => {
1408
- const alias = `${mutationName}${index + 1}`;
1409
- const aliasData = data[alias];
1410
- const aliasErrors = errors.filter((e: unknown) =>
1411
- e && typeof e === 'object' && 'path' in e && Array.isArray((e as any).path) && (e as any).path.includes(alias)
1412
- );
1413
-
1414
- if (aliasData && !aliasErrors.length) {
1415
- result.executed++;
1416
- } else {
1417
- result.failed++;
1418
- const errorMsg = aliasErrors[0] && typeof aliasErrors[0] === 'object' && 'message' in aliasErrors[0]
1419
- ? String((aliasErrors[0] as any).message)
1420
- : 'Mutation failed';
1421
- const productRef = price.input?.productRef || 'unknown';
1422
- result.errors.push(`Failed to update price for ${productRef}: ${errorMsg}`);
1423
- }
1424
- });
1425
-
1426
- return result;
1427
- }
1428
-
1429
- // ============================================================================
1430
- // Service Function 3: Write Mutation Log (S3 Upload)
1431
- // ============================================================================
1432
-
1433
- /**
1434
- * Write mutation execution log to S3
1435
- *
1436
- * @param s3 - S3DataSource instance
1437
- * @param fileName - Original CSV file name
1438
- * @param result - FileProcessingResult
1439
- * @param logPrefix - S3 prefix for logs
1440
- * @param log - Logger instance
1441
- */
1442
- async function writeMutationLog(
1443
- s3: S3DataSource,
1444
- fileName: string,
1445
- result: FileProcessingResult,
1446
- logPrefix: string,
1447
- log: any
1448
- ): Promise<void> {
1449
- const logFileName = `${fileName.replace('.csv', '')}-log.json`;
1450
- const logPath = `${logPrefix}${logFileName}`;
1451
-
1452
- const logData = {
1453
- fileName,
1454
- timestamp: new Date().toISOString(),
1455
- summary: {
1456
- successful: result.successful,
1457
- failed: result.failed,
1458
- validationFailed: result.validationFailed,
1459
- priceChanges: result.priceChanges.length,
1460
- },
1461
- priceChanges: result.priceChanges,
1462
- errors: result.errors,
1463
- };
1464
-
1465
- const logContent = JSON.stringify(logData, null, 2);
1466
-
1467
- // Upload log to S3 (uploadFile accepts string or Buffer)
1468
- await s3.uploadFile(logPath, logContent);
1469
-
1470
- log.info('Mutation log written', { logPath });
1471
- }
1472
-
1473
- // ============================================================================
1474
- // Main Workflow Function (Per-File Processing)
1475
- // ============================================================================
1476
-
1477
- async function executePriceSync(ctx: any, jobId: string, tracker: any) {
1478
- const { log, activation, openKv } = ctx;
1479
- const startTime = Date.now();
1480
-
1481
- log.info('🔄 [PriceSync] Starting scheduled price sync from S3');
1482
-
1483
- // ✅ CRITICAL: Declare S3 outside try block for safe disposal
1484
- let s3: S3DataSource | undefined;
1485
-
1486
- try {
1487
- // ========================================
1488
- // STEP 1: CONFIGURATION & VALIDATION
1489
- // ========================================
1490
-
1491
- // Read activation variables
1492
- const s3Bucket = activation?.getVariable('s3BucketName');
1493
- const s3Region = activation?.getVariable('awsRegion') || 'us-east-1';
1494
- const s3AccessKeyId = activation?.getVariable('awsAccessKeyId');
1495
- const s3SecretAccessKey = activation?.getVariable('awsSecretAccessKey');
1496
- const s3Prefix = activation?.getVariable('s3Prefix') || 'prices/';
1497
- const retailerId = activation?.getVariable('retailerId');
1498
- const maxFiles = parseInt(activation?.getVariable('maxFilesToProcess') || '10', 10);
1499
- const filePattern = (activation?.getVariable('filePattern') || '.csv').toLowerCase();
1500
- const enableArchival = activation?.getVariable('enableArchival') !== 'false';
1501
- const archivePrefix = activation?.getVariable('archivePrefix') || 'processed/';
1502
- const errorPrefix = activation?.getVariable('errorPrefix') || 'errors/';
1503
- const logPrefix = activation?.getVariable('logPrefix') || 'logs/';
1504
-
1505
- // Configuration with defaults
1506
- const mutationBatchSize = parseInt(
1507
- activation?.getVariable('mutationBatchSize') || '1', // Default: 1 (sequential)
1508
- 10
1509
- );
1510
-
1511
- const mutationsPerAliasBatch = activation?.getVariable('mutationsPerAliasBatch')
1512
- ? parseInt(activation?.getVariable('mutationsPerAliasBatch') || '1', 10)
1513
- : undefined; // Default: undefined (disabled)
1514
-
1515
- const minPrice = parseFloat(activation?.getVariable('minPrice') || '0.01');
1516
- const maxPrice = parseFloat(activation?.getVariable('maxPrice') || '999999.99');
1517
- const validateProducts = activation?.getVariable('validateProducts') !== 'false';
1518
- const validateConnection = activation?.getVariable('validateConnection') !== 'false';
1519
- const enableFileTracking = activation?.getVariable('enableFileTracking') !== 'false';
1520
-
1521
- // Validate required variables
1522
- const missingVars: string[] = [];
1523
- if (!s3Bucket) missingVars.push('s3BucketName');
1524
- if (!s3AccessKeyId) missingVars.push('awsAccessKeyId');
1525
- if (!s3SecretAccessKey) missingVars.push('awsSecretAccessKey');
1526
- if (!retailerId) missingVars.push('retailerId');
1527
-
1528
- if (missingVars.length > 0) {
1529
- const errorMsg = `Missing required variables: ${missingVars.join(', ')}`;
1530
- log.error('❌ [PriceSync] Configuration error', { error: errorMsg });
1531
- return {
1532
- success: false,
1533
- error: errorMsg,
1534
- recommendation: `Please configure these activation variables: ${missingVars.join(', ')}`,
1535
- processed: 0,
1536
- duration: Date.now() - startTime,
1537
- };
1538
- }
1539
-
1540
- // ========================================
1541
- // STEP 2: CLIENT INITIALIZATION
1542
- // ========================================
1543
-
1544
- const client = await createClient(ctx);
1545
- if (!client) {
1546
- throw new Error('Failed to create Fluent Commerce client');
1547
- }
1548
-
1549
- // ✅ CORRECT: GraphQL mutations don't need setRetailerId()
1550
- // Check your GraphQL schema to determine retailerId handling:
1551
- // - Mandatory retailerId → Must pass it in mutation input
1552
- // - Optional retailerId → Can pass it if needed
1553
- // - No retailerId field → Don't pass it
1554
- // See: fc-connect-sdk/docs/00-START-HERE/retailerid-configuration.md
1555
-
1556
- log.info('✅ [PriceSync] Fluent client initialized');
1557
-
1558
- // ========================================
1559
- // STEP 3: SERVICE INITIALIZATION
1560
- // ========================================
1561
-
1562
- s3 = new S3DataSource(
1563
- {
1564
- type: 'S3_CSV',
1565
- connectionId: 's3-price-sync',
1566
- name: 'Source S3',
1567
- s3Config: {
1568
- bucket: s3Bucket,
1569
- region: s3Region,
1570
- accessKeyId: s3AccessKeyId,
1571
- secretAccessKey: s3SecretAccessKey,
1572
- },
1573
- },
1574
- log
1575
- );
1576
-
1577
- // Validate S3 connection if enabled
1578
- if (validateConnection) {
1579
- try {
1580
- await s3.listFiles({ prefix: s3Prefix, maxKeys: 1 });
1581
- log.info('✅ [PriceSync] S3 connection validated');
1582
- } catch (error: any) {
1583
- log.error('❌ [PriceSync] S3 connection validation failed', {
1584
- error: error instanceof Error ? error.message : String(error),
1585
- });
1586
- return {
1587
- success: false,
1588
- error: 'S3 connection validation failed',
1589
- details: error?.message,
1590
- recommendation: 'Please verify S3 credentials and bucket access permissions',
1591
- duration: Date.now() - startTime,
1592
- };
1593
- }
1594
- }
1595
-
1596
- const parser = new CSVParserService();
1597
- const kv = openKv(':project:');
1598
- const fileTracker = enableFileTracking
1599
- ? new VersoriFileTracker(kv, 's3-csv-price-sync')
1600
- : null;
1601
-
1602
- // Create custom resolvers for price validation
1603
- const customResolvers = {
1604
- 'custom.validatePriceRange': (value: any) => {
1605
- const price = parseFloat(value);
1606
- if (isNaN(price)) {
1607
- throw new Error(`Invalid price: ${value}`);
1608
- }
1609
- if (price < minPrice) {
1610
- throw new Error(`Price ${price} below minimum ${minPrice}`);
1611
- }
1612
- if (price > maxPrice) {
1613
- throw new Error(`Price ${price} exceeds maximum ${maxPrice}`);
1614
- }
1615
- return price;
1616
- },
1617
-
1618
- 'custom.normalizePriceType': (value: any) => {
1619
- const type = String(value || 'DEFAULT').toUpperCase().trim();
1620
- const validTypes = ['DEFAULT', 'SALE', 'CLEARANCE', 'BULK', 'PROMOTIONAL', 'MEMBER'];
1621
- if (!validTypes.includes(type)) {
1622
- throw new Error(`Invalid price type: ${type}`);
1623
- }
1624
- return type;
1625
- },
1626
-
1627
- 'custom.normalizeQuantity': (value: any) => {
1628
- if (!value || value === '') return undefined;
1629
- const qty = parseInt(value, 10);
1630
- if (isNaN(qty) || qty < 0) {
1631
- throw new Error(`Invalid quantity: ${value}`);
1632
- }
1633
- return qty;
1634
- },
1635
- };
1636
-
1637
- // ✅ CRITICAL: Load mapping config from external JSON file
1638
- // Mapping config uses GraphQLMutationMapper structure (nested objects, not dot notation)
1639
- // File: src/config/price-mapping.json
1640
- const mappingConfigJson = await import('../config/price-mapping.json', { assert: { type: 'json' } });
1641
- const mappingConfig = mappingConfigJson.default;
1642
-
1643
- // Initialize GraphQLMutationMapper with client for schema introspection
1644
- const mapper = new GraphQLMutationMapper(mappingConfig, log, { fluentClient: client });
1645
-
1646
- // ========================================
1647
- // STEP 4: FILE DISCOVERY
1648
- // ========================================
1649
-
1650
- try {
1651
- // List files (pattern filtering handled by listFiles)
1652
- const files = await s3.listFiles({
1653
- prefix: s3Prefix,
1654
- pattern: filePattern,
1655
- maxKeys: 1000,
1656
- });
1657
-
1658
- const csvFiles = files
1659
- .sort((a, b) => {
1660
- const dateA = a.lastModified ? new Date(a.lastModified).getTime() : 0;
1661
- const dateB = b.lastModified ? new Date(b.lastModified).getTime() : 0;
1662
- return dateA - dateB;
1663
- })
1664
- .slice(0, maxFiles);
1665
-
1666
- log.info('📂 [PriceSync] Files discovered', {
1667
- total: files.length,
1668
- toProcess: csvFiles.length,
1669
- maxFiles,
1670
- });
1671
-
1672
- const results = {
1673
- processed: 0,
1674
- skipped: 0,
1675
- failed: 0,
1676
- totalRecords: 0,
1677
- pricesUpdated: 0,
1678
- pricesSkipped: 0,
1679
- validationErrors: 0,
1680
- priceChanges: [] as Array<{
1681
- sku: string;
1682
- type: string;
1683
- oldPrice?: number;
1684
- newPrice: number;
1685
- }>,
1686
- errors: [] as string[],
1687
- };
1688
-
1689
- // ========================================
1690
- // STEP 5: FILE PROCESSING LOOP
1691
- // ========================================
1692
-
1693
- // Process each file using service functions
1694
- for (const file of csvFiles) {
1695
- const filePath = file.path;
1696
- const fileName = file.name;
1697
- const fileStartTime = Date.now();
1698
-
1699
- log.info('📄 [PriceSync] Processing file', { fileName });
1700
-
1701
- // Duplicate prevention via file tracker
1702
- if (fileTracker && (await fileTracker.wasFileProcessed(fileName))) {
1703
- log.info('⏭️ [PriceSync] Skipping already processed file', { fileName });
1704
- results.skipped++;
1705
- continue;
1706
- }
1707
-
1708
- try {
1709
- // SERVICE FUNCTION 1: Process file (S3 + CSV + Mapper)
1710
- const { records: mappedRecords, errors: mappingErrors } = await processFile(
1711
- s3,
1712
- parser,
1713
- mapper,
1714
- filePath,
1715
- fileName,
1716
- log
1717
- );
1718
-
1719
- if (!mappedRecords.length) {
1720
- log.warn('No valid records after mapping, archiving', { fileName });
1721
- if (enableArchival) {
1722
- await s3.moveFile(filePath, `${archivePrefix}${fileName}`);
1723
- }
1724
- results.skipped++;
1725
- continue;
1726
- }
1727
-
1728
- // SERVICE FUNCTION 2: Execute mutations
1729
- // ✅ Configuration with defaults
1730
- const mutationBatchSize = parseInt(
1731
- ctx.activation?.getVariable('mutationBatchSize') || '1', // ✅ Default: 1 (sequential)
1732
- 10
1733
- );
1734
-
1735
- const mutationsPerAliasBatch = ctx.activation?.getVariable('mutationsPerAliasBatch')
1736
- ? parseInt(ctx.activation?.getVariable('mutationsPerAliasBatch') || '1', 10)
1737
- : undefined; // ✅ Default: undefined (disabled, use separate requests)
1738
-
1739
- // ? Enhanced: Extract context for progress logging
1740
- const sampleProductRefs = mappedRecords.slice(0, 5).map((r: any) => r.input?.skuRef || r.input?.productRef || 'unknown');
1741
- const mutationType = mapper?.mutationName || 'updatePrice';
1742
-
1743
- // ? Enhanced: Start logging with context
1744
- log.info(`[GraphQLMutations] Sending price mutations for file "${fileName}"`, {
1745
- totalMutations: mappedRecords.length,
1746
- mutationType,
1747
- batchSize: mutationBatchSize,
1748
- batchMode: mutationBatchSize === 1 ? 'sequential' : `parallel (${mutationBatchSize})`,
1749
- sampleProductRefs: sampleProductRefs.join(', '),
1750
- aliasBatching: mutationsPerAliasBatch ? `enabled (${mutationsPerAliasBatch} per alias)` : 'disabled',
1751
- validateProducts
1752
- });
1753
-
1754
- const mutationResult = await executeMutations(
1755
- client,
1756
- mapper,
1757
- mappedRecords,
1758
- retailerId,
1759
- validateProducts,
1760
- mutationBatchSize, // Concurrency control (default: 1)
1761
- mutationsPerAliasBatch, // ✅ NEW: Alias batching (default: undefined)
1762
- log
1763
- );
1764
-
1765
- // ? Enhanced: Completion logging with summary
1766
- log.info(`[GraphQLMutations] Price mutation submission completed for file "${fileName}"`, {
1767
- totalMutations: mappedRecords.length,
1768
- successful: mutationResult.successful,
1769
- failed: mutationResult.failed,
1770
- successRate: mappedRecords.length > 0 ? `${Math.round((mutationResult.successful / mappedRecords.length) * 100)}%` : '0%',
1771
- mutationType,
1772
- priceChanges: mutationResult.priceChanges?.length || 0
1773
- });
1774
-
1775
- // Aggregate results
1776
- results.processed++;
1777
- results.totalRecords += mappedRecords.length;
1778
- results.pricesUpdated += mutationResult.successful;
1779
- results.validationErrors += mappingErrors.length;
1780
- results.priceChanges.push(...mutationResult.priceChanges);
1781
-
1782
- // Build file processing result for logging
1783
- const fileResult: FileProcessingResult = {
1784
- fileName,
1785
- successful: mutationResult.successful,
1786
- failed: mutationResult.failed,
1787
- validationFailed: mappingErrors.length,
1788
- priceChanges: mutationResult.priceChanges,
1789
- errors: mappingErrors,
1790
- };
1791
-
1792
- // SERVICE FUNCTION 3: Write mutation log to S3
1793
- await writeMutationLog(s3, fileName, fileResult, logPrefix, log);
1794
-
1795
- // Mark processed with file tracker
1796
- if (fileTracker) {
1797
- await fileTracker.markFileProcessed(fileName, {
1798
- successful: mutationResult.successful,
1799
- failed: mutationResult.failed,
1800
- validationFailed: mappingErrors.length,
1801
- recordCount: mappedRecords.length,
1802
- duration: Date.now() - fileStartTime,
1803
- });
1804
- }
1805
-
1806
- if (enableArchival) {
1807
- await s3.moveFile(filePath, `${archivePrefix}${fileName}`);
1808
- }
1809
-
1810
- const fileDuration = Date.now() - fileStartTime;
1811
- log.info('✅ [PriceSync] File processed successfully', {
1812
- fileName,
1813
- recordCount: mappedRecords.length,
1814
- successful: mutationResult.successful,
1815
- failed: mutationResult.failed,
1816
- duration: `${fileDuration}ms`,
1817
- });
1818
-
1819
- if (mappingErrors.length > 0) {
1820
- results.errors.push(`${fileName}: ${mappingErrors.length} mapping errors`);
1821
- }
1822
- } catch (error: unknown) {
1823
- // ✅ Enhanced error logging: Extract all error details for visibility
1824
- const errorMsg = error instanceof Error ? error.message : String(error);
1825
- const errorDetails = {
1826
- message: errorMsg,
1827
- stack: error instanceof Error ? error.stack : undefined,
1828
- errorType: error instanceof Error ? error.constructor.name : 'Error',
1829
- };
1830
- log.error('❌ [PriceSync] File processing failed', {
1831
- fileName,
1832
- ...errorDetails,
1833
- });
1834
- results.failed++;
1835
- results.errors.push(`${fileName}: ${errorMsg}`);
1836
-
1837
- // Attempt to move to error directory
1838
- try {
1839
- await s3.moveFile(filePath, `${errorPrefix}${fileName}`);
1840
- log.info('📁 [PriceSync] Moved failed file to error directory', { fileName });
1841
- } catch (moveError) {
1842
- log.error('❌ [PriceSync] Failed to move error file', {
1843
- fileName,
1844
- error: moveError instanceof Error ? moveError.message : String(moveError),
1845
- });
1846
- }
1847
- }
1848
- }
1849
-
1850
- // ========================================
1851
- // STEP 6: SUMMARY & COMPLETION
1852
- // ========================================
1853
-
1854
- const totalDuration = Date.now() - startTime;
1855
- const summary = {
1856
- success: true,
1857
- processed: results.processed,
1858
- skipped: results.skipped,
1859
- failed: results.failed,
1860
- totalRecords: results.totalRecords,
1861
- pricesUpdated: results.pricesUpdated,
1862
- pricesSkipped: results.pricesSkipped,
1863
- validationErrors: results.validationErrors,
1864
- priceChanges: results.priceChanges.length,
1865
- errors: results.errors.length > 0 ? results.errors : undefined,
1866
- duration: totalDuration,
1867
- timestamp: new Date().toISOString(),
1868
- };
1869
-
1870
- log.info('🎉 [PriceSync] Price sync completed', {
1871
- ...summary,
1872
- duration: `${totalDuration}ms`,
1873
- });
1874
-
1875
- // Log significant price changes
1876
- if (results.priceChanges.length > 0) {
1877
- log.info('💰 [PriceSync] Price changes detected', {
1878
- count: results.priceChanges.length,
1879
- changes: results.priceChanges.slice(0, 10),
1880
- });
1881
- }
1882
-
1883
- return summary;
1884
- } catch (error: unknown) {
1885
- // ✅ Enhanced error logging: Extract all error details for visibility
1886
- const errorMsg = error instanceof Error ? error.message : String(error);
1887
- const errorDetails = {
1888
- message: errorMsg,
1889
- stack: error instanceof Error ? error.stack : undefined,
1890
- errorType: error instanceof Error ? error.constructor.name : 'Error',
1891
- };
1892
- log.error('❌ [PriceSync] Fatal error', errorDetails);
1893
- return {
1894
- success: false,
1895
- error: errorMsg,
1896
- recommendation: 'Check error logs for details and verify configuration',
1897
- processed: 0,
1898
- duration: Date.now() - startTime,
1899
- timestamp: new Date().toISOString(),
1900
- };
1901
- }
1902
- } catch (error: unknown) {
1903
- const errorMsg = error instanceof Error ? error.message : String(error);
1904
- log.error('❌ [PriceSync] Initialization failed', {
1905
- error: errorMsg,
1906
- stack: error instanceof Error ? error.stack : undefined,
1907
- });
1908
- return {
1909
- success: false,
1910
- error: errorMsg,
1911
- recommendation: 'Check configuration and service initialization',
1912
- duration: Date.now() - startTime,
1913
- };
1914
- } finally {
1915
- // ✅ CRITICAL: Always dispose S3 connection to prevent connection pool exhaustion
1916
- if (s3) {
1917
- try {
1918
- await s3.dispose();
1919
- log.info('✅ [PriceSync] S3 connection disposed successfully');
1920
- } catch (disposeError: any) {
1921
- log.error('⚠️ [PriceSync] Failed to dispose S3 connection', {
1922
- error: disposeError instanceof Error ? disposeError.message : String(disposeError),
1923
- stack: disposeError instanceof Error ? disposeError.stack : undefined,
1924
- });
1925
- // Don't throw - disposal failure shouldn't prevent workflow completion
1926
- }
1927
- }
1928
- }
1929
- }
1930
- ```
1931
-
1932
- **Note:** The `runPriceSync` function above should be renamed to `executePriceSync` and moved to `src/services/price-sync.service.ts` to match the new workflow structure.
1933
-
1934
- ## Key Patterns Explained
1935
-
1936
- ### Pattern 1: Service Function Architecture
1937
-
1938
- **Three modular service functions for clean separation of concerns:**
1939
-
1940
- ```typescript
1941
- // SERVICE FUNCTION 1: processFile()
1942
- // Handles: S3 download + CSV parsing + field mapping
1943
- const { records: mappedRecords, errors: mappingErrors } = await processFile(
1944
- s3,
1945
- parser,
1946
- mapper,
1947
- filePath,
1948
- fileName,
1949
- log
1950
- );
1951
-
1952
- // SERVICE FUNCTION 2: executeMutations()
1953
- // Handles: GraphQL mutations + product validation + concurrency control + price change tracking
1954
- const mutationResult = await executeMutations(
1955
- client,
1956
- mapper,
1957
- mappedRecords,
1958
- retailerId,
1959
- validateProducts,
1960
- mutationBatchSize, // Concurrency control (default: 1)
1961
- mutationsPerAliasBatch, // Alias batching (default: undefined)
1962
- log
1963
- );
1964
-
1965
- // SERVICE FUNCTION 3: writeMutationLog()
1966
- // Handles: Write execution log to S3 (with Buffer for Versori/Deno)
1967
- await writeMutationLog(s3, fileName, fileResult, logPrefix, log);
1968
- ```
1969
-
1970
- **Why this pattern?**
1971
- - ✅ **Testability**: Each function can be unit tested independently
1972
- - ✅ **Reusability**: Functions can be reused in different workflows
1973
- - ✅ **Clarity**: Clear responsibilities for each step
1974
- - ✅ **Error handling**: Isolated error boundaries per function
1975
- - ✅ **Maintainability**: Easy to modify single concerns without affecting others
1976
-
1977
- ### Pattern 2: Buffer Import for S3 Upload (Versori/Deno)
1978
-
1979
- **CRITICAL for Versori/Deno runtime:**
1980
-
1981
- ```typescript
1982
- // MUST import Buffer explicitly (not global like Node.js)
1983
- import { Buffer } from 'node:buffer';
1984
-
1985
- // writeMutationLog() function
1986
- async function writeMutationLog(s3, fileName, result, logPrefix, log) {
1987
- const logContent = JSON.stringify(logData, null, 2);
1988
-
1989
- // ✅ CORRECT: Use Buffer.from() for S3 upload
1990
- await s3.uploadFile(logPath, logContent);
1991
-
1992
- // ✅ CORRECT: uploadFile accepts string directly (no Buffer needed)
1993
- // await s3.uploadFile(logPath, logContent);
1994
- }
1995
- ```
1996
-
1997
- **Why?** Deno runtime requires explicit `Buffer` import from `node:buffer`. Without it, you'll get `Buffer is not defined` errors.
1998
-
1999
- ### Pattern 3: Direct KV State Management
2000
-
2001
- ```typescript
2002
- const kv = new VersoriKVAdapter(ctx.openKv(':project:'));
2003
-
2004
- // Check if file already processed
2005
- const stateKey = ['processed-files', 's3-price-sync', fileName];
2006
- const existing = await kv.get(stateKey);
2007
- if (existing) {
2008
- log.info('Skipping already processed file', { fileName });
2009
- continue;
2010
- }
2011
-
2012
- // Mark as processed after success
2013
- await kv.set(stateKey, {
2014
- successful,
2015
- failed,
2016
- validationFailed,
2017
- processedAt: new Date().toISOString(),
2018
- });
2019
- ```
2020
-
2021
- ### Pattern 4: Custom Price Validation Resolvers
2022
-
2023
- ```typescript
2024
- const customResolvers = {
2025
- 'custom.validatePriceRange': (value: any) => {
2026
- const price = parseFloat(value);
2027
- if (isNaN(price)) {
2028
- throw new Error(`Invalid price: ${value}`);
2029
- }
2030
- if (price < minPrice) {
2031
- throw new Error(`Price ${price} below minimum ${minPrice}`);
2032
- }
2033
- if (price > maxPrice) {
2034
- throw new Error(`Price ${price} exceeds maximum ${maxPrice}`);
2035
- }
2036
- return price;
2037
- },
2038
-
2039
- 'custom.normalizePriceType': (value: any) => {
2040
- const type = String(value || 'DEFAULT').toUpperCase().trim();
2041
- const validTypes = ['DEFAULT', 'SALE', 'CLEARANCE', 'BULK', 'PROMOTIONAL', 'MEMBER'];
2042
- if (!validTypes.includes(type)) {
2043
- throw new Error(`Invalid price type: ${type}`);
2044
- }
2045
- return type;
2046
- },
2047
- };
2048
- ```
2049
-
2050
- **Why this matters**: Price data requires strict validation to prevent:
2051
-
2052
- - Negative prices
2053
- - Prices outside business rules (too low/high)
2054
- - Invalid price tiers
2055
- - Data entry errors
2056
-
2057
- ### Pattern 5: Product Existence Validation
2058
-
2059
- ```typescript
2060
- async function validateProductExists(
2061
- client: any,
2062
- productRef: string,
2063
- retailerId: string,
2064
- log: any
2065
- ): Promise<boolean> {
2066
- const query = `
2067
- query GetProduct($ref: String!, $retailerId: ID!) {
2068
- products(first: 1, ref: [$ref], retailerId: $retailerId) {
2069
- edges {
2070
- node {
2071
- id
2072
- ref
2073
- status
2074
- }
2075
- }
2076
- }
2077
- }
2078
- `;
2079
-
2080
- const result = await client.graphql({ query, variables: { ref: productRef, retailerId } });
2081
- const product = result?.data?.products?.edges?.[0]?.node;
2082
-
2083
- if (!product || product.status !== 'ACTIVE') {
2084
- return false;
2085
- }
2086
-
2087
- return true;
2088
- }
2089
- ```
2090
-
2091
- **Why this matters**: Prevents price updates for non-existent products, which would fail at GraphQL level.
2092
-
2093
- ### Pattern 6: Price Change Tracking
2094
-
2095
- ```typescript
2096
- // Get current price before update
2097
- const currentPrice = await getCurrentPrice(
2098
- client,
2099
- priceData.productRef,
2100
- priceData.type,
2101
- priceData.currency,
2102
- retailerId
2103
- );
2104
-
2105
- // Track change after update
2106
- if (currentPrice !== undefined && currentPrice !== priceData.value) {
2107
- results.priceChanges.push({
2108
- sku: priceData.productRef,
2109
- type: priceData.type,
2110
- oldPrice: currentPrice,
2111
- newPrice: priceData.value,
2112
- });
2113
- }
2114
-
2115
- // Log changes at end
2116
- if (results.priceChanges.length > 0) {
2117
- log.info('Price changes detected', {
2118
- count: results.priceChanges.length,
2119
- changes: results.priceChanges.slice(0, 10),
2120
- });
2121
- }
2122
- ```
2123
-
2124
- **Why this matters**: Provides audit trail and monitoring for price changes.
2125
-
2126
- ### Pattern 7: Concurrency Control & Alias Batching
2127
-
2128
- ```typescript
2129
- // ✅ Configuration with defaults
2130
- const mutationBatchSize = parseInt(
2131
- activation?.getVariable('mutationBatchSize') || '1', // Default: 1 (sequential)
2132
- 10
2133
- );
2134
-
2135
- const mutationsPerAliasBatch = activation?.getVariable('mutationsPerAliasBatch')
2136
- ? parseInt(activation?.getVariable('mutationsPerAliasBatch') || '1', 10)
2137
- : undefined; // Default: undefined (disabled)
2138
-
2139
- // Execute with bounded concurrency + optional alias batching
2140
- const mutationResult = await executeMutations(
2141
- client,
2142
- mapper,
2143
- priceRecords,
2144
- retailerId,
2145
- validateProducts,
2146
- mutationBatchSize, // 1=sequential, 3-10=parallel
2147
- mutationsPerAliasBatch, // Optional: Group mutations (e.g., 5)
2148
- log
2149
- );
2150
- ```
2151
-
2152
- **Performance Modes:**
2153
- - `mutationBatchSize: 1` → Sequential (safe default, ~1 mutation/sec)
2154
- - `mutationBatchSize: 3-5` → Balanced (~3-5 mutations/sec)
2155
- - `mutationBatchSize: 10` → High-volume (~10 mutations/sec)
2156
- - `mutationsPerAliasBatch: 5` → Alias batching (reduces network overhead by ~80%)
2157
-
2158
- ## Advanced: Multi-Currency Price Management
2159
-
2160
- ### External Mapping File for Multi-Currency
2161
-
2162
- **File**: `config/price-mapping.json`
2163
-
2164
- ```json
2165
- {
2166
- "version": "1.0.0",
2167
- "description": "Multi-currency price mapping",
2168
- "fields": {
2169
- "productRef": {
2170
- "source": "sku",
2171
- "required": true,
2172
- "resolver": "sdk.trim"
2173
- },
2174
- "type": {
2175
- "source": "price_type",
2176
- "required": true,
2177
- "resolver": "custom.normalizePriceType"
2178
- },
2179
- "value": {
2180
- "source": "amount",
2181
- "required": true,
2182
- "resolver": "custom.validatePriceRange"
2183
- },
2184
- "currency": {
2185
- "source": "currency_code",
2186
- "required": true,
2187
- "resolver": "custom.normalizeCurrency"
2188
- },
2189
- "effectiveFrom": {
2190
- "source": "start_date",
2191
- "resolver": "sdk.formatDate"
2192
- },
2193
- "effectiveTo": {
2194
- "source": "end_date",
2195
- "resolver": "sdk.formatDate"
2196
- }
2197
- }
2198
- }
2199
- ```
2200
-
2201
- ### Custom Resolvers for Multi-Currency
2202
-
2203
- ```typescript
2204
- const customResolvers = {
2205
- 'custom.normalizeCurrency': (value: string) => {
2206
- // Normalize currency codes to ISO 4217
2207
- const currencyMap: Record<string, string> = {
2208
- usd: 'USD',
2209
- us: 'USD',
2210
- dollar: 'USD',
2211
- eur: 'EUR',
2212
- euro: 'EUR',
2213
- gbp: 'GBP',
2214
- pound: 'GBP',
2215
- cad: 'CAD',
2216
- aud: 'AUD',
2217
- };
2218
-
2219
- const normalized = currencyMap[value.toLowerCase()] || value.toUpperCase();
2220
-
2221
- // Validate against known currencies
2222
- const validCurrencies = ['USD', 'EUR', 'GBP', 'CAD', 'AUD', 'JPY', 'CNY'];
2223
- if (!validCurrencies.includes(normalized)) {
2224
- throw new Error(`Invalid currency code: ${value}`);
2225
- }
2226
-
2227
- return normalized;
2228
- },
2229
-
2230
- 'custom.convertToBaseCurrency': (value: any, sourceRecord: any) => {
2231
- // Convert foreign currency to base currency
2232
- const amount = parseFloat(value);
2233
- const currency = sourceRecord.currency_code;
2234
-
2235
- // Exchange rates (in production, fetch from API)
2236
- const exchangeRates: Record<string, number> = {
2237
- USD: 1.0,
2238
- EUR: 0.85,
2239
- GBP: 0.73,
2240
- CAD: 1.35,
2241
- AUD: 1.45,
2242
- };
2243
-
2244
- const rate = exchangeRates[currency?.toUpperCase()] || 1.0;
2245
- return amount * rate;
2246
- },
2247
-
2248
- 'custom.buildPriceAttributes': (_: any, sourceRecord: any) => {
2249
- // Build price attributes array
2250
- const attributes: Array<{ name: string; type: string; value: any }> = [];
2251
-
2252
- if (sourceRecord.cost_basis) {
2253
- attributes.push({
2254
- name: 'costBasis',
2255
- type: 'Float',
2256
- value: parseFloat(sourceRecord.cost_basis),
2257
- });
2258
- }
2259
-
2260
- if (sourceRecord.margin_percent) {
2261
- attributes.push({
2262
- name: 'marginPercent',
2263
- type: 'Float',
2264
- value: parseFloat(sourceRecord.margin_percent),
2265
- });
2266
- }
2267
-
2268
- if (sourceRecord.competitor_price) {
2269
- attributes.push({
2270
- name: 'competitorPrice',
2271
- type: 'Float',
2272
- value: parseFloat(sourceRecord.competitor_price),
2273
- });
2274
- }
2275
-
2276
- if (sourceRecord.price_source) {
2277
- attributes.push({
2278
- name: 'priceSource',
2279
- type: 'String',
2280
- value: sourceRecord.price_source,
2281
- });
2282
- }
2283
-
2284
- return attributes;
2285
- },
2286
- };
2287
- ```
2288
-
2289
- ## Schema Validation (Before Deployment)
2290
-
2291
- Use SDK CLI tools to validate your GraphQL schema and mapping configuration:
2292
-
2293
- ```bash
2294
- # Install SDK globally (or use npx)
2295
- npm install -g @fluentcommerce/fc-connect-sdk
2296
-
2297
- # 1. Introspect Fluent GraphQL schema
2298
- fc-connect introspect-schema \
2299
- --url https://api.fluentcommerce.com/graphql \
2300
- --client-id YOUR_CLIENT_ID \
2301
- --client-secret YOUR_CLIENT_SECRET \
2302
- --output fluent-schema.json
2303
-
2304
- # 2. Create mutation file (price-mutation.graphql)
2305
- cat > price-mutation.graphql << 'EOF'
2306
- mutation UpdateProductPrice($input: UpdateProductInput!) {
2307
- updateProduct(input: $input) {
2308
- id
2309
- ref
2310
- prices {
2311
- type
2312
- value
2313
- currency
2314
- }
2315
- }
2316
- }
2317
- EOF
2318
-
2319
- # 3. Generate mapping from mutation (validates structure)
2320
- fc-connect generate-mutation-mapping \
2321
- --file price-mutation.graphql \
2322
- --output price-mapping.json
2323
-
2324
- # 4. Validate mapping against schema
2325
- fc-connect validate-schema \
2326
- --mapping price-mapping.json \
2327
- --schema fluent-schema.json
2328
-
2329
- # 5. Analyze field coverage
2330
- fc-connect analyze-coverage \
2331
- --mapping price-mapping.json \
2332
- --schema fluent-schema.json
2333
- ```
2334
-
2335
- **Output Example**:
2336
-
2337
- ```bash
2338
- ✓ Schema validation passed
2339
- ✓ All required fields present: ref, retailerId, prices[].type, prices[].value, prices[].currency
2340
- ✓ Mutation structure matches GraphQL schema
2341
- ⚠ Optional fields not mapped: prices[].taxType, prices[].taxRate
2342
- ```
2343
-
2344
- ## Testing the Workflow
2345
-
2346
- ### 1. Upload Test CSV to S3
2347
-
2348
- ```bash
2349
- # Using AWS CLI
2350
- aws s3 cp product-prices-test.csv s3://my-price-bucket/prices/
2351
-
2352
- # Or using S3 Console
2353
- # Navigate to bucket → prices/ → Upload
2354
- ```
2355
-
2356
- ### 2. Deploy to Versori
2357
-
2358
- ```bash
2359
- npm run deploy
2360
- ```
2361
-
2362
- ### 3. Manual Testing via Webhook
2363
-
2364
- ```bash
2365
- curl -X POST https://your-workspace.versori.run/sync-prices-now \
2366
- -H "Content-Type: application/json"
2367
- ```
2368
-
2369
- ### 4. Check Status
2370
-
2371
- ```bash
2372
- curl https://your-workspace.versori.run/check-status
2373
- ```
2374
-
2375
- ### 5. Monitor Logs
2376
-
2377
- ```bash
2378
- npm run logs
2379
- # Or via Versori dashboard
2380
- ```
2381
-
2382
- ---
2383
-
2384
- ## Monitoring
2385
-
2386
- ### Success Response
2387
-
2388
- ```json
2389
- {
2390
- "success": true,
2391
- "filesProcessed": 1,
2392
- "filesSkipped": 0,
2393
- "filesFailed": 0,
2394
- "totalRecords": 100,
2395
- "mutationsExecuted": 100,
2396
- "mutationsFailed": 0,
2397
- "results": [
2398
- {
2399
- "file": "prices_2025-01-22.csv",
2400
- "success": true,
2401
- "recordsProcessed": 100,
2402
- "mutationsExecuted": 100,
2403
- "mutationsFailed": 0
2404
- }
2405
- ],
2406
- "duration": 12345
2407
- }
2408
- ```
2409
-
2410
- ### Partial Success Response
2411
-
2412
- ```json
2413
- {
2414
- "success": true,
2415
- "filesProcessed": 1,
2416
- "filesSkipped": 0,
2417
- "filesFailed": 0,
2418
- "totalRecords": 100,
2419
- "mutationsExecuted": 95,
2420
- "mutationsFailed": 5,
2421
- "results": [
2422
- {
2423
- "file": "prices_2025-01-22.csv",
2424
- "success": true,
2425
- "recordsProcessed": 100,
2426
- "mutationsExecuted": 95,
2427
- "mutationsFailed": 5,
2428
- "errors": ["PRICE-001: Invalid price value", "PRICE-002: Missing currency"]
2429
- }
2430
- ],
2431
- "duration": 12345
2432
- }
2433
- ```
2434
-
2435
- ### Error Response
2436
-
2437
- ```json
2438
- {
2439
- "success": false,
2440
- "filesProcessed": 0,
2441
- "filesFailed": 1,
2442
- "totalRecords": 0,
2443
- "mutationsExecuted": 0,
2444
- "mutationsFailed": 0,
2445
- "results": [
2446
- {
2447
- "file": "prices_2025-01-22.csv",
2448
- "success": false,
2449
- "error": "CSV parse error: Invalid structure"
2450
- }
2451
- ],
2452
- "duration": 876
2453
- }
2454
- ```
2455
-
2456
- ### Monitoring Metrics
2457
-
2458
- Monitor these metrics via Versori logs filtered by `jobId` or `stage`:
2459
-
2460
- - **Files Processed** - Total files successfully processed
2461
- - **Mutations Executed** - Total GraphQL mutations executed successfully
2462
- - **Mutations Failed** - Mutations that failed (check error logs)
2463
- - **Processing Duration** - Time taken for complete workflow
2464
- - **Rate Limiting** - Watch for 429 errors indicating GraphQL throttling
2465
-
2466
- Use the status webhook for dashboards and automated monitoring.
2467
-
2468
- ---
2469
-
2470
- ### Issue 1: Product Not Found
2471
-
2472
- **Error**: `Skipping price update for non-existent product: PROD-XYZ`
2473
-
2474
- **Solution**:
2475
-
2476
- - Verify product exists in Fluent Commerce
2477
- - Check SKU matches exactly (case-sensitive)
2478
- - Ensure product status is `ACTIVE`
2479
- - Disable validation temporarily for testing: `validateProducts=false`
2480
-
2481
- ### Issue 2: Price Validation Failures
2482
-
2483
- **Error**: `Price 0.00 below minimum 0.01`
2484
-
2485
- **Solution**:
2486
-
2487
- ```bash
2488
- # Adjust validation rules in activation variables
2489
- minPrice=0.00 # Allow zero prices
2490
- maxPrice=9999999.99 # Increase max
2491
- ```
2492
-
2493
- Or fix source data:
2494
-
2495
- ```csv
2496
- # ❌ WRONG
2497
- PROD-001,DEFAULT,0,USD
2498
-
2499
- # ✅ CORRECT
2500
- PROD-001,DEFAULT,0.01,USD
2501
- ```
2502
-
2503
- ### Issue 3: Invalid Price Type
2504
-
2505
- **Error**: `Invalid price type: PROMO`
2506
-
2507
- **Solution**: Update custom resolver to include new type:
2508
-
2509
- ```typescript
2510
- 'custom.normalizePriceType': (value: any) => {
2511
- const type = String(value || 'DEFAULT').toUpperCase().trim();
2512
- const validTypes = [
2513
- 'DEFAULT',
2514
- 'SALE',
2515
- 'CLEARANCE',
2516
- 'BULK',
2517
- 'PROMOTIONAL',
2518
- 'MEMBER',
2519
- 'PROMO', // Add new type
2520
- ];
2521
- if (!validTypes.includes(type)) {
2522
- throw new Error(`Invalid price type: ${type}`);
2523
- }
2524
- return type;
2525
- };
2526
- ```
2527
-
2528
- ### Issue 4: Duplicate Price Updates
2529
-
2530
- **Symptom**: Same file processed multiple times
2531
-
2532
- **Solution**: VersoriKVAdapter already prevents this:
2533
-
2534
- ```typescript
2535
- const stateKey = ['processed-files', 's3-price-sync', fileName];
2536
- const existing = await kv.get(stateKey);
2537
- if (existing) {
2538
- log.info('Skipping already processed file', { fileName });
2539
- continue;
2540
- }
2541
- ```
2542
-
2543
- Verify KV storage is working:
2544
-
2545
- ```bash
2546
- # Check Versori logs for:
2547
- [INFO] Skipping already processed file: product-prices-20250122-001.csv
2548
- ```
2549
-
2550
- ### Issue 5: S3 Access Denied
2551
-
2552
- Required IAM Permissions:
2553
-
2554
- ```json
2555
- {
2556
- "Version": "2012-10-17",
2557
- "Statement": [
2558
- {
2559
- "Effect": "Allow",
2560
- "Action": ["s3:ListBucket", "s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
2561
- "Resource": [
2562
- "arn:aws:s3:::my-price-bucket",
2563
- "arn:aws:s3:::my-price-bucket/*"
2564
- ]
2565
- }
2566
- ]
2567
- }
2568
- ```
2569
-
2570
- ### Issue 6: GraphQL Mutation Failures
2571
-
2572
- **Common Causes**:
2573
-
2574
- - Invalid price type (check Fluent schema for valid types)
2575
- - Missing required fields (ref, retailerId, prices)
2576
- - Invalid currency code (must be ISO 4217: USD, EUR, GBP)
2577
- - Product not found or inactive
2578
-
2579
- **Debugging**:
2580
-
2581
- ```typescript
2582
- // Log mutation input before execution
2583
- log.info('GraphQL mutation input', { input });
2584
-
2585
- // Validate required fields
2586
- if (!priceData.productRef || !priceData.type || !priceData.value) {
2587
- log.error('Missing required fields', { priceData });
2588
- continue;
2589
- }
2590
- ```
2591
-
2592
- ## Production Checklist
2593
-
2594
- - [ ] S3 credentials validated with correct IAM permissions
2595
- - [ ] Activation secrets stored securely (not in code)
2596
- - [ ] GraphQL schema validated using CLI tools
2597
- - [ ] Mapping configuration uses GraphQLMutationMapper structure (NOT UniversalMapper)
2598
- - [ ] Mapping configuration tested with sample data
2599
- - [ ] NO setRetailerId() call in code (only for Job/Event API)
2600
- - [ ] Price validation rules configured per business policy
2601
- - [ ] Product existence validation enabled (`validateProducts=true`)
2602
- - [ ] File duplicate prevention working via KV state
2603
- - [ ] Concurrency control configured (mutationBatchSize)
2604
- - [ ] Optional alias batching tested if enabled (mutationsPerAliasBatch)
2605
- - [ ] Error handling tested with malformed CSV
2606
- - [ ] Retry logic tested with transient failures
2607
- - [ ] File archival working (processed and error directories)
2608
- - [ ] Price change tracking verified in logs
2609
- - [ ] Monitoring/alerting configured for price update failures
2610
- - [ ] Clear runbook for error recovery
2611
-
2612
- ## Related Guides
2613
-
2614
- - **GraphQL Mutation Mapping**: `docs/02-CORE-GUIDES/mapping/graphql-mutation-mapping/`
2615
- - **GraphQL Alias Batching**: `docs/02-CORE-GUIDES/mapping/graphql-alias-batching-guide.md`
2616
- - **retailerId Configuration**: `docs/00-START-HERE/retailerid-configuration.md`
2617
- - **CLI Tools**: `fc-connect-sdk/bin/readme.md`
2618
- - **State & KV patterns**: `docs/03-PATTERN-GUIDES/file-operations/`
2619
- - **Error handling**: `docs/03-PATTERN-GUIDES/error-handling/`
1
+ ---
2
+ template_id: tpl-ingest-s3-csv-to-price-graphql
3
+ canonical_filename: template-ingestion-s3-csv-price-graphql.md
4
+ version: 2.0.0
5
+ sdk_version: ^0.1.39
6
+ runtime: versori
7
+ direction: ingestion
8
+ source: s3-csv
9
+ destination: fluent-graphql
10
+ entity: price
11
+ format: csv
12
+ logging: versori
13
+ status: stable
14
+ features:
15
+ - graphql-mutation-mapper
16
+ - memory-management
17
+ - enhanced-logging
18
+ - attribute-transformation
19
+ compliance: gold-standard
20
+ ---
21
+
22
+ # Template: Ingestion - S3 CSV to Price GraphQL
23
+ n**Template Version:** 2.0.0
24
+ **SDK Version:** @fluentcommerce/fc-connect-sdk@^0.1.39
25
+ **Last Updated:** 2025-01-24
26
+
27
+ **🆕 Version 2.0.0 Enhancements:**
28
+ - ✅ **GraphQL Mutation Mapper** - Direct field mapping to mutation variables
29
+ - ✅ **Memory Management** - Clear large arrays after processing batches
30
+ - ✅ **Enhanced Logging** - Track mutation execution with emoji indicators
31
+ - ✅ **Attribute Transformation** - Handle complex nested data structures
32
+
33
+ ## STEP 1: What This Template Does
34
+
35
+ This template provides a **scheduled Versori workflow** that:
36
+
37
+ 1. **Discovers CSV files** from S3 bucket with price data
38
+ 2. **Parses CSV** using `CSVParserService` with validation
39
+ 3. **Validates products exist** in Fluent Commerce before updating prices
40
+ 4. **Maps fields** using `GraphQLMutationMapper` with custom price validation resolvers
41
+ 5. **Updates prices** via direct GraphQL mutations (`updateProduct`)
42
+ 6. **Tracks price changes** for audit trail
43
+ 7. **Archives processed files** to prevent duplicates
44
+ 8. **Uses JobTracker** for job lifecycle management
45
+
46
+ **Key Features:**
47
+ - ✅ Direct GraphQL mutations (NOT Batch API)
48
+ - ✅ Product existence validation before price updates
49
+ - ✅ Price range validation (min/max checks)
50
+ - ✅ Multi-tier pricing support (DEFAULT, SALE, CLEARANCE, BULK)
51
+ - ✅ Configurable concurrency control (sequential or parallel)
52
+ - ✅ Optional GraphQL alias batching for high-volume scenarios
53
+ - ✅ Price change tracking and reporting
54
+ - ✅ Versori KV state management (duplicate prevention)
55
+ - ✅ File archival with error handling
56
+
57
+ **Use Cases:**
58
+ - Daily product price synchronization
59
+ - Promotional price updates
60
+ - Multi-currency pricing management
61
+ - Quantity-based pricing tiers
62
+
63
+ ## STEP 2: Understanding This Template (AI Agent Guide)
64
+
65
+ **🎯 Template Purpose:** Scheduled price synchronization from S3 CSV files to Fluent Commerce using direct GraphQL mutations.
66
+
67
+ ### Architecture Pattern
68
+
69
+ ```
70
+ S3 CSV Files → CSVParserService → GraphQLMutationMapper → Product Validation → GraphQL Mutations → Archive
71
+ ↓ ↓ ↓ ↓ ↓ ↓
72
+ File Discovery Parsing Field Mapping Existence Check updateProduct Processed/
73
+ Price Data Validation Price Validation Active Status Concurrency Ctrl Error Dirs
74
+ ```
75
+
76
+ ### SDK Services Used
77
+
78
+ | Service | Purpose | Key Methods |
79
+ |---------|---------|-------------|
80
+ | `createClient(ctx)` | Fluent API client | `client.graphql()` |
81
+ | `S3DataSource` | S3 operations | `listFiles()`, `downloadFile()`, `moveFile()` |
82
+ | `CSVParserService` | CSV parsing | `parse()` |
83
+ | `GraphQLMutationMapper` | Field transformation | `map()` with custom resolvers |
84
+ | `VersoriKVAdapter` | State management | `get()`, `set()`, `list()` |
85
+ | `JobTracker` | Job lifecycle | `createJob()`, `updateJob()`, `markCompleted()`, `markFailed()`, `getJob()` |
86
+
87
+ ### Price Entity Structure
88
+
89
+ **Fluent Commerce Price Schema:**
90
+ ```typescript
91
+ {
92
+ productRef: string; // SKU reference (required)
93
+ type: string; // DEFAULT, SALE, CLEARANCE, BULK (required)
94
+ value: number; // Price amount (required, validated)
95
+ currency: string; // ISO 4217 code (USD, EUR, GBP)
96
+ effectiveFrom?: string; // ISO 8601 date
97
+ effectiveTo?: string; // ISO 8601 date
98
+ minQuantity?: number; // Min qty for tier pricing
99
+ maxQuantity?: number; // Max qty for tier pricing
100
+ }
101
+ ```
102
+
103
+ ### GraphQL Mutation Pattern
104
+
105
+ **Mutation Used:** `updateProduct` (NOT Batch API)
106
+
107
+ ```graphql
108
+ mutation UpdateProductPrice($input: UpdateProductInput!) {
109
+ updateProduct(input: $input) {
110
+ id
111
+ ref
112
+ prices {
113
+ type
114
+ value
115
+ currency
116
+ effectiveFrom
117
+ effectiveTo
118
+ minQuantity
119
+ maxQuantity
120
+ }
121
+ }
122
+ }
123
+ ```
124
+
125
+ **Why Direct Mutations?**
126
+ - ✅ Immediate price updates (no batch processing delay)
127
+ - ✅ Product validation before update
128
+ - ✅ Price change tracking
129
+ - ✅ Suitable for low-volume price updates (<1000/day)
130
+
131
+ **NO BPP (Batch Pre-Processing):** Direct GraphQL mutations bypass batch workflows entirely.
132
+
133
+ ### Custom Resolvers
134
+
135
+ **Three price-specific resolvers:**
136
+
137
+ 1. **`custom.validatePriceRange`** - Ensures price within min/max bounds
138
+ 2. **`custom.normalizePriceType`** - Validates price tier types
139
+ 3. **`custom.normalizeQuantity`** - Validates quantity ranges for BULK pricing
140
+
141
+ ### Product Validation Pattern
142
+
143
+ **Critical Step:** Validate product exists and is ACTIVE before price update:
144
+
145
+ ```typescript
146
+ async function validateProductExists(client, productRef, retailerId, log) {
147
+ const query = `
148
+ query GetProduct($ref: String!, $retailerId: ID!) {
149
+ products(first: 1, ref: [$ref], retailerId: $retailerId) {
150
+ edges {
151
+ node { id ref status }
152
+ }
153
+ }
154
+ }
155
+ `;
156
+ const result = await client.graphql({ query, variables: { ref: productRef, retailerId } });
157
+ const product = result?.data?.products?.edges?.[0]?.node;
158
+ return product && product.status === 'ACTIVE';
159
+ }
160
+ ```
161
+
162
+ **Why?** Prevents GraphQL errors for non-existent products.
163
+
164
+ ### Concurrency Control & Performance
165
+
166
+ **Mutation execution** supports two performance modes:
167
+
168
+ **Mode 1: Concurrency Control (default)**
169
+ ```typescript
170
+ // Configuration: Control concurrent mutations
171
+ const mutationBatchSize = parseInt(
172
+ activation?.getVariable('mutationBatchSize') || '1', // Default: 1 (sequential)
173
+ 10
174
+ );
175
+
176
+ // Execute with bounded concurrency
177
+ await executeMutations(
178
+ client,
179
+ mapper,
180
+ priceRecords,
181
+ retailerId,
182
+ validateProducts,
183
+ mutationBatchSize, // 1=sequential, 3-10=parallel
184
+ undefined, // No alias batching
185
+ log
186
+ );
187
+ ```
188
+
189
+ **Mode 2: GraphQL Alias Batching (optional, high-volume)**
190
+ ```typescript
191
+ // Configuration: Group mutations into aliased requests
192
+ const mutationsPerAliasBatch = activation?.getVariable('mutationsPerAliasBatch')
193
+ ? parseInt(activation?.getVariable('mutationsPerAliasBatch') || '1', 10)
194
+ : undefined; // Default: undefined (disabled)
195
+
196
+ // Execute with alias batching (reduces network overhead by ~80%)
197
+ await executeMutations(
198
+ client,
199
+ mapper,
200
+ priceRecords,
201
+ retailerId,
202
+ validateProducts,
203
+ mutationBatchSize, // Concurrency control
204
+ mutationsPerAliasBatch, // Alias batching (e.g., 5)
205
+ log
206
+ );
207
+ ```
208
+
209
+ **Activation Variables:**
210
+ - `mutationBatchSize=1` - Default: sequential (safe)
211
+ - `mutationBatchSize=3-5` - Balanced throughput
212
+ - `mutationBatchSize=10` - High-volume (100+ locations)
213
+ - `mutationsPerAliasBatch` - Optional: Group mutations (e.g., 5 per request)
214
+
215
+ ### State Management
216
+
217
+ **VersoriKVAdapter** prevents duplicate file processing:
218
+
219
+ ```typescript
220
+ const stateKey = ['processed-files', 's3-price-sync', fileName];
221
+ const existing = await kv.get(stateKey);
222
+ if (existing) {
223
+ log.info('Skipping already processed file', { fileName });
224
+ continue;
225
+ }
226
+ // After success:
227
+ await kv.set(stateKey, { successful, failed, processedAt: new Date().toISOString() });
228
+ ```
229
+
230
+ ### JobTracker Integration
231
+
232
+ **Job lifecycle tracking:**
233
+
234
+ ```typescript
235
+ const tracker = new JobTracker(openKv(':project:'), log);
236
+ await tracker.startJob(jobId, { mode: 'scheduled' });
237
+ try {
238
+ const result = await runPriceCsvWorkflow(ctx, jobId, tracker);
239
+ await tracker.completeJob(jobId, { result });
240
+ } catch (e) {
241
+ await tracker.failJob(jobId, { error: e.message });
242
+ }
243
+ ```
244
+
245
+ ### Price Change Tracking
246
+
247
+ **Audit trail for price updates:**
248
+
249
+ ```typescript
250
+ const currentPrice = await getCurrentPrice(client, productRef, priceType, currency, retailerId);
251
+ // ... after update ...
252
+ if (currentPrice !== undefined && currentPrice !== priceData.value) {
253
+ results.priceChanges.push({
254
+ sku: priceData.productRef,
255
+ type: priceData.type,
256
+ oldPrice: currentPrice,
257
+ newPrice: priceData.value,
258
+ });
259
+ }
260
+ // Log changes at end:
261
+ log.info('Price changes detected', { count: results.priceChanges.length, changes });
262
+ ```
263
+
264
+ ### Schema Validation (CLI Commands)
265
+
266
+ **Before deployment, validate GraphQL schema:**
267
+
268
+ ```bash
269
+ # 1. Introspect Fluent schema
270
+ fc-connect introspect-schema \
271
+ --url https://api.fluentcommerce.com/graphql \
272
+ --client-id CLIENT_ID \
273
+ --client-secret CLIENT_SECRET \
274
+ --output fluent-schema.json
275
+
276
+ # 2. Create mutation file
277
+ cat > price-mutation.graphql << 'EOF'
278
+ mutation UpdateProductPrice($input: UpdateProductInput!) {
279
+ updateProduct(input: $input) {
280
+ id
281
+ ref
282
+ prices { type value currency }
283
+ }
284
+ }
285
+ EOF
286
+
287
+ # 3. Generate mapping from mutation
288
+ fc-connect generate-mutation-mapping \
289
+ --file price-mutation.graphql \
290
+ --output price-mapping.json
291
+
292
+ # 4. Validate mapping against schema
293
+ fc-connect validate-schema \
294
+ --mapping price-mapping.json \
295
+ --schema fluent-schema.json
296
+
297
+ # 5. Analyze field coverage
298
+ fc-connect analyze-coverage \
299
+ --mapping price-mapping.json \
300
+ --schema fluent-schema.json
301
+ ```
302
+
303
+ ### Configuration File Structure
304
+
305
+ **Mapping Config:** `config/price-mapping.json`
306
+
307
+ **✅ PRODUCTION STANDARD:** External JSON file with GraphQLMutationMapper structure
308
+
309
+ ```json
310
+ {
311
+ "mutation": "updateProduct",
312
+ "sourceFormat": "csv",
313
+ "version": "1.0.0",
314
+ "arguments": {
315
+ "input": {
316
+ "ref": {
317
+ "source": "sku",
318
+ "required": true,
319
+ "resolver": "trim"
320
+ },
321
+ "prices": {
322
+ "type": {
323
+ "source": "priceType",
324
+ "required": true,
325
+ "resolver": "normalizePriceType"
326
+ },
327
+ "value": {
328
+ "source": "amount",
329
+ "required": true,
330
+ "resolver": "validatePriceRange"
331
+ },
332
+ "currency": {
333
+ "source": "currency",
334
+ "required": true,
335
+ "resolver": "toUpperCase"
336
+ },
337
+ "effectiveFrom": {
338
+ "source": "effectiveDate",
339
+ "resolver": "formatDate"
340
+ },
341
+ "effectiveTo": {
342
+ "source": "expiryDate",
343
+ "resolver": "formatDate"
344
+ },
345
+ "minQuantity": {
346
+ "source": "minQuantity",
347
+ "resolver": "normalizeQuantity"
348
+ },
349
+ "maxQuantity": {
350
+ "source": "maxQuantity",
351
+ "resolver": "normalizeQuantity"
352
+ }
353
+ }
354
+ }
355
+ }
356
+ }
357
+ ```
358
+
359
+ **Key Differences from UniversalMapper:**
360
+ - ✅ `mutation` property (not `mutationName`)
361
+ - ✅ `arguments.input` structure (matches GraphQL schema)
362
+ - ✅ Nested `prices` object (not flat productRef/type/value)
363
+ - ✅ Resolver names without `sdk.` or `custom.` prefix (`trim`, `toUpperCase`, `formatDate`)
364
+ - ✅ Used with `GraphQLMutationMapper.map()` method (not `UniversalMapper.map()`)
365
+
366
+ ### Activation Variables Required
367
+
368
+ **Minimum Required:**
369
+ - `s3BucketName` - S3 bucket with price CSV files
370
+ - `awsAccessKeyId` - AWS credentials
371
+ - `awsSecretAccessKey` - AWS credentials
372
+
373
+ **Optional (with defaults):**
374
+ - `awsRegion=us-east-1`
375
+ - `s3Prefix=prices/`
376
+ - `archivePrefix=processed/`
377
+ - `errorPrefix=errors/`
378
+ - `logPrefix=logs/` - Where to write mutation logs
379
+ - `maxFilesToProcess=10`
380
+ - `mutationBatchSize=1` - Number of concurrent mutations (1=sequential, 3-10=parallel)
381
+ - `mutationsPerAliasBatch` - Optional: Number of mutations per aliased request (default: undefined = disabled)
382
+ - `minPrice=0.01`
383
+ - `maxPrice=999999.99`
384
+ - `validateProducts=true`
385
+ - `validateConnection=true` - Validate S3 connection at startup
386
+ - `enableFileTracking=true` - Enable file tracking via VersoriFileTracker
387
+ - `fluentRetailerId` - Optional: Only if mutation schema requires retailerId in input (most Price mutations don't need it)
388
+
389
+ ### Error Handling Patterns
390
+
391
+ **Three-level error handling:**
392
+
393
+ 1. **Mapping errors** - Invalid price data (validation failures)
394
+ 2. **GraphQL errors** - Mutation failures (product not found, invalid fields)
395
+ 3. **File processing errors** - S3 access issues, CSV parsing failures
396
+
397
+ **Error state tracking with exponential backoff:**
398
+ ```typescript
399
+ const errorKey = ['error-state', fileName];
400
+ const attempts = (prev?.attemptCount || 0) + 1;
401
+ const backoffMinutes = Math.min(Math.pow(2, attempts) * 5, 24 * 60);
402
+ const nextRetryAt = new Date(Date.now() + backoffMinutes * 60000).toISOString();
403
+ ```
404
+
405
+ ### Common Pitfalls
406
+
407
+ **❌ WRONG Patterns:**
408
+ - Using Batch API for price updates (overhead, delay)
409
+ - Using `UniversalMapper` instead of `GraphQLMutationMapper`
410
+ - Calling `client.setRetailerId()` for GraphQL mutations (only needed for Job/Event API)
411
+ - Skipping product validation (causes GraphQL errors)
412
+ - No concurrency control (API throttling)
413
+ - Hardcoded price validation rules (not configurable)
414
+ - Missing price change tracking (no audit trail)
415
+
416
+ **✅ CORRECT Patterns:**
417
+ - Direct GraphQL mutations for immediate updates
418
+ - `GraphQLMutationMapper` with nested mapping structure
419
+ - NO `setRetailerId()` call (pass in mutation input if schema requires it)
420
+ - Product existence validation before updates
421
+ - Configurable concurrency control (mutationBatchSize)
422
+ - Optional alias batching for high-volume scenarios
423
+ - Custom resolvers for price validation
424
+ - Price change tracking for audit
425
+ - VersoriKVAdapter for state management
426
+ - JobTracker for job lifecycle
427
+
428
+ ### Testing Checklist
429
+
430
+ **Before deployment:**
431
+ - [ ] S3 credentials validated (IAM permissions: ListBucket, GetObject, PutObject, DeleteObject)
432
+ - [ ] GraphQL schema validated using CLI tools
433
+ - [ ] Mapping configuration uses GraphQLMutationMapper structure (nested objects, not flat fields)
434
+ - [ ] Mapping configuration tested with sample CSV
435
+ - [ ] Product validation working (queries products endpoint)
436
+ - [ ] Price validation rules configured (min/max bounds)
437
+ - [ ] Concurrency control configured (mutationBatchSize)
438
+ - [ ] Optional alias batching tested if enabled (mutationsPerAliasBatch)
439
+ - [ ] NO setRetailerId() call present (only for Job/Event API)
440
+ - [ ] File duplicate prevention working (KV state)
441
+ - [ ] Price change tracking verified (logs price changes)
442
+ - [ ] Error handling tested (malformed CSV, invalid products)
443
+ - [ ] File archival working (processed/ and errors/ directories)
444
+
445
+ ### Related Documentation
446
+
447
+ - **Core Guides:** `docs/02-CORE-GUIDES/ingestion/modules/`
448
+ - **Universal Mapping:** `docs/02-CORE-GUIDES/mapping/modules/`
449
+ - **CLI Tools:** `fc-connect-sdk/bin/readme.md`
450
+ - **State Management:** `docs/03-PATTERN-GUIDES/file-operations/`
451
+ - **Error Handling:** `docs/03-PATTERN-GUIDES/error-handling/`
452
+
453
+ **FC Connect SDK Use Case Guide**
454
+
455
+ > **SDK**: [@fluentcommerce/fc-connect-sdk](https://www.npmjs.com/package/@fluentcommerce/fc-connect-sdk)
456
+ > **Version**: Use latest - `npm install @fluentcommerce/fc-connect-sdk@latest`
457
+
458
+ **Context**: Scheduled Versori workflow that reads product price CSV files from S3 and creates/updates Fluent Commerce product prices via GraphQL mutations
459
+
460
+ **Complexity**: Medium-High
461
+
462
+ **Runtime**: Versori Platform (Scheduled)
463
+
464
+ **Estimated Lines**: ~700 lines
465
+
466
+ ## What You'll Build
467
+
468
+ - Versori scheduled workflow (cron trigger)
469
+ - S3 file listing and download with retry logic
470
+ - CSV parsing with validation
471
+ - UniversalMapper for price field transformations
472
+ - Product existence validation before price updates
473
+ - GraphQL mutations for price upserts with rate limiting
474
+ - Versori KV state management (duplicate prevention)
475
+ - Price change tracking and reporting
476
+ - File archival after processing
477
+
478
+ ## Versori Workflows Structure
479
+
480
+ **Key Concept**: Versori workflows are organized by **trigger type** at the first level, then by **specific workflow** with descriptive file names.
481
+
482
+ **Trigger Types:**
483
+ - **`schedule()`** → Time-based triggers (cron expressions) - NOT exposed as HTTP endpoints
484
+ - **`webhook()`** → HTTP-based triggers (event-driven) - Creates HTTP endpoints
485
+ - **`workflow()`** → Durable workflows (advanced, rarely used)
486
+
487
+ **Execution Steps (chained to triggers):**
488
+ - **`http()`** → External API calls (chained from schedule/webhook)
489
+ - **`fn()`** → Internal processing (chained from schedule/webhook)
490
+
491
+ ### Recommended Project Structure
492
+
493
+ ```
494
+ s3-csv-price-graphql/
495
+ ├── index.ts # Entry point - exports all workflows
496
+ └── src/
497
+ ├── workflows/
498
+ │ ├── scheduled/
499
+ │ │ └── daily-price-sync.ts # Scheduled: Daily price sync
500
+ │ │
501
+ │ └── webhook/
502
+ │ ├── adhoc-price-sync.ts # Webhook: Manual trigger
503
+ │ └── job-status-check.ts # Webhook: Status query
504
+
505
+ ├── services/
506
+ │ └── price-sync.service.ts # Shared orchestration logic (reusable)
507
+
508
+ └── config/
509
+ └── price-mapping.json # GraphQL mapping config
510
+ ```
511
+
512
+ ---
513
+
514
+ ## Workflow Files
515
+
516
+ ### 1. Scheduled Workflows (`src/workflows/scheduled/`)
517
+
518
+ All time-based triggers that run automatically on cron schedules.
519
+
520
+ #### `src/workflows/scheduled/daily-price-sync.ts`
521
+
522
+ **Purpose**: Automatic daily price sync
523
+ **Trigger**: Cron schedule (`0 2 * * *`)
524
+ **Exposed as Endpoint**: ❌ NO - Runs automatically
525
+
526
+ ```typescript
527
+ import { schedule, http } from '@versori/run';
528
+ import { JobTracker } from '@fluentcommerce/fc-connect-sdk';
529
+ import { executePriceSync } from '../../services/price-sync.service';
530
+
531
+ /**
532
+ * Scheduled Workflow: Daily Price Sync
533
+ *
534
+ * Runs automatically daily at 2 AM UTC
535
+ * NOT exposed as HTTP endpoint - Versori executes on schedule
536
+ *
537
+ * Uses shared service: price-sync.service.ts
538
+ */
539
+ export const dailyPriceSync = schedule(
540
+ 'price-sync-scheduled',
541
+ '0 2 * * *' // Daily at 2 AM UTC
542
+ ).then(
543
+ http('run-price-sync', { connection: 'fluent_commerce' }, async (ctx: any) => {
544
+ const { log, openKv } = ctx;
545
+ const jobId = `price-sync-${Date.now()}`;
546
+ const executionStartTime = Date.now();
547
+ const tracker = new JobTracker(openKv(':project:'), log);
548
+
549
+ log.info('🔄 [WORKFLOW] Starting price sync', { jobId });
550
+
551
+ await tracker.createJob(jobId, {
552
+ triggeredBy: 'schedule',
553
+ stage: 'initialization',
554
+ startTime: executionStartTime,
555
+ });
556
+ await tracker.updateJob(jobId, { status: 'processing' });
557
+
558
+ try {
559
+ const result = await executePriceSync(ctx, jobId, tracker);
560
+
561
+ if (result.success) {
562
+ await tracker.markCompleted(jobId, result);
563
+ log.info('✅ [WORKFLOW] Price sync completed', {
564
+ jobId,
565
+ filesProcessed: result.filesProcessed,
566
+ duration: result.duration,
567
+ });
568
+ } else {
569
+ await tracker.markFailed(jobId, result.error || 'Unknown error');
570
+ log.error('❌ [WORKFLOW] Price sync failed', { jobId, error: result.error });
571
+ }
572
+
573
+ return { success: true, jobId, ...result };
574
+ } catch (e: any) {
575
+ const errorMessage = e instanceof Error ? e.message : String(e);
576
+ await tracker.markFailed(jobId, errorMessage);
577
+ log.error('❌ [WORKFLOW] Price sync failed with exception', {
578
+ jobId,
579
+ error: errorMessage,
580
+ stack: e instanceof Error ? e.stack : undefined,
581
+ });
582
+ return { success: false, jobId, error: errorMessage };
583
+ }
584
+ })
585
+ );
586
+ ```
587
+
588
+ ---
589
+
590
+ ### 2. Webhook Workflows (`src/workflows/webhook/`)
591
+
592
+ All HTTP-based triggers that create webhook endpoints.
593
+
594
+ #### `src/workflows/webhook/adhoc-price-sync.ts`
595
+
596
+ **Purpose**: Manual price sync trigger (on-demand)
597
+ **Trigger**: HTTP POST
598
+ **Endpoint**: `POST https://{workspace}.versori.run/price-sync-adhoc`
599
+ **Use Cases**: Testing, priority processing, ad-hoc runs
600
+
601
+ ```typescript
602
+ import { webhook, http } from '@versori/run';
603
+ import { JobTracker } from '@fluentcommerce/fc-connect-sdk';
604
+ import { executePriceSync } from '../../services/price-sync.service';
605
+
606
+ /**
607
+ * Webhook: Manual Price Sync Trigger
608
+ *
609
+ * Endpoint: POST https://{workspace}.versori.run/price-sync-adhoc
610
+ * Request body (optional): { filePattern: "urgent_*.csv", maxFiles: 5 }
611
+ *
612
+ * Pattern: webhook().then(http()) - needs Fluent API access
613
+ * Uses shared service: price-sync.service.ts
614
+ *
615
+ * SECURITY: Authentication handled via connection parameter
616
+ * No manual API key validation needed - Versori manages this via connection auth
617
+ */
618
+ export const adhocPriceSync = webhook('price-sync-adhoc', {
619
+ response: { mode: 'sync' },
620
+ connection: 'price-sync-adhoc', // Versori validates API key
621
+ }).then(
622
+ http('run-price-sync-adhoc', { connection: 'fluent_commerce' }, async (ctx: any) => {
623
+ const { log, openKv, data } = ctx;
624
+ const jobId = `price-sync-adhoc-${Date.now()}`;
625
+ const executionStartTime = Date.now();
626
+ const tracker = new JobTracker(openKv(':project:'), log);
627
+
628
+ log.info('🔄 [WEBHOOK] Starting adhoc price sync', { jobId });
629
+
630
+ await tracker.createJob(jobId, {
631
+ triggeredBy: 'manual',
632
+ stage: 'initialization',
633
+ startTime: executionStartTime,
634
+ options: data // Optional: filePattern, maxFiles, etc.
635
+ });
636
+ await tracker.updateJob(jobId, { status: 'processing' });
637
+
638
+ try {
639
+ const result = await executePriceSync(ctx, jobId, tracker);
640
+
641
+ if (result.success) {
642
+ await tracker.markCompleted(jobId, result);
643
+ log.info('✅ [WEBHOOK] Adhoc price sync completed', {
644
+ jobId,
645
+ filesProcessed: result.filesProcessed,
646
+ duration: result.duration,
647
+ });
648
+ } else {
649
+ await tracker.markFailed(jobId, result.error || 'Unknown error');
650
+ log.error('❌ [WEBHOOK] Adhoc price sync failed', { jobId, error: result.error });
651
+ }
652
+
653
+ return { success: true, jobId, ...result };
654
+ } catch (e: any) {
655
+ const errorMessage = e instanceof Error ? e.message : String(e);
656
+ await tracker.markFailed(jobId, errorMessage);
657
+ log.error('❌ [WEBHOOK] Adhoc price sync failed with exception', {
658
+ jobId,
659
+ error: errorMessage,
660
+ stack: e instanceof Error ? e.stack : undefined,
661
+ });
662
+ return { success: false, jobId, error: errorMessage };
663
+ }
664
+ })
665
+ );
666
+ ```
667
+
668
+ ---
669
+
670
+ #### `src/workflows/webhook/job-status-check.ts`
671
+
672
+ **Purpose**: Query job status
673
+ **Trigger**: HTTP POST
674
+ **Endpoint**: `POST https://{workspace}.versori.run/price-sync-job-status`
675
+ **Request body**: `{ "jobId": "price-sync-1234567890" }`
676
+
677
+ ```typescript
678
+ import { webhook, fn } from '@versori/run';
679
+ import { JobTracker } from '@fluentcommerce/fc-connect-sdk';
680
+
681
+ /**
682
+ * Webhook: Job Status Check
683
+ *
684
+ * Endpoint: POST https://{workspace}.versori.run/price-sync-job-status
685
+ * Request body: { "jobId": "price-sync-1234567890" }
686
+ *
687
+ * Pattern: webhook().then(fn()) - no external API needed, only KV storage
688
+ * Lightweight: Only queries KV store, no Fluent API calls
689
+ *
690
+ * SECURITY: Authentication handled via connection parameter
691
+ * No manual API key validation needed - Versori manages this via connection auth
692
+ */
693
+ export const priceSyncJobStatus = webhook('price-sync-job-status', {
694
+ response: { mode: 'sync' },
695
+ connection: 'price-sync-job-status',
696
+ }).then(
697
+ fn('status', async (ctx: any) => {
698
+ const { data, log, openKv } = ctx;
699
+ const jobId = data?.jobId as string;
700
+
701
+ log.info('🔍 [STATUS] Checking job status', { jobId });
702
+
703
+ if (!jobId) {
704
+ log.warn('❌ [STATUS] Missing jobId in request');
705
+ return { success: false, error: 'jobId required' };
706
+ }
707
+
708
+ const tracker = new JobTracker(openKv(':project:'), log);
709
+ const status = await tracker.getJob(jobId);
710
+
711
+ if (status) {
712
+ log.info('✅ [STATUS] Job found', { jobId, status: status.status });
713
+ return { success: true, jobId, ...status };
714
+ } else {
715
+ log.warn('❌ [STATUS] Job not found', { jobId });
716
+ return { success: false, error: 'Job not found', jobId };
717
+ }
718
+ })
719
+ );
720
+ ```
721
+
722
+ ---
723
+
724
+ ### 3. Entry Point (`index.ts`)
725
+
726
+ **Purpose**: Register all workflows with Versori platform
727
+
728
+ ```typescript
729
+ /**
730
+ * Entry Point - Registers all workflows with Versori platform
731
+ *
732
+ * Versori automatically discovers and registers exported workflows
733
+ *
734
+ * File Structure:
735
+ * - src/workflows/scheduled/ → Time-based triggers (cron)
736
+ * - src/workflows/webhook/ → HTTP-based triggers (webhooks)
737
+ */
738
+
739
+ // Import scheduled workflows
740
+ import { dailyPriceSync } from './workflows/scheduled/daily-price-sync';
741
+
742
+ // Import webhook workflows
743
+ import { adhocPriceSync } from './workflows/webhook/adhoc-price-sync';
744
+ import { priceSyncJobStatus } from './workflows/webhook/job-status-check';
745
+
746
+ // Register all workflows
747
+ export {
748
+ // Scheduled (time-based triggers)
749
+ dailyPriceSync,
750
+
751
+ // Webhooks (HTTP-based triggers)
752
+ adhocPriceSync,
753
+ priceSyncJobStatus,
754
+ };
755
+ ```
756
+
757
+ **What Gets Exposed:**
758
+ - ✅ `adhocPriceSync` → `https://{workspace}.versori.run/price-sync-adhoc`
759
+ - ✅ `priceSyncJobStatus` → `https://{workspace}.versori.run/price-sync-job-status`
760
+ - ❌ `dailyPriceSync` → NOT exposed (runs automatically on cron)
761
+
762
+ ---
763
+
764
+ ## SDK Methods Used
765
+
766
+ **Core Services:**
767
+ - `createClient(ctx)` - Create Fluent client (auto-detects Versori context)
768
+ - `S3DataSource(config, log)` - S3 operations (list, download, upload, move)
769
+ - `CSVParserService()` - CSV parsing with validation
770
+ - `GraphQLMutationMapper(mappingConfig, log, { fluentClient: client })` - Field mapping with schema validation
771
+ - `VersoriKVAdapter(openKv(':project:'))` - KV state management for duplicate prevention
772
+
773
+ **Key Methods:**
774
+ - `s3.listFiles({ prefix, maxKeys })` - List S3 files
775
+ - `s3.downloadFile(path, options)` - Download file content
776
+ - `s3.uploadFile(path, content)` - Upload file (accepts string or Buffer)
777
+ - `s3.moveFile(source, dest)` - Archive or move file
778
+ - `parser.parse(content, options)` - Parse CSV to records
779
+ - `mapper.map(record)` - Transform single record
780
+ - `client.graphql({ query, variables })` - Execute GraphQL mutations/queries
781
+ - `kv.get(key)` / `kv.set(key, value)` - State management
782
+
783
+ **Critical Imports:**
784
+ - `Buffer` - **Required for Versori/Deno runtime** (S3 upload operations)
785
+ ```typescript
786
+ import { Buffer } from 'node:buffer';
787
+ ```
788
+ **Why:** Deno runtime (used by Versori) does not have `Buffer` as a global. However, `uploadFile()` accepts string or Buffer, so you can use strings directly without Buffer conversion.
789
+
790
+ **Service Functions (User-Defined):**
791
+ - `processFile()` - S3 download + CSV parsing + field mapping
792
+ - `executeMutations()` - GraphQL price updates with validation + rate limiting
793
+ - `writeMutationLog()` - Write execution log to S3 (uses Buffer)
794
+
795
+ ## Sample CSV Input Data
796
+
797
+ **File**: `product-prices-20250122-001.csv`
798
+
799
+ ```csv
800
+ sku,priceType,amount,currency,effectiveDate,expiryDate,minQuantity,maxQuantity
801
+ PROD-001,DEFAULT,29.99,USD,2025-01-22T00:00:00Z,,1,
802
+ PROD-002,SALE,19.99,USD,2025-01-22T00:00:00Z,2025-02-15T23:59:59Z,1,
803
+ PROD-003,CLEARANCE,9.99,USD,2025-01-22T00:00:00Z,2025-01-31T23:59:59Z,1,
804
+ PROD-004,DEFAULT,149.99,USD,2025-01-22T00:00:00Z,,1,
805
+ PROD-004,BULK,129.99,USD,2025-01-22T00:00:00Z,,10,99
806
+ PROD-004,BULK,119.99,USD,2025-01-22T00:00:00Z,,100,
807
+ ```
808
+
809
+ **Note**: Products can have multiple price tiers. Same SKU with different `priceType` or quantity ranges = multiple rows.
810
+
811
+ **Field Mapping**:
812
+
813
+ - `sku` → Product reference (required) - must exist in Fluent
814
+ - `priceType` → Price tier (DEFAULT, SALE, CLEARANCE, BULK, etc.) (required)
815
+ - `amount` → Price value (required, must be > 0)
816
+ - `currency` → Currency code (USD, EUR, GBP, etc.) (required)
817
+ - `effectiveDate` → When price becomes active (ISO 8601 format)
818
+ - `expiryDate` → When price expires (optional, ISO 8601 format)
819
+ - `minQuantity` → Minimum quantity for this price tier (optional, default: 1)
820
+ - `maxQuantity` → Maximum quantity for this price tier (optional)
821
+
822
+ **Price Tier Logic**:
823
+
824
+ - **DEFAULT**: Standard retail price (most products)
825
+ - **SALE**: Temporary promotional price
826
+ - **CLEARANCE**: Final markdown price
827
+ - **BULK**: Quantity-based pricing (requires minQuantity/maxQuantity)
828
+
829
+ ## Project Setup
830
+
831
+ ```bash
832
+ mkdir versori-s3-csv-price-sync && cd $_
833
+ npm init -y
834
+ npm install @fluentcommerce/fc-connect-sdk@latest @versori/run
835
+ mkdir -p src
836
+ ```
837
+
838
+ ### Package Configuration (package.json)
839
+
840
+ ```json
841
+ {
842
+ "name": "versori-s3-csv-price-sync",
843
+ "version": "1.0.0",
844
+ "description": "Versori workflow: S3 CSV price sync to Fluent GraphQL",
845
+ "versori": {
846
+ "workflows": "./src/index.ts"
847
+ },
848
+ "type": "module",
849
+ "scripts": {
850
+ "deploy": "versori deploy",
851
+ "logs": "versori logs"
852
+ },
853
+ "dependencies": {
854
+ "@fluentcommerce/fc-connect-sdk": "^0.1.39",
855
+ "@versori/run": "latest"
856
+ },
857
+ "devDependencies": {
858
+ "typescript": "^5.0.0",
859
+ "@types/node": "^20.0.0"
860
+ }
861
+ }
862
+ ```
863
+
864
+ ### Activation Variables (Versori)
865
+
866
+ ```bash
867
+ # Required Variables
868
+ s3BucketName=my-price-bucket
869
+ awsAccessKeyId=AKIAXXXXXXXXXXXX
870
+ awsSecretAccessKey=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
871
+ retailerId=your-retailer-id
872
+
873
+ # Optional Variables (with defaults shown)
874
+ awsRegion=us-east-1
875
+ s3Prefix=prices/
876
+ archivePrefix=processed/
877
+ errorPrefix=errors/
878
+ logPrefix=logs/ # Where to write mutation logs
879
+ maxFilesToProcess=10
880
+ filePattern=.csv
881
+ enableArchival=true
882
+ mutationRateLimit=10
883
+
884
+ # Price Validation Rules
885
+ minPrice=0.01 # Minimum allowed price
886
+ maxPrice=999999.99 # Maximum allowed price
887
+ validateProducts=true # Validate products exist before updating prices
888
+ ```
889
+
890
+ ## Complete Workflow (src/index.ts)
891
+
892
+ ```typescript
893
+ import { schedule, webhook, http, fn } from '@versori/run';
894
+ import { Buffer } from 'node:buffer'; // Required for Versori/Deno runtime
895
+ import {
896
+ createClient,
897
+ S3DataSource,
898
+ CSVParserService,
899
+ GraphQLMutationMapper,
900
+ VersoriFileTracker,
901
+ JobTracker,
902
+ } from '@fluentcommerce/fc-connect-sdk';
903
+
904
+ // ============================================================================
905
+ // Type Definitions
906
+ // ============================================================================
907
+
908
+ interface FileProcessingResult {
909
+ fileName: string;
910
+ successful: number;
911
+ failed: number;
912
+ validationFailed: number;
913
+ priceChanges: Array<{
914
+ sku: string;
915
+ type: string;
916
+ oldPrice?: number;
917
+ newPrice: number;
918
+ }>;
919
+ errors: string[];
920
+ }
921
+
922
+ interface MutationResult {
923
+ successful: number;
924
+ failed: number;
925
+ priceChanges: Array<{
926
+ sku: string;
927
+ type: string;
928
+ oldPrice?: number;
929
+ newPrice: number;
930
+ }>;
931
+ }
932
+
933
+ interface MappedPriceData {
934
+ productRef: string;
935
+ type: string;
936
+ value: number;
937
+ currency: string;
938
+ effectiveFrom?: string;
939
+ effectiveTo?: string;
940
+ minQuantity?: number;
941
+ maxQuantity?: number;
942
+ }
943
+
944
+ // ============================================================================
945
+ // Utility Functions
946
+ // ============================================================================
947
+
948
+ /**
949
+ * Retry utility with exponential backoff
950
+ * (User-defined - not part of SDK public API)
951
+ */
952
+ async function retryWithBackoff<T>(
953
+ operation: () => Promise<T>,
954
+ maxRetries = 3,
955
+ baseDelayMs = 1000
956
+ ): Promise<T> {
957
+ let lastError: any;
958
+ for (let attempt = 0; attempt < maxRetries; attempt++) {
959
+ try {
960
+ return await operation();
961
+ } catch (error) {
962
+ lastError = error;
963
+ if (attempt < maxRetries - 1) {
964
+ const delay = baseDelayMs * Math.pow(2, attempt) + Math.floor(Math.random() * 200);
965
+ await new Promise(resolve => setTimeout(resolve, delay));
966
+ }
967
+ }
968
+ }
969
+ throw lastError;
970
+ }
971
+
972
+ /**
973
+ * Validate product exists in Fluent Commerce
974
+ */
975
+ async function validateProductExists(
976
+ client: any,
977
+ productRef: string,
978
+ retailerId: string,
979
+ log: any
980
+ ): Promise<boolean> {
981
+ const query = `
982
+ query GetProduct($ref: String!, $retailerId: ID!) {
983
+ products(first: 1, ref: [$ref], retailerId: $retailerId) {
984
+ edges {
985
+ node {
986
+ id
987
+ ref
988
+ status
989
+ }
990
+ }
991
+ }
992
+ }
993
+ `;
994
+
995
+ try {
996
+ const result = await client.graphql({
997
+ query,
998
+ variables: { ref: productRef, retailerId },
999
+ });
1000
+
1001
+ const product = result?.data?.products?.edges?.[0]?.node;
1002
+ if (!product) {
1003
+ log.warn(`Product not found: ${productRef}`);
1004
+ return false;
1005
+ }
1006
+
1007
+ if (product.status !== 'ACTIVE') {
1008
+ log.warn(`Product not active: ${productRef} (status: ${product.status})`);
1009
+ return false;
1010
+ }
1011
+
1012
+ return true;
1013
+ } catch (error: unknown) {
1014
+ // ✅ Enhanced error logging: Extract all error details for visibility
1015
+ const errorMsg = error instanceof Error ? error.message : String(error);
1016
+ const errorDetails = {
1017
+ message: errorMsg,
1018
+ stack: error instanceof Error ? error.stack : undefined,
1019
+ errorType: error instanceof Error ? error.constructor.name : 'Error',
1020
+ };
1021
+ log.error(`Failed to validate product: ${productRef}`, errorDetails);
1022
+ return false;
1023
+ }
1024
+ }
1025
+
1026
+ /**
1027
+ * Get current product price for change tracking
1028
+ */
1029
+ async function getCurrentPrice(
1030
+ client: any,
1031
+ productRef: string,
1032
+ priceType: string,
1033
+ currency: string,
1034
+ retailerId: string
1035
+ ): Promise<number | undefined> {
1036
+ try {
1037
+ const query = `
1038
+ query GetProductPrice($ref: String!, $retailerId: ID!) {
1039
+ products(first: 1, ref: [$ref], retailerId: $retailerId) {
1040
+ edges {
1041
+ node {
1042
+ id
1043
+ ref
1044
+ prices {
1045
+ type
1046
+ value
1047
+ currency
1048
+ }
1049
+ }
1050
+ }
1051
+ }
1052
+ }
1053
+ `;
1054
+
1055
+ const result = await client.graphql({
1056
+ query,
1057
+ variables: { ref: productRef, retailerId },
1058
+ });
1059
+
1060
+ const product = result?.data?.products?.edges?.[0]?.node;
1061
+ if (product?.prices) {
1062
+ const existingPrice = product.prices.find(
1063
+ (p: any) => p.type === priceType && p.currency === currency
1064
+ );
1065
+ if (existingPrice) {
1066
+ return existingPrice.value;
1067
+ }
1068
+ }
1069
+ } catch (err) {
1070
+ // Ignore - price change tracking is optional
1071
+ }
1072
+ return undefined;
1073
+ }
1074
+
1075
+ // ============================================================================
1076
+ // Service Function 1: Process File (S3 + CSV + Mapper)
1077
+ // ============================================================================
1078
+
1079
+ /**
1080
+ * Process single CSV file: Download from S3, parse CSV, map fields to Price schema
1081
+ *
1082
+ * @param s3 - S3DataSource instance
1083
+ * @param parser - CSVParserService instance
1084
+ * @param mapper - UniversalMapper instance (with custom price resolvers)
1085
+ * @param filePath - S3 file path
1086
+ * @param fileName - File name
1087
+ * @param log - Logger instance
1088
+ * @returns FileProcessingResult with parsed and mapped price data
1089
+ */
1090
+ async function processFile(
1091
+ s3: S3DataSource,
1092
+ parser: CSVParserService,
1093
+ mapper: GraphQLMutationMapper,
1094
+ filePath: string,
1095
+ fileName: string,
1096
+ log: any
1097
+ ): Promise<{ records: Array<{ query: string; variables: any; input: any }>; errors: string[] }> {
1098
+ log.info('Processing file', { fileName });
1099
+
1100
+ // Download CSV from S3 with retry
1101
+ const content = await retryWithBackoff(
1102
+ () => s3.downloadFile(filePath, { encoding: 'utf8' }) as Promise<string>
1103
+ );
1104
+
1105
+ // Parse CSV
1106
+ const rawRecords = await parser.parse(content, {
1107
+ columns: true,
1108
+ skip_empty_lines: true,
1109
+ trim: true,
1110
+ });
1111
+
1112
+ if (!rawRecords.length) {
1113
+ log.warn('Empty CSV file', { fileName });
1114
+ return { records: [], errors: [] };
1115
+ }
1116
+
1117
+ log.info('CSV parsed', { fileName, recordCount: rawRecords.length });
1118
+
1119
+ // Map CSV records to Price schema
1120
+ const mappedRecords: Array<{ query: string; variables: any; input: any }> = [];
1121
+ const errors: string[] = [];
1122
+
1123
+ // ✅ PRODUCTION ENHANCEMENT: Log transformation start
1124
+ log.info('Transforming records to GraphQL mutations', {
1125
+ fileName,
1126
+ totalRecords: rawRecords.length,
1127
+ });
1128
+
1129
+ for (let i = 0; i < rawRecords.length; i++) {
1130
+ const rec = rawRecords[i];
1131
+ const recordNumber = i + 1;
1132
+
1133
+ // ✅ PRODUCTION ENHANCEMENT: Log progress every 50 records
1134
+ if (recordNumber % 50 === 0) {
1135
+ log.info(`📤 Transforming record ${recordNumber}/${rawRecords.length}`, {
1136
+ fileName,
1137
+ recordNumber,
1138
+ totalRecords: rawRecords.length,
1139
+ validSoFar: mappedRecords.length,
1140
+ errorsSoFar: errors.length,
1141
+ progress: `${((recordNumber / rawRecords.length) * 100).toFixed(1)}%`,
1142
+ });
1143
+ }
1144
+
1145
+ try {
1146
+ // GraphQLMutationMapper returns { query, variables } directly
1147
+ const mapped = await mapper.map(rec);
1148
+
1149
+ mappedRecords.push({
1150
+ query: mapped.query,
1151
+ variables: mapped.variables,
1152
+ input: mapped.variables.input || mapped.variables,
1153
+ });
1154
+ } catch (error: unknown) {
1155
+ const errorMsg = error instanceof Error ? error.message : String(error);
1156
+ errors.push(`Row ${recordNumber}: ${errorMsg}`);
1157
+ }
1158
+ }
1159
+
1160
+ log.info('Mapping completed', {
1161
+ fileName,
1162
+ successful: mappedRecords.length,
1163
+ failed: errors.length,
1164
+ totalRecords: rawRecords.length,
1165
+ successRate: `${((mappedRecords.length / rawRecords.length) * 100).toFixed(1)}%`,
1166
+ });
1167
+
1168
+ return { records: mappedRecords, errors };
1169
+ }
1170
+
1171
+ // ============================================================================
1172
+ // Service Function 2: Execute Mutations (GraphQL Price Updates)
1173
+ // ============================================================================
1174
+
1175
+ /**
1176
+ * Execute GraphQL mutations for price updates with validation and rate limiting
1177
+ * ✅ Supports alias batching for improved performance
1178
+ *
1179
+ * @param client - FluentClient instance
1180
+ * @param mapper - GraphQLMutationMapper instance
1181
+ * @param priceRecords - Mapped price data with query and variables
1182
+ * @param retailerId - Fluent retailer ID
1183
+ * @param validateProducts - Whether to validate product existence
1184
+ * @param batchSize - Number of concurrent requests (concurrency control)
1185
+ * @param mutationsPerAliasBatch - Optional: Number of mutations per aliased request (alias batching)
1186
+ * @param log - Logger instance
1187
+ * @returns MutationResult with success/failure counts and price changes
1188
+ */
1189
+ async function executeMutations(
1190
+ client: any,
1191
+ mapper: GraphQLMutationMapper,
1192
+ priceRecords: Array<{ query: string; variables: any; input: any }>,
1193
+ retailerId: string,
1194
+ validateProducts: boolean,
1195
+ batchSize: number = 1, // ✅ Default: 1 (sequential)
1196
+ mutationsPerAliasBatch?: number, // ✅ NEW: Alias batching parameter (default: undefined = disabled)
1197
+ log: any
1198
+ ): Promise<MutationResult> {
1199
+ let successful = 0;
1200
+ let failed = 0;
1201
+ const priceChanges: Array<{
1202
+ sku: string;
1203
+ type: string;
1204
+ oldPrice?: number;
1205
+ newPrice: number;
1206
+ }> = [];
1207
+
1208
+ // ✅ Use alias batching if enabled
1209
+ const useAliases = mutationsPerAliasBatch && mutationsPerAliasBatch > 1;
1210
+
1211
+ if (useAliases) {
1212
+ // Process with alias batching
1213
+ const results = await executeMutationsWithAliases(
1214
+ priceRecords,
1215
+ client,
1216
+ mapper,
1217
+ log,
1218
+ retailerId,
1219
+ batchSize,
1220
+ mutationsPerAliasBatch,
1221
+ 'updateProduct'
1222
+ );
1223
+ successful = results.executed;
1224
+ failed = results.failed;
1225
+ // Note: Price changes tracking would need to be done separately if needed
1226
+ } else {
1227
+ // Process individually with validation
1228
+ for (const priceData of priceRecords) {
1229
+ try {
1230
+ // Validate product exists (if enabled)
1231
+ if (validateProducts) {
1232
+ const productExists = await validateProductExists(
1233
+ client,
1234
+ priceData.input.productRef,
1235
+ retailerId,
1236
+ log
1237
+ );
1238
+
1239
+ if (!productExists) {
1240
+ log.warn(`Skipping price update for non-existent product: ${priceData.input.productRef}`);
1241
+ failed++;
1242
+ continue;
1243
+ }
1244
+ }
1245
+
1246
+ // Get current price for change tracking
1247
+ const currentPrice = await getCurrentPrice(
1248
+ client,
1249
+ priceData.input.productRef,
1250
+ priceData.input.type,
1251
+ priceData.input.currency,
1252
+ retailerId
1253
+ );
1254
+
1255
+ // Execute mutation using pre-generated query and variables
1256
+ await retryWithBackoff(() =>
1257
+ client.graphql({
1258
+ query: priceData.query,
1259
+ variables: priceData.variables,
1260
+ })
1261
+ );
1262
+
1263
+ successful++;
1264
+
1265
+ // Track price change
1266
+ if (currentPrice !== undefined && currentPrice !== priceData.input.value) {
1267
+ priceChanges.push({
1268
+ sku: priceData.input.productRef,
1269
+ type: priceData.input.type,
1270
+ oldPrice: currentPrice,
1271
+ newPrice: priceData.input.value,
1272
+ });
1273
+ }
1274
+
1275
+ log.info('Price updated', {
1276
+ ref: priceData.input.productRef,
1277
+ type: priceData.input.type,
1278
+ oldPrice: currentPrice,
1279
+ newPrice: priceData.input.value,
1280
+ currency: priceData.input.currency,
1281
+ });
1282
+ } catch (error: unknown) {
1283
+ failed++;
1284
+ const errorMsg = error instanceof Error ? error.message : String(error);
1285
+ const errorDetails = {
1286
+ message: errorMsg,
1287
+ stack: error instanceof Error ? error.stack : undefined,
1288
+ errorType: error instanceof Error ? error.constructor.name : 'Error',
1289
+ };
1290
+ log.error('Failed to update price', errorDetails, {
1291
+ ref: priceData.input?.productRef,
1292
+ type: priceData.input?.type,
1293
+ });
1294
+ }
1295
+ }
1296
+ }
1297
+
1298
+ return { successful, failed, priceChanges };
1299
+ }
1300
+
1301
+ /**
1302
+ * ✅ NEW: Execute mutations with GraphQL alias batching
1303
+ */
1304
+ async function executeMutationsWithAliases(
1305
+ priceRecords: Array<{ query: string; variables: any; input: any }>,
1306
+ client: any,
1307
+ mapper: GraphQLMutationMapper,
1308
+ log: any,
1309
+ retailerId: string,
1310
+ maxParallel: number,
1311
+ mutationsPerAliasBatch: number,
1312
+ mutationName: string
1313
+ ): Promise<{ executed: number; failed: number; errors: string[] }> {
1314
+ const results = { executed: 0, failed: 0, errors: [] as string[] };
1315
+
1316
+ const aliasBatches: Array<Array<typeof priceRecords[0]>> = [];
1317
+ for (let i = 0; i < priceRecords.length; i += mutationsPerAliasBatch) {
1318
+ aliasBatches.push(priceRecords.slice(i, i + mutationsPerAliasBatch));
1319
+ }
1320
+
1321
+ log.info(`Processing ${aliasBatches.length} alias batches`, {
1322
+ totalPrices: priceRecords.length,
1323
+ maxParallel,
1324
+ });
1325
+
1326
+ for (let i = 0; i < aliasBatches.length; i += maxParallel) {
1327
+ const concurrentBatches = aliasBatches.slice(i, i + maxParallel);
1328
+
1329
+ const batchResults = await Promise.allSettled(
1330
+ concurrentBatches.map(async (batch) => {
1331
+ const { query, variables } = buildAliasedBatch(batch, mutationName, retailerId);
1332
+ const response = await retryWithBackoff(() => client.graphql({ query, variables }), log);
1333
+ return parseAliasResponse(response, batch, mutationName);
1334
+ })
1335
+ );
1336
+
1337
+ batchResults.forEach((result, idx) => {
1338
+ if (result.status === 'fulfilled') {
1339
+ const batchResult = result.value;
1340
+ results.executed += batchResult.executed;
1341
+ results.failed += batchResult.failed;
1342
+ results.errors.push(...batchResult.errors);
1343
+ } else {
1344
+ const batch = concurrentBatches[idx];
1345
+ const errorMsg = result.reason instanceof Error ? result.reason.message : String(result.reason);
1346
+ batch.forEach(price => {
1347
+ results.failed++;
1348
+ const productRef = price.input?.productRef || 'unknown';
1349
+ results.errors.push(`Failed to update price for ${productRef}: ${errorMsg}`);
1350
+ });
1351
+ }
1352
+ });
1353
+
1354
+ if (i + maxParallel < aliasBatches.length) {
1355
+ await new Promise(resolve => setTimeout(resolve, 500));
1356
+ }
1357
+ }
1358
+
1359
+ return results;
1360
+ }
1361
+
1362
+ /**
1363
+ * ✅ NEW: Build aliased batch query and variables
1364
+ */
1365
+ function buildAliasedBatch(
1366
+ batch: Array<{ query: string; variables: any; input: any }>,
1367
+ mutationName: string,
1368
+ retailerId: string
1369
+ ): { query: string; variables: Record<string, any> } {
1370
+ const batchSize = batch.length;
1371
+ const inputTypeName = `${mutationName.charAt(0).toUpperCase() + mutationName.slice(1)}Input`;
1372
+
1373
+ const variables = Array.from({ length: batchSize }, (_, i) =>
1374
+ `$input${i + 1}: ${inputTypeName}!`
1375
+ ).join(', ');
1376
+
1377
+ const aliasedMutations = Array.from({ length: batchSize }, (_, i) => {
1378
+ const alias = `${mutationName}${i + 1}`;
1379
+ return ` ${alias}: ${mutationName}(input: $input${i + 1}) { id ref }`;
1380
+ }).join('\n');
1381
+
1382
+ const operationName = `Batch${mutationName.charAt(0).toUpperCase() + mutationName.slice(1)}s`;
1383
+ const query = `mutation ${operationName}(${variables}) {\n${aliasedMutations}\n}`;
1384
+
1385
+ const variablesObj: Record<string, any> = {};
1386
+ batch.forEach((price, index) => {
1387
+ const input = price.variables.input || price.variables;
1388
+ variablesObj[`input${index + 1}`] = input;
1389
+ });
1390
+
1391
+ return { query, variables: variablesObj };
1392
+ }
1393
+
1394
+ /**
1395
+ * ✅ NEW: Parse aliased GraphQL response
1396
+ */
1397
+ function parseAliasResponse(
1398
+ response: any,
1399
+ batch: Array<{ query: string; variables: any; input: any }>,
1400
+ mutationName: string
1401
+ ): { executed: number; failed: number; errors: string[] } {
1402
+ const result = { executed: 0, failed: 0, errors: [] as string[] };
1403
+
1404
+ const data = response.data || {};
1405
+ const errors = response.errors || [];
1406
+
1407
+ batch.forEach((price, index) => {
1408
+ const alias = `${mutationName}${index + 1}`;
1409
+ const aliasData = data[alias];
1410
+ const aliasErrors = errors.filter((e: unknown) =>
1411
+ e && typeof e === 'object' && 'path' in e && Array.isArray((e as any).path) && (e as any).path.includes(alias)
1412
+ );
1413
+
1414
+ if (aliasData && !aliasErrors.length) {
1415
+ result.executed++;
1416
+ } else {
1417
+ result.failed++;
1418
+ const errorMsg = aliasErrors[0] && typeof aliasErrors[0] === 'object' && 'message' in aliasErrors[0]
1419
+ ? String((aliasErrors[0] as any).message)
1420
+ : 'Mutation failed';
1421
+ const productRef = price.input?.productRef || 'unknown';
1422
+ result.errors.push(`Failed to update price for ${productRef}: ${errorMsg}`);
1423
+ }
1424
+ });
1425
+
1426
+ return result;
1427
+ }
1428
+
1429
+ // ============================================================================
1430
+ // Service Function 3: Write Mutation Log (S3 Upload)
1431
+ // ============================================================================
1432
+
1433
+ /**
1434
+ * Write mutation execution log to S3
1435
+ *
1436
+ * @param s3 - S3DataSource instance
1437
+ * @param fileName - Original CSV file name
1438
+ * @param result - FileProcessingResult
1439
+ * @param logPrefix - S3 prefix for logs
1440
+ * @param log - Logger instance
1441
+ */
1442
+ async function writeMutationLog(
1443
+ s3: S3DataSource,
1444
+ fileName: string,
1445
+ result: FileProcessingResult,
1446
+ logPrefix: string,
1447
+ log: any
1448
+ ): Promise<void> {
1449
+ const logFileName = `${fileName.replace('.csv', '')}-log.json`;
1450
+ const logPath = `${logPrefix}${logFileName}`;
1451
+
1452
+ const logData = {
1453
+ fileName,
1454
+ timestamp: new Date().toISOString(),
1455
+ summary: {
1456
+ successful: result.successful,
1457
+ failed: result.failed,
1458
+ validationFailed: result.validationFailed,
1459
+ priceChanges: result.priceChanges.length,
1460
+ },
1461
+ priceChanges: result.priceChanges,
1462
+ errors: result.errors,
1463
+ };
1464
+
1465
+ const logContent = JSON.stringify(logData, null, 2);
1466
+
1467
+ // Upload log to S3 (uploadFile accepts string or Buffer)
1468
+ await s3.uploadFile(logPath, logContent);
1469
+
1470
+ log.info('Mutation log written', { logPath });
1471
+ }
1472
+
1473
+ // ============================================================================
1474
+ // Main Workflow Function (Per-File Processing)
1475
+ // ============================================================================
1476
+
1477
+ async function executePriceSync(ctx: any, jobId: string, tracker: any) {
1478
+ const { log, activation, openKv } = ctx;
1479
+ const startTime = Date.now();
1480
+
1481
+ log.info('🔄 [PriceSync] Starting scheduled price sync from S3');
1482
+
1483
+ // ✅ CRITICAL: Declare S3 outside try block for safe disposal
1484
+ let s3: S3DataSource | undefined;
1485
+
1486
+ try {
1487
+ // ========================================
1488
+ // STEP 1: CONFIGURATION & VALIDATION
1489
+ // ========================================
1490
+
1491
+ // Read activation variables
1492
+ const s3Bucket = activation?.getVariable('s3BucketName');
1493
+ const s3Region = activation?.getVariable('awsRegion') || 'us-east-1';
1494
+ const s3AccessKeyId = activation?.getVariable('awsAccessKeyId');
1495
+ const s3SecretAccessKey = activation?.getVariable('awsSecretAccessKey');
1496
+ const s3Prefix = activation?.getVariable('s3Prefix') || 'prices/';
1497
+ const retailerId = activation?.getVariable('retailerId');
1498
+ const maxFiles = parseInt(activation?.getVariable('maxFilesToProcess') || '10', 10);
1499
+ const filePattern = (activation?.getVariable('filePattern') || '.csv').toLowerCase();
1500
+ const enableArchival = activation?.getVariable('enableArchival') !== 'false';
1501
+ const archivePrefix = activation?.getVariable('archivePrefix') || 'processed/';
1502
+ const errorPrefix = activation?.getVariable('errorPrefix') || 'errors/';
1503
+ const logPrefix = activation?.getVariable('logPrefix') || 'logs/';
1504
+
1505
+ // Configuration with defaults
1506
+ const mutationBatchSize = parseInt(
1507
+ activation?.getVariable('mutationBatchSize') || '1', // Default: 1 (sequential)
1508
+ 10
1509
+ );
1510
+
1511
+ const mutationsPerAliasBatch = activation?.getVariable('mutationsPerAliasBatch')
1512
+ ? parseInt(activation?.getVariable('mutationsPerAliasBatch') || '1', 10)
1513
+ : undefined; // Default: undefined (disabled)
1514
+
1515
+ const minPrice = parseFloat(activation?.getVariable('minPrice') || '0.01');
1516
+ const maxPrice = parseFloat(activation?.getVariable('maxPrice') || '999999.99');
1517
+ const validateProducts = activation?.getVariable('validateProducts') !== 'false';
1518
+ const validateConnection = activation?.getVariable('validateConnection') !== 'false';
1519
+ const enableFileTracking = activation?.getVariable('enableFileTracking') !== 'false';
1520
+
1521
+ // Validate required variables
1522
+ const missingVars: string[] = [];
1523
+ if (!s3Bucket) missingVars.push('s3BucketName');
1524
+ if (!s3AccessKeyId) missingVars.push('awsAccessKeyId');
1525
+ if (!s3SecretAccessKey) missingVars.push('awsSecretAccessKey');
1526
+ if (!retailerId) missingVars.push('retailerId');
1527
+
1528
+ if (missingVars.length > 0) {
1529
+ const errorMsg = `Missing required variables: ${missingVars.join(', ')}`;
1530
+ log.error('❌ [PriceSync] Configuration error', { error: errorMsg });
1531
+ return {
1532
+ success: false,
1533
+ error: errorMsg,
1534
+ recommendation: `Please configure these activation variables: ${missingVars.join(', ')}`,
1535
+ processed: 0,
1536
+ duration: Date.now() - startTime,
1537
+ };
1538
+ }
1539
+
1540
+ // ========================================
1541
+ // STEP 2: CLIENT INITIALIZATION
1542
+ // ========================================
1543
+
1544
+ const client = await createClient(ctx);
1545
+ if (!client) {
1546
+ throw new Error('Failed to create Fluent Commerce client');
1547
+ }
1548
+
1549
+ // ✅ CORRECT: GraphQL mutations don't need setRetailerId()
1550
+ // Check your GraphQL schema to determine retailerId handling:
1551
+ // - Mandatory retailerId → Must pass it in mutation input
1552
+ // - Optional retailerId → Can pass it if needed
1553
+ // - No retailerId field → Don't pass it
1554
+ // See: fc-connect-sdk/docs/00-START-HERE/retailerid-configuration.md
1555
+
1556
+ log.info('✅ [PriceSync] Fluent client initialized');
1557
+
1558
+ // ========================================
1559
+ // STEP 3: SERVICE INITIALIZATION
1560
+ // ========================================
1561
+
1562
+ s3 = new S3DataSource(
1563
+ {
1564
+ type: 'S3_CSV',
1565
+ connectionId: 's3-price-sync',
1566
+ name: 'Source S3',
1567
+ s3Config: {
1568
+ bucket: s3Bucket,
1569
+ region: s3Region,
1570
+ accessKeyId: s3AccessKeyId,
1571
+ secretAccessKey: s3SecretAccessKey,
1572
+ },
1573
+ },
1574
+ log
1575
+ );
1576
+
1577
+ // Validate S3 connection if enabled
1578
+ if (validateConnection) {
1579
+ try {
1580
+ await s3.listFiles({ prefix: s3Prefix, maxKeys: 1 });
1581
+ log.info('✅ [PriceSync] S3 connection validated');
1582
+ } catch (error: any) {
1583
+ log.error('❌ [PriceSync] S3 connection validation failed', {
1584
+ error: error instanceof Error ? error.message : String(error),
1585
+ });
1586
+ return {
1587
+ success: false,
1588
+ error: 'S3 connection validation failed',
1589
+ details: error?.message,
1590
+ recommendation: 'Please verify S3 credentials and bucket access permissions',
1591
+ duration: Date.now() - startTime,
1592
+ };
1593
+ }
1594
+ }
1595
+
1596
+ const parser = new CSVParserService();
1597
+ const kv = openKv(':project:');
1598
+ const fileTracker = enableFileTracking
1599
+ ? new VersoriFileTracker(kv, 's3-csv-price-sync')
1600
+ : null;
1601
+
1602
+ // Create custom resolvers for price validation
1603
+ const customResolvers = {
1604
+ 'custom.validatePriceRange': (value: any) => {
1605
+ const price = parseFloat(value);
1606
+ if (isNaN(price)) {
1607
+ throw new Error(`Invalid price: ${value}`);
1608
+ }
1609
+ if (price < minPrice) {
1610
+ throw new Error(`Price ${price} below minimum ${minPrice}`);
1611
+ }
1612
+ if (price > maxPrice) {
1613
+ throw new Error(`Price ${price} exceeds maximum ${maxPrice}`);
1614
+ }
1615
+ return price;
1616
+ },
1617
+
1618
+ 'custom.normalizePriceType': (value: any) => {
1619
+ const type = String(value || 'DEFAULT').toUpperCase().trim();
1620
+ const validTypes = ['DEFAULT', 'SALE', 'CLEARANCE', 'BULK', 'PROMOTIONAL', 'MEMBER'];
1621
+ if (!validTypes.includes(type)) {
1622
+ throw new Error(`Invalid price type: ${type}`);
1623
+ }
1624
+ return type;
1625
+ },
1626
+
1627
+ 'custom.normalizeQuantity': (value: any) => {
1628
+ if (!value || value === '') return undefined;
1629
+ const qty = parseInt(value, 10);
1630
+ if (isNaN(qty) || qty < 0) {
1631
+ throw new Error(`Invalid quantity: ${value}`);
1632
+ }
1633
+ return qty;
1634
+ },
1635
+ };
1636
+
1637
+ // ✅ CRITICAL: Load mapping config from external JSON file
1638
+ // Mapping config uses GraphQLMutationMapper structure (nested objects, not dot notation)
1639
+ // File: src/config/price-mapping.json
1640
+ const mappingConfigJson = await import('../config/price-mapping.json', { assert: { type: 'json' } });
1641
+ const mappingConfig = mappingConfigJson.default;
1642
+
1643
+ // Initialize GraphQLMutationMapper with client for schema introspection
1644
+ const mapper = new GraphQLMutationMapper(mappingConfig, log, { fluentClient: client });
1645
+
1646
+ // ========================================
1647
+ // STEP 4: FILE DISCOVERY
1648
+ // ========================================
1649
+
1650
+ try {
1651
+ // List files (pattern filtering handled by listFiles)
1652
+ const files = await s3.listFiles({
1653
+ prefix: s3Prefix,
1654
+ pattern: filePattern,
1655
+ maxKeys: 1000,
1656
+ });
1657
+
1658
+ const csvFiles = files
1659
+ .sort((a, b) => {
1660
+ const dateA = a.lastModified ? new Date(a.lastModified).getTime() : 0;
1661
+ const dateB = b.lastModified ? new Date(b.lastModified).getTime() : 0;
1662
+ return dateA - dateB;
1663
+ })
1664
+ .slice(0, maxFiles);
1665
+
1666
+ log.info('📂 [PriceSync] Files discovered', {
1667
+ total: files.length,
1668
+ toProcess: csvFiles.length,
1669
+ maxFiles,
1670
+ });
1671
+
1672
+ const results = {
1673
+ processed: 0,
1674
+ skipped: 0,
1675
+ failed: 0,
1676
+ totalRecords: 0,
1677
+ pricesUpdated: 0,
1678
+ pricesSkipped: 0,
1679
+ validationErrors: 0,
1680
+ priceChanges: [] as Array<{
1681
+ sku: string;
1682
+ type: string;
1683
+ oldPrice?: number;
1684
+ newPrice: number;
1685
+ }>,
1686
+ errors: [] as string[],
1687
+ };
1688
+
1689
+ // ========================================
1690
+ // STEP 5: FILE PROCESSING LOOP
1691
+ // ========================================
1692
+
1693
+ // Process each file using service functions
1694
+ for (const file of csvFiles) {
1695
+ const filePath = file.path;
1696
+ const fileName = file.name;
1697
+ const fileStartTime = Date.now();
1698
+
1699
+ log.info('📄 [PriceSync] Processing file', { fileName });
1700
+
1701
+ // Duplicate prevention via file tracker
1702
+ if (fileTracker && (await fileTracker.wasFileProcessed(fileName))) {
1703
+ log.info('⏭️ [PriceSync] Skipping already processed file', { fileName });
1704
+ results.skipped++;
1705
+ continue;
1706
+ }
1707
+
1708
+ try {
1709
+ // SERVICE FUNCTION 1: Process file (S3 + CSV + Mapper)
1710
+ const { records: mappedRecords, errors: mappingErrors } = await processFile(
1711
+ s3,
1712
+ parser,
1713
+ mapper,
1714
+ filePath,
1715
+ fileName,
1716
+ log
1717
+ );
1718
+
1719
+ if (!mappedRecords.length) {
1720
+ log.warn('No valid records after mapping, archiving', { fileName });
1721
+ if (enableArchival) {
1722
+ await s3.moveFile(filePath, `${archivePrefix}${fileName}`);
1723
+ }
1724
+ results.skipped++;
1725
+ continue;
1726
+ }
1727
+
1728
+ // SERVICE FUNCTION 2: Execute mutations
1729
+ // ✅ Configuration with defaults
1730
+ const mutationBatchSize = parseInt(
1731
+ ctx.activation?.getVariable('mutationBatchSize') || '1', // ✅ Default: 1 (sequential)
1732
+ 10
1733
+ );
1734
+
1735
+ const mutationsPerAliasBatch = ctx.activation?.getVariable('mutationsPerAliasBatch')
1736
+ ? parseInt(ctx.activation?.getVariable('mutationsPerAliasBatch') || '1', 10)
1737
+ : undefined; // ✅ Default: undefined (disabled, use separate requests)
1738
+
1739
+ // ? Enhanced: Extract context for progress logging
1740
+ const sampleProductRefs = mappedRecords.slice(0, 5).map((r: any) => r.input?.skuRef || r.input?.productRef || 'unknown');
1741
+ const mutationType = mapper?.mutationName || 'updatePrice';
1742
+
1743
+ // ? Enhanced: Start logging with context
1744
+ log.info(`[GraphQLMutations] Sending price mutations for file "${fileName}"`, {
1745
+ totalMutations: mappedRecords.length,
1746
+ mutationType,
1747
+ batchSize: mutationBatchSize,
1748
+ batchMode: mutationBatchSize === 1 ? 'sequential' : `parallel (${mutationBatchSize})`,
1749
+ sampleProductRefs: sampleProductRefs.join(', '),
1750
+ aliasBatching: mutationsPerAliasBatch ? `enabled (${mutationsPerAliasBatch} per alias)` : 'disabled',
1751
+ validateProducts
1752
+ });
1753
+
1754
+ const mutationResult = await executeMutations(
1755
+ client,
1756
+ mapper,
1757
+ mappedRecords,
1758
+ retailerId,
1759
+ validateProducts,
1760
+ mutationBatchSize, // Concurrency control (default: 1)
1761
+ mutationsPerAliasBatch, // ✅ NEW: Alias batching (default: undefined)
1762
+ log
1763
+ );
1764
+
1765
+ // ? Enhanced: Completion logging with summary
1766
+ log.info(`[GraphQLMutations] Price mutation submission completed for file "${fileName}"`, {
1767
+ totalMutations: mappedRecords.length,
1768
+ successful: mutationResult.successful,
1769
+ failed: mutationResult.failed,
1770
+ successRate: mappedRecords.length > 0 ? `${Math.round((mutationResult.successful / mappedRecords.length) * 100)}%` : '0%',
1771
+ mutationType,
1772
+ priceChanges: mutationResult.priceChanges?.length || 0
1773
+ });
1774
+
1775
+ // Aggregate results
1776
+ results.processed++;
1777
+ results.totalRecords += mappedRecords.length;
1778
+ results.pricesUpdated += mutationResult.successful;
1779
+ results.validationErrors += mappingErrors.length;
1780
+ results.priceChanges.push(...mutationResult.priceChanges);
1781
+
1782
+ // Build file processing result for logging
1783
+ const fileResult: FileProcessingResult = {
1784
+ fileName,
1785
+ successful: mutationResult.successful,
1786
+ failed: mutationResult.failed,
1787
+ validationFailed: mappingErrors.length,
1788
+ priceChanges: mutationResult.priceChanges,
1789
+ errors: mappingErrors,
1790
+ };
1791
+
1792
+ // SERVICE FUNCTION 3: Write mutation log to S3
1793
+ await writeMutationLog(s3, fileName, fileResult, logPrefix, log);
1794
+
1795
+ // Mark processed with file tracker
1796
+ if (fileTracker) {
1797
+ await fileTracker.markFileProcessed(fileName, {
1798
+ successful: mutationResult.successful,
1799
+ failed: mutationResult.failed,
1800
+ validationFailed: mappingErrors.length,
1801
+ recordCount: mappedRecords.length,
1802
+ duration: Date.now() - fileStartTime,
1803
+ });
1804
+ }
1805
+
1806
+ if (enableArchival) {
1807
+ await s3.moveFile(filePath, `${archivePrefix}${fileName}`);
1808
+ }
1809
+
1810
+ const fileDuration = Date.now() - fileStartTime;
1811
+ log.info('✅ [PriceSync] File processed successfully', {
1812
+ fileName,
1813
+ recordCount: mappedRecords.length,
1814
+ successful: mutationResult.successful,
1815
+ failed: mutationResult.failed,
1816
+ duration: `${fileDuration}ms`,
1817
+ });
1818
+
1819
+ if (mappingErrors.length > 0) {
1820
+ results.errors.push(`${fileName}: ${mappingErrors.length} mapping errors`);
1821
+ }
1822
+ } catch (error: unknown) {
1823
+ // ✅ Enhanced error logging: Extract all error details for visibility
1824
+ const errorMsg = error instanceof Error ? error.message : String(error);
1825
+ const errorDetails = {
1826
+ message: errorMsg,
1827
+ stack: error instanceof Error ? error.stack : undefined,
1828
+ errorType: error instanceof Error ? error.constructor.name : 'Error',
1829
+ };
1830
+ log.error('❌ [PriceSync] File processing failed', {
1831
+ fileName,
1832
+ ...errorDetails,
1833
+ });
1834
+ results.failed++;
1835
+ results.errors.push(`${fileName}: ${errorMsg}`);
1836
+
1837
+ // Attempt to move to error directory
1838
+ try {
1839
+ await s3.moveFile(filePath, `${errorPrefix}${fileName}`);
1840
+ log.info('📁 [PriceSync] Moved failed file to error directory', { fileName });
1841
+ } catch (moveError) {
1842
+ log.error('❌ [PriceSync] Failed to move error file', {
1843
+ fileName,
1844
+ error: moveError instanceof Error ? moveError.message : String(moveError),
1845
+ });
1846
+ }
1847
+ }
1848
+ }
1849
+
1850
+ // ========================================
1851
+ // STEP 6: SUMMARY & COMPLETION
1852
+ // ========================================
1853
+
1854
+ const totalDuration = Date.now() - startTime;
1855
+ const summary = {
1856
+ success: true,
1857
+ processed: results.processed,
1858
+ skipped: results.skipped,
1859
+ failed: results.failed,
1860
+ totalRecords: results.totalRecords,
1861
+ pricesUpdated: results.pricesUpdated,
1862
+ pricesSkipped: results.pricesSkipped,
1863
+ validationErrors: results.validationErrors,
1864
+ priceChanges: results.priceChanges.length,
1865
+ errors: results.errors.length > 0 ? results.errors : undefined,
1866
+ duration: totalDuration,
1867
+ timestamp: new Date().toISOString(),
1868
+ };
1869
+
1870
+ log.info('🎉 [PriceSync] Price sync completed', {
1871
+ ...summary,
1872
+ duration: `${totalDuration}ms`,
1873
+ });
1874
+
1875
+ // Log significant price changes
1876
+ if (results.priceChanges.length > 0) {
1877
+ log.info('💰 [PriceSync] Price changes detected', {
1878
+ count: results.priceChanges.length,
1879
+ changes: results.priceChanges.slice(0, 10),
1880
+ });
1881
+ }
1882
+
1883
+ return summary;
1884
+ } catch (error: unknown) {
1885
+ // ✅ Enhanced error logging: Extract all error details for visibility
1886
+ const errorMsg = error instanceof Error ? error.message : String(error);
1887
+ const errorDetails = {
1888
+ message: errorMsg,
1889
+ stack: error instanceof Error ? error.stack : undefined,
1890
+ errorType: error instanceof Error ? error.constructor.name : 'Error',
1891
+ };
1892
+ log.error('❌ [PriceSync] Fatal error', errorDetails);
1893
+ return {
1894
+ success: false,
1895
+ error: errorMsg,
1896
+ recommendation: 'Check error logs for details and verify configuration',
1897
+ processed: 0,
1898
+ duration: Date.now() - startTime,
1899
+ timestamp: new Date().toISOString(),
1900
+ };
1901
+ }
1902
+ } catch (error: unknown) {
1903
+ const errorMsg = error instanceof Error ? error.message : String(error);
1904
+ log.error('❌ [PriceSync] Initialization failed', {
1905
+ error: errorMsg,
1906
+ stack: error instanceof Error ? error.stack : undefined,
1907
+ });
1908
+ return {
1909
+ success: false,
1910
+ error: errorMsg,
1911
+ recommendation: 'Check configuration and service initialization',
1912
+ duration: Date.now() - startTime,
1913
+ };
1914
+ } finally {
1915
+ // ✅ CRITICAL: Always dispose S3 connection to prevent connection pool exhaustion
1916
+ if (s3) {
1917
+ try {
1918
+ await s3.dispose();
1919
+ log.info('✅ [PriceSync] S3 connection disposed successfully');
1920
+ } catch (disposeError: any) {
1921
+ log.error('⚠️ [PriceSync] Failed to dispose S3 connection', {
1922
+ error: disposeError instanceof Error ? disposeError.message : String(disposeError),
1923
+ stack: disposeError instanceof Error ? disposeError.stack : undefined,
1924
+ });
1925
+ // Don't throw - disposal failure shouldn't prevent workflow completion
1926
+ }
1927
+ }
1928
+ }
1929
+ }
1930
+ ```
1931
+
1932
+ **Note:** The `runPriceSync` function above should be renamed to `executePriceSync` and moved to `src/services/price-sync.service.ts` to match the new workflow structure.
1933
+
1934
+ ## Key Patterns Explained
1935
+
1936
+ ### Pattern 1: Service Function Architecture
1937
+
1938
+ **Three modular service functions for clean separation of concerns:**
1939
+
1940
+ ```typescript
1941
+ // SERVICE FUNCTION 1: processFile()
1942
+ // Handles: S3 download + CSV parsing + field mapping
1943
+ const { records: mappedRecords, errors: mappingErrors } = await processFile(
1944
+ s3,
1945
+ parser,
1946
+ mapper,
1947
+ filePath,
1948
+ fileName,
1949
+ log
1950
+ );
1951
+
1952
+ // SERVICE FUNCTION 2: executeMutations()
1953
+ // Handles: GraphQL mutations + product validation + concurrency control + price change tracking
1954
+ const mutationResult = await executeMutations(
1955
+ client,
1956
+ mapper,
1957
+ mappedRecords,
1958
+ retailerId,
1959
+ validateProducts,
1960
+ mutationBatchSize, // Concurrency control (default: 1)
1961
+ mutationsPerAliasBatch, // Alias batching (default: undefined)
1962
+ log
1963
+ );
1964
+
1965
+ // SERVICE FUNCTION 3: writeMutationLog()
1966
+ // Handles: Write execution log to S3 (with Buffer for Versori/Deno)
1967
+ await writeMutationLog(s3, fileName, fileResult, logPrefix, log);
1968
+ ```
1969
+
1970
+ **Why this pattern?**
1971
+ - ✅ **Testability**: Each function can be unit tested independently
1972
+ - ✅ **Reusability**: Functions can be reused in different workflows
1973
+ - ✅ **Clarity**: Clear responsibilities for each step
1974
+ - ✅ **Error handling**: Isolated error boundaries per function
1975
+ - ✅ **Maintainability**: Easy to modify single concerns without affecting others
1976
+
1977
+ ### Pattern 2: Buffer Import for S3 Upload (Versori/Deno)
1978
+
1979
+ **CRITICAL for Versori/Deno runtime:**
1980
+
1981
+ ```typescript
1982
+ // MUST import Buffer explicitly (not global like Node.js)
1983
+ import { Buffer } from 'node:buffer';
1984
+
1985
+ // writeMutationLog() function
1986
+ async function writeMutationLog(s3, fileName, result, logPrefix, log) {
1987
+ const logContent = JSON.stringify(logData, null, 2);
1988
+
1989
+ // ✅ CORRECT: Use Buffer.from() for S3 upload
1990
+ await s3.uploadFile(logPath, logContent);
1991
+
1992
+ // ✅ CORRECT: uploadFile accepts string directly (no Buffer needed)
1993
+ // await s3.uploadFile(logPath, logContent);
1994
+ }
1995
+ ```
1996
+
1997
+ **Why?** Deno runtime requires explicit `Buffer` import from `node:buffer`. Without it, you'll get `Buffer is not defined` errors.
1998
+
1999
+ ### Pattern 3: Direct KV State Management
2000
+
2001
+ ```typescript
2002
+ const kv = new VersoriKVAdapter(ctx.openKv(':project:'));
2003
+
2004
+ // Check if file already processed
2005
+ const stateKey = ['processed-files', 's3-price-sync', fileName];
2006
+ const existing = await kv.get(stateKey);
2007
+ if (existing) {
2008
+ log.info('Skipping already processed file', { fileName });
2009
+ continue;
2010
+ }
2011
+
2012
+ // Mark as processed after success
2013
+ await kv.set(stateKey, {
2014
+ successful,
2015
+ failed,
2016
+ validationFailed,
2017
+ processedAt: new Date().toISOString(),
2018
+ });
2019
+ ```
2020
+
2021
+ ### Pattern 4: Custom Price Validation Resolvers
2022
+
2023
+ ```typescript
2024
+ const customResolvers = {
2025
+ 'custom.validatePriceRange': (value: any) => {
2026
+ const price = parseFloat(value);
2027
+ if (isNaN(price)) {
2028
+ throw new Error(`Invalid price: ${value}`);
2029
+ }
2030
+ if (price < minPrice) {
2031
+ throw new Error(`Price ${price} below minimum ${minPrice}`);
2032
+ }
2033
+ if (price > maxPrice) {
2034
+ throw new Error(`Price ${price} exceeds maximum ${maxPrice}`);
2035
+ }
2036
+ return price;
2037
+ },
2038
+
2039
+ 'custom.normalizePriceType': (value: any) => {
2040
+ const type = String(value || 'DEFAULT').toUpperCase().trim();
2041
+ const validTypes = ['DEFAULT', 'SALE', 'CLEARANCE', 'BULK', 'PROMOTIONAL', 'MEMBER'];
2042
+ if (!validTypes.includes(type)) {
2043
+ throw new Error(`Invalid price type: ${type}`);
2044
+ }
2045
+ return type;
2046
+ },
2047
+ };
2048
+ ```
2049
+
2050
+ **Why this matters**: Price data requires strict validation to prevent:
2051
+
2052
+ - Negative prices
2053
+ - Prices outside business rules (too low/high)
2054
+ - Invalid price tiers
2055
+ - Data entry errors
2056
+
2057
+ ### Pattern 5: Product Existence Validation
2058
+
2059
+ ```typescript
2060
+ async function validateProductExists(
2061
+ client: any,
2062
+ productRef: string,
2063
+ retailerId: string,
2064
+ log: any
2065
+ ): Promise<boolean> {
2066
+ const query = `
2067
+ query GetProduct($ref: String!, $retailerId: ID!) {
2068
+ products(first: 1, ref: [$ref], retailerId: $retailerId) {
2069
+ edges {
2070
+ node {
2071
+ id
2072
+ ref
2073
+ status
2074
+ }
2075
+ }
2076
+ }
2077
+ }
2078
+ `;
2079
+
2080
+ const result = await client.graphql({ query, variables: { ref: productRef, retailerId } });
2081
+ const product = result?.data?.products?.edges?.[0]?.node;
2082
+
2083
+ if (!product || product.status !== 'ACTIVE') {
2084
+ return false;
2085
+ }
2086
+
2087
+ return true;
2088
+ }
2089
+ ```
2090
+
2091
+ **Why this matters**: Prevents price updates for non-existent products, which would fail at GraphQL level.
2092
+
2093
+ ### Pattern 6: Price Change Tracking
2094
+
2095
+ ```typescript
2096
+ // Get current price before update
2097
+ const currentPrice = await getCurrentPrice(
2098
+ client,
2099
+ priceData.productRef,
2100
+ priceData.type,
2101
+ priceData.currency,
2102
+ retailerId
2103
+ );
2104
+
2105
+ // Track change after update
2106
+ if (currentPrice !== undefined && currentPrice !== priceData.value) {
2107
+ results.priceChanges.push({
2108
+ sku: priceData.productRef,
2109
+ type: priceData.type,
2110
+ oldPrice: currentPrice,
2111
+ newPrice: priceData.value,
2112
+ });
2113
+ }
2114
+
2115
+ // Log changes at end
2116
+ if (results.priceChanges.length > 0) {
2117
+ log.info('Price changes detected', {
2118
+ count: results.priceChanges.length,
2119
+ changes: results.priceChanges.slice(0, 10),
2120
+ });
2121
+ }
2122
+ ```
2123
+
2124
+ **Why this matters**: Provides audit trail and monitoring for price changes.
2125
+
2126
+ ### Pattern 7: Concurrency Control & Alias Batching
2127
+
2128
+ ```typescript
2129
+ // ✅ Configuration with defaults
2130
+ const mutationBatchSize = parseInt(
2131
+ activation?.getVariable('mutationBatchSize') || '1', // Default: 1 (sequential)
2132
+ 10
2133
+ );
2134
+
2135
+ const mutationsPerAliasBatch = activation?.getVariable('mutationsPerAliasBatch')
2136
+ ? parseInt(activation?.getVariable('mutationsPerAliasBatch') || '1', 10)
2137
+ : undefined; // Default: undefined (disabled)
2138
+
2139
+ // Execute with bounded concurrency + optional alias batching
2140
+ const mutationResult = await executeMutations(
2141
+ client,
2142
+ mapper,
2143
+ priceRecords,
2144
+ retailerId,
2145
+ validateProducts,
2146
+ mutationBatchSize, // 1=sequential, 3-10=parallel
2147
+ mutationsPerAliasBatch, // Optional: Group mutations (e.g., 5)
2148
+ log
2149
+ );
2150
+ ```
2151
+
2152
+ **Performance Modes:**
2153
+ - `mutationBatchSize: 1` → Sequential (safe default, ~1 mutation/sec)
2154
+ - `mutationBatchSize: 3-5` → Balanced (~3-5 mutations/sec)
2155
+ - `mutationBatchSize: 10` → High-volume (~10 mutations/sec)
2156
+ - `mutationsPerAliasBatch: 5` → Alias batching (reduces network overhead by ~80%)
2157
+
2158
+ ## Advanced: Multi-Currency Price Management
2159
+
2160
+ ### External Mapping File for Multi-Currency
2161
+
2162
+ **File**: `config/price-mapping.json`
2163
+
2164
+ ```json
2165
+ {
2166
+ "version": "1.0.0",
2167
+ "description": "Multi-currency price mapping",
2168
+ "fields": {
2169
+ "productRef": {
2170
+ "source": "sku",
2171
+ "required": true,
2172
+ "resolver": "sdk.trim"
2173
+ },
2174
+ "type": {
2175
+ "source": "price_type",
2176
+ "required": true,
2177
+ "resolver": "custom.normalizePriceType"
2178
+ },
2179
+ "value": {
2180
+ "source": "amount",
2181
+ "required": true,
2182
+ "resolver": "custom.validatePriceRange"
2183
+ },
2184
+ "currency": {
2185
+ "source": "currency_code",
2186
+ "required": true,
2187
+ "resolver": "custom.normalizeCurrency"
2188
+ },
2189
+ "effectiveFrom": {
2190
+ "source": "start_date",
2191
+ "resolver": "sdk.formatDate"
2192
+ },
2193
+ "effectiveTo": {
2194
+ "source": "end_date",
2195
+ "resolver": "sdk.formatDate"
2196
+ }
2197
+ }
2198
+ }
2199
+ ```
2200
+
2201
+ ### Custom Resolvers for Multi-Currency
2202
+
2203
+ ```typescript
2204
+ const customResolvers = {
2205
+ 'custom.normalizeCurrency': (value: string) => {
2206
+ // Normalize currency codes to ISO 4217
2207
+ const currencyMap: Record<string, string> = {
2208
+ usd: 'USD',
2209
+ us: 'USD',
2210
+ dollar: 'USD',
2211
+ eur: 'EUR',
2212
+ euro: 'EUR',
2213
+ gbp: 'GBP',
2214
+ pound: 'GBP',
2215
+ cad: 'CAD',
2216
+ aud: 'AUD',
2217
+ };
2218
+
2219
+ const normalized = currencyMap[value.toLowerCase()] || value.toUpperCase();
2220
+
2221
+ // Validate against known currencies
2222
+ const validCurrencies = ['USD', 'EUR', 'GBP', 'CAD', 'AUD', 'JPY', 'CNY'];
2223
+ if (!validCurrencies.includes(normalized)) {
2224
+ throw new Error(`Invalid currency code: ${value}`);
2225
+ }
2226
+
2227
+ return normalized;
2228
+ },
2229
+
2230
+ 'custom.convertToBaseCurrency': (value: any, sourceRecord: any) => {
2231
+ // Convert foreign currency to base currency
2232
+ const amount = parseFloat(value);
2233
+ const currency = sourceRecord.currency_code;
2234
+
2235
+ // Exchange rates (in production, fetch from API)
2236
+ const exchangeRates: Record<string, number> = {
2237
+ USD: 1.0,
2238
+ EUR: 0.85,
2239
+ GBP: 0.73,
2240
+ CAD: 1.35,
2241
+ AUD: 1.45,
2242
+ };
2243
+
2244
+ const rate = exchangeRates[currency?.toUpperCase()] || 1.0;
2245
+ return amount * rate;
2246
+ },
2247
+
2248
+ 'custom.buildPriceAttributes': (_: any, sourceRecord: any) => {
2249
+ // Build price attributes array
2250
+ const attributes: Array<{ name: string; type: string; value: any }> = [];
2251
+
2252
+ if (sourceRecord.cost_basis) {
2253
+ attributes.push({
2254
+ name: 'costBasis',
2255
+ type: 'Float',
2256
+ value: parseFloat(sourceRecord.cost_basis),
2257
+ });
2258
+ }
2259
+
2260
+ if (sourceRecord.margin_percent) {
2261
+ attributes.push({
2262
+ name: 'marginPercent',
2263
+ type: 'Float',
2264
+ value: parseFloat(sourceRecord.margin_percent),
2265
+ });
2266
+ }
2267
+
2268
+ if (sourceRecord.competitor_price) {
2269
+ attributes.push({
2270
+ name: 'competitorPrice',
2271
+ type: 'Float',
2272
+ value: parseFloat(sourceRecord.competitor_price),
2273
+ });
2274
+ }
2275
+
2276
+ if (sourceRecord.price_source) {
2277
+ attributes.push({
2278
+ name: 'priceSource',
2279
+ type: 'String',
2280
+ value: sourceRecord.price_source,
2281
+ });
2282
+ }
2283
+
2284
+ return attributes;
2285
+ },
2286
+ };
2287
+ ```
2288
+
2289
+ ## Schema Validation (Before Deployment)
2290
+
2291
+ Use SDK CLI tools to validate your GraphQL schema and mapping configuration:
2292
+
2293
+ ```bash
2294
+ # Install SDK globally (or use npx)
2295
+ npm install -g @fluentcommerce/fc-connect-sdk
2296
+
2297
+ # 1. Introspect Fluent GraphQL schema
2298
+ fc-connect introspect-schema \
2299
+ --url https://api.fluentcommerce.com/graphql \
2300
+ --client-id YOUR_CLIENT_ID \
2301
+ --client-secret YOUR_CLIENT_SECRET \
2302
+ --output fluent-schema.json
2303
+
2304
+ # 2. Create mutation file (price-mutation.graphql)
2305
+ cat > price-mutation.graphql << 'EOF'
2306
+ mutation UpdateProductPrice($input: UpdateProductInput!) {
2307
+ updateProduct(input: $input) {
2308
+ id
2309
+ ref
2310
+ prices {
2311
+ type
2312
+ value
2313
+ currency
2314
+ }
2315
+ }
2316
+ }
2317
+ EOF
2318
+
2319
+ # 3. Generate mapping from mutation (validates structure)
2320
+ fc-connect generate-mutation-mapping \
2321
+ --file price-mutation.graphql \
2322
+ --output price-mapping.json
2323
+
2324
+ # 4. Validate mapping against schema
2325
+ fc-connect validate-schema \
2326
+ --mapping price-mapping.json \
2327
+ --schema fluent-schema.json
2328
+
2329
+ # 5. Analyze field coverage
2330
+ fc-connect analyze-coverage \
2331
+ --mapping price-mapping.json \
2332
+ --schema fluent-schema.json
2333
+ ```
2334
+
2335
+ **Output Example**:
2336
+
2337
+ ```bash
2338
+ ✓ Schema validation passed
2339
+ ✓ All required fields present: ref, retailerId, prices[].type, prices[].value, prices[].currency
2340
+ ✓ Mutation structure matches GraphQL schema
2341
+ ⚠ Optional fields not mapped: prices[].taxType, prices[].taxRate
2342
+ ```
2343
+
2344
+ ## Testing the Workflow
2345
+
2346
+ ### 1. Upload Test CSV to S3
2347
+
2348
+ ```bash
2349
+ # Using AWS CLI
2350
+ aws s3 cp product-prices-test.csv s3://my-price-bucket/prices/
2351
+
2352
+ # Or using S3 Console
2353
+ # Navigate to bucket → prices/ → Upload
2354
+ ```
2355
+
2356
+ ### 2. Deploy to Versori
2357
+
2358
+ ```bash
2359
+ npm run deploy
2360
+ ```
2361
+
2362
+ ### 3. Manual Testing via Webhook
2363
+
2364
+ ```bash
2365
+ curl -X POST https://your-workspace.versori.run/sync-prices-now \
2366
+ -H "Content-Type: application/json"
2367
+ ```
2368
+
2369
+ ### 4. Check Status
2370
+
2371
+ ```bash
2372
+ curl https://your-workspace.versori.run/check-status
2373
+ ```
2374
+
2375
+ ### 5. Monitor Logs
2376
+
2377
+ ```bash
2378
+ npm run logs
2379
+ # Or via Versori dashboard
2380
+ ```
2381
+
2382
+ ---
2383
+
2384
+ ## Monitoring
2385
+
2386
+ ### Success Response
2387
+
2388
+ ```json
2389
+ {
2390
+ "success": true,
2391
+ "filesProcessed": 1,
2392
+ "filesSkipped": 0,
2393
+ "filesFailed": 0,
2394
+ "totalRecords": 100,
2395
+ "mutationsExecuted": 100,
2396
+ "mutationsFailed": 0,
2397
+ "results": [
2398
+ {
2399
+ "file": "prices_2025-01-22.csv",
2400
+ "success": true,
2401
+ "recordsProcessed": 100,
2402
+ "mutationsExecuted": 100,
2403
+ "mutationsFailed": 0
2404
+ }
2405
+ ],
2406
+ "duration": 12345
2407
+ }
2408
+ ```
2409
+
2410
+ ### Partial Success Response
2411
+
2412
+ ```json
2413
+ {
2414
+ "success": true,
2415
+ "filesProcessed": 1,
2416
+ "filesSkipped": 0,
2417
+ "filesFailed": 0,
2418
+ "totalRecords": 100,
2419
+ "mutationsExecuted": 95,
2420
+ "mutationsFailed": 5,
2421
+ "results": [
2422
+ {
2423
+ "file": "prices_2025-01-22.csv",
2424
+ "success": true,
2425
+ "recordsProcessed": 100,
2426
+ "mutationsExecuted": 95,
2427
+ "mutationsFailed": 5,
2428
+ "errors": ["PRICE-001: Invalid price value", "PRICE-002: Missing currency"]
2429
+ }
2430
+ ],
2431
+ "duration": 12345
2432
+ }
2433
+ ```
2434
+
2435
+ ### Error Response
2436
+
2437
+ ```json
2438
+ {
2439
+ "success": false,
2440
+ "filesProcessed": 0,
2441
+ "filesFailed": 1,
2442
+ "totalRecords": 0,
2443
+ "mutationsExecuted": 0,
2444
+ "mutationsFailed": 0,
2445
+ "results": [
2446
+ {
2447
+ "file": "prices_2025-01-22.csv",
2448
+ "success": false,
2449
+ "error": "CSV parse error: Invalid structure"
2450
+ }
2451
+ ],
2452
+ "duration": 876
2453
+ }
2454
+ ```
2455
+
2456
+ ### Monitoring Metrics
2457
+
2458
+ Monitor these metrics via Versori logs filtered by `jobId` or `stage`:
2459
+
2460
+ - **Files Processed** - Total files successfully processed
2461
+ - **Mutations Executed** - Total GraphQL mutations executed successfully
2462
+ - **Mutations Failed** - Mutations that failed (check error logs)
2463
+ - **Processing Duration** - Time taken for complete workflow
2464
+ - **Rate Limiting** - Watch for 429 errors indicating GraphQL throttling
2465
+
2466
+ Use the status webhook for dashboards and automated monitoring.
2467
+
2468
+ ---
2469
+
2470
+ ### Issue 1: Product Not Found
2471
+
2472
+ **Error**: `Skipping price update for non-existent product: PROD-XYZ`
2473
+
2474
+ **Solution**:
2475
+
2476
+ - Verify product exists in Fluent Commerce
2477
+ - Check SKU matches exactly (case-sensitive)
2478
+ - Ensure product status is `ACTIVE`
2479
+ - Disable validation temporarily for testing: `validateProducts=false`
2480
+
2481
+ ### Issue 2: Price Validation Failures
2482
+
2483
+ **Error**: `Price 0.00 below minimum 0.01`
2484
+
2485
+ **Solution**:
2486
+
2487
+ ```bash
2488
+ # Adjust validation rules in activation variables
2489
+ minPrice=0.00 # Allow zero prices
2490
+ maxPrice=9999999.99 # Increase max
2491
+ ```
2492
+
2493
+ Or fix source data:
2494
+
2495
+ ```csv
2496
+ # ❌ WRONG
2497
+ PROD-001,DEFAULT,0,USD
2498
+
2499
+ # ✅ CORRECT
2500
+ PROD-001,DEFAULT,0.01,USD
2501
+ ```
2502
+
2503
+ ### Issue 3: Invalid Price Type
2504
+
2505
+ **Error**: `Invalid price type: PROMO`
2506
+
2507
+ **Solution**: Update custom resolver to include new type:
2508
+
2509
+ ```typescript
2510
+ 'custom.normalizePriceType': (value: any) => {
2511
+ const type = String(value || 'DEFAULT').toUpperCase().trim();
2512
+ const validTypes = [
2513
+ 'DEFAULT',
2514
+ 'SALE',
2515
+ 'CLEARANCE',
2516
+ 'BULK',
2517
+ 'PROMOTIONAL',
2518
+ 'MEMBER',
2519
+ 'PROMO', // Add new type
2520
+ ];
2521
+ if (!validTypes.includes(type)) {
2522
+ throw new Error(`Invalid price type: ${type}`);
2523
+ }
2524
+ return type;
2525
+ };
2526
+ ```
2527
+
2528
+ ### Issue 4: Duplicate Price Updates
2529
+
2530
+ **Symptom**: Same file processed multiple times
2531
+
2532
+ **Solution**: VersoriKVAdapter already prevents this:
2533
+
2534
+ ```typescript
2535
+ const stateKey = ['processed-files', 's3-price-sync', fileName];
2536
+ const existing = await kv.get(stateKey);
2537
+ if (existing) {
2538
+ log.info('Skipping already processed file', { fileName });
2539
+ continue;
2540
+ }
2541
+ ```
2542
+
2543
+ Verify KV storage is working:
2544
+
2545
+ ```bash
2546
+ # Check Versori logs for:
2547
+ [INFO] Skipping already processed file: product-prices-20250122-001.csv
2548
+ ```
2549
+
2550
+ ### Issue 5: S3 Access Denied
2551
+
2552
+ Required IAM Permissions:
2553
+
2554
+ ```json
2555
+ {
2556
+ "Version": "2012-10-17",
2557
+ "Statement": [
2558
+ {
2559
+ "Effect": "Allow",
2560
+ "Action": ["s3:ListBucket", "s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
2561
+ "Resource": [
2562
+ "arn:aws:s3:::my-price-bucket",
2563
+ "arn:aws:s3:::my-price-bucket/*"
2564
+ ]
2565
+ }
2566
+ ]
2567
+ }
2568
+ ```
2569
+
2570
+ ### Issue 6: GraphQL Mutation Failures
2571
+
2572
+ **Common Causes**:
2573
+
2574
+ - Invalid price type (check Fluent schema for valid types)
2575
+ - Missing required fields (ref, retailerId, prices)
2576
+ - Invalid currency code (must be ISO 4217: USD, EUR, GBP)
2577
+ - Product not found or inactive
2578
+
2579
+ **Debugging**:
2580
+
2581
+ ```typescript
2582
+ // Log mutation input before execution
2583
+ log.info('GraphQL mutation input', { input });
2584
+
2585
+ // Validate required fields
2586
+ if (!priceData.productRef || !priceData.type || !priceData.value) {
2587
+ log.error('Missing required fields', { priceData });
2588
+ continue;
2589
+ }
2590
+ ```
2591
+
2592
+ ## Production Checklist
2593
+
2594
+ - [ ] S3 credentials validated with correct IAM permissions
2595
+ - [ ] Activation secrets stored securely (not in code)
2596
+ - [ ] GraphQL schema validated using CLI tools
2597
+ - [ ] Mapping configuration uses GraphQLMutationMapper structure (NOT UniversalMapper)
2598
+ - [ ] Mapping configuration tested with sample data
2599
+ - [ ] NO setRetailerId() call in code (only for Job/Event API)
2600
+ - [ ] Price validation rules configured per business policy
2601
+ - [ ] Product existence validation enabled (`validateProducts=true`)
2602
+ - [ ] File duplicate prevention working via KV state
2603
+ - [ ] Concurrency control configured (mutationBatchSize)
2604
+ - [ ] Optional alias batching tested if enabled (mutationsPerAliasBatch)
2605
+ - [ ] Error handling tested with malformed CSV
2606
+ - [ ] Retry logic tested with transient failures
2607
+ - [ ] File archival working (processed and error directories)
2608
+ - [ ] Price change tracking verified in logs
2609
+ - [ ] Monitoring/alerting configured for price update failures
2610
+ - [ ] Clear runbook for error recovery
2611
+
2612
+ ## Related Guides
2613
+
2614
+ - **GraphQL Mutation Mapping**: `docs/02-CORE-GUIDES/mapping/graphql-mutation-mapping/`
2615
+ - **GraphQL Alias Batching**: `docs/02-CORE-GUIDES/mapping/graphql-alias-batching-guide.md`
2616
+ - **retailerId Configuration**: `docs/00-START-HERE/retailerid-configuration.md`
2617
+ - **CLI Tools**: `fc-connect-sdk/bin/readme.md`
2618
+ - **State & KV patterns**: `docs/03-PATTERN-GUIDES/file-operations/`
2619
+ - **Error handling**: `docs/03-PATTERN-GUIDES/error-handling/`