@jackchuka/gql-ingest 4.0.0 → 4.2.0

package/dist/index.js

@@ -14314,6 +14314,12 @@ var MetricsCollector = class {
 };
 
 // src/lib/mapper.ts
+var RefResolutionError = class extends Error {
+  constructor(message) {
+    super(message);
+    this.name = "RefResolutionError";
+  }
+};
 var DataMapper = class {
   constructor(client, basePath = process.cwd(), metrics, logger = noopLogger, formatOverride) {
     this.client = client;
@@ -14331,7 +14337,7 @@ var DataMapper = class {
   /**
    * Process an entity with event callbacks and abort support
    */
-  async processEntityWithEvents(configPath, parallelConfig, retryConfig, signal, callbacks) {
+  async processEntityWithEvents(configPath, parallelConfig, retryConfig, signal, callbacks, outputStore) {
     const entityStartTime = Date.now();
     const configFullPath = path.resolve(this.basePath, configPath);
     const config2 = JSON.parse(fs6.readFileSync(configFullPath, "utf8"));
@@ -14359,7 +14365,9 @@ var DataMapper = class {
         parallelConfig,
         retryConfig,
         signal,
-        callbacks
+        callbacks,
+        config2.outputCapture,
+        outputStore
       );
     } else {
       await this.processRowsSequentiallyWithEvents(
@@ -14369,7 +14377,9 @@ var DataMapper = class {
         entityName,
         retryConfig,
         signal,
-        callbacks
+        callbacks,
+        config2.outputCapture,
+        outputStore
       );
     }
     this.metrics.finishEntityProcessing(entityName);
@@ -14393,7 +14403,7 @@ var DataMapper = class {
       durationMs: duration3
     });
   }
-  async processRowsSequentiallyWithEvents(data, mutation, mapping, entityName, retryConfig, signal, callbacks) {
+  async processRowsSequentiallyWithEvents(data, mutation, mapping, entityName, retryConfig, signal, callbacks, outputCapture, outputStore) {
     const totalRows = data.length;
     const variableTypes = this.extractVariableTypes(mutation);
     for (let i = 0; i < data.length; i++) {
@@ -14402,11 +14412,12 @@ var DataMapper = class {
         return;
       }
       const row = data[i];
-      const variables = this.mapRowToVariables(row, mapping, variableTypes);
       const rowStartTime = Date.now();
       try {
+        const variables = this.mapRowToVariables(row, mapping, variableTypes, outputStore);
         const result = await this.client.executeMutation(mutation, variables, retryConfig, signal);
         this.metrics.recordSuccess(entityName);
+        this.captureOutput(row, result, entityName, outputCapture, outputStore);
         callbacks?.onRowSuccess?.({
           entityName,
           rowIndex: i,
@@ -14435,7 +14446,7 @@ var DataMapper = class {
       }
     }
   }
-  async processRowsConcurrentlyWithEvents(data, mutation, mapping, entityName, parallelConfig, retryConfig, signal, callbacks) {
+  async processRowsConcurrentlyWithEvents(data, mutation, mapping, entityName, parallelConfig, retryConfig, signal, callbacks, outputCapture, outputStore) {
     const concurrency = parallelConfig.concurrency;
     this.logger.info(`Processing ${data.length} rows with concurrency: ${concurrency}`);
     const variableTypes = this.extractVariableTypes(mutation);
@@ -14451,9 +14462,9 @@ var DataMapper = class {
       const chunkStartIndex = chunkIndex * concurrency;
       const promises = chunk.map(async (row, index) => {
         const rowIndex = chunkStartIndex + index;
-        const variables = this.mapRowToVariables(row, mapping, variableTypes);
         const rowStartTime = Date.now();
         try {
+          const variables = this.mapRowToVariables(row, mapping, variableTypes, outputStore);
           const result = await this.client.executeMutation(
             mutation,
             variables,
@@ -14461,6 +14472,7 @@ var DataMapper = class {
             signal
           );
           this.metrics.recordSuccess(entityName);
+          this.captureOutput(row, result, entityName, outputCapture, outputStore);
           callbacks?.onRowSuccess?.({
             entityName,
             rowIndex,
@@ -14516,13 +14528,13 @@ var DataMapper = class {
     }
     return chunks;
   }
-  mapRowToVariables(row, mapping, variableTypes) {
+  mapRowToVariables(row, mapping, variableTypes, outputStore) {
     const variables = {};
     for (const [graphqlVar, mappingValue] of Object.entries(mapping)) {
       if (mappingValue === "$") {
         variables[graphqlVar] = row;
       } else if (typeof mappingValue === "string" && mappingValue.startsWith("$.")) {
-        const dataPath = mappingValue.substring(2);
+        const dataPath = this.stripJsonPathPrefix(mappingValue);
         const value = this.getValueByPath(row, dataPath);
         if (value !== void 0) {
           const type = variableTypes[graphqlVar];
@@ -14532,8 +14544,15 @@ var DataMapper = class {
         const rawValue = row[mappingValue];
         const type = variableTypes[graphqlVar];
         variables[graphqlVar] = this.convertValue(rawValue, type, graphqlVar);
+      } else if (this.isRefMapping(mappingValue)) {
+        variables[graphqlVar] = this.resolveRef(row, mappingValue, outputStore);
       } else if (typeof mappingValue === "object" && mappingValue !== null) {
-        variables[graphqlVar] = this.mapNestedObject(row, mappingValue, variableTypes);
+        variables[graphqlVar] = this.mapNestedObject(row, mappingValue, variableTypes, outputStore);
+      }
+    }
+    if (outputStore) {
+      for (const [key, value] of Object.entries(variables)) {
+        variables[key] = this.resolveRefsInData(value, row, outputStore);
       }
     }
     return variables;
@@ -14550,15 +14569,18 @@ var DataMapper = class {
     }
     return current;
   }
-  mapNestedObject(row, mappingObj, variableTypes) {
+  mapNestedObject(row, mappingObj, variableTypes, outputStore) {
     if (Array.isArray(mappingObj)) {
-      return mappingObj.map((item) => this.mapNestedObject(row, item, variableTypes));
+      return mappingObj.map((item) => this.mapNestedObject(row, item, variableTypes, outputStore));
     }
     if (typeof mappingObj === "object" && mappingObj !== null) {
+      if (this.isRefMapping(mappingObj)) {
+        return this.resolveRef(row, mappingObj, outputStore);
+      }
       const result = {};
       for (const [key, value] of Object.entries(mappingObj)) {
         if (typeof value === "string" && value.startsWith("$.")) {
-          const dataPath = value.substring(2);
+          const dataPath = this.stripJsonPathPrefix(value);
           let fieldValue = this.getValueByPath(row, dataPath);
           if (key === "values" && typeof fieldValue === "string" && fieldValue.includes(",")) {
             fieldValue = fieldValue.split(",").map((v) => v.trim());
@@ -14566,8 +14588,10 @@ var DataMapper = class {
           result[key] = fieldValue;
         } else if (typeof value === "string" && row[value] !== void 0) {
           result[key] = row[value];
+        } else if (this.isRefMapping(value)) {
+          result[key] = this.resolveRef(row, value, outputStore);
         } else if (typeof value === "object") {
-          result[key] = this.mapNestedObject(row, value, variableTypes);
+          result[key] = this.mapNestedObject(row, value, variableTypes, outputStore);
         } else {
           result[key] = value;
         }
@@ -14576,6 +14600,113 @@ var DataMapper = class {
     }
     return mappingObj;
   }
+  /**
+   * Recursively walk a resolved value and resolve any $ref objects found in data.
+   * Unlike mapping-level $ref (where key is a JSONPath into the row), data-level
+   * $ref uses key as a literal lookup value unless it starts with "$.".
+   */
+  resolveRefsInData(value, row, outputStore) {
+    if (Array.isArray(value)) {
+      let changed = false;
+      const mapped = value.map((item) => {
+        const resolved = this.resolveRefsInData(item, row, outputStore);
+        if (resolved !== item) changed = true;
+        return resolved;
+      });
+      return changed ? mapped : value;
+    }
+    if (typeof value === "object" && value !== null) {
+      if (this.isRefMapping(value)) {
+        const keyStr = value.key.startsWith("$.") ? String(this.getValueByPath(row, this.stripJsonPathPrefix(value.key)) ?? value.key) : value.key;
+        return this.lookupRef(outputStore, value.$ref, keyStr, value.field);
+      }
+      let changed = false;
+      const result = {};
+      for (const [k, v] of Object.entries(value)) {
+        const resolved = this.resolveRefsInData(v, row, outputStore);
+        if (resolved !== v) changed = true;
+        result[k] = resolved;
+      }
+      return changed ? result : value;
+    }
+    return value;
+  }
+  stripJsonPathPrefix(jsonPath) {
+    return jsonPath.startsWith("$.") ? jsonPath.substring(2) : jsonPath;
+  }
+  isRefMapping(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value) && "$ref" in value && "key" in value && "field" in value;
+  }
+  /** Resolve a mapping-level $ref — throws on any failure so the row is skipped. */
+  resolveRef(row, ref, outputStore) {
+    if (!outputStore) {
+      throw new RefResolutionError(`$ref to "${ref.$ref}" found but no output store is available`);
+    }
+    const keyPath = this.stripJsonPathPrefix(ref.key);
+    const lookupKeyValue = this.getValueByPath(row, keyPath);
+    if (lookupKeyValue === void 0) {
+      throw new RefResolutionError(`$ref lookup key "${ref.key}" not found in current row`);
+    }
+    const keyStr = String(lookupKeyValue);
+    const value = this.lookupRef(outputStore, ref.$ref, keyStr, ref.field);
+    if (value === void 0) {
+      throw new RefResolutionError(
+        `$ref resolution failed for "${ref.$ref}[${keyStr}].${ref.field}"`
+      );
+    }
+    return value;
+  }
+  /**
+   * Core ref lookup: entityName -> key -> field.
+   * Returns undefined with a warning on miss (caller decides whether to throw).
+   */
+  lookupRef(outputStore, entityName, keyStr, field) {
+    const entityStore = outputStore.get(entityName);
+    if (!entityStore) {
+      this.logger.warn(`$ref entity "${entityName}" not found in output store`);
+      return void 0;
+    }
+    const captured = entityStore.get(keyStr);
+    if (!captured) {
+      this.logger.warn(`$ref key "${keyStr}" not found in entity "${entityName}" output store`);
+      return void 0;
+    }
+    const value = captured[field];
+    if (value === void 0) {
+      this.logger.warn(
+        `$ref field "${field}" not found in captured output for "${entityName}[${keyStr}]"`
+      );
+    }
+    return value;
+  }
+  captureOutput(row, result, entityName, outputCapture, outputStore) {
+    if (!outputCapture || !outputStore) return;
+    const keyPath = this.stripJsonPathPrefix(outputCapture.key);
+    const keyValue = this.getValueByPath(row, keyPath);
+    if (keyValue === void 0) {
+      this.logger.warn(
+        `outputCapture key "${outputCapture.key}" not found in input row for entity "${entityName}". Skipping capture.`
+      );
+      return;
+    }
+    const keyStr = String(keyValue);
+    const captured = {};
+    for (const [fieldName, responsePath] of Object.entries(outputCapture.fields)) {
+      const dataPath = this.stripJsonPathPrefix(responsePath);
+      captured[fieldName] = this.getValueByPath(result, dataPath);
+    }
+    let entityStore = outputStore.get(entityName);
+    if (!entityStore) {
+      entityStore = /* @__PURE__ */ new Map();
+      outputStore.set(entityName, entityStore);
+    }
+    if (entityStore.has(keyStr)) {
+      this.logger.warn(
+        `outputCapture: duplicate key "${keyStr}" for entity "${entityName}" \u2014 overwriting previous value`
+      );
+    }
+    entityStore.set(keyStr, captured);
+  }
   extractVariableTypes(mutation) {
     const types = {};
     try {
@@ -14742,6 +14873,7 @@ var DEFAULT_EVENT_OPTIONS = {
 var GQLIngest = class extends EventEmitter {
   constructor(options) {
     super();
+    this.outputStore = /* @__PURE__ */ new Map();
     // Cancellation state
     this.abortController = null;
     this.isProcessing = false;
@@ -14884,6 +15016,7 @@ var GQLIngest = class extends EventEmitter {
         return this.handleCancellation(signal.reason);
       }
       this.metrics = new MetricsCollector();
+      this.outputStore = /* @__PURE__ */ new Map();
       this.client = new GraphQLClientWrapper(
         this.endpoint,
         this.headers,
@@ -14961,7 +15094,8 @@ var GQLIngest = class extends EventEmitter {
         this.mapper,
         config2,
         this.logger,
-        signal
+        signal,
+        this.outputStore
       );
       if (signal.aborted) {
         return this.handleCancellation(signal.reason);
@@ -15067,7 +15201,7 @@ var GQLIngest = class extends EventEmitter {
   /**
    * Process entities in dependency-aware waves with abort support
    */
-  async processEntitiesInWaves(pathMap, resolver, mapper, config2, logger, signal) {
+  async processEntitiesInWaves(pathMap, resolver, mapper, config2, logger, signal, outputStore) {
     const waves = resolver.resolveExecutionOrder();
     logger.info(`Processing ${waves.length} dependency waves...`);
     for (const wave of waves) {
@@ -15092,18 +15226,25 @@ var GQLIngest = class extends EventEmitter {
             try {
               const entityConfig = getEntityConfig(entityName, config2, logger);
               const retryConfig = getRetryConfig(entityName, config2);
-              await mapper.processEntityWithEvents(configPath, entityConfig, retryConfig, signal, {
-                onEntityStart: (payload) => this.safeEmit("entityStart", {
-                  ...payload,
-                  waveIndex: wave.wave
-                }),
-                onEntityComplete: (payload) => {
-                  this.entitiesCompleted++;
-                  this.safeEmit("entityComplete", payload);
+              await mapper.processEntityWithEvents(
+                configPath,
+                entityConfig,
+                retryConfig,
+                signal,
+                {
+                  onEntityStart: (payload) => this.safeEmit("entityStart", {
+                    ...payload,
+                    waveIndex: wave.wave
+                  }),
+                  onEntityComplete: (payload) => {
+                    this.entitiesCompleted++;
+                    this.safeEmit("entityComplete", payload);
+                  },
+                  onRowSuccess: this.eventOptions.emitRowEvents ? (payload) => this.safeEmit("rowSuccess", payload) : void 0,
+                  onRowFailure: this.eventOptions.emitRowEvents ? (payload) => this.safeEmit("rowFailure", payload) : void 0
                 },
-                onRowSuccess: this.eventOptions.emitRowEvents ? (payload) => this.safeEmit("rowSuccess", payload) : void 0,
-                onRowFailure: this.eventOptions.emitRowEvents ? (payload) => this.safeEmit("rowFailure", payload) : void 0
-              });
+                outputStore
+              );
             } catch (error48) {
               const message = error48 instanceof Error ? error48.message : String(error48);
               logger.warn(`Warning: Could not process ${configPath}: ${message}`);