@jackchuka/gql-ingest 2.0.2 → 2.2.0

package/src/config.ts

@@ -1,125 +0,0 @@
-import fs from "fs";
-import path from "path";
-import * as yaml from "js-yaml";
-
-export interface ParallelProcessingConfig {
-  concurrency: number;
-  entityConcurrency: number;
-  preserveRowOrder: boolean;
-}
-
-export interface RetryConfig {
-  maxAttempts: number;
-  baseDelay: number;
-  maxDelay: number;
-  exponentialBackoff: boolean;
-  retryableStatusCodes: number[];
-}
-
-export interface EntityConfig {
-  concurrency?: number;
-  preserveRowOrder?: boolean;
-  retry?: Partial<RetryConfig>;
-}
-
-export interface ProcessingConfig {
-  retry: RetryConfig;
-  parallelProcessing: ParallelProcessingConfig;
-  entityConfig: Record<string, EntityConfig>;
-  entityDependencies: Record<string, string[]>;
-}
-
-export interface FullConfig extends ProcessingConfig {
-  // Future: additional config sections can be added here
-}
-
-export const DEFAULT_RETRY_CONFIG: RetryConfig = {
-  maxAttempts: 3,
-  baseDelay: 1000,
-  maxDelay: 30000,
-  exponentialBackoff: true,
-  retryableStatusCodes: [408, 429, 500, 502, 503, 504],
-};
-
-export const DEFAULT_PARALLEL_CONFIG: ParallelProcessingConfig = {
-  concurrency: 1,
-  entityConcurrency: 1,
-  preserveRowOrder: true,
-};
-
-export const DEFAULT_CONFIG: ProcessingConfig = {
-  retry: DEFAULT_RETRY_CONFIG,
-  parallelProcessing: DEFAULT_PARALLEL_CONFIG,
-  entityConfig: {},
-  entityDependencies: {},
-};
-
-export function loadConfig(configDir: string): ProcessingConfig {
-  const configPath = path.join(configDir, "config.yaml");
-
-  try {
-    if (!fs.existsSync(configPath)) {
-      console.log("No config.yaml found, using default sequential processing");
-      return DEFAULT_CONFIG;
-    }
-
-    const fileContents = fs.readFileSync(configPath, "utf8");
-    const yamlConfig = yaml.load(fileContents) as Partial<FullConfig>;
-
-    return mergeWithDefaults(yamlConfig);
-  } catch (error) {
-    console.warn(
-      `Warning: Failed to parse config.yaml: ${error}. Using defaults.`
-    );
-    return DEFAULT_CONFIG;
-  }
-}
-
-function mergeWithDefaults(yamlConfig: Partial<FullConfig>): ProcessingConfig {
-  return {
-    retry: {
-      ...DEFAULT_RETRY_CONFIG,
-      ...(yamlConfig.retry || {}),
-    },
-    parallelProcessing: {
-      ...DEFAULT_PARALLEL_CONFIG,
-      ...(yamlConfig.parallelProcessing || {}),
-    },
-    entityConfig: yamlConfig.entityConfig || {},
-    entityDependencies: yamlConfig.entityDependencies || {},
-  };
-}
-
-export function getEntityConfig(
-  entityName: string,
-  globalConfig: ProcessingConfig
-): ParallelProcessingConfig {
-  const entityOverrides = globalConfig.entityConfig[entityName] || {};
-
-  let finalConfig = {
-    ...globalConfig.parallelProcessing,
-    ...entityOverrides,
-  };
-
-  // Apply constraint: preserveRowOrder forces concurrency = 1
-  if (finalConfig.preserveRowOrder && finalConfig.concurrency > 1) {
-    console.warn(
-      `Entity '${entityName}': preserveRowOrder=true forces concurrency=1 (was ${finalConfig.concurrency})`
-    );
-    finalConfig.concurrency = 1;
-  }
-
-  return finalConfig;
-}
-
-export function getRetryConfig(
-  entityName: string,
-  globalConfig: ProcessingConfig
-): RetryConfig {
-  const entityOverrides = globalConfig.entityConfig[entityName]?.retry || {};
-
-  return {
-    ...globalConfig.retry,
-    ...entityOverrides,
-  };
-}

package/src/dependency-resolver.test.ts

@@ -1,211 +0,0 @@
-import { DependencyResolver } from "./dependency-resolver";
-
-describe("DependencyResolver", () => {
-  describe("resolveExecutionOrder", () => {
-    it("should handle entities with no dependencies", () => {
-      const entities = ["users", "products", "orders"];
-      const dependencies = {};
-      const resolver = new DependencyResolver(entities, dependencies);
-
-      const waves = resolver.resolveExecutionOrder();
-
-      expect(waves).toHaveLength(1);
-      expect(waves[0].wave).toBe(0);
-      expect(waves[0].entities).toEqual(["users", "products", "orders"]);
-    });
-
-    it("should resolve simple linear dependencies", () => {
-      const entities = ["users", "products", "orders"];
-      const dependencies = {
-        products: ["users"],
-        orders: ["products"],
-      };
-      const resolver = new DependencyResolver(entities, dependencies);
-
-      const waves = resolver.resolveExecutionOrder();
-
-      expect(waves).toHaveLength(3);
-      expect(waves[0].entities).toEqual(["users"]);
-      expect(waves[1].entities).toEqual(["products"]);
-      expect(waves[2].entities).toEqual(["orders"]);
-    });
-
-    it("should resolve complex dependency graph", () => {
-      const entities = ["users", "categories", "products", "orders", "reviews"];
-      const dependencies = {
-        products: ["users", "categories"],
-        orders: ["users", "products"],
-        reviews: ["users", "products"],
-      };
-      const resolver = new DependencyResolver(entities, dependencies);
-
-      const waves = resolver.resolveExecutionOrder();
-
-      expect(waves).toHaveLength(3);
-
-      // Wave 0: users and categories (no dependencies)
-      expect(waves[0].entities.sort()).toEqual(["categories", "users"]);
-
-      // Wave 1: products (depends on users and categories)
-      expect(waves[1].entities).toEqual(["products"]);
-
-      // Wave 2: orders and reviews (both depend on users and products)
-      expect(waves[2].entities.sort()).toEqual(["orders", "reviews"]);
-    });
-
-    it("should handle entities with multiple dependencies", () => {
-      const entities = ["a", "b", "c", "d"];
-      const dependencies = {
-        c: ["a", "b"],
-        d: ["a", "b"],
-      };
-      const resolver = new DependencyResolver(entities, dependencies);
-
-      const waves = resolver.resolveExecutionOrder();
-
-      expect(waves).toHaveLength(2);
-      expect(waves[0].entities.sort()).toEqual(["a", "b"]);
-      expect(waves[1].entities.sort()).toEqual(["c", "d"]);
-    });
-
-    it("should detect circular dependencies", () => {
-      const entities = ["a", "b", "c"];
-      const dependencies = {
-        a: ["b"],
-        b: ["c"],
-        c: ["a"], // circular
-      };
-      const resolver = new DependencyResolver(entities, dependencies);
-
-      expect(() => resolver.resolveExecutionOrder()).toThrow(
-        "Circular dependency detected or missing dependencies for entities: a, b, c"
-      );
-    });
-
-    it("should detect missing dependencies", () => {
-      const entities = ["a", "b"];
-      const dependencies = {
-        a: ["missing"],
-        b: ["a"],
-      };
-      const resolver = new DependencyResolver(entities, dependencies, false);
-
-      expect(() => resolver.resolveExecutionOrder()).toThrow(
-        "Circular dependency detected or missing dependencies for entities: a, b"
-      );
-    });
-
-    it("should allow entities with dependencies not in the entity list when partial resolution is enabled", () => {
-      const entities = ["a", "b"];
-      const dependencies = {
-        a: ["missing"],
-        b: ["a"],
-      };
-      const resolver = new DependencyResolver(entities, dependencies, true);
-
-      const waves = resolver.resolveExecutionOrder();
-      expect(waves).toHaveLength(2);
-      expect(waves[0].entities).toEqual(["a"]);
-      expect(waves[1].entities).toEqual(["b"]);
-    });
-  });
-
-  describe("validateDependencies", () => {
-    it("should return no errors for valid dependencies", () => {
-      const entities = ["users", "products", "orders"];
-      const dependencies = {
-        products: ["users"],
-        orders: ["users", "products"],
-      };
-      const resolver = new DependencyResolver(entities, dependencies);
-
-      const errors = resolver.validateDependencies();
-
-      expect(errors).toHaveLength(0);
-    });
-
-    it("should detect entity with dependencies not in entity list", () => {
-      const entities = ["users", "products"];
-      const dependencies = {
-        products: ["users"],
-        orders: ["products"], // orders not in entities list
-      };
-      const resolver = new DependencyResolver(entities, dependencies);
-
-      const errors = resolver.validateDependencies();
-
-      expect(errors).toContain(
-        "Entity 'orders' has dependencies but is not in the entity list"
-      );
-    });
-
-    it("should detect dependencies on non-existent entities", () => {
-      const entities = ["users", "products"];
-      const dependencies = {
-        products: ["users", "categories"], // categories doesn't exist
-      };
-      const resolver = new DependencyResolver(entities, dependencies);
-
-      const errors = resolver.validateDependencies();
-
-      expect(errors).toContain(
-        "Entity 'products' depends on 'categories' which does not exist"
-      );
-    });
-
-    it("should detect multiple validation errors", () => {
-      const entities = ["users"];
-      const dependencies = {
-        products: ["categories"], // products not in list, categories doesn't exist
-        orders: ["missing"], // orders not in list, missing doesn't exist
-      };
-      const resolver = new DependencyResolver(entities, dependencies);
-
-      const errors = resolver.validateDependencies();
-
-      expect(errors).toHaveLength(2);
-      expect(errors).toContain(
-        "Entity 'products' has dependencies but is not in the entity list"
-      );
-      expect(errors).toContain(
-        "Entity 'orders' has dependencies but is not in the entity list"
-      );
-    });
-  });
-
-  describe("getDependents", () => {
-    it("should return entities that depend on the given entity", () => {
-      const entities = ["users", "products", "orders", "reviews"];
-      const dependencies = {
-        products: ["users"],
-        orders: ["users", "products"],
-        reviews: ["users", "products"],
-      };
-      const resolver = new DependencyResolver(entities, dependencies);
-
-      const usersDependents = resolver.getDependents("users");
-      const productsDependents = resolver.getDependents("products");
-      const ordersDependents = resolver.getDependents("orders");
-
-      expect(usersDependents.sort()).toEqual(["orders", "products", "reviews"]);
-      expect(productsDependents.sort()).toEqual(["orders", "reviews"]);
-      expect(ordersDependents).toEqual([]);
-    });
-  });
-
-  describe("getDependencies", () => {
-    it("should return dependencies for the given entity", () => {
-      const entities = ["users", "products", "orders"];
-      const dependencies = {
-        products: ["users"],
-        orders: ["users", "products"],
-      };
-      const resolver = new DependencyResolver(entities, dependencies);
-
-      expect(resolver.getDependencies("users")).toEqual([]);
-      expect(resolver.getDependencies("products")).toEqual(["users"]);
-      expect(resolver.getDependencies("orders")).toEqual(["users", "products"]);
-      expect(resolver.getDependencies("nonexistent")).toEqual([]);
-    });
-  });
-});

package/src/dependency-resolver.ts

@@ -1,102 +0,0 @@
-export interface DependencyGraph {
-  [entityName: string]: string[];
-}
-
-export interface ExecutionWave {
-  entities: string[];
-  wave: number;
-}
-
-export class DependencyResolver {
-  private dependencies: DependencyGraph;
-  private entities: string[];
-  private allowPartialResolution: boolean;
-
-  constructor(entities: string[], dependencies: DependencyGraph = {}, allowPartialResolution: boolean = false) {
-    this.entities = entities;
-    this.dependencies = dependencies;
-    this.allowPartialResolution = allowPartialResolution;
-  }
-
-  resolveExecutionOrder(): ExecutionWave[] {
-    const waves: ExecutionWave[] = [];
-    const processed = new Set<string>();
-    const inProgress = new Set<string>();
-    let waveNumber = 0;
-
-    while (processed.size < this.entities.length) {
-      const currentWave: string[] = [];
-
-      for (const entity of this.entities) {
-        if (processed.has(entity) || inProgress.has(entity)) {
-          continue;
-        }
-
-        const deps = this.dependencies[entity] || [];
-        const canProcess = deps.every((dep) => 
-          processed.has(dep) || (this.allowPartialResolution && !this.entities.includes(dep))
-        );
-
-        if (canProcess) {
-          currentWave.push(entity);
-          inProgress.add(entity);
-        }
-      }
-
-      if (currentWave.length === 0) {
-        const remaining = this.entities.filter((e) => !processed.has(e));
-        throw new Error(
-          `Circular dependency detected or missing dependencies for entities: ${remaining.join(
-            ", "
-          )}`
-        );
-      }
-
-      waves.push({
-        entities: currentWave,
-        wave: waveNumber++,
-      });
-
-      currentWave.forEach((entity) => {
-        processed.add(entity);
-        inProgress.delete(entity);
-      });
-    }
-
-    return waves;
-  }
-
-  validateDependencies(): string[] {
-    const errors: string[] = [];
-    const entitySet = new Set(this.entities);
-
-    for (const [entity, deps] of Object.entries(this.dependencies)) {
-      if (!entitySet.has(entity)) {
-        errors.push(
-          `Entity '${entity}' has dependencies but is not in the entity list`
-        );
-        continue;
-      }
-
-      for (const dep of deps) {
-        if (!entitySet.has(dep)) {
-          errors.push(
-            `Entity '${entity}' depends on '${dep}' which does not exist`
-          );
-        }
-      }
-    }
-
-    return errors;
-  }
-
-  getDependents(entityName: string): string[] {
-    return Object.entries(this.dependencies)
-      .filter(([_, deps]) => deps.includes(entityName))
-      .map(([entity, _]) => entity);
-  }
-
-  getDependencies(entityName: string): string[] {
-    return this.dependencies[entityName] || [];
-  }
-}

package/src/graphql-client.test.ts

@@ -1,219 +0,0 @@
-import { GraphQLClientWrapper } from "./graphql-client";
-
-const mockRequest = jest.fn();
-const mockSetHeaders = jest.fn();
-
-jest.mock("graphql-request", () => ({
-  GraphQLClient: jest.fn().mockImplementation(() => ({
-    request: mockRequest,
-    setHeaders: mockSetHeaders,
-  })),
-}));
-
-describe("GraphQLClientWrapper", () => {
-  let clientWrapper: GraphQLClientWrapper;
-
-  beforeEach(() => {
-    jest.clearAllMocks();
-    clientWrapper = new GraphQLClientWrapper("https://api.example.com/graphql");
-  });
-
-  it("should create GraphQLClient with endpoint and default headers", () => {
-    const { GraphQLClient } = require("graphql-request");
-    expect(GraphQLClient).toHaveBeenCalledWith(
-      "https://api.example.com/graphql",
-      { headers: {} }
-    );
-  });
-
-  it("should create GraphQLClient with custom headers", () => {
-    const headers = { Authorization: "Bearer token123" };
-    new GraphQLClientWrapper("https://api.example.com/graphql", headers);
-
-    const { GraphQLClient } = require("graphql-request");
-    expect(GraphQLClient).toHaveBeenCalledWith(
-      "https://api.example.com/graphql",
-      { headers }
-    );
-  });
-
-  it("should execute mutation successfully", async () => {
-    const mutation = "mutation { createUser(name: $name) { id } }";
-    const variables = { name: "John" };
-    const expectedResult = { createUser: { id: "123" } };
-
-    mockRequest.mockResolvedValue(expectedResult);
-
-    const result = await clientWrapper.executeMutation(mutation, variables);
-
-    expect(mockRequest).toHaveBeenCalledWith(mutation, variables);
-    expect(result).toEqual(expectedResult);
-  });
-
-  it("should handle GraphQL errors", async () => {
-    const mutation = "mutation { createUser(name: $name) { id } }";
-    const variables = { name: "John" };
-    const error = new Error("GraphQL error");
-
-    mockRequest.mockRejectedValue(error);
-
-    const consoleSpy = jest.spyOn(console, "error").mockImplementation();
-
-    await expect(
-      clientWrapper.executeMutation(mutation, variables)
-    ).rejects.toThrow("GraphQL error");
-
-    expect(consoleSpy).toHaveBeenCalledWith(
-      "GraphQL mutation failed after 3 attempts:",
-      error
-    );
-
-    consoleSpy.mockRestore();
-  });
-
-  it("should set headers on the client", () => {
-    const newHeaders = { "X-API-Key": "api123" };
-
-    clientWrapper.setHeaders(newHeaders);
-
-    expect(mockSetHeaders).toHaveBeenCalledWith(newHeaders);
-  });
-
-  describe("retry functionality", () => {
-    it("should retry on retryable status codes", async () => {
-      const mutation = "mutation { createUser(name: $name) { id } }";
-      const variables = { name: "John" };
-      const retryConfig = {
-        maxAttempts: 3,
-        baseDelay: 100,
-        maxDelay: 1000,
-        exponentialBackoff: false,
-        retryableStatusCodes: [500],
-      };
-
-      const serverError = new Error("Server Error");
-      (serverError as any).response = { status: 500 };
-
-      mockRequest
-        .mockRejectedValueOnce(serverError)
-        .mockRejectedValueOnce(serverError)
-        .mockResolvedValueOnce({ data: { result: "success" } });
-
-      const result = await clientWrapper.executeMutation(
-        mutation,
-        variables,
-        retryConfig
-      );
-
-      expect(mockRequest).toHaveBeenCalledTimes(3);
-      expect(result).toEqual({ data: { result: "success" } });
-    });
-
-    it("should not retry on non-retryable status codes", async () => {
-      const mutation = "mutation { createUser(name: $name) { id } }";
-      const variables = { name: "John" };
-      const retryConfig = {
-        maxAttempts: 3,
-        baseDelay: 100,
-        maxDelay: 1000,
-        exponentialBackoff: false,
-        retryableStatusCodes: [500],
-      };
-
-      const clientError = new Error("Bad Request");
-      (clientError as any).response = { status: 400 };
-
-      mockRequest.mockRejectedValueOnce(clientError);
-
-      await expect(
-        clientWrapper.executeMutation(mutation, variables, retryConfig)
-      ).rejects.toThrow("Bad Request");
-
-      expect(mockRequest).toHaveBeenCalledTimes(1);
-    });
-
-    it("should retry on network errors (no response)", async () => {
-      const mutation = "mutation { createUser(name: $name) { id } }";
-      const variables = { name: "John" };
-      const retryConfig = {
-        maxAttempts: 2,
-        baseDelay: 100,
-        maxDelay: 1000,
-        exponentialBackoff: false,
-        retryableStatusCodes: [500],
-      };
-
-      const networkError = new Error("Network Error");
-      // No response property = network error
-
-      mockRequest
-        .mockRejectedValueOnce(networkError)
-        .mockResolvedValueOnce({ data: { result: "success" } });
-
-      const result = await clientWrapper.executeMutation(
-        mutation,
-        variables,
-        retryConfig
-      );
-
-      expect(mockRequest).toHaveBeenCalledTimes(2);
-      expect(result).toEqual({ data: { result: "success" } });
-    });
-
-    it("should fail after max attempts are exhausted", async () => {
-      const mutation = "mutation { createUser(name: $name) { id } }";
-      const variables = { name: "John" };
-      const retryConfig = {
-        maxAttempts: 2,
-        baseDelay: 100,
-        maxDelay: 1000,
-        exponentialBackoff: false,
-        retryableStatusCodes: [500],
-      };
-
-      const serverError = new Error("Server Error");
-      (serverError as any).response = { status: 500 };
-
-      mockRequest.mockRejectedValue(serverError);
-
-      await expect(
-        clientWrapper.executeMutation(mutation, variables, retryConfig)
-      ).rejects.toThrow("Server Error");
-
-      expect(mockRequest).toHaveBeenCalledTimes(2);
-    });
-
-    it("should use exponential backoff when configured", async () => {
-      const mutation = "mutation { createUser(name: $name) { id } }";
-      const variables = { name: "John" };
-      const retryConfig = {
-        maxAttempts: 3,
-        baseDelay: 100,
-        maxDelay: 1000,
-        exponentialBackoff: true,
-        retryableStatusCodes: [500],
-      };
-
-      const serverError = new Error("Server Error");
-      (serverError as any).response = { status: 500 };
-
-      mockRequest
-        .mockRejectedValueOnce(serverError)
-        .mockRejectedValueOnce(serverError)
-        .mockResolvedValueOnce({ data: { result: "success" } });
-
-      const startTime = Date.now();
-      const result = await clientWrapper.executeMutation(
-        mutation,
-        variables,
-        retryConfig
-      );
-      const totalTime = Date.now() - startTime;
-
-      expect(mockRequest).toHaveBeenCalledTimes(3);
-      expect(result).toEqual({ data: { result: "success" } });
-      // Should have some delay for retries (base + exponential)
-      expect(totalTime).toBeGreaterThan(200); // At least 100ms base + 200ms exponential
-    });
-  });
-});