@machinemetrics/mm-erp-sdk 0.1.1-beta.1

package/src/services/caching-service/batch-cache-manager.ts

@@ -0,0 +1,111 @@
+import { HashedCacheManager } from "./hashed-cache-manager";
+import { ERPObjType } from "../../types/erp-types";
+
+export type BatchResult<T> = {
+  duplicates: T[];
+  nonDuplicates: T[];
+};
+
+type BatchCacheManagerOptions = {
+  ttl?: number;
+  tableName?: string;
+};
+
+export class BatchCacheManager {
+  private cacheManager: HashedCacheManager;
+
+  constructor(options?: BatchCacheManagerOptions) {
+    this.cacheManager = new HashedCacheManager({
+      ttl: options?.ttl,
+      tableName: options?.tableName,
+    });
+  }
+
+  /**
+   * Checks a batch of objects against the cache and separates them into duplicates and non-duplicates
+   * @param type The type of objects
+   * @param objects Array of objects to check
+   * @returns Object containing arrays of duplicate and non-duplicate objects
+   */
+  async dedupeBatch(
+    type: ERPObjType,
+    objects: object[]
+  ): Promise<BatchResult<object>> {
+    const result: BatchResult<object> = {
+      duplicates: [],
+      nonDuplicates: [],
+    };
+
+    // Check each object individually
+    for (const object of objects) {
+      const hasChanged = await this.cacheManager.hasChanged(type, object);
+
+      if (hasChanged) {
+        result.nonDuplicates.push(object);
+      } else {
+        result.duplicates.push(object);
+      }
+    }
+
+    return result;
+  }
+
+  /**
+   * Stores a batch of objects in the cache
+   * @param type The type of objects
+   * @param objects Array of objects to store
+   */
+  async storeBatch(
+    type: ERPObjType,
+    objects: object[]
+  ): Promise<{ totalInserted: number }> {
+    // Store each object individually
+    let totalInserted = 0;
+    for (const object of objects) {
+      if (await this.cacheManager.store(type, object)) {
+        totalInserted++;
+      }
+    }
+    return { totalInserted };
+  }
+
+  /**
+   * Removes objects from the cache
+   * @param type The type of objects
+   * @param objects Array of objects to remove
+   */
+  async removeObjects(type: ERPObjType, objects: object[]): Promise<void> {
+    // Remove each object individually
+    for (const object of objects) {
+      await this.cacheManager.removeRecord(type, object);
+    }
+  }
+
+  /**
+   * Gets all records of a specific type
+   */
+  async getRecordsByType(type: ERPObjType): Promise<string[]> {
+    return this.cacheManager.getRecordsByType(type);
+  }
+
+  /**
+   * Removes all records of a specific type
+   */
+  async removeRecordsByType(type: ERPObjType): Promise<void> {
+    return this.cacheManager.removeRecordsByType(type);
+  }
+
+  /**
+   * Clears all records from the cache
+   */
+  async clear(): Promise<void> {
+    return this.cacheManager.clear();
+  }
+
+  /**
+   * Cleans up resources
+   */
+  async destroy(): Promise<void> {
+    return this.cacheManager.destroy();
+  }
+}

package/src/services/caching-service/hashed-cache-manager.ts

@@ -0,0 +1,253 @@
+import knex, { Knex } from "knex";
+import config from "../../knexfile";
+import stringify from "json-stable-stringify";
+import XXH from "xxhashjs";
+import { ERPObjType } from "../../types/erp-types";
+import { CacheMetrics } from "./index";
+import { CoreConfiguration } from "../data-sync-service/configuration-manager";
+import { logger } from "../reporting-service";
+
+type HashedCacheManagerOptions = {
+  ttl?: number;
+  tableName?: string;
+};
+
+export class HashedCacheManager {
+  private static TABLE_NAME = "sdk_cache";
+  private db: Knex;
+  private options: HashedCacheManagerOptions;
+  private static readonly SEED = 0xabcd; // Arbitrary seed for hashing
+  private isDestroyed: boolean = false;
+  private metrics: CacheMetrics = {
+    recordCounts: {},
+  };
+
+  constructor(options?: HashedCacheManagerOptions) {
+    this.options = {
+      ttl: options?.ttl || CoreConfiguration.inst().cacheTTL,
+      tableName: options?.tableName || HashedCacheManager.TABLE_NAME,
+    };
+    this.db = knex({
+      ...config.local,
+      pool: {
+        min: 0,
+        max: 10,
+      },
+    });
+  }
+
+  /**
+   * Checks if the cache manager is still valid
+   * @throws Error if the cache manager has been destroyed
+   */
+  private checkValid(): void {
+    if (this.isDestroyed) {
+      throw new Error("Cache manager has been destroyed");
+    }
+  }
+
+  /**
+   * Generates a stable hash of a record using JSON stringify + xxhash
+   */
+  public static hashRecord(record: object): string {
+    try {
+      const serialized = stringify(record);
+      if (!serialized) {
+        throw new Error("Failed to serialize record for hashing");
+      }
+      const hash = XXH.h64(serialized, HashedCacheManager.SEED).toString(16);
+      return hash;
+    } catch (error) {
+      if (error instanceof Error && error.message.includes("circular")) {
+        throw new Error("Failed to serialize record for hashing");
+      }
+      throw error;
+    }
+  }
+
+  /**
+   * Gets a record from the cache
+   * @param type The type of record
+   * @param hash The hash of the record
+   * @returns The record if it exists, null otherwise
+   */
+  private async getRecord(
+    type: ERPObjType,
+    hash: string
+  ): Promise<{ key: string } | null> {
+    this.checkValid();
+    return this.db(this.options.tableName)
+      .select("key")
+      .where({ type, key: hash })
+      .first();
+  }
+
+  /**
+   * Stores a record in the cache
+   * @param type The type of record
+   * @param record The record to store
+   * @returns true if a new record was created, false if an existing record was updated
+   */
+  public async store(type: ERPObjType, record: object): Promise<boolean> {
+    if (!this.isDestroyed && record) {
+      try {
+        const hash = HashedCacheManager.hashRecord(record);
+        const now = new Date();
+
+        // First check if record exists with same type and hash
+        const existing = await this.db(this.options.tableName)
+          .where({
+            type,
+            key: hash,
+          })
+          .first();
+
+        if (existing) {
+          return false; // No need to update, hash hasn't changed
+        } else {
+          // Insert new record with minimal data
+          const result = await this.db(this.options.tableName)
+            .insert({
+              type,
+              key: hash,
+              created_at: now,
+            })
+            .returning("id");
+          return result.length > 0;
+        }
+      } catch (error) {
+        logger.error("Error storing record:", error);
+        throw error;
+      }
+    }
+    return false;
+  }
+
+  /**
+   * Checks if a record has changed since last seen
+   * @param type The type of record
+   * @param record The record to check
+   * @returns true if the record has changed or is new
+   */
+  async hasChanged(type: ERPObjType, record: object): Promise<boolean> {
+    this.checkValid();
+    const newHash = HashedCacheManager.hashRecord(record);
+    const existing = await this.getRecord(type, newHash);
+    return !existing;
+  }
+
+  /**
+   * Checks if a record has changed and stores it if it has
+   * @param type The type of record
+   * @param record The record to check and store
+   * @returns true if the record was changed or is new
+   */
+  async upsert(type: ERPObjType, record: object): Promise<boolean> {
+    this.checkValid();
+    const hasChanged = await this.hasChanged(type, record);
+    if (hasChanged) {
+      await this.store(type, record as Record<string, unknown>);
+    }
+    return hasChanged;
+  }
+
+  /**
+   * Removes expired records based on TTL
+   */
+  async removeExpiredObjects(): Promise<void> {
+    this.checkValid();
+    const ttl = this.options.ttl;
+    if (!ttl) return;
+
+    const ttlMilliseconds = ttl * 1000;
+    const expirationLimitDate = new Date(Date.now() - ttlMilliseconds);
+    const expirationLimit = expirationLimitDate
+      .toISOString()
+      .slice(0, 19)
+      .replace("T", " ");
+
+    await this.db(this.options.tableName)
+      .where("created_at", "<", expirationLimit)
+      .del();
+  }
+
+  /**
+   * Gets all records of a specific type
+   */
+  async getRecordsByType(type: ERPObjType): Promise<string[]> {
+    this.checkValid();
+    const records = await this.db(this.options.tableName)
+      .select("key")
+      .where({ type });
+
+    return records.map((record) => record.key);
+  }
+
+  /**
+   * Removes all records of a specific type
+   */
+  async removeRecordsByType(type: ERPObjType): Promise<void> {
+    this.checkValid();
+    await this.db(this.options.tableName).where({ type }).del();
+  }
+
+  /**
+   * Removes a specific record
+   */
+  public async removeRecord(type: ERPObjType, record: object): Promise<void> {
+    if (!this.isDestroyed) {
+      try {
+        const hash = HashedCacheManager.hashRecord(record);
+        await this.db(this.options.tableName)
+          .where({ type, key: hash }) // Use key for deletion
+          .del();
+      } catch (error) {
+        logger.error("Error removing record:", error);
+        throw error;
+      }
+    }
+  }
+
+  /**
+   * Clears all records from the cache
+   */
+  async clear(): Promise<void> {
+    this.checkValid();
+    await this.db(this.options.tableName).del();
+  }
+
+  /**
+   * Cleans up database connection and marks the cache manager as destroyed
+   */
+  async destroy(): Promise<void> {
+    if (!this.isDestroyed) {
+      await this.db.destroy();
+      this.isDestroyed = true;
+    }
+  }
+
+  /**
+   * Gets the current cache metrics
+   * @returns The current cache metrics
+   */
+  async getMetrics(): Promise<CacheMetrics> {
+    this.checkValid();
+
+    // Get counts for each type
+    const counts = (await this.db(this.options.tableName)
+      .select("type")
+      .count("* as count")
+      .groupBy("type")) as Array<{ type: string; count: string }>;
+
+    // Update metrics
+    this.metrics.recordCounts = counts.reduce(
+      (acc, row) => {
+        acc[row.type] = parseInt(row.count, 10);
+        return acc;
+      },
+      {} as Record<string, number>
+    );
+
+    return this.metrics;
+  }
+}

package/src/services/caching-service/index.ts

@@ -0,0 +1,114 @@
+import { ERPObjType, ERPObject } from "../../types/erp-types";
+
+export interface CacheServiceConfig {
+  tableName: string;
+  ttl?: number;
+}
+
+export interface CacheMetrics {
+  /**
+   * Count of records by type in the cache
+   * Key is the ERPObjType string value
+   */
+  recordCounts: {
+    [type: string]: number;
+  };
+}
+
+export interface CacheService {
+  /**
+   * Checks if a record has changed since last seen
+   * @param type The type of record
+   * @param key The unique key for the record
+   * @param record The record to check
+   * @returns true if the record has changed or is new
+   */
+  hasChanged(
+    type: ERPObjType,
+    key: string,
+    record: ERPObject
+  ): Promise<boolean>;
+
+  /**
+   * Stores a record in the cache
+   * @param type The type of record
+   * @param key The unique key for the record
+   * @param record The record to store
+   */
+  store(type: ERPObjType, key: string, record: ERPObject): Promise<void>;
+
+  /**
+   * Checks if a record has changed and stores it if it has
+   * @param type The type of record
+   * @param key The unique key for the record
+   * @param record The record to check and store
+   * @returns true if the record was changed or is new
+   */
+  upsert(type: ERPObjType, key: string, record: ERPObject): Promise<boolean>;
+
+  /**
+   * Checks a batch of objects against the cache and separates them into duplicates and non-duplicates
+   * @param type The type of objects
+   * @param objects Array of objects to check
+   * @param idField The field to use as the unique identifier
+   * @returns Object containing arrays of duplicate and non-duplicate objects
+   */
+  checkBatch(
+    type: ERPObjType,
+    objects: ERPObject[],
+    idField: string
+  ): Promise<{ duplicates: ERPObject[]; nonDuplicates: ERPObject[] }>;
+
+  /**
+   * Stores a batch of objects in the cache
+   * @param type The type of objects
+   * @param objects Array of objects to store
+   * @param idField The field to use as the unique identifier
+   */
+  storeBatch(
+    type: ERPObjType,
+    objects: ERPObject[],
+    idField: string
+  ): Promise<void>;
+
+  /**
+   * Gets all records of a specific type
+   * @param type The type of records to get
+   * @returns Array of records with their values
+   */
+  getRecordsByType(type: ERPObjType): Promise<string[]>;
+
+  /**
+   * Removes all records of a specific type
+   * @param type The type of records to remove
+   */
+  removeRecordsByType(type: ERPObjType): Promise<void>;
+
+  /**
+   * Removes a specific record
+   * @param type The type of record
+   * @param key The unique key for the record
+   */
+  removeRecord(type: ERPObjType, key: string): Promise<void>;
+
+  /**
+   * Removes expired records based on TTL
+   */
+  removeExpiredObjects(): Promise<void>;
+
+  /**
+   * Clears all records from the cache
+   */
+  clear(): Promise<void>;
+
+  /**
+   * Gets the current cache metrics
+   * @returns The current cache metrics
+   */
+  getMetrics(): CacheMetrics;
+
+  /**
+   * Cleans up resources and marks the service as destroyed
+   */
+  destroy(): Promise<void>;
+}

package/src/services/caching-service/record-tracking-manager.ts

@@ -0,0 +1,41 @@
+import knex, { Knex } from "knex";
+import config from "../../knexfile";
+import { ERPObjType } from "../../types/erp-types";
+
+const DEFAULT_RECORD_TRACKING_TABLE_NAME = "record_tracking";
+
+export type RecordTrackingObject = {
+  entityType?: ERPObjType;
+  lastValue: string; // Typically date/time string
+  recordId: string; // A value to disambiguate records with the same lastValue
+};
+
+export class RecordTrackingManager {
+  db: Knex;
+
+  constructor() {
+    this.db = knex(config.local);
+  }
+
+  async updateRecord(record: RecordTrackingObject) {
+    if (record.lastValue && record.recordId) {
+      const recordUpdated = await this.db(DEFAULT_RECORD_TRACKING_TABLE_NAME)
+        .where({ entityType: record.entityType })
+        .update(record);
+
+      if (recordUpdated < 1) {
+        await this.db(DEFAULT_RECORD_TRACKING_TABLE_NAME).insert(record);
+      }
+    }
+  }
+
+  async getTrackingRecord(type: ERPObjType) {
+    return await this.db(DEFAULT_RECORD_TRACKING_TABLE_NAME)
+      .select("*")
+      .where({ entityType: type.toString() });
+  }
+
+  async destroy() {
+    return this.db.destroy();
+  }
+}

package/src/services/data-sync-service/configuration-manager.ts

@@ -0,0 +1,153 @@
+import "dotenv/config";
+import { configureLogger } from "../reporting-service/logger";
+import { SQLServerConfiguration } from "../sql-server-erp-service/configuration";
+
+export class CoreConfiguration {
+  private static instance: CoreConfiguration;
+
+  // General Configuration
+  public readonly logLevel: string;
+  public readonly erpSystem: string;
+  public readonly nodeEnv: string;
+
+  // MM API (aka "Mapping") Service
+  public readonly mmERPSvcApiBaseUrl: string;
+  public readonly mmApiBaseUrl: string;
+  public readonly mmApiAuthToken: string;
+  public readonly mmApiRetryAttempts: number;
+
+  // Caching (optionally used for interacting with the MM API)
+  public readonly cacheTTL: number;
+
+  // ERP API Service
+  public readonly erpApiRetryAttemptsDef: number; //Retry attempts for ERP API
+  public readonly erpApiPagingLimit: number; //Pagination limit for ERP API
+
+  // Job timing Intervals
+  public readonly fromErpInterval: string;
+  public readonly toErpInterval: string;
+  public readonly retryLaborTicketsInterval: string;
+  public readonly cacheExpirationCheckInterval: string;
+
+  private constructor() {
+    this.logLevel = process.env.LOG_LEVEL || "info";
+    this.erpSystem = process.env.ERP_SYSTEM || "template";
+    this.nodeEnv = process.env.NODE_ENV || "development";
+
+    //#region MM API (aka "Mapping") Service
+    /**
+     * MM ERP Service REST API URL (typically https://erp-api.svc.machinemetrics.com)
+     */
+    this.mmERPSvcApiBaseUrl = process.env.MM_MAPPING_SERVICE_URL || "";
+
+    /**
+     * MM REST API URL  (typically https://api.machinemetrics.com)
+     */
+    this.mmApiBaseUrl = process.env.MM_MAPPING_AUTH_SERVICE_URL || "";
+
+    /**
+     * Company Auth Token
+     */
+    this.mmApiAuthToken = process.env.MM_MAPPING_SERVICE_TOKEN || "";
+
+    /**
+     * Number of retry attempts for MM API calls
+     */
+    this.mmApiRetryAttempts = parseInt(process.env.RETRY_ATTEMPTS || "0");
+    //#endregion MM API (aka "Mapping") Service
+
+    //#region ERP API Service
+    /**
+     * Number of retry attempts for ERP API calls
+     */
+    this.erpApiRetryAttemptsDef = parseInt(
+      process.env.ERP_API_RETRY_ATTEMPTS_DEF || "3"
+    );
+
+    /**
+     * Default pagination limit for ERP API
+     */
+    this.erpApiPagingLimit = parseInt(process.env.ERP_PAGINATION_LIMIT || "0");
+    //#endregion ERP API Service
+
+    /**
+     * For how to define the intervals, see Bree's documentation: https://github.com/breejs/bree
+     */
+    this.fromErpInterval =
+      process.env.FROM_ERP_INTERVAL || process.env.POLL_INTERVAL || "5 min";
+    this.toErpInterval = process.env.TO_ERP_INTERVAL || "5 min";
+    this.retryLaborTicketsInterval =
+      process.env.RETRY_LABOR_TICKETS_INTERVAL || "30 min";
+    this.cacheExpirationCheckInterval =
+      process.env.CACHE_EXPIRATION_CHECK_INTERVAL || "5 min";
+
+    /**
+     * Cache TTL (in seconds)
+     */
+    const cacheTTLDef = 7 * 24 * 60 * 60; // 7 days
+    this.cacheTTL = parseInt(process.env.CACHE_TTL || cacheTTLDef.toString());
+
+    // Configure the logger with our settings
+    configureLogger(this.logLevel, this.nodeEnv);
+  }
+
+  public static inst(): CoreConfiguration {
+    if (!CoreConfiguration.instance) {
+      CoreConfiguration.instance = new CoreConfiguration();
+    }
+    return CoreConfiguration.instance;
+  }
+}
+
+/**
+ * Helper function to get the SQL Server Configuration for collectors that use SQL Server to interact with the ERP
+ */
+export const getSQLServerConfiguration = (): SQLServerConfiguration => {
+  return {
+    username: process.env.ERP_SQLSERVER_USERNAME || "",
+    password: process.env.ERP_SQLSERVER_PASSWORD || "",
+    database: process.env.ERP_SQLSERVER_DATABASE || "",
+    host:
+      process.env.ERP_SQLSERVER_HOST || process.env.ERP_SQLSERVER_SERVER || "",
+    port: process.env.ERP_SQLSERVER_PORT || "1433",
+    connectionTimeout: process.env.ERP_SQLSERVER_CONNECTION_TIMEOUT || "30000",
+    requestTimeout: process.env.ERP_SQLSERVER_REQUEST_TIMEOUT || "60000",
+    poolMax: process.env.ERP_SQLSERVER_MAX || "10",
+    poolMin: process.env.ERP_SQLSERVER_MIN || "0",
+    idleTimeoutMillis:
+      process.env.ERP_SQLSERVER_IDLE_TIMEOUT_MMILLIS || "30000",
+    encrypt: process.env.ERP_SQLSERVER_ENCRYPT === "true",
+    trustServer: process.env.ERP_SQLSERVER_TRUST_SERVER === "true",
+  };
+};
+
+/**
+ * Parameters required to connect to an ERP system via its API.
+ * Contains all the necessary settings to establish a connection and authenticate with an ERP system's API.
+ */
+export class ErpApiConnectionParams {
+  constructor(
+    public readonly erpApiUrl: string, // Base url of ERP
+    public readonly erpApiClientId: string, // Client ID to authenticate with ERP
+    public readonly erpApiClientSecret: string, // Client Secret to authenticate with ERP
+    public readonly erpApiOrganizationId: string, // Organization / tenant Id
+    public readonly erpAuthBaseUrl: string, // Auth base url
+    public readonly retryAttempts: number = 3 // Number of retry attempts for API calls
+  ) {}
+}
+
+/**
+ * Helper function to get the ERP API Connection Parameters
+ * Not all connectors use these, but keeping these commonly values in one place may
+ * make it easier to set and understand env var names set in App.
+ */
+export const getErpApiConnectionParams = (): ErpApiConnectionParams => {
+  return new ErpApiConnectionParams(
+    process.env.ERP_API_URL || "",
+    process.env.ERP_API_CLIENT_ID || "",
+    process.env.ERP_API_CLIENT_SECRET || "",
+    process.env.ERP_API_ORGANIZATION_ID || "",
+    process.env.ERP_AUTH_BASE_URL || "",
+    parseInt(process.env.ERP_API_RETRY_ATTEMPTS || "3")
+  );
+};