@machinemetrics/mm-erp-sdk 0.1.2 → 0.1.4-beta.0

package/src/utils/connector-log/log-deduper.ts

@@ -0,0 +1,282 @@
+import fs from 'fs';
+import path from 'path';
+import logger from '../../services/reporting-service/logger';
+import type { LogEntry, LogLevelString } from './mm-connector-logger';
+
+/**
+ * Implement a custom LogDeduper or use the provided FileLogDeduper
+ */
+export interface LogDeduper {
+  decide(entry: LogEntry, now: number): Promise<string | null>;
+  onSuccess(entry: LogEntry, now: number): Promise<void>;
+  retryFailedTransmissions?(send: (entry: LogEntry, message: string) => Promise<void>): Promise<void>;
+}
+
+interface LogDedupeEntry {
+  lastTransmitted: number;
+  suppressedCount: number;
+  firstUnsentEventTs: number; // Earliest unsent occurrence time for the current unsent batch (if any)
+  lastEventTs: number;        // Most recent occurrence time (allowed or suppressed)
+  level: LogLevelString;
+  message: string;
+}
+
+/**
+ * LogDedupeStore is the store for of the most recent log entries for each dedupeKey
+ */
+interface LogDedupeStore {
+  [key: string]: LogDedupeEntry;
+}
+
+/**
+ * FileLogDeduper implements the LogDeduper interface to deduplicate
+ * logs by storing them in a file; this allows deduping across jobs.
+ */
+export class FileLogDeduper implements LogDeduper {
+  private readonly storeFilePath: string;
+  private readonly windowMs: number;
+  private readonly ttlMs: number;
+  private readonly sweepIntervalMs: number;
+  private lastSweepTsMs: number;
+
+  private readonly DEFAULT_WINDOW_TEN_MINS = 600;
+  private readonly DEFAULT_TTL_ONE_HOUR = 3600;
+  private readonly DEFAULT_SWEEP_INTERVAL_FIVE_MINS = 300;
+  private readonly DEFAULT_STORE_FILE_PATH = path.join('/tmp', 'log-deduplication.json');
+
+  /**
+   * Ctor.
+   * @param storeFilePath:         The path to the file where the deduplication store is stored; recommended is to use the default
+   * @param windowSeconds:         Suppression window. Duplicates within this period are suppressed.
+   * @param ttlSeconds:            Eviction TTL. Store entries for keys inactive beyond this are removed. Enforced to be ≥ windowSeconds.
+   * @param sweepIntervalSeconds:  Efficiency parameter. How often (min interval) to run opportunistic eviction; retry always sweeps
+   *                               The sweep is lazy, used only when the store is accessed
+   */
+  constructor({
+    storeFilePath = this.DEFAULT_STORE_FILE_PATH,
+    windowSeconds = this.DEFAULT_WINDOW_TEN_MINS,
+    ttlSeconds = this.DEFAULT_TTL_ONE_HOUR,
+    sweepIntervalSeconds = this.DEFAULT_SWEEP_INTERVAL_FIVE_MINS
+  } : {
+    storeFilePath?: string;
+    windowSeconds?: number;
+    ttlSeconds?: number;
+    sweepIntervalSeconds?: number;
+  } = {}) {
+
+    this.storeFilePath = storeFilePath;
+    this.windowMs = Math.max(1, windowSeconds) * 1000;
+    this.ttlMs = Math.max(this.windowMs, Math.max(1, ttlSeconds) * 1000);
+    this.sweepIntervalMs = Math.max(1, sweepIntervalSeconds) * 1000;
+    this.lastSweepTsMs = 0;
+
+    this.ensureStoreFileExists();
+  }
+
+  /** 
+   * Deduplication gating function
+   * Returns the formatted message to send, or null to suppress
+   * Decision is based on the dedupeKey and the time of the entry
+  */
+  async decide(entry: LogEntry, now: number): Promise<string | null> {
+    if (!entry.dedupeKey || typeof entry.dedupeKey !== 'string' || entry.dedupeKey.trim().length === 0) {
+      throw new Error('dedupeKey is required and must be a non-empty string');
+    }
+    const key = entry.dedupeKey;
+
+    return this.withLock(async () => {
+      const store = this.readStore();
+
+      // Opportunistic eviction gated by sweep interval
+      if (now - this.lastSweepTsMs >= this.sweepIntervalMs) {
+        this.evictExpiredInStore(store, now);
+        this.lastSweepTsMs = now;
+        this.writeStore(store);
+      }
+      const existing = store[key];
+
+      if (existing) {
+        const withinWindow = existing.lastTransmitted > 0 && (existing.lastTransmitted + this.windowMs) > now;
+        if (withinWindow) {
+          // Suppress and increment count
+          store[key] = {
+            ...existing,
+            suppressedCount: existing.suppressedCount + 1,
+            lastEventTs: entry.eventTime ?? now,
+            level: entry.level,
+            message: entry.message
+          };
+          this.writeStore(store);
+          return null;
+        }
+
+        // Outside window: allow, include timestamp and prior suppression count
+        const messageToSend = this.formatMessage(entry.message, entry.eventTime ?? now, existing.suppressedCount);
+        store[key] = {
+          ...existing,
+          suppressedCount: 0,
+          lastEventTs: entry.eventTime ?? now,
+          level: entry.level,
+          message: entry.message
+        };
+        this.writeStore(store);
+        return messageToSend;
+      }
+
+      // New entry: allow, timestamp, no suppression count
+      const messageToSend = this.formatMessage(entry.message, entry.eventTime ?? now, 0);
+      store[key] = {
+        lastTransmitted: 0,
+        suppressedCount: 0,
+        firstUnsentEventTs: entry.eventTime ?? now,
+        lastEventTs: entry.eventTime ?? now,
+        level: entry.level,
+        message: entry.message
+      };
+      this.writeStore(store);
+      return messageToSend;
+    });
+  }
+
+  async onSuccess(entry: LogEntry, now: number): Promise<void> {
+    if (!entry.dedupeKey || typeof entry.dedupeKey !== 'string' || entry.dedupeKey.trim().length === 0) {
+      throw new Error('dedupeKey is required and must be a non-empty string');
+    }
+    const key = entry.dedupeKey;
+
+    await this.withLock(async () => {
+      const store = this.readStore();
+      const existing = store[key];
+      if (existing) {
+        store[key] = {
+          ...existing,
+          lastTransmitted: now,
+          firstUnsentEventTs: 0,
+          suppressedCount: 0
+        };
+        this.writeStore(store);
+      }
+    });
+  }
+
+  async retryFailedTransmissions(send: (entry: LogEntry, message: string) => Promise<void>): Promise<void> {
+    // Best-effort retry: first sweep expired entries, then iterate over entries with lastTransmitted === 0
+    const now = Date.now();
+    const entries = await this.withLock(async () => {
+      const store = this.readStore();
+      // Always sweep on retry
+      this.evictExpiredInStore(store, now);
+      this.lastSweepTsMs = now;
+      this.writeStore(store); 
+      return Object.entries(store)
+        .filter(([, rec]) => rec.lastTransmitted === 0)
+        .map(([key, rec]) => ({ key, rec }));
+    });
+
+    for (const { key, rec } of entries) {
+      try {
+        const message = this.formatMessage(rec.message, rec.lastEventTs, rec.suppressedCount, rec.firstUnsentEventTs);
+        await send({ level: rec.level, message: rec.message, dedupeKey: key, eventTime: rec.lastEventTs }, message);
+        // Mark success
+        await this.withLock(async () => {
+          const store = this.readStore();
+          const current = store[key];
+          if (current) {
+            store[key] = {
+              ...current,
+              lastTransmitted: Date.now(),
+              suppressedCount: 0
+            };
+            this.writeStore(store);
+          }
+        });
+      } catch (err) {
+        // Leave entry for future retries; do not throw
+        logger.error('Failed to retry failed transmission', { key, rec, error: err });
+        return;
+      }
+    }
+  }
+
+  // --- Internals ---
+
+  private ensureStoreFileExists(): void {
+    try {
+      if (!fs.existsSync(this.storeFilePath)) {
+        fs.writeFileSync(this.storeFilePath, JSON.stringify({}), 'utf-8');
+      }
+    } catch {
+      // ignore
+    }
+  }
+
+  private readStore(): LogDedupeStore {
+    try {
+      if (!fs.existsSync(this.storeFilePath)) return {};
+      const content = fs.readFileSync(this.storeFilePath, 'utf-8');
+      return content ? (JSON.parse(content) as LogDedupeStore) : {};
+    } catch {
+      return {};
+    }
+  }
+
+  private writeStore(store: LogDedupeStore): void {
+    try {
+      fs.writeFileSync(this.storeFilePath, JSON.stringify(store, null, 2), 'utf-8');
+    } catch {
+      // ignore
+    }
+  }
+
+  private formatMessage(message: string, eventTs: number, suppressedCount: number, firstUnsentEventTs?: number): string {
+    const timestamp = new Date(eventTs).toISOString();
+    const base = `${timestamp} | ${message}`;
+    if (suppressedCount > 0) {
+      const since = firstUnsentEventTs && firstUnsentEventTs > 0 ? ` since ${new Date(firstUnsentEventTs).toISOString()}` : '';
+      return `${base} (${suppressedCount} suppressed${since})`;
+    }
+    return base;
+  }
+
+  private async withLock<T>(fn: () => Promise<T>): Promise<T> {
+    const lockPath = `${this.storeFilePath}.lock`;
+    const start = Date.now();
+
+    // Try to acquire the lock with retries (best effort)
+    // Avoid infinite waits; cap at ~3 seconds
+    while (true) {
+      try {
+        const fd = fs.openSync(lockPath, 'wx');
+        try {
+          const result = await fn();
+          return result;
+        } finally {
+          try { fs.closeSync(fd); } catch {/* ignore */}
+          try { fs.unlinkSync(lockPath); } catch {/* ignore */}
+        }
+      } catch {
+        // Lock exists
+        if (Date.now() - start > 3000) {
+          // Timed out; proceed without lock to avoid deadlock
+          return await fn();
+        }
+        await new Promise(resolve => setTimeout(resolve, 50));
+      }
+    }
+  }
+
+  /**
+   * Evict expired entries from the store based on the TTL and the key's last transmitted time
+   */
+  private evictExpiredInStore(store: LogDedupeStore, now: number): void {
+    const keys = Object.keys(store);
+    if (keys.length === 0) return;
+    for (const key of keys) {
+      const rec = store[key];
+      const referenceTs = rec.lastTransmitted > 0 ? rec.lastTransmitted : rec.lastEventTs;
+      if (now - referenceTs > this.ttlMs) {
+        delete store[key];
+      }
+    }
+  }
+}

package/src/utils/connector-log/mm-connector-logger-example.ts

@@ -0,0 +1,97 @@
+// import { FileLogDeduper } from './log-deduper';
+// import { MMConnectorLogger, LogEntry } from './mm-connector-logger';
+// import { HTTPError } from '../http-client';
+// import { getErrorType } from '../error-utils';
+// /**
+//  * Example usage of the MM Connector Logger with deduplication
+//  * 
+//  * This example demonstrates how to use the MMConnectorLogger class to send
+//  * progress updates and error logs to the MM cloud. Deduplication is configured
+//  * via environment variables and applied consistently to all log messages.
+//  */
+
+// async function exampleUsage() {
+//   // Initialize logger with source identifier and deduplication settings
+//   const logger = new MMConnectorLogger('proshop', new FileLogDeduper({ 
+//     windowSeconds: 300,    // 5 minute deduplication window
+//     ttlSeconds: 3600,      // 1 hour TTL for store cleanup
+//     sweepIntervalSeconds: 600  // 10 minute sweep interval
+//   }));
+
+//   try {
+//     // Sends initial sync start message
+//     await logger.sendLog(new LogEntry({
+//       level: 'info',
+//       dedupeKey: 'SyncStart',
+//       message: 'Starting sync process'
+//     }));
+
+//     // Simulates some work with progress updates
+//     for (let i = 1; i <= 3; i++) {
+//       await logger.sendLog(new LogEntry({
+//         level: 'info',
+//         dedupeKey: 'SyncProgress',
+//         message: `Processing batch ${i}/3`
+//       }));
+//       await new Promise(resolve => setTimeout(resolve, 100)); // Simulate work
+//     }
+
+//     // Sends completion message
+//     await logger.sendLog(new LogEntry({
+//       level: 'info',
+//       dedupeKey: 'SyncComplete',
+//       message: 'Sync process completed successfully'
+//     }));
+
+//     // Demonstrates error logging with error type extraction
+//     try {
+//       throw new Error('Simulated database connection failure');
+//     } catch (error) {
+//       const errorType = getErrorType(error);
+//       await logger.sendLog(new LogEntry({
+//         level: 'error',
+//         dedupeKey: `DatabaseError_${errorType}`,
+//         message: `Database operation failed: ${error instanceof Error ? error.message : 'Unknown error'}`
+//       }));
+//     }
+
+//     // Demonstrates getErrorType with different error types
+//     const httpError = new HTTPError('Bad Request', 400, 'INVALID_REQUEST');
+//     const errorTypeFromCode = getErrorType(httpError); // Returns 'INVALID_REQUEST'
+    
+//     const typeError = new TypeError('Cannot read property of undefined');
+//     const errorTypeFromName = getErrorType(typeError); // Returns 'TypeError'
+    
+//     await logger.sendLog(new LogEntry({
+//       level: 'error',
+//       dedupeKey: `HTTPError_${errorTypeFromCode}`,
+//       message: `HTTP request failed with code: ${errorTypeFromCode}`
+//     }));
+//     await logger.sendLog(new LogEntry({
+//       level: 'error',
+//       dedupeKey: `TypeError_${errorTypeFromName}`,
+//       message: `Type error occurred: ${errorTypeFromName}`
+//     }));
+
+//     // Demonstrate deduplication - this message will be suppressed if sent within 5 minutes
+//     await logger.sendLog(new LogEntry({
+//       level: 'warn',
+//       dedupeKey: 'RateLimit',
+//       message: 'API rate limit approaching'
+//     }));
+//     await logger.sendLog(new LogEntry({
+//       level: 'warn',
+//       dedupeKey: 'RateLimit',
+//       message: 'API rate limit approaching'
+//     })); // This will be deduplicated
+
+//     // Retry any failed transmissions
+//     await logger.retryFailedTransmissions();
+ 
+//   } catch (error) {
+//     console.error('Failed to send logs:', error);
+//   } finally {
+//     // Clean up resources
+//     await logger.destroy();
+//   }
+// }

package/src/utils/connector-log/mm-connector-logger.ts

@@ -0,0 +1,177 @@
+import { MMApiClient } from '../../services/mm-api-service/mm-api-service';
+import logger from '../../services/reporting-service/logger';
+import type { LogDeduper } from './log-deduper';
+
+export type LogLevelString = 'info' | 'warn' | 'error';
+
+/**
+ * dedupeKey - key used to deduplicate log entries when a LogDeduper is injected into the MMConnectorLogger
+ * 
+ * A recommended practice is to use a meaningful unique identifier for this log entry type.
+ * Examples: SyncFromERPCompleted
+ *           SyncFromERPError:HTTPError
+ */
+export class LogEntry {
+  level: LogLevelString;
+  message: string;
+  dedupeKey: string;
+  eventTime: number;
+
+  constructor(params: { level: LogLevelString; message: string; dedupeKey: string}) {
+    this.level = params.level;
+    this.message = params.message;
+    this.dedupeKey = params.dedupeKey;
+    this.eventTime =  Date.now();
+  }
+}
+
+/**
+ * The response from the MM connector logging API
+ */
+export interface LogResponse {
+  message: string;
+  processed?: number;
+}
+
+/**
+ * Helper function to check if a value is a LogResponse
+ */
+function isLogResponse(value: unknown): value is LogResponse {
+  if (value === null || typeof value !== 'object') return false;
+  const v = value as Record<string, unknown>;
+  if (typeof v.message !== 'string') return false;
+  if ('processed' in v && typeof v.processed !== 'number') return false;
+  return true;
+}
+
+/**
+ * MM Connector Logger for sending progress and error updates to the MM cloud
+ * 
+ * This class provides helper methods to send individual logs to the
+ * MM connector logging API, providing support for validation, retries, and error handling
+ * according to the API specification. 
+ * 
+ * By injecting a LogDeduper, the log entries are deduplicated across jobs over a specified time window
+ */
+export class MMConnectorLogger {
+  private readonly MAX_MSG_LEN = 2000;
+  
+  private mmApiClient: MMApiClient;
+  private readonly deduper?: LogDeduper;
+  private readonly source: string;
+
+  constructor(source: string, deduper?: LogDeduper) {
+    if (source.length < 1 || source.length > 64) {
+      throw new Error('source must be 1-64 characters');
+    }
+
+    this.mmApiClient = new MMApiClient();
+    this.deduper = deduper;
+    this.source = source;
+  }
+
+  // Deduplication helpers are delegated to injected FileLogDeduper
+
+  /**
+   * Send a single log entry to the MM cloud with deduplication.
+   * 
+   * The deduplication is handled by the injected LogDeduper.
+   * If no deduper is injected, the log entry is sent without deduplication.
+   * 
+   * The standard deduper, FileLogDeduper, stores the deduplication state in a file, 
+   * allowing deduplication across jobs, 
+   * 
+   * @param logEntry - The log entry to send
+   * @returns Promise resolving to the API response or null if suppressed
+   * @throws HTTPError if the request fails or Error if the log entry is invalid
+   */
+  async sendLog(logEntry: LogEntry): Promise<LogResponse | null> {
+    this.validateLogEntry(logEntry);
+    
+    const now = Date.now();
+    let messageToSend = logEntry.message;
+    if (this.deduper) {
+      const decision = await this.deduper.decide(logEntry, now);
+      if (decision === null) return null; // suppressed
+      messageToSend = decision;
+    }
+
+    try {
+      const logEntryToSend = {
+        source: this.source,
+        level: logEntry.level,
+        message: messageToSend
+      };
+
+      const response = await this.mmApiClient.sendConnectorLog(logEntryToSend);
+      if (this.deduper) {
+        await this.deduper.onSuccess(logEntry, now);
+      }
+
+      if (!isLogResponse(response)) {
+        logger.warn('Unexpected success response format from MM API for connector log', { response });
+        return { message: 'Unexpected success response format when sending log' };
+      }
+      return { message: response.message };
+
+    } catch (error) {
+      logger.error('Failed to send log to MM cloud', { 
+        level: logEntry.level,
+        error: error instanceof Error ? error.message : 'Unknown error'
+      });
+      throw error;
+    }
+  }
+
+  /**
+   * @throws Error if validation fails
+   */
+  private validateLogEntry(logEntry: LogEntry): void {
+    const allowedLevels: ReadonlyArray<LogLevelString> = ['info', 'warn', 'error'];
+    if (!logEntry.level || !allowedLevels.includes(logEntry.level)) {
+      throw new Error(`level must be one of: ${allowedLevels.join(', ')}`);
+    }
+
+    if (!logEntry.message || typeof logEntry.message !== 'string') {
+      throw new Error('message is required and must be a string');
+    }
+
+    // Truncate message to MAX_MSG_LEN characters
+    logEntry.message = logEntry.message.slice(0, this.MAX_MSG_LEN);
+
+    if (!logEntry.dedupeKey || typeof logEntry.dedupeKey !== 'string') {
+      throw new Error('dedupeKey is required and must be a string');
+    }
+
+    if (logEntry.dedupeKey.trim().length < 1) {
+      throw new Error('dedupeKey must be a non-empty string');
+    }
+  }
+
+  /**
+   * Retry all failed transmissions silently
+   * This method attempts to retry all messages that failed to transmit
+   * and removes them from the failed list if successful, else leaves them for the client to retry
+   * 
+   * Expected usage is by a client to call this as part of its own retry mechanism
+   */
+  async retryFailedTransmissions(): Promise<void> {
+    if (!this.deduper || !this.deduper.retryFailedTransmissions) {
+      return;
+    }
+    await this.deduper.retryFailedTransmissions(async (entry, message) => {
+      await this.mmApiClient.sendConnectorLog({
+        source: this.source,
+        level: entry.level,
+        message
+      });
+    });
+  }
+
+  /**
+   * Clean up resources
+   */
+  async destroy(): Promise<void> {
+    await this.mmApiClient.destroy();
+  }
+}

package/src/utils/error-utils.ts

@@ -0,0 +1,18 @@
+/** A utility to get the type of an error, particularly
+ *  useful for the purpose of error deduplication
+ * 
+ *  Note: prefers 'code' over 'name' in the expectation that this better
+ *  represents the error type
+ */
+const isNonEmptyString = (v: unknown): v is string => typeof v === 'string' && v.trim().length > 0;
+
+export function getErrorType(error: unknown): string {
+  if (error && typeof error === 'object') {
+    const o = error as { code?: unknown; name?: unknown; constructor?: { name?: unknown } };
+    if (isNonEmptyString(o.code)) return o.code;
+    if (isNonEmptyString(o.name)) return o.name;
+    const ctorName = o.constructor?.name;
+    if (isNonEmptyString(ctorName) && ctorName !== 'Object') return ctorName;
+  }
+  return 'Error';
+}

package/src/utils/index.ts

@@ -31,10 +31,8 @@ export { BatchCacheManager } from "../services/caching-service/batch-cache-manag
  * Process and data transformation utilities
  */
 export { StandardProcessDrivers } from "./standard-process-drivers/";
-export type {
-  WriteEntitiesToMMResult,
-  MMBatchValidationError,
-} from "./standard-process-drivers/";
+export type { WriteEntitiesToMMResult } from "./standard-process-drivers/";
+export { MMBatchValidationError } from "./standard-process-drivers/";
 export { getCachedTimezoneOffset } from "./local-data-store/jobs-shared-data";
 
 // Local data store
@@ -54,11 +52,19 @@ export { getERPAPITypeFromEntity } from "./erp-type-from-entity";
  * HTTP Client utilities
  */
 export { HTTPClientFactory } from "./http-client";
+export type { HTTPClient, HTTPRequestConfig, HTTPResponse, HTTPError} from "./http-client";
 
 /**
  * MM Labor Ticket utilities
  */
 export * from "./mm-labor-ticket-helpers";
+export { getErrorType } from './error-utils';
+
+/**
+ * MM Connector Logger utilities
+ */
+export { MMConnectorLogger, type LogResponse, type LogEntry, type LogLevelString } from './connector-log/mm-connector-logger';
+export { FileLogDeduper } from './connector-log/log-deduper';
 
 /**
  * Application initializer

package/src/utils/standard-process-drivers/index.ts

@@ -1,6 +1,4 @@
 // Public exports for standard-process-drivers
 export { StandardProcessDrivers } from "./standard-process-drivers";
-export type {
-  WriteEntitiesToMMResult,
-  MMBatchValidationError,
-} from "./standard-process-drivers";
+export type { WriteEntitiesToMMResult } from "./standard-process-drivers";
+export { MMBatchValidationError } from "./standard-process-drivers";