@devopsplaybook.io/common-utils 1.0.0-beta.5.18ca754

package/src/ConfigBase.spec.ts

@@ -0,0 +1,108 @@
+import * as fse from "fs-extra";
+import * as path from "path";
+import * as os from "os";
+
+jest.mock("uuid", () => ({
+  v4: () => "mock-uuid-1234",
+}));
+
+import { ConfigBase } from "./ConfigBase";
+
+/** Concrete subclass for testing. */
+class TestConfig extends ConfigBase {
+  public MY_SETTING = "default_value";
+  public SECRET_KEY = "secret_default";
+
+  constructor(configFile?: string) {
+    super("test-service", configFile);
+    this.addConfigField({ field: "MY_SETTING" });
+    this.addConfigField({ field: "SECRET_KEY", sensitive: true });
+  }
+}
+
+describe("ConfigBase", () => {
+  let tmpDir: string;
+  let configPath: string;
+
+  beforeEach(() => {
+    tmpDir = fse.mkdtempSync(path.join(os.tmpdir(), "config-test-"));
+    configPath = path.join(tmpDir, "config.json");
+  });
+
+  afterEach(() => {
+    fse.removeSync(tmpDir);
+    // Clean up env vars that tests may have set
+    delete process.env.MY_SETTING;
+    delete process.env.SECRET_KEY;
+    delete process.env.LOG_LEVEL;
+    delete process.env.DATA_DIR;
+  });
+
+  it("should initialise with default values", () => {
+    const config = new TestConfig(configPath);
+    expect(config.SERVICE_ID).toBe("test-service");
+    expect(config.MY_SETTING).toBe("default_value");
+    expect(config.SECRET_KEY).toBe("secret_default");
+    expect(config.LOG_LEVEL).toBe("info");
+    expect(config.DATABASE_TYPE).toBe("sqlite");
+  });
+
+  it("should load values from config.json", async () => {
+    fse.writeJsonSync(configPath, {
+      MY_SETTING: "from_file",
+      LOG_LEVEL: "debug",
+    });
+    const config = new TestConfig(configPath);
+    await config.reload();
+    expect(config.MY_SETTING).toBe("from_file");
+    expect(config.LOG_LEVEL).toBe("debug");
+  });
+
+  it("should prefer environment variables over config.json", async () => {
+    fse.writeJsonSync(configPath, {
+      MY_SETTING: "from_file",
+    });
+    process.env.MY_SETTING = "from_env";
+    const config = new TestConfig(configPath);
+    await config.reload();
+    expect(config.MY_SETTING).toBe("from_env");
+  });
+
+  it("should mask sensitive fields in log output", async () => {
+    const logs: string[] = [];
+    const config = new TestConfig(configPath);
+    await config.reload((msg) => logs.push(msg));
+    const secretLog = logs.find((l) => l.includes("SECRET_KEY"));
+    expect(secretLog).toContain("********************");
+    expect(secretLog).not.toContain("secret_default");
+  });
+
+  it("should not mask non-sensitive fields in log output", async () => {
+    const logs: string[] = [];
+    const config = new TestConfig(configPath);
+    await config.reload((msg) => logs.push(msg));
+    const settingLog = logs.find((l) => l.includes("MY_SETTING"));
+    expect(settingLog).toContain("default_value");
+  });
+
+  it("should survive missing config.json gracefully", async () => {
+    const config = new TestConfig("/nonexistent/path/config.json");
+    await config.reload();
+    // Should keep defaults
+    expect(config.MY_SETTING).toBe("default_value");
+  });
+
+  it("should register additional fields via addConfigField", async () => {
+    fse.writeJsonSync(configPath, { MY_SETTING: "updated" });
+    const config = new TestConfig(configPath);
+    await config.reload();
+    expect(config.MY_SETTING).toBe("updated");
+  });
+
+  it("should handle DATABASE_TYPE field", async () => {
+    fse.writeJsonSync(configPath, { DATABASE_TYPE: "postgres" });
+    const config = new TestConfig(configPath);
+    await config.reload();
+    expect(config.DATABASE_TYPE).toBe("postgres");
+  });
+});

package/src/ConfigBase.ts

@@ -0,0 +1,213 @@
+import { ConfigOTelInterface } from "@devopsplaybook.io/otel-utils";
+import * as fse from "fs-extra";
+import { v4 as uuidv4 } from "uuid";
+import path from "path";
+
+/**
+ * Configuration field descriptor used by {@link ConfigBase.addConfigField}.
+ */
+export interface ConfigFieldDef {
+  /** Property name on the config instance. */
+  field: string;
+  /** When `true` the value is masked in log output. */
+  sensitive?: boolean;
+}
+
+/**
+ * Database-specific configuration fields shared by every project that
+ * supports both SQLite and PostgreSQL backends.
+ */
+export interface ConfigDatabaseInterface {
+  DATABASE_TYPE: "sqlite" | "postgres";
+  DATABASE_POSTGRES_HOST: string;
+  DATABASE_POSTGRES_PORT: number;
+  DATABASE_POSTGRES_USER: string;
+  DATABASE_POSTGRES_PASSWORD: string;
+  DATABASE_POSTGRES_DATABASE: string;
+}
+
+/**
+ * Common server configuration fields shared across projects.
+ */
+export interface ConfigCommonInterface
+  extends ConfigOTelInterface, ConfigDatabaseInterface {
+  CONFIG_FILE: string;
+  API_PORT: number;
+  JWT_VALIDITY_DURATION: number;
+  CORS_POLICY_ORIGIN: string;
+  DATA_DIR: string;
+  JWT_KEY: string;
+  LOG_LEVEL: string;
+}
+
+/**
+ * Abstract base class for project configuration.
+ *
+ * Implements the three-layer override strategy used across all
+ * devopsplaybook.io server projects:
+ *   1. **Environment variable** (highest priority)
+ *   2. **config.json** file value
+ *   3. **Default** declared on the class property
+ *
+ * Subclasses add project-specific fields and call {@link addConfigField}
+ * inside their constructor so that {@link reload} picks them up.
+ *
+ * @example
+ * ```ts
+ * class MyConfig extends ConfigBase {
+ *   public MY_SETTING = "default";
+ *   constructor() {
+ *     super("my-service");
+ *     this.addConfigField({ field: "MY_SETTING" });
+ *   }
+ * }
+ * ```
+ */
+export abstract class ConfigBase implements ConfigCommonInterface {
+  // -- OTel fields (ConfigOTelInterface) --
+  public SERVICE_ID: string;
+  public VERSION = "1";
+  public OPENTELEMETRY_COLLECTOR_HTTP_TRACES = "";
+  public OPENTELEMETRY_COLLECTOR_HTTP_METRICS = "";
+  public OPENTELEMETRY_COLLECTOR_HTTP_LOGS = "";
+  public OPENTELEMETRY_COLLECTOR_AWS = false;
+  public OPENTELEMETRY_COLLECTOR_EXPORT_LOGS_INTERVAL_SECONDS = 60;
+  public OPENTELEMETRY_COLLECTOR_EXPORT_METRICS_INTERVAL_SECONDS = 60;
+  public OPENTELEMETRY_COLLECT_AUTHORIZATION_HEADER = "";
+
+  // -- Common server fields --
+  public CONFIG_FILE: string;
+  public API_PORT = 8080;
+  public JWT_VALIDITY_DURATION = 3 * 31 * 24 * 3600;
+  public CORS_POLICY_ORIGIN = "";
+  public DATA_DIR = process.env.DATA_DIR || "/data";
+  public JWT_KEY: string = uuidv4();
+  public LOG_LEVEL = "info";
+
+  // -- Database fields --
+  public DATABASE_TYPE: "sqlite" | "postgres" = "sqlite";
+  public DATABASE_POSTGRES_HOST = "";
+  public DATABASE_POSTGRES_PORT = 5432;
+  public DATABASE_POSTGRES_USER = "";
+  public DATABASE_POSTGRES_PASSWORD = "";
+  public DATABASE_POSTGRES_DATABASE = "";
+
+  /**
+   * Fields registered by subclasses (or the base) that {@link reload}
+   * should process.  Common / DB / OTel fields are pre-registered.
+   */
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  private _fields: { field: string; sensitive: boolean; defaultValue: any }[] =
+    [];
+
+  /**
+   * @param serviceId  Unique service identifier (e.g. `"cryptotrader-server"`).
+   * @param configFile Optional path to the JSON config file. Defaults to `"config.json"`.
+   */
+  constructor(serviceId: string, configFile?: string) {
+    this.SERVICE_ID = serviceId;
+    this.CONFIG_FILE = configFile || process.env.CONFIG_FILE || "config.json";
+
+    // Auto-detect version from nearest package.json
+    try {
+      const pkg = fse.readJsonSync(path.resolve(__dirname, "../package.json"));
+      if (pkg && pkg.version) {
+        this.VERSION = pkg.version;
+      }
+      // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    } catch (_e) {
+      // fallback to "1"
+    }
+
+    // Pre-register base + DB + OTel fields so reload() handles them
+    const baseFields: ConfigFieldDef[] = [
+      { field: "JWT_VALIDITY_DURATION" },
+      { field: "CORS_POLICY_ORIGIN" },
+      { field: "DATA_DIR" },
+      { field: "JWT_KEY", sensitive: true },
+      { field: "LOG_LEVEL" },
+      { field: "DATABASE_TYPE" },
+      { field: "DATABASE_POSTGRES_HOST" },
+      { field: "DATABASE_POSTGRES_PORT" },
+      { field: "DATABASE_POSTGRES_USER" },
+      { field: "DATABASE_POSTGRES_PASSWORD", sensitive: true },
+      { field: "DATABASE_POSTGRES_DATABASE" },
+      { field: "OPENTELEMETRY_COLLECTOR_HTTP_TRACES" },
+      { field: "OPENTELEMETRY_COLLECTOR_HTTP_METRICS" },
+      { field: "OPENTELEMETRY_COLLECTOR_HTTP_LOGS" },
+      { field: "OPENTELEMETRY_COLLECTOR_AWS" },
+      {
+        field: "OPENTELEMETRY_COLLECTOR_EXPORT_LOGS_INTERVAL_SECONDS",
+      },
+      {
+        field: "OPENTELEMETRY_COLLECTOR_EXPORT_METRICS_INTERVAL_SECONDS",
+      },
+      {
+        field: "OPENTELEMETRY_COLLECT_AUTHORIZATION_HEADER",
+        sensitive: true,
+      },
+    ];
+    for (const f of baseFields) {
+      this.addConfigField(f);
+    }
+  }
+
+  /**
+   * Register a configuration field so that {@link reload} processes it.
+   * Call this in your subclass constructor for every project-specific field.
+   */
+  public addConfigField(def: ConfigFieldDef): void {
+    this._fields.push({
+      field: def.field,
+      sensitive: def.sensitive ?? false,
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      defaultValue: (this as any)[def.field],
+    });
+  }
+
+  /**
+   * Load (or reload) configuration from the JSON file and environment variables.
+   * Environment variables always take precedence over file values.
+   *
+   * @param logger  Optional log callback `(message: string) => void`.
+   *                When omitted nothing is logged (useful in tests).
+   */
+  public async reload(logger?: (message: string) => void): Promise<void> {
+    // eslint-disable-next-line @typescript-eslint/no-empty-function
+    const log = logger ?? (() => {});
+    let content: Record<string, unknown> = {};
+    try {
+      content = await fse.readJson(this.CONFIG_FILE);
+      // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    } catch (_e) {
+      // config file is optional – fall back to env + defaults
+    }
+
+    log(`Configuration Value: CONFIG_FILE: ${this.CONFIG_FILE}`);
+    log(`Configuration Value: SERVICE_ID: ${this.SERVICE_ID}`);
+    log(`Configuration Value: VERSION: ${this.VERSION}`);
+
+    for (const { field, sensitive } of this._fields) {
+      let from = "defaults";
+      if (process.env[field] !== undefined) {
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        (this as any)[field] = process.env[field];
+        from = "environment";
+      } else if (content[field] !== undefined) {
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        (this as any)[field] = content[field];
+        from = "config";
+      }
+      if (sensitive) {
+        log(
+          `Configuration Value: ${field}: ******************** (from ${from})`,
+        );
+      } else {
+        log(
+          // eslint-disable-next-line @typescript-eslint/no-explicit-any
+          `Configuration Value: ${field}: ${(this as any)[field]} (from ${from})`,
+        );
+      }
+    }
+  }
+}

package/src/DbUtils.spec.ts

@@ -0,0 +1,23 @@
+import { convertToPostgresPlaceholders } from "./DbUtils";
+
+describe("convertToPostgresPlaceholders", () => {
+  it("should convert single ?", () => {
+    expect(convertToPostgresPlaceholders("SELECT * FROM t WHERE id = ?")).toBe(
+      "SELECT * FROM t WHERE id = $1",
+    );
+  });
+
+  it("should convert multiple ? to $1, $2, ...", () => {
+    expect(
+      convertToPostgresPlaceholders("INSERT INTO t (a,b,c) VALUES (?,?,?)"),
+    ).toBe("INSERT INTO t (a,b,c) VALUES ($1,$2,$3)");
+  });
+
+  it("should return SQL unchanged when no ? present", () => {
+    expect(convertToPostgresPlaceholders("SELECT 1")).toBe("SELECT 1");
+  });
+
+  it("should handle empty string", () => {
+    expect(convertToPostgresPlaceholders("")).toBe("");
+  });
+});

package/src/DbUtils.ts

@@ -0,0 +1,118 @@
+import { Span } from "@opentelemetry/sdk-trace-base";
+import { StandardTracer, StandardLogger } from "@devopsplaybook.io/otel-utils";
+import * as SqlDbUtils from "./SqlDbUtils";
+import * as PostgresDbUtils from "./PostgresDbUtils";
+
+/**
+ * Configuration subset required by the unified DB facade.
+ */
+export interface DbUtilsConfig
+  extends SqlDbUtils.SqlDbConfig, PostgresDbUtils.PostgresDbConfig {
+  DATABASE_TYPE: "sqlite" | "postgres";
+}
+
+let databaseType: "sqlite" | "postgres" = "sqlite";
+
+/**
+ * Injects the OTel tracer and logger instances used by the DB layer.
+ * Must be called once at startup, before {@link DbUtilsInit}.
+ */
+export function DbUtilsSetOTel(
+  tracer: StandardTracer,
+  logger: StandardLogger,
+): void {
+  SqlDbUtils.SqlDbUtilsSetOTel(tracer, logger);
+  PostgresDbUtils.PostgresDbUtilsSetOTel(tracer, logger);
+}
+
+/**
+ * Initialise the database layer.
+ *
+ * Dispatches to the SQLite or Postgres backend depending on
+ * `config.DATABASE_TYPE` and runs pending migration files from `sqlDir`.
+ *
+ * @param context  Parent OTel span.
+ * @param config   Server configuration.
+ * @param sqlDir   Absolute path to the directory containing SQL migration files.
+ */
+export async function DbUtilsInit(
+  context: Span,
+  config: DbUtilsConfig,
+  sqlDir: string,
+): Promise<void> {
+  databaseType = config.DATABASE_TYPE;
+  if (databaseType === "postgres") {
+    await PostgresDbUtils.PostgresDbUtilsInit(context, config, sqlDir);
+  } else {
+    await SqlDbUtils.SqlDbUtilsInit(context, config, sqlDir);
+  }
+}
+
+/**
+ * Returns the native database handle.
+ * - SQLite: `better-sqlite3` `Database` instance
+ * - Postgres: `pg` `Pool` instance
+ */
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export function DbUtilsGetDatabase(): any {
+  if (databaseType === "postgres") {
+    return PostgresDbUtils.PostgresDbUtilsGetPool();
+  }
+  return SqlDbUtils.SqlDbUtilsGetDatabase();
+}
+
+/** Convert SQLite `?` placeholders to PostgreSQL `$1, $2, ...` numbering. */
+export function convertToPostgresPlaceholders(sql: string): string {
+  let paramIndex = 1;
+  return sql.replace(/\?/g, () => `$${paramIndex++}`);
+}
+
+/**
+ * Execute a write SQL statement with OTel tracing.
+ * Automatically converts `?` placeholders to `$N` when using Postgres.
+ *
+ * @returns Number of rows changed.
+ */
+export function DbUtilsExecSQL(
+  context: Span,
+  sql: string,
+  params: unknown[] = [],
+): number | Promise<number> {
+  if (databaseType === "postgres") {
+    return PostgresDbUtils.PostgresDbUtilsExecSQL(
+      context,
+      convertToPostgresPlaceholders(sql),
+      params,
+    );
+  }
+  return SqlDbUtils.SqlDbUtilsExecSQL(context, sql, params);
+}
+
+/**
+ * Execute a read SQL query with OTel tracing.
+ * Automatically converts `?` placeholders to `$N` when using Postgres.
+ *
+ * @returns Array of row objects.
+ */
+export function DbUtilsQuerySQL(
+  context: Span,
+  sql: string,
+  params: unknown[] = [],
+  debug = false,
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+): any[] | Promise<any[]> {
+  if (databaseType === "postgres") {
+    return PostgresDbUtils.PostgresDbUtilsQuerySQL(
+      context,
+      convertToPostgresPlaceholders(sql),
+      params,
+      debug,
+    );
+  }
+  return SqlDbUtils.SqlDbUtilsQuerySQL(context, sql, params, debug);
+}
+
+/** Returns the active database type (`"sqlite"` or `"postgres"`). */
+export function DbUtilsGetType(): "sqlite" | "postgres" {
+  return databaseType;
+}

package/src/DbUtilsNoTelemetry.spec.ts

@@ -0,0 +1,174 @@
+import {
+  DbUtilsNoTelemetryBatchInsert,
+  DbUtilsNoTelemetryExecSQL,
+  DbUtilsNoTelemetryQuerySQL,
+} from "./DbUtilsNoTelemetry";
+
+/* eslint-disable @typescript-eslint/no-unsafe-function-type */
+
+// Shared mock DB handle – defined BEFORE jest.mock factory so it's hoisted correctly.
+// We use jest.fn() at module scope; the mock factory captures the same reference.
+const mockPrepare = jest.fn(() => ({
+  run: jest.fn().mockReturnValue({ changes: 0 }),
+  all: jest.fn().mockReturnValue([]),
+}));
+const mockQuery = jest.fn();
+const mockDbHandle = { prepare: mockPrepare, query: mockQuery };
+
+let currentDbType: "sqlite" | "postgres" = "sqlite";
+
+jest.mock("./DbUtils", () => ({
+  DbUtilsGetDatabase: jest.fn(() => mockDbHandle),
+  DbUtilsGetType: jest.fn(() => currentDbType),
+  convertToPostgresPlaceholders: jest.fn((sql: string) => {
+    let idx = 1;
+    return sql.replace(/\?/g, () => `$${idx++}`);
+  }),
+}));
+
+jest.mock("@devopsplaybook.io/otel-utils", () => ({
+  ModuleLogger: jest.fn(),
+  StandardLogger: jest.fn(),
+}));
+
+import * as DbUtilsNoTelemetryModule from "./DbUtilsNoTelemetry";
+
+beforeAll(() => {
+  const mockLogger = {
+    error: jest.fn(),
+    info: jest.fn(),
+    warn: jest.fn(),
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  } as any;
+  DbUtilsNoTelemetryModule.DbUtilsNoTelemetrySetLogger({
+    createModuleLogger: () => mockLogger,
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  } as any);
+});
+
+beforeEach(() => {
+  jest.clearAllMocks();
+  currentDbType = "sqlite";
+});
+
+describe("DbUtilsNoTelemetryBatchInsert", () => {
+  it("returns 0 for empty rows", () => {
+    const result = DbUtilsNoTelemetryBatchInsert("INTO t (c)", 1, []);
+    expect(result).toBe(0);
+  });
+
+  it("generates correct multi-row VALUES SQL (sqlite)", () => {
+    currentDbType = "sqlite";
+    mockPrepare.mockReturnValue({
+      run: jest.fn().mockReturnValue({ changes: 2 }),
+      all: jest.fn(),
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    } as any);
+
+    const rows = [
+      ["a1", "b1"],
+      ["a2", "b2"],
+    ];
+    const result = DbUtilsNoTelemetryBatchInsert("INTO t (c1,c2)", 2, rows);
+    expect(result).toBe(2);
+  });
+});
+
+describe("DbUtilsNoTelemetryExecSQL (sqlite)", () => {
+  beforeEach(() => {
+    currentDbType = "sqlite";
+  });
+
+  it("resolves with changes count on success", () => {
+    mockPrepare.mockReturnValue({
+      run: jest.fn().mockReturnValue({ changes: 3 }),
+      all: jest.fn(),
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    } as any);
+
+    const result = DbUtilsNoTelemetryExecSQL("INSERT INTO t (c) VALUES (?)", [
+      "x",
+    ]);
+    expect(result).toBe(3);
+  });
+});
+
+describe("DbUtilsNoTelemetryExecSQL (postgres)", () => {
+  beforeEach(() => {
+    currentDbType = "postgres";
+  });
+
+  it("resolves with rowCount on success", async () => {
+    mockQuery.mockImplementation(
+      (_sql: string, _params: unknown[], cb: Function) => {
+        cb(null, { rowCount: 5 });
+      },
+    );
+
+    const result = await DbUtilsNoTelemetryExecSQL(
+      "INSERT INTO t (c) VALUES (?)",
+      ["x"],
+    );
+    expect(result).toBe(5);
+  });
+
+  it("rejects on error", async () => {
+    mockQuery.mockImplementation(
+      (_sql: string, _params: unknown[], cb: Function) => {
+        cb(new Error("deadlock detected"));
+      },
+    );
+
+    await expect(
+      DbUtilsNoTelemetryExecSQL("INSERT INTO t (c) VALUES (?)", ["x"]),
+    ).rejects.toThrow("deadlock detected");
+  });
+});
+
+describe("DbUtilsNoTelemetryQuerySQL (sqlite)", () => {
+  beforeEach(() => {
+    currentDbType = "sqlite";
+  });
+
+  it("returns rows on success", () => {
+    const expectedRows = [{ id: 1 }, { id: 2 }];
+    mockPrepare.mockReturnValue({
+      run: jest.fn(),
+      all: jest.fn().mockReturnValue(expectedRows),
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    } as any);
+
+    const result = DbUtilsNoTelemetryQuerySQL("SELECT * FROM t");
+    expect(result).toEqual(expectedRows);
+  });
+});
+
+describe("DbUtilsNoTelemetryQuerySQL (postgres)", () => {
+  beforeEach(() => {
+    currentDbType = "postgres";
+  });
+
+  it("returns rows on success", async () => {
+    const expectedRows = [{ id: 1 }, { id: 2 }];
+    mockQuery.mockImplementation(
+      (_sql: string, _params: unknown[], cb: Function) => {
+        cb(null, { rows: expectedRows });
+      },
+    );
+
+    const result = await DbUtilsNoTelemetryQuerySQL("SELECT * FROM t");
+    expect(result).toEqual(expectedRows);
+  });
+
+  it("rejects on error", async () => {
+    mockQuery.mockImplementation(
+      (_sql: string, _params: unknown[], cb: Function) => {
+        cb(new Error("connection lost"));
+      },
+    );
+
+    await expect(DbUtilsNoTelemetryQuerySQL("SELECT * FROM t")).rejects.toThrow(
+      "connection lost",
+    );
+  });
+});