@etcsec-com/etc-collector 1.4.0

package/eslint.config.js

@@ -0,0 +1,54 @@
+const tsPlugin = require('@typescript-eslint/eslint-plugin');
+const tsParser = require('@typescript-eslint/parser');
+const prettierConfig = require('eslint-config-prettier');
+
+module.exports = [
+  {
+    ignores: ['node_modules/', 'dist/', 'coverage/', '.github/', 'docs/', 'scripts/'],
+  },
+  {
+    files: ['src/**/*.ts'],
+    languageOptions: {
+      parser: tsParser,
+      parserOptions: {
+        ecmaVersion: 2022,
+        sourceType: 'module',
+        project: './tsconfig.json',
+      },
+      globals: {
+        console: 'readonly',
+        process: 'readonly',
+        Buffer: 'readonly',
+        __dirname: 'readonly',
+        __filename: 'readonly',
+      },
+    },
+    plugins: {
+      '@typescript-eslint': tsPlugin,
+    },
+    rules: {
+      ...tsPlugin.configs['recommended'].rules,
+      ...prettierConfig.rules,
+      // Relaxed temporarily - TODO: fix and re-enable
+      '@typescript-eslint/no-explicit-any': 'warn',
+      '@typescript-eslint/no-unsafe-assignment': 'off',
+      '@typescript-eslint/no-unsafe-member-access': 'off',
+      '@typescript-eslint/no-unsafe-call': 'off',
+      '@typescript-eslint/no-unsafe-return': 'off',
+      '@typescript-eslint/no-unsafe-argument': 'off',
+      '@typescript-eslint/explicit-function-return-type': 'off',
+      '@typescript-eslint/no-unused-vars': [
+        'error',
+        { argsIgnorePattern: '^_', varsIgnorePattern: '^_', caughtErrors: 'none' },
+      ],
+      '@typescript-eslint/no-floating-promises': 'error',
+      '@typescript-eslint/no-misused-promises': 'error',
+      '@typescript-eslint/require-await': 'off',
+      'max-lines': ['error', { max: 2000, skipBlankLines: true, skipComments: true }],
+      complexity: ['warn', 20],
+      'max-depth': ['warn', 5],
+      'max-nested-callbacks': ['error', 3],
+      'no-console': 'off',
+    },
+  },
+];

package/jest.config.js

@@ -0,0 +1,38 @@
+/** @type {import('jest').Config} */
+module.exports = {
+  preset: 'ts-jest',
+  testEnvironment: 'node',
+  roots: ['<rootDir>/tests'],
+  testMatch: ['**/*.test.ts'],
+  // TODO: Fix these tests and remove from ignore list
+  testPathIgnorePatterns: [
+    '/node_modules/',
+    'tests/unit/services/audit/detectors/',
+    'tests/unit/services/export/',
+    'tests/integration/auth/',
+    'tests/integration/api/',
+  ],
+  collectCoverageFrom: [
+    'src/**/*.ts',
+    '!src/**/*.types.ts',
+    '!src/types/**',
+  ],
+  coverageThreshold: {
+    global: {
+      branches: 30,
+      functions: 30,
+      lines: 30,
+      statements: 30,
+    },
+  },
+  coverageReporters: ['text', 'lcov', 'html'],
+  setupFilesAfterEnv: ['<rootDir>/tests/setup.ts'],
+  moduleFileExtensions: ['ts', 'js', 'json'],
+  transform: {
+    '^.+\\.ts$': ['ts-jest', {
+      diagnostics: false, // Disable type checking in tests for now
+      isolatedModules: true,
+    }],
+  },
+  verbose: true,
+};

package/package.json

@@ -0,0 +1,97 @@
+{
+  "name": "@etcsec-com/etc-collector",
+  "version": "1.4.0",
+  "description": "Active Directory and Azure AD security auditing tool",
+  "main": "dist/server.js",
+  "bin": {
+    "etc-collector": "dist/server.js"
+  },
+  "directories": {
+    "doc": "docs",
+    "test": "tests"
+  },
+  "pkg": {
+    "scripts": "dist/**/*.js",
+    "assets": [
+      "dist/**/*",
+      "docs/api/**/*"
+    ],
+    "targets": [
+      "node20-linux-x64",
+      "node20-macos-x64",
+      "node20-macos-arm64",
+      "node20-win-x64"
+    ],
+    "outputPath": "binaries"
+  },
+  "scripts": {
+    "dev": "ts-node src/server.ts",
+    "build": "tsc",
+    "start": "node dist/server.js",
+    "test": "jest",
+    "test:watch": "jest --watch",
+    "test:coverage": "jest --coverage",
+    "lint": "eslint src/",
+    "lint:fix": "eslint src/ --fix",
+    "format": "prettier --write \"src/**/*.ts\"",
+    "format:check": "prettier --check \"src/**/*.ts\"",
+    "typecheck": "tsc --noEmit"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/etcsec-com/etc-collector.git"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "Apache-2.0",
+  "type": "commonjs",
+  "bugs": {
+    "url": "https://github.com/etcsec-com/etc-collector/issues"
+  },
+  "homepage": "https://github.com/etcsec-com/etc-collector#readme",
+  "devDependencies": {
+    "@types/better-sqlite3": "^7.6.13",
+    "@types/cors": "^2.8.19",
+    "@types/express": "^5.0.6",
+    "@types/isomorphic-fetch": "^0.0.39",
+    "@types/jest": "^30.0.0",
+    "@types/js-yaml": "^4.0.9",
+    "@types/jsonwebtoken": "^9.0.10",
+    "@types/microsoft-graph": "^2.40.1",
+    "@types/node": "^25.0.6",
+    "@types/supertest": "^6.0.3",
+    "@types/swagger-ui-express": "^4.1.8",
+    "@types/uuid": "^10.0.0",
+    "@typescript-eslint/eslint-plugin": "^8.52.0",
+    "@typescript-eslint/parser": "^8.52.0",
+    "eslint": "^9.39.2",
+    "eslint-config-prettier": "^10.1.8",
+    "eslint-plugin-prettier": "^5.5.4",
+    "jest": "^30.2.0",
+    "prettier": "^3.7.4",
+    "supertest": "^7.2.2",
+    "ts-jest": "^29.4.6",
+    "ts-node": "^10.9.2",
+    "typescript": "^5.9.3"
+  },
+  "dependencies": {
+    "@azure/identity": "^4.13.0",
+    "@marsaud/smb2": "^0.18.0",
+    "@microsoft/microsoft-graph-client": "^3.0.7",
+    "better-sqlite3": "^12.6.0",
+    "cors": "^2.8.5",
+    "dotenv": "^17.2.3",
+    "express": "^5.2.1",
+    "express-rate-limit": "^8.2.1",
+    "helmet": "^8.1.0",
+    "isomorphic-fetch": "^3.0.0",
+    "js-yaml": "^4.1.1",
+    "jsonwebtoken": "^9.0.3",
+    "ldapts": "^8.1.3",
+    "swagger-ui-express": "^5.0.1",
+    "uuid": "^13.0.0",
+    "winston": "^3.19.0",
+    "winston-daily-rotate-file": "^5.0.0",
+    "zod": "^4.3.5"
+  }
+}

package/scripts/fetch-ad-cert.sh

@@ -0,0 +1,142 @@
+#!/bin/bash
+################################################################################
+# Fetch AD Root CA Certificate
+# Retrieves the Root CA certificate from Active Directory for LDAPS
+################################################################################
+
+set -e
+
+# Colors
+GREEN='\033[0;32m'
+RED='\033[0;31m'
+YELLOW='\033[1;33m'
+CYAN='\033[0;36m'
+NC='\033[0m'
+
+print_success() { echo -e "${GREEN}✅ $1${NC}"; }
+print_error() { echo -e "${RED}❌ ERROR: $1${NC}"; }
+print_warning() { echo -e "${YELLOW}⚠️  WARNING: $1${NC}"; }
+print_info() { echo -e "${CYAN}ℹ️  $1${NC}"; }
+
+# Check if ldapsearch is available
+if ! command -v ldapsearch &> /dev/null; then
+    print_error "ldapsearch not found. Please install openldap-clients (RHEL/CentOS) or ldap-utils (Debian/Ubuntu)"
+    exit 1
+fi
+
+# Check if openssl is available
+if ! command -v openssl &> /dev/null; then
+    print_error "openssl not found. Please install openssl"
+    exit 1
+fi
+
+# Load environment variables from .env if available
+if [ -f ".env" ]; then
+    print_info "Loading LDAP configuration from .env"
+    set -a
+    source <(grep -E "^LDAP_" .env | sed 's/^/export /')
+    set +a
+fi
+
+# Get LDAP connection details
+LDAP_URL=${LDAP_URL:-"ldaps://dc.example.com:636"}
+LDAP_BIND_DN=${LDAP_BIND_DN:-"CN=ldap-reader,OU=Service Accounts,DC=example,DC=com"}
+LDAP_BIND_PASSWORD=${LDAP_BIND_PASSWORD:-""}
+
+if [ -z "$LDAP_BIND_PASSWORD" ]; then
+    print_error "LDAP_BIND_PASSWORD not set in .env"
+    exit 1
+fi
+
+# Extract host and port
+LDAP_HOST=$(echo "$LDAP_URL" | sed -E 's|ldaps?://([^:]+).*|\1|')
+LDAP_PORT=$(echo "$LDAP_URL" | sed -E 's|ldaps?://[^:]+:?([0-9]+)?.*|\1|')
+LDAP_PORT=${LDAP_PORT:-636}
+
+# Determine protocol
+if [[ "$LDAP_URL" =~ ^ldaps:// ]]; then
+    LDAP_PROTOCOL="ldaps"
+else
+    LDAP_PROTOCOL="ldap"
+fi
+
+# Extract domain DN from bind DN (everything starting with DC=)
+DOMAIN_DN=$(echo "$LDAP_BIND_DN" | sed 's/^[^D]*\(DC=.*\)$/\1/')
+
+print_info "Fetching certificate from $LDAP_HOST:$LDAP_PORT"
+print_info "Domain DN: $DOMAIN_DN"
+
+# Create certs directory
+mkdir -p ./certs
+
+# Try to find the CA certificate in AD
+CA_BASES=(
+    "CN=Certification Authorities,CN=Public Key Services,CN=Services,CN=Configuration,$DOMAIN_DN"
+    "CN=NTAuthCertificates,CN=Public Key Services,CN=Services,CN=Configuration,$DOMAIN_DN"
+)
+
+CERT_FETCHED=false
+
+for CA_BASE in "${CA_BASES[@]}"; do
+    print_info "Searching at: $CA_BASE"
+
+    # Use LDAPTLS_REQCERT=never to bypass cert verification during fetch
+    # Use -LLL for LDIF output without comments, -o ldif-wrap=no for no line wrapping
+    CERT_DATA=$(LDAPTLS_REQCERT=never ldapsearch -x -LLL -o ldif-wrap=no \
+        -H "${LDAP_PROTOCOL}://${LDAP_HOST}:${LDAP_PORT}" \
+        -D "$LDAP_BIND_DN" \
+        -w "$LDAP_BIND_PASSWORD" \
+        -b "$CA_BASE" \
+        -s sub \
+        "(objectClass=*)" \
+        cACertificate \
+        2>/dev/null | \
+        grep "^cACertificate::" | \
+        head -1 | \
+        sed 's/^cACertificate:: //')
+
+    if [ -n "$CERT_DATA" ]; then
+        print_success "Found certificate at: $CA_BASE"
+
+        # Decode and convert to PEM
+        echo "$CERT_DATA" | base64 -d | \
+            openssl x509 -inform DER -outform PEM > ./certs/ad-root-ca.crt 2>/dev/null
+
+        if [ -f "./certs/ad-root-ca.crt" ] && [ -s "./certs/ad-root-ca.crt" ]; then
+            # Verify it's a valid certificate
+            CERT_SUBJECT=$(openssl x509 -in ./certs/ad-root-ca.crt -noout -subject 2>/dev/null | sed 's/subject=//')
+            CERT_ISSUER=$(openssl x509 -in ./certs/ad-root-ca.crt -noout -issuer 2>/dev/null | sed 's/issuer=//')
+            CERT_EXPIRY=$(openssl x509 -in ./certs/ad-root-ca.crt -noout -enddate 2>/dev/null | sed 's/notAfter=//')
+
+            print_success "Certificate retrieved and saved successfully!"
+            echo ""
+            echo "Certificate Details:"
+            echo "  Subject: $CERT_SUBJECT"
+            echo "  Issuer:  $CERT_ISSUER"
+            echo "  Expires: $CERT_EXPIRY"
+            echo ""
+            echo "Certificate saved to: ./certs/ad-root-ca.crt"
+
+            chmod 644 ./certs/ad-root-ca.crt
+            CERT_FETCHED=true
+            break
+        fi
+    fi
+done
+
+if [ "$CERT_FETCHED" = false ]; then
+    print_error "Could not automatically fetch the certificate"
+    echo ""
+    echo "Possible reasons:"
+    echo "  • Your AD doesn't store certs in standard locations"
+    echo "  • LDAP bind credentials don't have read access to PKI containers"
+    echo "  • Certificate container is in a non-standard location"
+    echo ""
+    print_info "You can manually export the Root CA certificate from your AD server:"
+    echo "  1. On your AD server, run: certutil -ca.cert ad-root-ca.cer"
+    echo "  2. Convert to PEM: openssl x509 -inform DER -in ad-root-ca.cer -out ad-root-ca.crt"
+    echo "  3. Copy to ./certs/ad-root-ca.crt"
+    exit 1
+fi
+
+exit 0

package/src/api/controllers/audit.controller.ts

@@ -0,0 +1,313 @@
+/**
+ * Audit Controller
+ *
+ * Handles AD and Azure audit operations.
+ * Story 1.10: API Controllers & Routes
+ *
+ * Endpoints:
+ * - POST /api/v1/audit/ad - Run AD audit (sync or async with ?async=true)
+ * - GET /api/v1/audit/ad/status - Test LDAP connection
+ * - POST /api/v1/audit/azure - Run Azure audit
+ * - GET /api/v1/audit/azure/status - Test Graph connection
+ */
+
+import { Request, Response, NextFunction } from 'express';
+import { ADAuditService } from '../../services/audit/ad-audit.service';
+import { AzureAuditService } from '../../services/audit/azure-audit.service';
+import { LDAPProvider } from '../../providers/ldap/ldap.provider';
+import { GraphProvider } from '../../providers/azure/graph.provider';
+import { loadConfig } from '../../services/config/config.service';
+import { formatADAuditResponse } from '../../services/audit/response-formatter';
+import { JobRunner } from '../../services/jobs/job-runner';
+import { AzureJobRunner } from '../../services/jobs/azure-job-runner';
+
+/**
+ * AD Audit request body
+ */
+interface ADAuditRequest {
+  includeDetails?: boolean;
+  maxUsers?: number;
+  maxGroups?: number;
+  maxComputers?: number;
+}
+
+/**
+ * Azure Audit request body
+ */
+interface AzureAuditRequest {
+  includeDetails?: boolean;
+  maxUsers?: number;
+  maxGroups?: number;
+  maxApps?: number;
+}
+
+/**
+ * Audit Controller
+ */
+export class AuditController {
+  /**
+   * Run AD audit
+   *
+   * POST /api/v1/audit/ad
+   * Query params:
+   * - async=true: Run audit asynchronously, returns job_id for polling
+   */
+  async runADAudit(
+    req: Request<object, object, ADAuditRequest>,
+    res: Response,
+    next: NextFunction
+  ): Promise<void> {
+    try {
+      const { includeDetails = false, maxUsers, maxGroups, maxComputers } = req.body;
+      const asyncMode = req.query['async'] === 'true';
+
+      // Load configuration
+      const config = await loadConfig();
+
+      // Create LDAP provider
+      const ldapProvider = new LDAPProvider(config.ldap);
+
+      // ASYNC MODE: Start job and return immediately
+      if (asyncMode) {
+        // Connect first to verify credentials work
+        await ldapProvider.connect();
+
+        // Create job runner with SMB config
+        const smbConfig = { smb: config.smb, ldap: config.ldap };
+        const jobRunner = new JobRunner(ldapProvider, smbConfig);
+        const job = jobRunner.startAudit({
+          includeDetails,
+          maxUsers,
+          maxGroups,
+          maxComputers,
+        });
+
+        // Return job info immediately (don't wait for audit)
+        res.status(202).json({
+          job_id: job.job_id,
+          status: job.status,
+          progress: job.progress,
+          current_step: job.current_step,
+          description: job.description,
+          started_at: job.started_at,
+          steps: job.steps,
+        });
+        return;
+      }
+
+      // SYNC MODE: Wait for audit completion (original behavior)
+      await ldapProvider.connect();
+
+      try {
+        // Create AD audit service with SMB config
+        const smbConfig = { smb: config.smb, ldap: config.ldap };
+        const adAuditService = new ADAuditService(ldapProvider, smbConfig);
+
+        // Run audit
+        const result = await adAuditService.runAudit({
+          includeDetails,
+          maxUsers,
+          maxGroups,
+          maxComputers,
+        });
+
+        // Format response to PRD FR55 structure
+        const response = formatADAuditResponse(
+          result.score,
+          result.findings,
+          result.stats,
+          {
+            domain: {
+              name: config.ldap.baseDN,
+              baseDN: config.ldap.baseDN,
+              ldapUrl: config.ldap.url,
+            },
+            options: {
+              includeDetails,
+              includeComputers: maxComputers !== 0,
+              includeConfig: true,
+            },
+            executionTimeMs: result.stats.executionTimeMs,
+            timestamp: result.timestamp,
+            domainConfig: result.domainConfig,
+            attackGraph: result.attackGraph,
+          }
+        );
+
+        res.json(response);
+      } finally {
+        // Ensure disconnection
+        await ldapProvider.disconnect();
+      }
+    } catch (error) {
+      next(error);
+    }
+  }
+
+  /**
+   * Test LDAP connection
+   *
+   * GET /api/v1/audit/ad/status
+   */
+  async getADStatus(
+    _req: Request,
+    res: Response,
+    next: NextFunction
+  ): Promise<void> {
+    try {
+      // Load configuration
+      const config = await loadConfig();
+
+      // Create LDAP provider
+      const ldapProvider = new LDAPProvider(config.ldap);
+
+      // Test connection
+      const result = await ldapProvider.testConnection();
+
+      // Disconnect
+      await ldapProvider.disconnect();
+
+      res.json({
+        success: result.success,
+        message: result.message,
+        details: result.details,
+      });
+    } catch (error) {
+      next(error);
+    }
+  }
+
+  /**
+   * Run Azure audit
+   *
+   * POST /api/v1/audit/azure
+   * Query params:
+   * - async=true: Run audit asynchronously, returns job_id for polling
+   */
+  async runAzureAudit(
+    req: Request<object, object, AzureAuditRequest>,
+    res: Response,
+    next: NextFunction
+  ): Promise<void> {
+    try {
+      const { includeDetails = false, maxUsers, maxGroups, maxApps } = req.body;
+      const asyncMode = req.query['async'] === 'true';
+
+      // Load configuration
+      const config = await loadConfig();
+
+      // Check if Azure is enabled
+      if (!config.azure.enabled || !config.azure.tenantId || !config.azure.clientId || !config.azure.clientSecret) {
+        res.status(400).json({
+          success: false,
+          error: {
+            code: 'AZURE_NOT_CONFIGURED',
+            message: 'Azure AD is not configured. Set AZURE_ENABLED=true and provide credentials.',
+          },
+        });
+        return;
+      }
+
+      // Create Graph provider
+      const graphProvider = new GraphProvider({
+        tenantId: config.azure.tenantId,
+        clientId: config.azure.clientId,
+        clientSecret: config.azure.clientSecret,
+      });
+
+      // ASYNC MODE: Start job and return immediately
+      if (asyncMode) {
+        // Test authentication first
+        await graphProvider.authenticate();
+
+        // Create Azure job runner
+        const azureJobRunner = new AzureJobRunner(graphProvider);
+        const job = azureJobRunner.startAudit({
+          includeDetails,
+          maxUsers,
+          maxGroups,
+          maxApps,
+        });
+
+        // Return job info immediately (don't wait for audit)
+        res.status(202).json({
+          job_id: job.job_id,
+          status: job.status,
+          progress: job.progress,
+          current_step: job.current_step,
+          description: job.description,
+          started_at: job.started_at,
+          steps: job.steps,
+        });
+        return;
+      }
+
+      // SYNC MODE: Wait for audit completion (original behavior)
+      // Create Azure audit service
+      const azureAuditService = new AzureAuditService(graphProvider);
+
+      // Run audit
+      const result = await azureAuditService.runAudit({
+        includeDetails,
+        maxUsers,
+        maxGroups,
+        maxApps,
+      });
+
+      res.json({
+        success: true,
+        data: result,
+      });
+    } catch (error) {
+      next(error);
+    }
+  }
+
+  /**
+   * Test Microsoft Graph connection
+   *
+   * GET /api/v1/audit/azure/status
+   */
+  async getAzureStatus(
+    _req: Request,
+    res: Response,
+    next: NextFunction
+  ): Promise<void> {
+    try {
+      // Load configuration
+      const config = await loadConfig();
+
+      // Check if Azure is enabled
+      if (!config.azure.enabled || !config.azure.tenantId || !config.azure.clientId || !config.azure.clientSecret) {
+        res.status(400).json({
+          success: false,
+          error: {
+            code: 'AZURE_NOT_CONFIGURED',
+            message: 'Azure AD is not configured. Set AZURE_ENABLED=true and provide credentials.',
+          },
+        });
+        return;
+      }
+
+      // Create Graph provider
+      const graphProvider = new GraphProvider({
+        tenantId: config.azure.tenantId,
+        clientId: config.azure.clientId,
+        clientSecret: config.azure.clientSecret,
+      });
+
+      // Create Azure audit service
+      const azureAuditService = new AzureAuditService(graphProvider);
+
+      // Test connection
+      const result = await azureAuditService.testConnection();
+
+      res.json({
+        success: result.success,
+        message: result.message,
+      });
+    } catch (error) {
+      next(error);
+    }
+  }
+}