@serve.zone/gitops 2.13.1

package/ts/cache/documents/classes.cached.secret.ts

@@ -0,0 +1,81 @@
+import * as plugins from '../../plugins.js';
+import { CacheDb } from '../classes.cachedb.js';
+import { CachedDocument, TTL } from '../classes.cached.document.js';
+import type { ISecret } from '../../../ts_interfaces/data/secret.js';
+
+/**
+ * Cached secret data from git providers. TTL: 24 hours.
+ */
+@plugins.smartdata.Collection(() => CacheDb.getInstance().getDb())
+export class CachedSecret extends CachedDocument<CachedSecret> {
+  @plugins.smartdata.unI()
+  public id: string = '';
+
+  @plugins.smartdata.svDb()
+  public connectionId: string = '';
+
+  @plugins.smartdata.svDb()
+  public scope: 'project' | 'group' = 'project';
+
+  @plugins.smartdata.svDb()
+  public scopeId: string = '';
+
+  @plugins.smartdata.svDb()
+  public scopeName: string = '';
+
+  @plugins.smartdata.svDb()
+  public key: string = '';
+
+  @plugins.smartdata.svDb()
+  public value: string = '';
+
+  @plugins.smartdata.svDb()
+  public protected: boolean = false;
+
+  @plugins.smartdata.svDb()
+  public masked: boolean = false;
+
+  @plugins.smartdata.svDb()
+  public environment: string = '';
+
+  constructor() {
+    super();
+    this.setTTL(TTL.HOURS_24);
+  }
+
+  /** Build the composite unique ID */
+  static buildId(connectionId: string, scope: string, scopeId: string, key: string): string {
+    return `${connectionId}:${scope}:${scopeId}:${key}`;
+  }
+
+  /** Create a CachedSecret from an ISecret */
+  static fromISecret(secret: ISecret): CachedSecret {
+    const doc = new CachedSecret();
+    doc.id = CachedSecret.buildId(secret.connectionId, secret.scope, secret.scopeId, secret.key);
+    doc.connectionId = secret.connectionId;
+    doc.scope = secret.scope;
+    doc.scopeId = secret.scopeId;
+    doc.scopeName = secret.scopeName;
+    doc.key = secret.key;
+    doc.value = secret.value;
+    doc.protected = secret.protected;
+    doc.masked = secret.masked;
+    doc.environment = secret.environment;
+    return doc;
+  }
+
+  /** Convert back to ISecret */
+  toISecret(): ISecret {
+    return {
+      connectionId: this.connectionId,
+      scope: this.scope,
+      scopeId: this.scopeId,
+      scopeName: this.scopeName,
+      key: this.key,
+      value: this.value,
+      protected: this.protected,
+      masked: this.masked,
+      environment: this.environment,
+    };
+  }
+}

package/ts/cache/documents/index.ts

@@ -0,0 +1,2 @@
+export { CachedProject } from './classes.cached.project.js';
+export { CachedSecret } from './classes.cached.secret.js';

package/ts/cache/index.ts

@@ -0,0 +1,7 @@
+export { CacheDb } from './classes.cachedb.js';
+export type { ICacheDbOptions } from './classes.cachedb.js';
+export { CachedDocument, TTL } from './classes.cached.document.js';
+export { CacheCleaner } from './classes.cache.cleaner.js';
+export { SecretsScanService } from './classes.secrets.scan.service.js';
+export type { IScanResult } from './classes.secrets.scan.service.js';
+export * from './documents/index.js';

package/ts/classes/actionlog.ts

@@ -0,0 +1,57 @@
+import { logger } from '../logging.js';
+import type * as interfaces from '../../ts_interfaces/index.js';
+import type { StorageManager } from '../storage/index.js';
+
+const ACTIONLOG_PREFIX = '/actionlog/';
+
+/**
+ * Persists and queries action log entries via StorageManager.
+ * Entries are stored as individual JSON files keyed by timestamp-id.
+ */
+export class ActionLog {
+  private storageManager: StorageManager;
+
+  constructor(storageManager: StorageManager) {
+    this.storageManager = storageManager;
+  }
+
+  async append(entry: Omit<interfaces.data.IActionLogEntry, 'id' | 'timestamp'>): Promise<interfaces.data.IActionLogEntry> {
+    const full: interfaces.data.IActionLogEntry = {
+      id: crypto.randomUUID(),
+      timestamp: Date.now(),
+      ...entry,
+    };
+    const key = `${ACTIONLOG_PREFIX}${String(full.timestamp).padStart(16, '0')}-${full.id}.json`;
+    await this.storageManager.setJSON(key, full);
+    logger.debug(`Action logged: ${full.actionType} ${full.entityType} "${full.entityName}"`);
+    return full;
+  }
+
+  async query(opts: {
+    limit?: number;
+    offset?: number;
+    entityType?: interfaces.data.TActionEntity;
+  } = {}): Promise<{ entries: interfaces.data.IActionLogEntry[]; total: number }> {
+    const limit = opts.limit ?? 50;
+    const offset = opts.offset ?? 0;
+
+    const keys = await this.storageManager.list(ACTIONLOG_PREFIX);
+    // Sort by key descending (newest first — keys are timestamp-prefixed)
+    keys.sort((a, b) => b.localeCompare(a));
+
+    // Load all entries (or filter by entityType)
+    let entries: interfaces.data.IActionLogEntry[] = [];
+    for (const key of keys) {
+      const entry = await this.storageManager.getJSON<interfaces.data.IActionLogEntry>(key);
+      if (entry) {
+        if (opts.entityType && entry.entityType !== opts.entityType) continue;
+        entries.push(entry);
+      }
+    }
+
+    const total = entries.length;
+    entries = entries.slice(offset, offset + limit);
+
+    return { entries, total };
+  }
+}

package/ts/classes/connectionmanager.ts

@@ -0,0 +1,263 @@
+import * as plugins from '../plugins.js';
+import { logger } from '../logging.js';
+import type * as interfaces from '../../ts_interfaces/index.js';
+import { BaseProvider, GiteaProvider, GitLabProvider } from '../providers/index.js';
+import type { StorageManager } from '../storage/index.js';
+
+const LEGACY_CONNECTIONS_FILE = './.nogit/connections.json';
+const CONNECTIONS_PREFIX = '/connections/';
+const KEYCHAIN_PREFIX = 'keychain:';
+
+/**
+ * Manages provider connections — persists each connection as an
+ * individual JSON file via StorageManager. Tokens are stored in
+ * the OS keychain (or encrypted file fallback) via SmartSecret.
+ */
+export class ConnectionManager {
+  private connections: interfaces.data.IProviderConnection[] = [];
+  private storageManager: StorageManager;
+  private smartSecret: plugins.smartsecret.SmartSecret;
+  /** Resolves when background connection health checks complete */
+  public healthCheckDone: Promise<void> = Promise.resolve();
+
+  constructor(storageManager: StorageManager, smartSecret: plugins.smartsecret.SmartSecret) {
+    this.storageManager = storageManager;
+    this.smartSecret = smartSecret;
+  }
+
+  async init(): Promise<void> {
+    await this.migrateLegacyFile();
+    await this.loadConnections();
+    // Auto-test all connections in the background
+    this.healthCheckDone = this.testAllConnections();
+  }
+
+  /**
+   * Tests all loaded connections in the background and updates their status.
+   * Fire-and-forget — does not block startup.
+   */
+  private async testAllConnections(): Promise<void> {
+    for (const conn of this.connections) {
+      if (conn.status === 'paused') continue;
+      try {
+        const provider = this.getProvider(conn.id);
+        const result = await provider.testConnection();
+        conn.status = result.ok ? 'connected' : 'error';
+        await this.persistConnection(conn);
+      } catch {
+        conn.status = 'error';
+      }
+    }
+  }
+
+  /**
+   * One-time migration from the legacy .nogit/connections.json file.
+   */
+  private async migrateLegacyFile(): Promise<void> {
+    try {
+      const text = await plugins.fs.readFile(LEGACY_CONNECTIONS_FILE, 'utf8');
+      const legacy: interfaces.data.IProviderConnection[] = JSON.parse(text);
+      if (legacy.length > 0) {
+        logger.info(`Migrating ${legacy.length} connection(s) from legacy file...`);
+        for (const conn of legacy) {
+          await this.storageManager.setJSON(`${CONNECTIONS_PREFIX}${conn.id}.json`, conn);
+        }
+        // Rename legacy file so migration doesn't repeat
+        await plugins.fs.rename(LEGACY_CONNECTIONS_FILE, LEGACY_CONNECTIONS_FILE + '.migrated');
+        logger.success('Legacy connections migrated successfully');
+      }
+    } catch {
+      // No legacy file or already migrated — nothing to do
+    }
+  }
+
+  private async loadConnections(): Promise<void> {
+    const keys = await this.storageManager.list(CONNECTIONS_PREFIX);
+    this.connections = [];
+    for (const key of keys) {
+      const conn = await this.storageManager.getJSON<interfaces.data.IProviderConnection>(key);
+      if (conn) {
+        // Migrate legacy baseGroup/baseGroupId property names
+        if ((conn as any).baseGroup !== undefined && conn.groupFilter === undefined) {
+          conn.groupFilter = (conn as any).baseGroup;
+          delete (conn as any).baseGroup;
+        }
+        if ((conn as any).baseGroupId !== undefined && conn.groupFilterId === undefined) {
+          conn.groupFilterId = (conn as any).baseGroupId;
+          delete (conn as any).baseGroupId;
+        }
+        if (conn.token.startsWith(KEYCHAIN_PREFIX)) {
+          // Token is in keychain — retrieve it
+          const realToken = await this.smartSecret.getSecret(conn.id);
+          if (realToken) {
+            conn.token = realToken;
+          } else {
+            logger.warn(`Could not retrieve token for connection ${conn.id} from keychain`);
+          }
+        } else if (conn.token && conn.token !== '***') {
+          // Plaintext token found — auto-migrate to keychain
+          await this.migrateTokenToKeychain(conn);
+        }
+        this.connections.push(conn);
+      }
+    }
+    if (this.connections.length > 0) {
+      logger.info(`Loaded ${this.connections.length} connection(s)`);
+    } else {
+      logger.debug('No existing connections found, starting fresh');
+    }
+  }
+
+  /**
+   * Migrates a plaintext token to keychain storage.
+   */
+  private async migrateTokenToKeychain(
+    conn: interfaces.data.IProviderConnection,
+  ): Promise<void> {
+    try {
+      await this.smartSecret.setSecret(conn.id, conn.token);
+      // Save sentinel to JSON file
+      const jsonConn = { ...conn, token: `${KEYCHAIN_PREFIX}${conn.id}` };
+      await this.storageManager.setJSON(`${CONNECTIONS_PREFIX}${conn.id}.json`, jsonConn);
+      logger.info(`Migrated token for connection "${conn.name}" to keychain`);
+    } catch (err) {
+      logger.warn(`Failed to migrate token for ${conn.id} to keychain: ${err}`);
+    }
+  }
+
+  private async persistConnection(conn: interfaces.data.IProviderConnection): Promise<void> {
+    // Store real token in keychain
+    await this.smartSecret.setSecret(conn.id, conn.token);
+    // Save JSON with sentinel value
+    const jsonConn = { ...conn, token: `${KEYCHAIN_PREFIX}${conn.id}` };
+    await this.storageManager.setJSON(`${CONNECTIONS_PREFIX}${conn.id}.json`, jsonConn);
+  }
+
+  private async removeConnection(id: string): Promise<void> {
+    await this.smartSecret.deleteSecret(id);
+    await this.storageManager.delete(`${CONNECTIONS_PREFIX}${id}.json`);
+  }
+
+  getConnections(): interfaces.data.IProviderConnection[] {
+    return this.connections.map((c) => ({ ...c, token: '***' }));
+  }
+
+  getConnection(id: string): interfaces.data.IProviderConnection | undefined {
+    return this.connections.find((c) => c.id === id);
+  }
+
+  async createConnection(
+    name: string,
+    providerType: interfaces.data.TProviderType,
+    baseUrl: string,
+    token: string,
+    groupFilter?: string,
+  ): Promise<interfaces.data.IProviderConnection> {
+    const connection: interfaces.data.IProviderConnection = {
+      id: crypto.randomUUID(),
+      name,
+      providerType,
+      baseUrl: baseUrl.replace(/\/+$/, ''),
+      token,
+      createdAt: Date.now(),
+      status: 'disconnected',
+      groupFilter: groupFilter || undefined,
+    };
+    this.connections.push(connection);
+    await this.persistConnection(connection);
+    logger.success(`Connection created: ${name} (${providerType})`);
+    return { ...connection, token: '***' };
+  }
+
+  async updateConnection(
+    id: string,
+    updates: { name?: string; baseUrl?: string; token?: string; groupFilter?: string },
+  ): Promise<interfaces.data.IProviderConnection> {
+    const conn = this.connections.find((c) => c.id === id);
+    if (!conn) throw new Error(`Connection not found: ${id}`);
+    if (updates.name) conn.name = updates.name;
+    if (updates.baseUrl) conn.baseUrl = updates.baseUrl.replace(/\/+$/, '');
+    if (updates.token) conn.token = updates.token;
+    if (updates.groupFilter !== undefined) {
+      conn.groupFilter = updates.groupFilter || undefined;
+      conn.groupFilterId = undefined; // Will be re-resolved on next test
+    }
+    await this.persistConnection(conn);
+    return { ...conn, token: '***' };
+  }
+
+  async deleteConnection(id: string): Promise<void> {
+    const idx = this.connections.findIndex((c) => c.id === id);
+    if (idx === -1) throw new Error(`Connection not found: ${id}`);
+    this.connections.splice(idx, 1);
+    await this.removeConnection(id);
+    logger.info(`Connection deleted: ${id}`);
+  }
+
+  async pauseConnection(id: string, paused: boolean): Promise<interfaces.data.IProviderConnection> {
+    const conn = this.connections.find((c) => c.id === id);
+    if (!conn) throw new Error(`Connection not found: ${id}`);
+    conn.status = paused ? 'paused' : 'disconnected';
+    await this.persistConnection(conn);
+    logger.info(`Connection ${paused ? 'paused' : 'resumed'}: ${conn.name}`);
+    return { ...conn, token: '***' };
+  }
+
+  async testConnection(id: string): Promise<{ ok: boolean; error?: string }> {
+    const conn = this.connections.find((c) => c.id === id)!;
+    if (conn.status === 'paused') {
+      return { ok: false, error: 'Connection is paused' };
+    }
+    const provider = this.getProvider(id);
+    const result = await provider.testConnection();
+    conn.status = result.ok ? 'connected' : 'error';
+    // Resolve group filter ID if connection has a groupFilter
+    if (result.ok && conn.groupFilter) {
+      await this.resolveGroupFilterId(conn);
+    }
+    await this.persistConnection(conn);
+    return result;
+  }
+
+  /**
+   * Resolves a human-readable groupFilter to the provider-specific group ID.
+   */
+  private async resolveGroupFilterId(conn: interfaces.data.IProviderConnection): Promise<void> {
+    if (!conn.groupFilter) {
+      conn.groupFilterId = undefined;
+      return;
+    }
+    try {
+      if (conn.providerType === 'gitlab') {
+        const gitlabClient = new plugins.gitlabClient.GitLabClient(conn.baseUrl, conn.token);
+        const group = await gitlabClient.getGroup(conn.groupFilter);
+        conn.groupFilterId = String(group.id);
+        logger.info(`Resolved group filter "${conn.groupFilter}" to ID ${conn.groupFilterId}`);
+      } else {
+        // For Gitea, the org name IS the ID
+        conn.groupFilterId = conn.groupFilter;
+        logger.info(`Group filter for Gitea connection set to org "${conn.groupFilterId}"`);
+      }
+    } catch (err) {
+      logger.warn(`Failed to resolve group filter "${conn.groupFilter}": ${err}`);
+      conn.groupFilterId = undefined;
+    }
+  }
+
+  /**
+   * Factory: returns the correct provider instance for a connection ID
+   */
+  getProvider(connectionId: string): BaseProvider {
+    const conn = this.connections.find((c) => c.id === connectionId);
+    if (!conn) throw new Error(`Connection not found: ${connectionId}`);
+
+    switch (conn.providerType) {
+      case 'gitea':
+        return new GiteaProvider(conn.id, conn.baseUrl, conn.token, conn.groupFilterId);
+      case 'gitlab':
+        return new GitLabProvider(conn.id, conn.baseUrl, conn.token, conn.groupFilterId);
+      default:
+        throw new Error(`Unknown provider type: ${conn.providerType}`);
+    }
+  }
+}

package/ts/classes/gitopsapp.ts

@@ -0,0 +1,128 @@
+import * as plugins from '../plugins.js';
+import { logger } from '../logging.js';
+import { ConnectionManager } from './connectionmanager.js';
+import { ActionLog } from './actionlog.js';
+import { SyncManager } from './syncmanager.js';
+import { ManagedSecretsManager } from './managedsecrets.manager.js';
+import { JobManager } from './jobmanager.js';
+import { OpsServer } from '../opsserver/index.js';
+import { StorageManager } from '../storage/index.js';
+import { CacheDb, CacheCleaner, CachedProject, CachedSecret, SecretsScanService } from '../cache/index.js';
+import { resolvePaths } from '../paths.js';
+import { unrefTimer } from '../timers.js';
+
+/**
+ * Main GitOps application orchestrator
+ */
+export class GitopsApp {
+  public storageManager: StorageManager;
+  public smartSecret: plugins.smartsecret.SmartSecret;
+  public connectionManager: ConnectionManager;
+  public actionLog: ActionLog;
+  public opsServer: OpsServer;
+  public cacheDb: CacheDb;
+  public cacheCleaner: CacheCleaner;
+  public syncManager!: SyncManager;
+  public managedSecretsManager!: ManagedSecretsManager;
+  public jobManager!: JobManager;
+  public secretsScanService!: SecretsScanService;
+  private scanIntervalId: ReturnType<typeof setInterval> | null = null;
+  private paths: ReturnType<typeof resolvePaths>;
+
+  constructor() {
+    const paths = resolvePaths();
+    this.paths = paths;
+    this.storageManager = new StorageManager({
+      backend: 'filesystem',
+      fsPath: paths.defaultStoragePath,
+    });
+    this.smartSecret = new plugins.smartsecret.SmartSecret({ service: 'gitops' });
+    this.connectionManager = new ConnectionManager(this.storageManager, this.smartSecret);
+    this.actionLog = new ActionLog(this.storageManager);
+
+    this.cacheDb = CacheDb.getInstance({
+      storagePath: paths.defaultTsmDbPath,
+      dbName: 'gitops_cache',
+    });
+    this.cacheCleaner = new CacheCleaner(this.cacheDb);
+    this.cacheCleaner.registerClass(CachedProject);
+    this.cacheCleaner.registerClass(CachedSecret);
+
+    this.opsServer = new OpsServer(this);
+  }
+
+  async start(port = 3000): Promise<void> {
+    logger.info('Initializing GitOps...');
+
+    // Start CacheDb
+    await this.cacheDb.start();
+
+    // Initialize connection manager (loads saved connections)
+    await this.connectionManager.init();
+
+    // Initialize managed secrets manager
+    this.managedSecretsManager = new ManagedSecretsManager(
+      this.storageManager,
+      this.smartSecret,
+      this.connectionManager,
+    );
+    await this.managedSecretsManager.init();
+
+    // Initialize sync manager
+    this.syncManager = new SyncManager(
+      this.storageManager,
+      this.connectionManager,
+      this.actionLog,
+    );
+    await this.syncManager.init();
+
+    // Initialize job manager
+    this.jobManager = new JobManager(
+      this.storageManager,
+      this.connectionManager,
+      this.actionLog,
+      this.smartSecret,
+    );
+    await this.jobManager.init();
+
+    // Initialize secrets scan service with 24h auto-scan
+    this.secretsScanService = new SecretsScanService(this.connectionManager);
+    const SCAN_INTERVAL_MS = 24 * 60 * 60 * 1000; // 24 hours
+    this.scanIntervalId = setInterval(() => {
+      this.secretsScanService.fullScan().catch((err) => {
+        logger.error(`Scheduled secrets scan failed: ${err}`);
+      });
+    }, SCAN_INTERVAL_MS);
+    unrefTimer(this.scanIntervalId);
+    // Fire-and-forget initial scan (doesn't block startup)
+    this.secretsScanService.fullScan().catch((err) => {
+      logger.error(`Initial secrets scan failed: ${err}`);
+    });
+
+    // Start CacheCleaner
+    this.cacheCleaner.start();
+
+    // Start OpsServer
+    await this.opsServer.start(port);
+
+    logger.success('GitOps initialized successfully');
+  }
+
+  async stop(): Promise<void> {
+    logger.info('Shutting down GitOps...');
+    if (this.scanIntervalId !== null) {
+      clearInterval(this.scanIntervalId);
+      this.scanIntervalId = null;
+    }
+    await this.opsServer.stop();
+    if (this.jobManager) {
+      await this.jobManager.stop();
+    }
+    if (this.syncManager) {
+      await this.syncManager.stop();
+    }
+    this.cacheCleaner.stop();
+    await this.cacheDb.stop();
+    logger.success('GitOps shutdown complete');
+  }
+}