@checkstack/gitops-backend 0.1.0

package/CHANGELOG.md

@@ -0,0 +1,67 @@
+# @checkstack/gitops-backend
+
+## 0.1.0
+
+### Minor Changes
+
+- 6c40b5b: feat: add GitOps Entity System foundation — entity envelope schema, Entity Kind Registry extension point, secret field utility, secret resolution engine, provenance tracking, and RPC contract
+- 6c40b5b: Generalized provenance system and GitOps frontend plugin
+
+  **Breaking**: `EntityKindDefinition.reconcile()` now returns `{ entityId: string }` instead of `void`. Plugins must return the plugin-specific entity ID (e.g., catalog system UUID) so the engine can store it in provenance.
+
+  - Added `entityId` column to the provenance table (non-nullable)
+  - Reconciler engine passes `existingEntityId` to plugins for updates
+  - `getProvenance` now supports lookup by `entityId` in addition to `entityName`
+  - Added provider CRUD endpoints: `createProvider`, `updateProvider`, `deleteProvider`
+  - Created `gitops-frontend` plugin with provider management, secret management, and sync status dashboard
+  - Removed `gitops_entity_name` metadata markers from catalog entities
+  - Removed `findSystemByGitOpsName`, `deleteSystemByGitOpsName` (and Group equivalents) from EntityService
+  - Added provenance-based UI locking in catalog-frontend: edit/delete/drag disabled for GitOps-managed systems and groups
+
+- 6c40b5b: ### GitOps Ecosystem: Healthcheck Kind Registration (Phase 5)
+
+  **gitops-common**: Added required `resolveEntityRef` to `ReconcileContext`, enabling extension reconcilers to resolve cross-kind entity references (e.g., healthcheck refs in System extensions).
+
+  **gitops-backend**: Updated reconciler to populate `resolveEntityRef` by querying local provenance — no RPC round-trip needed.
+
+  **healthcheck-backend**: Registered `kind: Healthcheck` and `System → healthchecks` extension with the EntityKindRegistry:
+
+  - Validates strategy configs against registered strategy schemas at reconcile time
+  - Validates collector configs against registered collector schemas at reconcile time
+  - Manages system ↔ healthcheck associations with automatic stale removal
+
+  **healthcheck-frontend**: Added GitOps provenance locking to the HealthCheck IDE editor — GitOps-managed health checks show a lock banner and disable editing.
+
+  **catalog-backend**: Updated test fixtures for new required `resolveEntityRef` context field.
+
+- 6c40b5b: Add GitOps discovery and sync engine (Phase 2)
+
+  - YAML document parser with multi-document support and SHA-256 content hashing for diff detection
+  - GitHub scraper: org/user repo enumeration, single-repo mode, default branch resolution, recursive Git Trees API, minimatch path filtering, Link header pagination
+  - GitLab scraper: group project enumeration (including subgroups), single-project mode, recursive tree walking, minimatch filtering, x-next-page pagination
+  - Configurable `baseUrl` per provider for GitHub Enterprise and self-managed GitLab instances
+  - Reconciliation orchestrator: scrape → parse → validate → resolve secrets → reconcile (base + extensions) → provenance tracking → orphan detection
+  - Sync worker: recurring queue jobs per provider, one-off manual trigger via triggerSync RPC
+  - Per-entity error isolation ensures individual failures don't halt the sync
+
+- 6c40b5b: Add Kind Registry browser and developer documentation
+
+  - Added `gitopsAccess.kinds.read` access rule for standalone Kind Registry access
+  - Added `describeKinds()` method to the internal entity kind registry, serializing Zod schemas to JSON Schema
+  - Added `listKinds` RPC endpoint gated by the new access rule
+  - Created standalone Kind Registry page with schema visualization, extension listing, and auto-generated YAML examples
+  - Added Kind Registry link to the user menu
+  - Created developer documentation for entity kind and extension registration in `docs/backend/gitops-entity-kinds.md`
+
+### Patch Changes
+
+- 6c40b5b: Register catalog System and Group as GitOps entity kinds
+
+  - **catalog-backend**: Registers `kind: System` and `kind: Group` with the GitOps Entity Kind Registry. The catalog now supports declarative management via YAML descriptors in Git repositories. Systems and groups are reconciled using the `metadata.gitops_entity_name` marker for cross-sync identity lookup.
+  - **gitops-backend**: Wires up the delete reconciler for orphan cleanup — both automatic deletion (via `deletionPolicy: "auto"`) and manual orphan confirmation now invoke the owning plugin's `delete()` handler before removing provenance entries.
+
+- Updated dependencies [6c40b5b]
+- Updated dependencies [6c40b5b]
+- Updated dependencies [6c40b5b]
+- Updated dependencies [6c40b5b]
+  - @checkstack/gitops-common@0.1.0

package/drizzle/0000_tense_stryfe.sql

@@ -0,0 +1,46 @@
+CREATE TYPE "deletion_policy" AS ENUM('orphan', 'auto');--> statement-breakpoint
+CREATE TYPE "provenance_status" AS ENUM('synced', 'error', 'orphaned');--> statement-breakpoint
+CREATE TYPE "provider_type" AS ENUM('github', 'gitlab');--> statement-breakpoint
+CREATE TABLE "provenance" (
+	"id" text PRIMARY KEY NOT NULL,
+	"api_version" text NOT NULL,
+	"kind" text NOT NULL,
+	"entity_name" text NOT NULL,
+	"entity_id" text NOT NULL,
+	"provider_id" text NOT NULL,
+	"repository" text NOT NULL,
+	"file_path" text NOT NULL,
+	"last_sync_hash" text NOT NULL,
+	"status" "provenance_status" DEFAULT 'synced' NOT NULL,
+	"error_message" text,
+	"last_synced_at" timestamp DEFAULT now() NOT NULL,
+	"created_at" timestamp DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE "providers" (
+	"id" text PRIMARY KEY NOT NULL,
+	"type" "provider_type" NOT NULL,
+	"target" text NOT NULL,
+	"path_pattern" text NOT NULL,
+	"auth_token" text,
+	"base_url" text,
+	"sync_interval" integer DEFAULT 300 NOT NULL,
+	"deletion_policy" "deletion_policy" DEFAULT 'orphan' NOT NULL,
+	"last_sync_at" timestamp,
+	"last_sync_error" text,
+	"created_at" timestamp DEFAULT now() NOT NULL,
+	"updated_at" timestamp DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE "secrets" (
+	"id" text PRIMARY KEY NOT NULL,
+	"name" text NOT NULL,
+	"encrypted_value" text NOT NULL,
+	"description" text,
+	"created_by" text,
+	"created_at" timestamp DEFAULT now() NOT NULL,
+	"updated_at" timestamp DEFAULT now() NOT NULL,
+	CONSTRAINT "secrets_name_unique" UNIQUE("name")
+);
+--> statement-breakpoint
+ALTER TABLE "provenance" ADD CONSTRAINT "provenance_provider_id_providers_id_fk" FOREIGN KEY ("provider_id") REFERENCES "providers"("id") ON DELETE cascade ON UPDATE no action;

package/drizzle/meta/0000_snapshot.json

@@ -0,0 +1,310 @@
+{
+  "id": "01bbd45b-d962-4e98-b25d-53310f487822",
+  "prevId": "00000000-0000-0000-0000-000000000000",
+  "version": "7",
+  "dialect": "postgresql",
+  "tables": {
+    "public.provenance": {
+      "name": "provenance",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "text",
+          "primaryKey": true,
+          "notNull": true
+        },
+        "api_version": {
+          "name": "api_version",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "kind": {
+          "name": "kind",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "entity_name": {
+          "name": "entity_name",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "entity_id": {
+          "name": "entity_id",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "provider_id": {
+          "name": "provider_id",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "repository": {
+          "name": "repository",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "file_path": {
+          "name": "file_path",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "last_sync_hash": {
+          "name": "last_sync_hash",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "status": {
+          "name": "status",
+          "type": "provenance_status",
+          "typeSchema": "public",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "'synced'"
+        },
+        "error_message": {
+          "name": "error_message",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "last_synced_at": {
+          "name": "last_synced_at",
+          "type": "timestamp",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "timestamp",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {
+        "provenance_provider_id_providers_id_fk": {
+          "name": "provenance_provider_id_providers_id_fk",
+          "tableFrom": "provenance",
+          "tableTo": "providers",
+          "columnsFrom": [
+            "provider_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "cascade",
+          "onUpdate": "no action"
+        }
+      },
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.providers": {
+      "name": "providers",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "text",
+          "primaryKey": true,
+          "notNull": true
+        },
+        "type": {
+          "name": "type",
+          "type": "provider_type",
+          "typeSchema": "public",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "target": {
+          "name": "target",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "path_pattern": {
+          "name": "path_pattern",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "auth_token": {
+          "name": "auth_token",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "base_url": {
+          "name": "base_url",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "sync_interval": {
+          "name": "sync_interval",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true,
+          "default": 300
+        },
+        "deletion_policy": {
+          "name": "deletion_policy",
+          "type": "deletion_policy",
+          "typeSchema": "public",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "'orphan'"
+        },
+        "last_sync_at": {
+          "name": "last_sync_at",
+          "type": "timestamp",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "last_sync_error": {
+          "name": "last_sync_error",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "timestamp",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "updated_at": {
+          "name": "updated_at",
+          "type": "timestamp",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.secrets": {
+      "name": "secrets",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "text",
+          "primaryKey": true,
+          "notNull": true
+        },
+        "name": {
+          "name": "name",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "encrypted_value": {
+          "name": "encrypted_value",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "description": {
+          "name": "description",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "created_by": {
+          "name": "created_by",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "timestamp",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "updated_at": {
+          "name": "updated_at",
+          "type": "timestamp",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {
+        "secrets_name_unique": {
+          "name": "secrets_name_unique",
+          "nullsNotDistinct": false,
+          "columns": [
+            "name"
+          ]
+        }
+      },
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    }
+  },
+  "enums": {
+    "public.deletion_policy": {
+      "name": "deletion_policy",
+      "schema": "public",
+      "values": [
+        "orphan",
+        "auto"
+      ]
+    },
+    "public.provenance_status": {
+      "name": "provenance_status",
+      "schema": "public",
+      "values": [
+        "synced",
+        "error",
+        "orphaned"
+      ]
+    },
+    "public.provider_type": {
+      "name": "provider_type",
+      "schema": "public",
+      "values": [
+        "github",
+        "gitlab"
+      ]
+    }
+  },
+  "schemas": {},
+  "sequences": {},
+  "roles": {},
+  "policies": {},
+  "views": {},
+  "_meta": {
+    "columns": {},
+    "schemas": {},
+    "tables": {}
+  }
+}

package/drizzle/meta/_journal.json

@@ -0,0 +1,13 @@
+{
+  "version": "7",
+  "dialect": "postgresql",
+  "entries": [
+    {
+      "idx": 0,
+      "version": "7",
+      "when": 1776797758378,
+      "tag": "0000_tense_stryfe",
+      "breakpoints": true
+    }
+  ]
+}

package/drizzle.config.ts

@@ -0,0 +1,7 @@
+import { defineConfig } from "drizzle-kit";
+
+export default defineConfig({
+  schema: "./src/schema.ts",
+  out: "./drizzle",
+  dialect: "postgresql",
+});

package/package.json

@@ -0,0 +1,37 @@
+{
+  "name": "@checkstack/gitops-backend",
+  "version": "0.1.0",
+  "type": "module",
+  "main": "src/index.ts",
+  "checkstack": {
+    "type": "backend"
+  },
+  "scripts": {
+    "typecheck": "tsc --noEmit",
+    "generate": "drizzle-kit generate",
+    "lint": "bun run lint:code",
+    "lint:code": "eslint . --max-warnings 0"
+  },
+  "dependencies": {
+    "@checkstack/backend-api": "0.12.0",
+    "@checkstack/gitops-common": "0.0.1",
+    "@checkstack/common": "0.6.5",
+    "@checkstack/command-backend": "0.1.19",
+    "@checkstack/queue-api": "0.2.13",
+    "@orpc/server": "^1.13.2",
+    "drizzle-orm": "^0.45.0",
+    "minimatch": "^10.0.0",
+    "uuid": "^13.0.0",
+    "zod": "^4.2.1",
+    "yaml": "^2.7.0"
+  },
+  "devDependencies": {
+    "@checkstack/drizzle-helper": "0.0.4",
+    "@checkstack/scripts": "0.1.2",
+    "@checkstack/tsconfig": "0.0.5",
+    "@types/bun": "^1.3.5",
+    "@types/node": "^20.0.0",
+    "@types/uuid": "^11.0.0",
+    "typescript": "^5.0.0"
+  }
+}

package/src/index.ts

@@ -0,0 +1,136 @@
+import {
+  createBackendPlugin,
+  createExtensionPoint,
+  coreServices,
+} from "@checkstack/backend-api";
+import type { SafeDatabase } from "@checkstack/backend-api";
+import {
+  pluginMetadata,
+  gitopsAccessRules,
+  gitopsContract,
+} from "@checkstack/gitops-common";
+import type {
+  EntityKindDefinition,
+  EntityKindExtensionDefinition,
+  EntityKindRegistry,
+} from "@checkstack/gitops-common";
+import { createEntityKindRegistry } from "./kind-registry";
+import { createGitOpsRouter } from "./router";
+import { setupSyncWorker } from "./sync/sync-worker";
+import { decrypt } from "@checkstack/backend-api";
+import * as schema from "./schema";
+import { eq } from "drizzle-orm";
+
+// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+// Extension Points
+// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+
+/**
+ * Extension point for the Entity Kind Registry.
+ * Plugins use this during their `register()` phase to register entity kinds
+ * and spec extensions for the GitOps system.
+ *
+ * @example
+ * ```typescript
+ * import { entityKindExtensionPoint } from "@checkstack/gitops-backend";
+ *
+ * // In your plugin's register() function:
+ * const registry = env.getExtensionPoint(entityKindExtensionPoint);
+ * registry.registerKind({
+ *   apiVersion: "checkstack.io/v1alpha1",
+ *   kind: "System",
+ *   specSchema: z.object({ description: z.string().optional() }),
+ *   reconcile: async ({ entity }) => { ... },
+ * });
+ * ```
+ */
+export const entityKindExtensionPoint =
+  createExtensionPoint<EntityKindRegistry>("gitops.entity-kind-registry");
+
+// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+// Plugin Definition
+// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+
+export default createBackendPlugin({
+  metadata: pluginMetadata,
+
+  register(env) {
+    // Create the kind registry
+    const kindRegistry = createEntityKindRegistry();
+
+    // Register access rules
+    env.registerAccessRules(gitopsAccessRules);
+
+    // Register the Entity Kind Extension Point
+    // Other plugins call this to register their entity kinds and extensions
+    env.registerExtensionPoint(entityKindExtensionPoint, {
+      registerKind<TSpec>(definition: EntityKindDefinition<TSpec>) {
+        kindRegistry.registerKind(definition);
+      },
+      registerKindExtension<TExtensionSpec>(
+        definition: EntityKindExtensionDefinition<TExtensionSpec>,
+      ) {
+        kindRegistry.registerKindExtension(definition);
+      },
+    });
+
+    env.registerInit({
+      schema,
+      deps: {
+        logger: coreServices.logger,
+        rpc: coreServices.rpc,
+        queueManager: coreServices.queueManager,
+      },
+      init: async ({ logger, database, rpc, queueManager }) => {
+        logger.debug("🔄 Initializing GitOps Backend...");
+
+        const db = database as SafeDatabase<typeof schema>;
+
+        const router = createGitOpsRouter({ database: db, queueManager, kindRegistry });
+        rpc.registerRouter(router, gitopsContract);
+
+        logger.debug("✅ GitOps Backend initialized.");
+      },
+      afterPluginsReady: async ({ logger, database, queueManager }) => {
+        const registeredKinds = kindRegistry.getKinds();
+        logger.debug(
+          `🔄 GitOps: ${registeredKinds.length} entity kinds registered: ${registeredKinds.map((k) => k.kind).join(", ")}`,
+        );
+
+        const db = database as SafeDatabase<typeof schema>;
+
+        // Create a SecretStore backed by the secrets table
+        const secretStore = {
+          resolve: async (name: string): Promise<string> => {
+            const rows = await db
+              .select()
+              .from(schema.secrets)
+              .where(eq(schema.secrets.name, name));
+            const secret = rows[0];
+            if (!secret) {
+              throw new Error(`Secret not found: ${name}`);
+            }
+            return decrypt(secret.encryptedValue);
+          },
+        };
+
+        // Bootstrap sync worker
+        await setupSyncWorker({
+          db,
+          logger,
+          queueManager,
+          kindRegistry,
+          secretStore,
+        });
+      },
+    });
+  },
+});
+
+// Re-export types for consumer plugins
+export type {
+  EntityKindDefinition,
+  EntityKindExtensionDefinition,
+  EntityKindRegistry,
+  ReconcileContext,
+} from "@checkstack/gitops-common";