@catladder/cli 0.0.0-vars-07a720d0 → 0.0.0-vitest-8a770211

package/src/apps/cli/commands/project/setup/setupAgents.ts

@@ -0,0 +1,323 @@
+import type { CommandInstance } from "vorpal";
+import { doGitlabRequest, getProjectInfo } from "../../../../../utils/gitlab";
+import { getProjectConfig } from "../../../../../config/getProjectConfig";
+import { getGitRemoteHostAndPath } from "../../../../../git/gitProjectInformation";
+import { getMainBranch } from "../../../../../git";
+import type { AgentConfig } from "@catladder/pipeline";
+
+type TriggerToken = {
+  id: number;
+  description: string;
+  created_at: string;
+  last_used: string | null;
+  token: string;
+  updated_at: string;
+  owner: any;
+};
+
+type Webhook = {
+  id: number;
+  url: string;
+  name: string;
+  description: string;
+  project_id: number;
+  push_events: boolean;
+  push_events_branch_filter: string;
+  issues_events: boolean;
+  confidential_issues_events: boolean;
+  merge_requests_events: boolean;
+  tag_push_events: boolean;
+  note_events: boolean;
+  confidential_note_events: boolean;
+  job_events: boolean;
+  pipeline_events: boolean;
+  wiki_page_events: boolean;
+  deployment_events: boolean;
+  releases_events: boolean;
+  milestone_events: boolean;
+  feature_flag_events: boolean;
+  enable_ssl_verification: boolean;
+  repository_update_events: boolean;
+  alert_status: string;
+  disabled_until: string | null;
+  url_variables: any[];
+  created_at: string;
+  resource_access_token_events: boolean;
+  custom_webhook_template: string;
+  custom_headers: Array<{ key: string; value?: string }>;
+};
+
+type WebhookData = {
+  url: string;
+  name: string;
+  description: string;
+  issues_events: boolean;
+  confidential_issues_events: boolean;
+  merge_requests_events: boolean;
+  note_events: boolean;
+  confidential_note_events: boolean;
+  custom_webhook_template: string;
+};
+
+const listTriggerTokens = async (
+  instance: CommandInstance,
+  projectId: string,
+): Promise<TriggerToken[]> => {
+  return await doGitlabRequest(instance, `projects/${projectId}/triggers`);
+};
+
+const deleteTriggerToken = async (
+  instance: CommandInstance,
+  projectId: string,
+  triggerId: number,
+): Promise<void> => {
+  await doGitlabRequest(
+    instance,
+    `projects/${projectId}/triggers/${triggerId}`,
+    undefined,
+    "DELETE",
+  );
+};
+
+const createTriggerToken = async (
+  instance: CommandInstance,
+  projectId: string,
+  description: string,
+): Promise<TriggerToken> => {
+  return await doGitlabRequest(
+    instance,
+    `projects/${projectId}/triggers`,
+    { description },
+    "POST",
+  );
+};
+
+const listWebhooks = async (
+  instance: CommandInstance,
+  projectId: string,
+): Promise<Webhook[]> => {
+  return await doGitlabRequest(instance, `projects/${projectId}/hooks`);
+};
+
+const createWebhook = async (
+  instance: CommandInstance,
+  projectId: string,
+  webhookData: WebhookData,
+): Promise<Webhook> => {
+  return await doGitlabRequest(
+    instance,
+    `projects/${projectId}/hooks`,
+    webhookData,
+    "POST",
+  );
+};
+
+const updateWebhook = async (
+  instance: CommandInstance,
+  projectId: string,
+  hookId: number,
+  webhookData: WebhookData,
+): Promise<Webhook> => {
+  return await doGitlabRequest(
+    instance,
+    `projects/${projectId}/hooks/${hookId}`,
+    webhookData,
+    "PUT",
+  );
+};
+
+const setupAgentWebhook = async (
+  instance: CommandInstance,
+  projectId: string,
+  agentName: string,
+  triggerToken: string,
+): Promise<Webhook> => {
+  const webhookName = `cl_agent_${agentName}_webhook`;
+  const { gitRemoteHost } = await getGitRemoteHostAndPath();
+  const mainBranch = await getMainBranch();
+  const webhookUrl = `https://${gitRemoteHost}/api/v4/projects/${projectId}/ref/${mainBranch}/trigger/pipeline?token=${triggerToken}`;
+
+  instance.log(`Setting up webhook for agent: ${agentName}`);
+
+  // List existing webhooks
+  const existingWebhooks = await listWebhooks(instance, projectId);
+
+  // Find existing webhook by name
+  const existingWebhook = existingWebhooks.find(
+    (webhook) => webhook.name === webhookName,
+  );
+
+  // Custom webhook template from the user's specification
+  const customWebhookTemplate = JSON.stringify({
+    variables: {
+      ASSIGNEE_USER_ID: "{{object_attributes.assignee_id}}",
+      OBJECT_DESCRIPTION: "{{object_attributes.description}}",
+    },
+    payload: {
+      object_kind: "{{object_kind}}",
+      event_type: "{{event_type}}",
+      user_id: "{{user.id}}",
+      user_name: "{{user.name}}",
+      user_username: "{{user.username}}",
+      user_avatar_url: "{{user.avatar_url}}",
+      user_email: "{{user.email}}",
+      project_id: "{{project.id}}",
+      project_name: "{{project.name}}",
+      project_web_url: "{{project.web_url}}",
+      project_namespace: "{{project.namespace}}",
+      project_path_with_namespace: "{{project.path_with_namespace}}",
+      object_attributes_id: "{{object_attributes.id}}",
+      object_attributes_iid: "{{object_attributes.iid}}",
+      object_attributes_title: "{{object_attributes.title}}",
+      object_attributes_description: "{{object_attributes.description}}",
+      object_attributes_state: "{{object_attributes.state}}",
+      object_attributes_created_at: "{{object_attributes.created_at}}",
+      object_attributes_updated_at: "{{object_attributes.updated_at}}",
+      object_attributes_url: "{{object_attributes.url}}",
+      object_attributes_action: "{{object_attributes.action}}",
+      object_attributes_noteable_type: "{{object_attributes.noteable_type}}",
+      merge_request_id: "{{merge_request.id}}",
+      merge_request_iid: "{{merge_request.iid}}",
+      merge_request_title: "{{merge_request.title}}",
+      merge_request_source_branch: "{{merge_request.source_branch}}",
+      merge_request_time_estimate: "{{merge_request.time_estimate}}",
+      merge_request_description: "{{merge_request.description}}",
+      merge_request_merge_status: "{{merge_request.merge_status}}",
+      merge_request_last_commit_title: "{{merge_request.last_commit.title}}",
+      merge_request_last_commit_message:
+        "{{merge_request.last_commit.message}}",
+      merge_request_last_commit_author_name:
+        "{{merge_request.last_commit.author.name}}",
+      merge_request_last_commit_author_email:
+        "{{merge_request.last_commit.author.email}}",
+    },
+  });
+
+  const webhookData: WebhookData = {
+    url: webhookUrl,
+    name: webhookName,
+    description: `Agent webhook for ${agentName} - triggers pipeline with custom payload`,
+    issues_events: true,
+    confidential_issues_events: false,
+    merge_requests_events: false,
+    note_events: true,
+    confidential_note_events: false,
+    custom_webhook_template: customWebhookTemplate,
+  };
+
+  let webhook: Webhook;
+
+  if (existingWebhook) {
+    instance.log(
+      `Updating existing webhook: ${webhookName} (ID: ${existingWebhook.id})`,
+    );
+    webhook = await updateWebhook(
+      instance,
+      projectId,
+      existingWebhook.id,
+      webhookData,
+    );
+    instance.log(`✅ Updated webhook: ${webhook.name}`);
+  } else {
+    instance.log(`Creating new webhook: ${webhookName}`);
+    webhook = await createWebhook(instance, projectId, webhookData);
+    instance.log(`✅ Created webhook: ${webhook.name}`);
+  }
+
+  instance.log(`   Webhook ID: ${webhook.id}`);
+  instance.log(`   Webhook URL: ${webhook.url}`);
+
+  return webhook;
+};
+
+const setupAgentTriggerToken = async (
+  instance: CommandInstance,
+  projectId: string,
+  agentName: string,
+): Promise<TriggerToken> => {
+  const tokenName = `cl_agent_${agentName}`;
+
+  instance.log(`Setting up trigger token for agent: ${agentName}`);
+
+  // List existing trigger tokens
+  const existingTokens = await listTriggerTokens(instance, projectId);
+
+  // Find and delete existing cl_agent_<agentName> tokens
+  const existingAgentTokens = existingTokens.filter((token) =>
+    token.description.startsWith(`cl_agent_${agentName}`),
+  );
+
+  for (const token of existingAgentTokens) {
+    instance.log(
+      `Deleting existing trigger token: ${token.description} (ID: ${token.id})`,
+    );
+    await deleteTriggerToken(instance, projectId, token.id);
+  }
+
+  // Create new trigger token
+  const description = `${tokenName} - Agent trigger token for ${agentName}`;
+  const newToken = await createTriggerToken(instance, projectId, description);
+
+  instance.log(`✅ Created new trigger token: ${newToken.description}`);
+  instance.log(`   Token ID: ${newToken.id}`);
+  instance.log(`   Token: ${newToken.token}`);
+
+  return newToken;
+};
+
+const setupAgent = async (
+  instance: CommandInstance,
+  agentName: string,
+  agentConfig: AgentConfig,
+) => {
+  const { id: projectId } = await getProjectInfo(instance);
+
+  instance.log(`🤖 Setting up agent: ${agentName}`);
+
+  // Setup trigger token for this agent
+  const triggerToken = await setupAgentTriggerToken(
+    instance,
+    projectId,
+    agentName,
+  );
+
+  // Setup webhook for this agent
+  await setupAgentWebhook(instance, projectId, agentName, triggerToken.token);
+
+  instance.log(`✅ Agent ${agentName} setup complete!`);
+};
+
+export const setupAgents = async (instance: CommandInstance) => {
+  const config = await getProjectConfig();
+
+  instance.log("");
+  instance.log("🤖 Setting up agents...");
+
+  const agents = config.agents ?? {};
+  const agentEntries = Object.entries(agents);
+
+  if (agentEntries.length === 0) {
+    instance.log("No agents configured in project config");
+    return;
+  }
+
+  for (const [agentName, agentConfig] of agentEntries) {
+    if (agentConfig.type === "claude" && agentConfig) {
+      await setupAgent(instance, agentName, agentConfig);
+    }
+  }
+
+  instance.log("✅ All agents setup complete!");
+  instance.log("");
+  instance.log("🔧 Manual setup required:");
+  instance.log(
+    "Please configure these environment variables in GitLab CI/CD unless they are already set (by group or global):",
+  );
+  instance.log("   Go to Settings > CI/CD > Variables and add:");
+  instance.log(
+    "   - CLAUDE_CODE_OAUTH_TOKEN or ANTHROPIC_API_KEY (your Anthropic API key, mark as Protected & Masked)",
+  );
+  instance.log(
+    "   - AGENT_GITLAB_PERSONAL_ACCESS_TOKEN (token from agent.claude user, mark as Protected & Masked)",
+  );
+};

package/src/apps/cli/commands/project/setup/setupCloudRun.ts

@@ -0,0 +1,84 @@
+import type { ComponentContext } from "@catladder/pipeline";
+import {
+  GCLOUD_DEPLOY_CREDENTIALS_KEY,
+  GCLOUD_RUN_CANONICAL_HOST_SUFFIX,
+  isOfDeployType,
+} from "@catladder/pipeline";
+
+import type { CommandInstance } from "vorpal";
+import { upsertGcloudArtifactsRegistry } from "../../../../../gcloud/artifactsRegistry";
+import { getCloudRunDomainSuffix } from "../../../../../gcloud/cloudRun/getCloudRunDomainSuffix";
+import { enableGCloudServices } from "../../../../../gcloud/enableServices";
+import { upsertGcloudServiceAccountAndSaveSecret } from "../../../../../gcloud/serviceAccounts";
+import { upsertAllVariables } from "../../../../../utils/gitlab";
+
+export const setupCloudRun = async (
+  instance: CommandInstance,
+  context: ComponentContext,
+) => {
+  if (!isOfDeployType(context.deploy?.config, "google-cloudrun")) {
+    throw new Error("deploy config is not of type 'google-cloudrun'");
+  }
+
+  const config = context.deploy?.config;
+
+  // enable services
+
+  instance.log("enable required services...");
+  await enableGCloudServices(
+    [
+      "run.googleapis.com",
+      "artifactregistry.googleapis.com",
+      "cloudscheduler.googleapis.com", // for scheduling jobs
+      "cloudresourcemanager.googleapis.com", // only required to get google cloud project number
+      ...(config.cloudSql
+        ? ["sqladmin.googleapis.com", "sql-component.googleapis.com"]
+        : []),
+    ],
+    config,
+  );
+  instance.log("upsert artifacts registry...");
+  await upsertGcloudArtifactsRegistry(config);
+
+  instance.log("upsert deploy service account...");
+  await upsertGcloudServiceAccountAndSaveSecret(
+    instance,
+    context,
+    {
+      projectId: config.projectId,
+      displayName: "Catladder gcloud deploy",
+      description: "This service account deploys to google cloud",
+      name: "cl-d",
+      // TODO: run.admin is a bit much, would be better to follow https://stackoverflow.com/questions/55788714/deploying-to-cloud-run-with-a-custom-service-account-failed-with-iam-serviceacco
+      roles: [
+        "roles/artifactregistry.repoAdmin",
+        "roles/run.admin",
+        "roles/iam.serviceAccountUser",
+        "roles/cloudscheduler.admin", // for scheduling
+        ...(config.cloudSql ? ["roles/cloudsql.admin"] : []),
+      ],
+    },
+    GCLOUD_DEPLOY_CREDENTIALS_KEY,
+  );
+
+  // gcloud run automatically gives us a hostname
+  // but the pipeline need to know it before deployment
+  // so we get this through some magic and add this as a ci/cd variable
+  // the pipeline than can use this to construct the canonical host
+  instance.log(
+    "get service domain suffix... that might take a while initially",
+  );
+  const suffix = await getCloudRunDomainSuffix(config);
+  instance.log("domain suffix: " + suffix);
+
+  await upsertAllVariables(
+    instance,
+    {
+      [GCLOUD_RUN_CANONICAL_HOST_SUFFIX]: suffix,
+    },
+    context.env,
+    context.name,
+    false, // backup
+    false, // masked
+  );
+};

package/src/apps/cli/commands/project/setup/setupContext.ts

@@ -1,37 +1,27 @@
-import { Context, isOfDeployType } from "@catladder/pipeline";
-import { CommandInstance } from "vorpal";
+import type { ComponentContext } from "@catladder/pipeline";
+import { isOfDeployType } from "@catladder/pipeline";
+import type { CommandInstance } from "vorpal";
+import { setupCloudRun } from "./setupCloudRun";
+
 import { setupKubernetes } from "./setupKubernetes";
+import { logSection } from "./logSection";
 
 export const setupContext = async (
   instance: CommandInstance,
-  context: Context
+  context: ComponentContext,
 ) => {
-  instance.log("");
-  instance.log(
-    "=================================================================================="
-  );
+  await logSection(
+    instance,
+    "setting up " + context.env + ":" + context.name,
+    async () => {
+      if (isOfDeployType(context.deploy?.config, "google-cloudrun")) {
+        await setupCloudRun(instance, context);
+      }
 
-  instance.log(
-    "🐱 🔧 setting up " +
-      context.environment.shortName +
-      ":" +
-      context.componentName +
-      "..."
+      const deployConfig = context.deploy?.config;
+      if (isOfDeployType(deployConfig, "kubernetes")) {
+        await setupKubernetes(instance, context);
+      }
+    },
   );
-  instance.log("");
-  const deployConfig = context.componentConfig.deploy;
-  if (isOfDeployType(deployConfig, "kubernetes")) {
-    await setupKubernetes(instance, context);
-  }
-
-  instance.log("");
-  instance.log(
-    "✅ " +
-      context.environment.shortName +
-      ":" +
-      context.componentName +
-      " done!"
-  );
-
-  instance.log("");
 };

package/src/apps/cli/commands/project/setup/setupKubernetes.ts

@@ -1,23 +1,54 @@
+import type { ComponentContext } from "@catladder/pipeline";
 import {
-  Context,
   getFullKubernetesClusterName,
   isOfDeployType,
 } from "@catladder/pipeline";
-import { CommandInstance } from "vorpal";
-import { $ } from "zx";
+import type { CommandInstance } from "vorpal";
+import { exec } from "child-process-promise";
 import { connectToCluster } from "../../../../../utils/cluster";
-import { upsertAllVariables } from "../../../../../utils/gitlab";
+import {
+  doGitlabRequest,
+  getProjectInfo,
+  upsertAllVariables,
+} from "../../../../../utils/gitlab";
 import ensureNamespace from "../utils/ensureNamespace";
 
 export const setupKubernetes = async (
   instance: CommandInstance,
-  context: Context
+  context: ComponentContext,
 ) => {
-  const deployConfig = context.componentConfig.deploy;
+  const deployConfig = context.deploy?.config;
   if (!isOfDeployType(deployConfig, "kubernetes")) {
     throw new Error("cannot run setupKubernetes on non-kubernetes deployments");
   }
 
+  const { id: projectId } = await getProjectInfo(instance);
+  const deploy_tokens = await doGitlabRequest(
+    instance,
+    `projects/${projectId}/deploy_tokens`,
+  );
+
+  if (
+    !deploy_tokens.find(
+      (v: { name: string }) => v.name === "gitlab-deploy-token",
+    )
+  ) {
+    instance.log(
+      "I will setup the 'GitLab Deploy Token', so Kubernetes can pull images from this project.",
+    );
+
+    await doGitlabRequest(
+      instance,
+      `projects/${projectId}/deploy_tokens`,
+      {
+        id: projectId,
+        name: "gitlab-deploy-token",
+        scopes: ["read_registry"],
+      },
+      "POST",
+    );
+  }
+
   const fullName = getFullKubernetesClusterName(deployConfig.cluster);
   instance.log(`cluster: ${fullName}`);
 
@@ -32,35 +63,42 @@ export const setupKubernetes = async (
   // we name the service account and the role and the role binding with the same name
   // we currently create one per component to better separate them
   instance.log("ensuring service accounts...");
-  const serviceAccountName = `cl-${context.componentName}-deploy`;
-  const KUBE_URL =
-    await $`TERM=dumb kubectl cluster-info | grep -E 'Kubernetes master|Kubernetes control plane' | awk '/http/ {print $NF}'`.then(
-      (s) => s.stdout.trim()
-    );
+  const serviceAccountName = `cl-${context.name}-deploy`;
+  const KUBE_URL = await exec(
+    `TERM=dumb kubectl cluster-info | grep -E 'Kubernetes master|Kubernetes control plane' | awk '/http/ {print $NF}'`,
+  ).then((s) => s.stdout.trim());
 
   // first upsert service acount in the ns
   try {
-    await $`kubectl delete serviceaccount --namespace ${namespace} ${serviceAccountName}`;
-    await $`kubectl delete rolebinding --namespace ${namespace} ${serviceAccountName}`;
-    await $`kubectl delete role --namespace ${namespace} ${serviceAccountName}`;
+    await exec(
+      `kubectl delete serviceaccount --namespace ${namespace} ${serviceAccountName}`,
+    );
+    await exec(
+      `kubectl delete rolebinding --namespace ${namespace} ${serviceAccountName}`,
+    );
+    await exec(
+      `kubectl delete role --namespace ${namespace} ${serviceAccountName}`,
+    );
   } catch (e) {
     // ignore
   }
 
-  await $`kubectl create serviceaccount --namespace ${namespace} ${serviceAccountName}`;
+  await exec(
+    `kubectl create serviceaccount --namespace ${namespace} ${serviceAccountName}`,
+  );
 
   // upsert role in the ns
 
-  await $`cat <<EOF | kubectl apply -f -
+  await exec(`cat <<EOF | kubectl apply -f -
 kind: Role
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   namespace: ${namespace}
   name: ${serviceAccountName}
 rules:
-- apiGroups: ["", "extensions", "apps", "networking.k8s.io", "batch"]
-  resources: ["deployments", "replicasets", "statefulsets", "pods", "secrets", "configmaps", "services", "ingresses", "serviceaccounts", "jobs", "cronjobs"]
-  verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] # You can also use ["*"]
+- apiGroups: ["", "extensions", "apps", "networking.k8s.io", "batch", "autoscaling", "rbac.authorization.k8s.io","snapshot.storage.k8s.io"]
+  resources: ["deployments", "replicasets", "statefulsets", "pods", "secrets", "configmaps", "services", "ingresses", "serviceaccounts", "roles", "rolebindings", "jobs", "cronjobs", "horizontalpodautoscalers", "persistentvolumeclaims", "volumesnapshots"]
+  verbs: ["*"]
 ---
 kind: RoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
@@ -76,20 +114,29 @@ roleRef:
   name: ${serviceAccountName}
   apiGroup: rbac.authorization.k8s.io
 EOF
-`;
+`);
 
-  // get token name
-  const tokenName =
-    await $`kubectl get serviceaccount --namespace ${namespace} ${serviceAccountName} -o jsonpath='{.secrets[0].name}'`;
+  const secretName = `${serviceAccountName}-secret`;
+  // create token for the service account
+  await exec(`
+kubectl apply -f - <<EOF
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ${secretName}
+  namespace: ${namespace}
+  annotations:
+    kubernetes.io/service-account.name: ${serviceAccountName}
+type: kubernetes.io/service-account-token
+EOF
+  `);
 
-  const KUBE_CA_PEM =
-    await $`kubectl get secret ${tokenName} --namespace ${namespace} -o jsonpath="{['data']['ca\\.crt']}"`.then(
-      (c) => c.stdout.trim()
-    );
-  const KUBE_TOKEN =
-    await $`kubectl get secret ${tokenName} --namespace ${namespace} -o jsonpath="{['data']['token']}" | base64 --decode`.then(
-      (c) => c.stdout.trim()
-    );
+  const KUBE_CA_PEM = await exec(
+    `kubectl get secret ${secretName} --namespace ${namespace} -o jsonpath="{['data']['ca\\.crt']}"`,
+  ).then((c) => c.stdout.trim());
+  const KUBE_TOKEN = await exec(
+    `kubectl get secret ${secretName} --namespace ${namespace} -o jsonpath="{['data']['token']}" | base64 --decode`,
+  ).then((c) => c.stdout.trim());
 
   const vars = {
     KUBE_TOKEN,
@@ -97,6 +144,17 @@ EOF
     KUBE_URL,
   };
 
+  const missing = Object.entries(vars).filter((e) => !e[1]);
+  if (missing.length > 0) {
+    throw new Error(
+      "could not setup credentials. Missing vars: " +
+        missing.map((m) => m[0]).join(", ") +
+        ". Check whether your local kubectl is still working for '" +
+        fullName +
+        "'",
+    );
+  }
+
   instance.log("service accounts created / updated!");
 
   instance.log("");
@@ -105,8 +163,9 @@ EOF
   await upsertAllVariables(
     instance,
     vars,
-    context.environment.shortName,
-    context.componentName
+    context.env,
+    context.name,
+    false, // no backup
   );
   instance.log("done!");
 };

package/src/apps/cli/commands/project/setup/setupTopic.ts

@@ -0,0 +1,17 @@
+import type { CommandInstance } from "vorpal";
+import { doGitlabRequest, getProjectInfo } from "../../../../../utils/gitlab";
+
+const catladderTopic = "catladder";
+
+export const setupTopic = async (instance: CommandInstance) => {
+  const { id: projectId } = await getProjectInfo(instance);
+
+  const { topics } = await doGitlabRequest(instance, `projects/${projectId}`);
+
+  await doGitlabRequest(
+    instance,
+    `projects/${projectId}`,
+    { topics: [catladderTopic, ...topics] },
+    "PUT",
+  );
+};

package/src/apps/cli/commands/project/utils/autocompletions.ts

@@ -12,12 +12,21 @@ export const allEnvs = async () => {
   return getAllEnvsInAllComponents(config);
 };
 
+export const allComponents = async () => {
+  const config = await getProjectConfig();
+  if (!config) {
+    return [];
+  }
+
+  return Object.keys(config.components);
+};
+
 export const envAndComponents = async () => {
   const allEnvAndcomponents = await getAllComponentsWithAllEnvsFlat();
 
   return allEnvAndcomponents.reduce<string[]>(
     (acc, { env, componentName }) => [...acc, env + ":" + componentName],
-    []
+    [],
   );
 };