duoops 0.1.0 → 0.1.4

package/README.md

@@ -158,6 +158,23 @@ duoops runner:logs --lines 300
 duoops runner:reinstall-duoops --version 0.1.0
 ```
 
+### 7. Use the GitLab CI Component
+
+Instead of copying `.duoops/measure-component.yml` into every project, you can reference the published component directly:
+
+```yaml
+include:
+  - component: gitlab.com/youneslaaroussi/duoops/templates/duoops-measure-component@v0.1.0
+    inputs:
+      gcp_project_id: "my-project"
+      gcp_instance_id: "1234567890123456789"
+      gcp_zone: "us-central1-a"
+      machine_type: "e2-standard-4"
+      tags: ["gcp"]
+```
+
+Make sure your runner already has DuoOps installed (the provisioning flow handles this) and that `GCP_SA_KEY_BASE64` plus any optional BigQuery variables are set in the project’s CI/CD settings. Pin to a version tag (`@v0.1.0`) and update when you’re ready to adopt new behavior.
+
 ## 🛠️ Development
 
 ### Project Structure
@@ -188,6 +205,13 @@ This compiles the TypeScript CLI and builds the React frontend, copying assets t
 3.  Inspect the publish payload locally with `pnpm pack` (this runs the `prepack` script, which now builds the CLI, portal, and Oclif manifest automatically). Untar the generated `.tgz` if you want to double-check the contents.
 4.  When you're satisfied, publish the package: `pnpm publish --access public`.
 
+## 📦 Publishing the CI Component
+
+1.  Update `templates/duoops-measure-component.yml` and commit the changes.
+2.  Let the `.gitlab-ci.yml` validation job pass (runs automatically on MRs, default branch, and tags).
+3.  Tag the repository (e.g., `git tag v0.1.0 && git push origin v0.1.0`). Consumers reference the component with the tag via the snippet above.
+4.  Update release notes/README with the new component version so teams know what changed.
+
 ## 📄 License
 
 MIT

package/dist/commands/init.js

@@ -2,11 +2,12 @@ import { BigQuery } from '@google-cloud/bigquery';
 import { confirm, input, password, select } from '@inquirer/prompts';
 import { Command } from '@oclif/core';
 import axios from 'axios';
-import { bold, gray, green } from 'kleur/colors';
+import { bold, gray, green, yellow } from 'kleur/colors';
 import { execSync } from 'node:child_process';
 import fs from 'node:fs';
 import os from 'node:os';
 import path from 'node:path';
+import { fileURLToPath } from 'node:url';
 import { configManager } from '../lib/config.js';
 const hasBinary = (command) => {
     try {
@@ -78,17 +79,23 @@ const createServiceAccountKey = (projectId, serviceAccount) => {
     fs.unlinkSync(tmpPath);
     return contents.toString('base64');
 };
-const ensureComputeApiEnabled = (projectId, log) => {
+const ensureServiceEnabled = (projectId, service, label, log) => {
     try {
-        log?.(gray('Ensuring Compute Engine API is enabled...'));
-        execSync(`gcloud services enable compute.googleapis.com --project=${projectId} --quiet`, {
+        log?.(gray(`Ensuring ${label} API is enabled...`));
+        execSync(`gcloud services enable ${service} --project=${projectId} --quiet`, {
             stdio: 'ignore',
         });
     }
     catch {
-        log?.(gray('Compute API enablement skipped (it may already be enabled).'));
+        log?.(gray(`${label} API enablement skipped (it may already be enabled).`));
     }
 };
+const ensureComputeApiEnabled = (projectId, log) => {
+    ensureServiceEnabled(projectId, 'compute.googleapis.com', 'Compute Engine', log);
+};
+const ensureMonitoringApiEnabled = (projectId, log) => {
+    ensureServiceEnabled(projectId, 'monitoring.googleapis.com', 'Cloud Monitoring', log);
+};
 const sleep = (ms) => new Promise((resolve) => {
     setTimeout(resolve, ms);
 });
@@ -184,6 +191,81 @@ const tryCreateRunnerToken = (projectPath, runnerDescription, runnerTag) => {
     catch {
     }
 };
+const updateGitLabCiYml = async (projectPath, config, runnerInfo, log) => {
+    const ciPath = path.join(process.cwd(), '.gitlab-ci.yml');
+    if (!fs.existsSync(ciPath)) {
+        log(yellow('No .gitlab-ci.yml found. Skipping automatic update.'));
+        return;
+    }
+    const useComponent = await confirm({
+        default: true,
+        message: 'Add the DuoOps measure component to your .gitlab-ci.yml?',
+    });
+    if (!useComponent) {
+        return;
+    }
+    const method = await select({
+        choices: [
+            {
+                description: 'Recommended: Automatically updated via GitLab Catalog',
+                name: 'Use Published Component (v0.1)',
+                value: 'published',
+            },
+            {
+                description: 'Copy template file into your repository',
+                name: 'Use Local File Copy',
+                value: 'local',
+            },
+        ],
+        message: 'How would you like to include the component?',
+    });
+    let includeBlock = '';
+    if (method === 'published') {
+        // Determine latest version or default to 0.1
+        const componentPath = `gitlab.com/youneslaaroussi/duoops/duoops-measure-component@0.1`;
+        includeBlock = `
+include:
+  - component: ${componentPath}
+    inputs:
+      gcp_project_id: "$GCP_PROJECT_ID"
+      gcp_instance_id: "$GCP_INSTANCE_ID"
+      gcp_zone: "$GCP_ZONE"
+      machine_type: "$MACHINE_TYPE"
+      tags: ["${runnerInfo.runnerTag}"]
+`;
+    }
+    else {
+        // Local copy
+        const targetDir = path.join(process.cwd(), '.duoops');
+        if (!fs.existsSync(targetDir)) {
+            fs.mkdirSync(targetDir, { recursive: true });
+        }
+        const templatePath = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../templates/duoops-measure-component.yml');
+        if (fs.existsSync(templatePath)) {
+            const targetFile = path.join(targetDir, 'measure-component.yml');
+            fs.copyFileSync(templatePath, targetFile);
+            log(green(`Copied component template to ${targetFile}`));
+        }
+        else {
+            log(yellow(`Could not find template at ${templatePath}. Using remote reference instead.`));
+        }
+        includeBlock = `
+include:
+  - local: .duoops/measure-component.yml
+`;
+    }
+    const currentContent = fs.readFileSync(ciPath, 'utf8');
+    if (currentContent.includes('duoops-measure-component')) {
+        log(yellow('It looks like DuoOps component is already included in .gitlab-ci.yml.'));
+        const overwrite = await confirm({
+            message: 'Append the new configuration anyway?',
+        });
+        if (!overwrite)
+            return;
+    }
+    fs.appendFileSync(ciPath, includeBlock);
+    log(green('Updated .gitlab-ci.yml with DuoOps component configuration.'));
+};
 export default class Init extends Command {
     static description = 'Initialize DuoOps, optionally wiring a GitLab project and GCP runner';
     async run() {
@@ -396,7 +478,8 @@ export default class Init extends Command {
                 this.warn(`  WARN Failed to set ${variable.key}: ${message}`);
             }
         }));
-        this.log(gray('\nAdd the duoops measure component to your .gitlab-ci.yml (demo/.duoops/measure-component.yml is a good starting point).'));
+        this.log('\n');
+        await updateGitLabCiYml(projectPath, configManager.get(), runnerInfo, (msg) => this.log(msg));
         this.log(gray(`Use the '${runnerInfo.runnerTag}' tag on jobs that should target this runner.`));
         this.log(green('Project wiring complete!'));
         const currentConfig = configManager.get();
@@ -440,6 +523,7 @@ export default class Init extends Command {
             message: 'Runner tag (GitLab jobs will use this)',
         });
         ensureComputeApiEnabled(gcpProjectId, (message) => this.log(message));
+        ensureMonitoringApiEnabled(gcpProjectId, (message) => this.log(message));
         let runnerToken = tryCreateRunnerToken(projectPath, vmName, runnerTag);
         if (!runnerToken) {
             this.log(gray('Unable to create a runner token automatically. Generate one in your GitLab project (Settings → CI/CD → Runners) and paste it below.'));

package/oclif.manifest.json

@@ -171,96 +171,6 @@
         "logs.js"
       ]
     },
-    "pipelines:list": {
-      "aliases": [],
-      "args": {
-        "project": {
-          "description": "Project ID or path (e.g. group/project)",
-          "name": "project",
-          "required": false
-        }
-      },
-      "description": "List GitLab CI pipelines for a project",
-      "examples": [
-        "<%= config.bin %> <%= command.id %> group/my-project",
-        "<%= config.bin %> <%= command.id %> 123 --limit 20 --ref main"
-      ],
-      "flags": {
-        "limit": {
-          "char": "n",
-          "description": "Maximum number of pipelines to return",
-          "name": "limit",
-          "default": 10,
-          "hasDynamicHelp": false,
-          "multiple": false,
-          "type": "option"
-        },
-        "ref": {
-          "description": "Filter by branch or tag",
-          "name": "ref",
-          "hasDynamicHelp": false,
-          "multiple": false,
-          "type": "option"
-        },
-        "status": {
-          "description": "Filter by status (created, pending, running, success, failed, canceled, skipped, manual, scheduled)",
-          "name": "status",
-          "hasDynamicHelp": false,
-          "multiple": false,
-          "type": "option"
-        }
-      },
-      "hasDynamicHelp": false,
-      "hiddenAliases": [],
-      "id": "pipelines:list",
-      "pluginAlias": "duoops",
-      "pluginName": "duoops",
-      "pluginType": "core",
-      "strict": true,
-      "enableJsonFlag": false,
-      "isESM": true,
-      "relativePath": [
-        "dist",
-        "commands",
-        "pipelines",
-        "list.js"
-      ]
-    },
-    "pipelines:show": {
-      "aliases": [],
-      "args": {
-        "project": {
-          "description": "Project ID (e.g. 123456)",
-          "name": "project",
-          "required": true
-        },
-        "pipeline_id": {
-          "description": "Pipeline ID",
-          "name": "pipeline_id",
-          "required": true
-        }
-      },
-      "description": "Show pipeline details and jobs",
-      "examples": [
-        "<%= config.bin %> <%= command.id %> 12345 67890"
-      ],
-      "flags": {},
-      "hasDynamicHelp": false,
-      "hiddenAliases": [],
-      "id": "pipelines:show",
-      "pluginAlias": "duoops",
-      "pluginName": "duoops",
-      "pluginType": "core",
-      "strict": true,
-      "enableJsonFlag": false,
-      "isESM": true,
-      "relativePath": [
-        "dist",
-        "commands",
-        "pipelines",
-        "show.js"
-      ]
-    },
     "measure:calculate": {
       "aliases": [],
       "args": {},
@@ -410,6 +320,96 @@
         "seed.js"
       ]
     },
+    "pipelines:list": {
+      "aliases": [],
+      "args": {
+        "project": {
+          "description": "Project ID or path (e.g. group/project)",
+          "name": "project",
+          "required": false
+        }
+      },
+      "description": "List GitLab CI pipelines for a project",
+      "examples": [
+        "<%= config.bin %> <%= command.id %> group/my-project",
+        "<%= config.bin %> <%= command.id %> 123 --limit 20 --ref main"
+      ],
+      "flags": {
+        "limit": {
+          "char": "n",
+          "description": "Maximum number of pipelines to return",
+          "name": "limit",
+          "default": 10,
+          "hasDynamicHelp": false,
+          "multiple": false,
+          "type": "option"
+        },
+        "ref": {
+          "description": "Filter by branch or tag",
+          "name": "ref",
+          "hasDynamicHelp": false,
+          "multiple": false,
+          "type": "option"
+        },
+        "status": {
+          "description": "Filter by status (created, pending, running, success, failed, canceled, skipped, manual, scheduled)",
+          "name": "status",
+          "hasDynamicHelp": false,
+          "multiple": false,
+          "type": "option"
+        }
+      },
+      "hasDynamicHelp": false,
+      "hiddenAliases": [],
+      "id": "pipelines:list",
+      "pluginAlias": "duoops",
+      "pluginName": "duoops",
+      "pluginType": "core",
+      "strict": true,
+      "enableJsonFlag": false,
+      "isESM": true,
+      "relativePath": [
+        "dist",
+        "commands",
+        "pipelines",
+        "list.js"
+      ]
+    },
+    "pipelines:show": {
+      "aliases": [],
+      "args": {
+        "project": {
+          "description": "Project ID (e.g. 123456)",
+          "name": "project",
+          "required": true
+        },
+        "pipeline_id": {
+          "description": "Pipeline ID",
+          "name": "pipeline_id",
+          "required": true
+        }
+      },
+      "description": "Show pipeline details and jobs",
+      "examples": [
+        "<%= config.bin %> <%= command.id %> 12345 67890"
+      ],
+      "flags": {},
+      "hasDynamicHelp": false,
+      "hiddenAliases": [],
+      "id": "pipelines:show",
+      "pluginAlias": "duoops",
+      "pluginName": "duoops",
+      "pluginType": "core",
+      "strict": true,
+      "enableJsonFlag": false,
+      "isESM": true,
+      "relativePath": [
+        "dist",
+        "commands",
+        "pipelines",
+        "show.js"
+      ]
+    },
     "runner:logs": {
       "aliases": [],
       "args": {},
@@ -480,5 +480,5 @@
       ]
     }
   },
-  "version": "0.1.0"
+  "version": "0.1.4"
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duoops",
   "description": "Toolset for Explainable and Sustainable CI on Gitlab.",
-  "version": "0.1.0",
+  "version": "0.1.4",
   "author": "Younes Laaroussi",
   "bin": {
     "duoops": "./bin/run.js"
@@ -61,7 +61,8 @@
   "files": [
     "./bin",
     "./dist",
-    "./oclif.manifest.json"
+    "./oclif.manifest.json",
+    "./templates"
   ],
   "homepage": "https://gitlab.com/youneslaaroussi/duoops",
   "keywords": [
@@ -88,16 +89,21 @@
       }
     }
   },
-  "repository": "youneslaaroussi/duoops",
-  "publishConfig": {
-    "access": "public"
+  "repository": {
+    "type": "git",
+    "url": "https://gitlab.com/youneslaaroussi/duoops.git"
   },
-  "types": "dist/index.d.ts",
   "scripts": {
     "build": "shx rm -rf dist && tsc -b && pnpm --filter portal run build && shx cp -r portal/dist dist/portal",
     "lint": "eslint --ignore-pattern \"portal/**\"",
+    "postpack": "shx rm -f oclif.manifest.json",
     "posttest": "pnpm run lint",
+    "prepack": "pnpm run build && oclif manifest && oclif readme",
     "test": "mocha --forbid-only \"test/**/*.test.ts\"",
     "version": "oclif readme && git add README.md"
-  }
-}
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "types": "dist/index.d.ts"
+}

package/templates/duoops-measure-component.yml

@@ -0,0 +1,149 @@
+spec:
+  inputs:
+    provider:
+      description: "Cloud provider (gcp or aws)"
+      type: string
+      default: "gcp"
+
+    # GCP inputs
+    gcp_project_id:
+      description: "GCP Project ID"
+      type: string
+      default: ""
+
+    gcp_instance_id:
+      description: "GCP Compute Engine instance ID (numeric)"
+      type: string
+      default: ""
+
+    gcp_zone:
+      description: "GCP zone (e.g., us-central1-a)"
+      type: string
+      default: ""
+
+    machine_type:
+      description: "Machine type (e.g., e2-standard-4)"
+      type: string
+
+    # Optional configuration
+    emission_budget_grams:
+      description: "Carbon/Emission budget in grams CO2e (optional)"
+      type: string
+      default: ""
+
+    fail_on_budget:
+      description: "Fail build if budget is exceeded"
+      type: boolean
+      default: false
+
+    duoops_version:
+      description: "duoops npm package version"
+      type: string
+      default: "latest"
+
+    stage:
+      description: "GitLab CI stage to run in"
+      type: string
+      default: ".post"
+
+    tags:
+      description: "Runner tags"
+      type: array
+      default: []
+
+---
+
+duoops-measure-analysis:
+  stage: $[[ inputs.stage ]]
+  tags: $[[ inputs.tags ]]
+  image: google/cloud-sdk:slim
+  variables:
+    PROVIDER: $[[ inputs.provider ]]
+    GCP_PROJECT_ID: $[[ inputs.gcp_project_id ]]
+    GCP_INSTANCE_ID: $[[ inputs.gcp_instance_id ]]
+    GCP_ZONE: $[[ inputs.gcp_zone ]]
+    MACHINE_TYPE: $[[ inputs.machine_type ]]
+    EMISSION_BUDGET_GRAMS: $[[ inputs.emission_budget_grams ]]
+    FAIL_ON_BUDGET: $[[ inputs.fail_on_budget ]]
+    DUOOPS_VERSION: $[[ inputs.duoops_version ]]
+  before_script:
+    - |
+      if ! command -v duoops &> /dev/null; then
+        echo "duoops binary not found on runner. Ensure the runner image installs duoops." >&2
+        exit 1
+      fi
+      echo "Using $(duoops --version)"
+  script:
+    - |
+      PROVIDER=${PROVIDER:-$[[ inputs.provider ]]}
+      START_TIME=$(date -u -d "$CI_PIPELINE_CREATED_AT" +%Y-%m-%dT%H:%M:%SZ)
+      END_TIME=$(date -u +%Y-%m-%dT%H:%M:%SZ)
+
+      if [ -z "$GCP_SA_KEY_BASE64" ]; then
+        echo "GCP_SA_KEY_BASE64 is required for provider=gcp" >&2
+        exit 1
+      fi
+
+      echo "$GCP_SA_KEY_BASE64" | base64 -d > /tmp/gcp-key.json
+      export GOOGLE_APPLICATION_CREDENTIALS=/tmp/gcp-key.json
+      gcloud auth activate-service-account --key-file=/tmp/gcp-key.json
+
+      TOKEN=$(gcloud auth print-access-token)
+
+      CPU_FILTER="resource.type=\"gce_instance\" AND resource.labels.instance_id=\"$GCP_INSTANCE_ID\" AND resource.labels.zone=\"$GCP_ZONE\" AND metric.type=\"compute.googleapis.com/instance/cpu/utilization\""
+      CPU_ENCODED=$(python3 -c "import urllib.parse; print(urllib.parse.quote('''$CPU_FILTER'''))")
+
+      RAM_USED_FILTER="resource.type=\"gce_instance\" AND resource.labels.instance_id=\"$GCP_INSTANCE_ID\" AND metric.type=\"compute.googleapis.com/instance/memory/balloon/ram_used\""
+      RAM_USED_ENCODED=$(python3 -c "import urllib.parse; print(urllib.parse.quote('''$RAM_USED_FILTER'''))")
+
+      RAM_SIZE_FILTER="resource.type=\"gce_instance\" AND resource.labels.instance_id=\"$GCP_INSTANCE_ID\" AND metric.type=\"compute.googleapis.com/instance/memory/balloon/ram_size\""
+      RAM_SIZE_ENCODED=$(python3 -c "import urllib.parse; print(urllib.parse.quote('''$RAM_SIZE_FILTER'''))")
+
+      START_SEC=$(date -d "$START_TIME" +%s)
+      NOW_SEC=$(date +%s)
+      DURATION_SEC=$((NOW_SEC - START_SEC))
+      EXPECTED_POINTS=$((DURATION_SEC / 60))
+      [ "$EXPECTED_POINTS" -lt 1 ] && EXPECTED_POINTS=1
+
+      echo "Pipeline duration: ${DURATION_SEC}s, expecting ~$EXPECTED_POINTS data points"
+      echo "Waiting for metrics data (GCP has ~3 min lag)..."
+
+      for i in $(seq 1 30); do
+        END_TIME=$(date -u +%Y-%m-%dT%H:%M:%SZ)
+        CPU_DATA=$(curl -s -H "Authorization: Bearer $TOKEN" \
+          "https://monitoring.googleapis.com/v3/projects/$GCP_PROJECT_ID/timeSeries?filter=$CPU_ENCODED&interval.startTime=$START_TIME&interval.endTime=$END_TIME")
+
+        POINTS=$(echo "$CPU_DATA" | python3 -c "import sys,json; d=json.load(sys.stdin); print(sum(len(ts.get('points',[])) for ts in d.get('timeSeries',[])))" 2>/dev/null || echo "0")
+
+        if [ "$POINTS" -ge "$EXPECTED_POINTS" ]; then
+          echo "Got $POINTS CPU data points (expected $EXPECTED_POINTS)"
+          break
+        fi
+        echo "Waiting for data... got $POINTS/$EXPECTED_POINTS (attempt $i/30)"
+        sleep 10
+      done
+
+      # Fetch RAM data
+      RAM_USED_DATA=$(curl -s -H "Authorization: Bearer $TOKEN" \
+        "https://monitoring.googleapis.com/v3/projects/$GCP_PROJECT_ID/timeSeries?filter=$RAM_USED_ENCODED&interval.startTime=$START_TIME&interval.endTime=$END_TIME")
+
+      RAM_SIZE_DATA=$(curl -s -H "Authorization: Bearer $TOKEN" \
+        "https://monitoring.googleapis.com/v3/projects/$GCP_PROJECT_ID/timeSeries?filter=$RAM_SIZE_ENCODED&interval.startTime=$START_TIME&interval.endTime=$END_TIME")
+
+      echo "$CPU_DATA" > /tmp/cpu_timeseries.json
+      echo "$RAM_USED_DATA" > /tmp/ram_used_timeseries.json
+      echo "$RAM_SIZE_DATA" > /tmp/ram_size_timeseries.json
+
+      CMD="duoops measure calculate --provider gcp --machine $MACHINE_TYPE --region $GCP_ZONE --cpu-timeseries /tmp/cpu_timeseries.json --ram-used-timeseries /tmp/ram_used_timeseries.json --ram-size-timeseries /tmp/ram_size_timeseries.json --out-md measure-report.md --out-json measure-report.json"
+      [ -n "${EMISSION_BUDGET_GRAMS:-}" ] && CMD="$CMD --budget $EMISSION_BUDGET_GRAMS"
+      [ "${FAIL_ON_BUDGET:-}" = "true" ] && CMD="$CMD --fail-on-budget"
+
+      eval "$CMD"
+      cat measure-report.md
+  artifacts:
+    paths:
+      - measure-report.md
+      - measure-report.json
+    expire_in: 30 days
+  rules:
+    - when: always