gitgreen 1.0.3 → 1.0.4

package/README.md

@@ -140,6 +140,29 @@ Pipeline starts → component script fetches CPU/RAM timeseries from GCP Monitor
               → optional MR note when CI_JOB_TOKEN is present
 ```
 
+## Output Integrations
+
+During `gitgreen init` you can opt into exporting GitGreen data to external systems. The wizard includes an integration step with two optional sinks:
+
+- **Per-job carbon data** – emissions, runtime, and runner tags for every CI job.
+- **Runner inventory** – the machine catalog that powers your GitLab runners, including machine type and scope 3 estimates.
+
+Built-in connectors today:
+- **MySQL** – populates `GITGREEN_JOB_MYSQL_*` / `GITGREEN_RUNNER_MYSQL_*` and inserts rows through a standard MySQL client.
+- **PostgreSQL** – captures host, port, credentials, schema, table, and SSL mode (`GITGREEN_JOB_POSTGRES_*` / `GITGREEN_RUNNER_POSTGRES_*`) for storage in Postgres.
+
+When you select either connector, the wizard captures host, port, username, password, database, and target table names and stores them in CI/CD variables. It immediately connects with those credentials to ensure the database, schema, and table exist (creating them with a generic `payload JSON/JSONB` + `ingested_at` structure when missing). The GitLab component then uses those variables to insert rows via the corresponding database driver.
+
+### Extending the interface
+
+Additional connectors can be added without touching the wizard logic. Each destination implements the `OutputIntegration` interface in `src/lib/integrations/output-integrations.ts`, which specifies:
+
+1. Display metadata (`id`, `name`, `description`)
+2. The data target it handles (`job` vs `runner`)
+3. Prompted credential fields (label, env var key, input type, default, mask flag)
+
+To add another sink (for example PostgreSQL or a webhook), create a new entry in that file with the fields your integration needs. Re-run `gitgreen init` and the option will automatically appear in the integration step.
+
 ## Adding a provider
 1. Extend `CloudProvider` and the provider guard in `src/index.ts` so the calculator accepts the new key.
 2. Add machine power data (`<provider>_machine_power_profiles.json`) and, if needed, CPU profiles to `data/`, then update `PowerProfileRepository.loadMachineData` to load it.

package/dist/init.js

@@ -11,6 +11,7 @@ const axios_1 = __importDefault(require("axios"));
 const prompts_1 = __importDefault(require("prompts"));
 const kleur_1 = require("kleur");
 const power_profile_repository_1 = require("./lib/carbon/power-profile-repository");
+const output_integrations_1 = require("./lib/integrations/output-integrations");
 const hasGlab = () => {
     try {
         (0, child_process_1.execSync)('glab --version', { stdio: 'ignore' });
@@ -81,6 +82,97 @@ const setVariable = async (auth, project, key, value, masked = false) => {
         return setVariableApi(auth.baseUrl, auth.pat, project, key, value, masked);
     }
 };
+const askIntegrationFieldValue = async (field) => {
+    const promptType = field.type === 'password' ? 'password' : field.type === 'number' ? 'number' : 'text';
+    if (field.helpText) {
+        console.log((0, kleur_1.gray)(field.helpText));
+    }
+    const envInitial = process.env[field.envKey];
+    const initialValue = envInitial ?? field.initial;
+    const initial = promptType === 'number' && initialValue !== undefined
+        ? (() => {
+            const parsed = Number(initialValue);
+            return Number.isFinite(parsed) ? parsed : undefined;
+        })()
+        : initialValue;
+    const answer = await (0, prompts_1.default)({
+        type: promptType,
+        name: 'value',
+        message: field.label,
+        initial
+    });
+    const rawValue = answer.value;
+    if (rawValue === undefined || rawValue === null || rawValue === '') {
+        if (field.required) {
+            console.log((0, kleur_1.red)(`${field.label} is required.`));
+            process.exit(1);
+        }
+        return undefined;
+    }
+    return String(rawValue);
+};
+const promptIntegrationFields = async (integration) => {
+    const collected = [];
+    const valuesByField = {};
+    for (const field of integration.fields) {
+        const value = await askIntegrationFieldValue(field);
+        if (value !== undefined) {
+            collected.push({ key: field.envKey, value, masked: Boolean(field.masked) });
+            valuesByField[field.id] = value;
+        }
+    }
+    return { variables: collected, values: valuesByField };
+};
+const selectIntegrationForTarget = async (target) => {
+    const available = output_integrations_1.outputIntegrations.filter(integration => integration.target === target);
+    if (!available.length)
+        return [];
+    const targetLabel = target === 'job' ? 'per-job carbon data' : 'runner inventory';
+    const { integrationId } = await (0, prompts_1.default)({
+        type: 'select',
+        name: 'integrationId',
+        message: `Output integration for ${targetLabel}`,
+        choices: [
+            { title: 'Skip', description: 'Do not export data', value: '_skip_' },
+            ...available.map(integration => ({
+                title: integration.name,
+                description: integration.description,
+                value: integration.id
+            }))
+        ]
+    });
+    if (!integrationId || integrationId === '_skip_') {
+        return [];
+    }
+    const integration = available.find(item => item.id === integrationId);
+    if (!integration)
+        return [];
+    console.log((0, kleur_1.gray)(`\n${integration.name}`));
+    if (integration.description) {
+        console.log((0, kleur_1.gray)(integration.description));
+    }
+    const { variables, values } = await promptIntegrationFields(integration);
+    if (integration.ensure) {
+        try {
+            await integration.ensure(values);
+        }
+        catch (errorState) {
+            console.log((0, kleur_1.red)(`Failed to verify ${integration.name}: ${errorState?.message || errorState}`));
+            process.exit(1);
+        }
+    }
+    return variables;
+};
+const promptOutputIntegrations = async (stepNumber) => {
+    if (!output_integrations_1.outputIntegrations.length)
+        return [];
+    console.log((0, kleur_1.gray)(`\nStep ${stepNumber}: Output Integrations`));
+    console.log((0, kleur_1.gray)('─'.repeat(40)));
+    console.log((0, kleur_1.gray)('Optionally send carbon job data or runner metadata to an external system.'));
+    const jobVars = await selectIntegrationForTarget('job');
+    const runnerVars = await selectIntegrationForTarget('runner');
+    return [...jobVars, ...runnerVars];
+};
 const generateCiJob = (opts) => {
     const { provider, runnerTag, carbonBudget, failOnBudget } = opts;
     let inputs = `      provider: ${provider}
@@ -259,6 +351,7 @@ const buildAwsUserData = (params) => {
         'dnf install -y gitlab-runner',
         'curl -fsSL https://rpm.nodesource.com/setup_20.x | bash -',
         'dnf install -y nodejs',
+        'npm install -g gitgreen@latest',
         '',
         'INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id)',
         '',
@@ -614,7 +707,9 @@ const runAwsInit = async (auth, projectPath) => {
         });
         failOnBudget = shouldFail;
     }
-    console.log((0, kleur_1.gray)('\nStep 8: Setting CI/CD Variables'));
+    // Step 8: Output integrations
+    const integrationVariables = await promptOutputIntegrations(8);
+    console.log((0, kleur_1.gray)('\nStep 9: Setting CI/CD Variables'));
     console.log((0, kleur_1.gray)('─'.repeat(40)));
     const variables = [
         { key: 'AWS_ACCESS_KEY_ID', value: accessKeyId, masked: true },
@@ -634,6 +729,9 @@ const runAwsInit = async (auth, projectPath) => {
     if (failOnBudget) {
         variables.push({ key: 'FAIL_ON_BUDGET', value: 'true', masked: false });
     }
+    if (integrationVariables.length) {
+        variables.push(...integrationVariables);
+    }
     for (const v of variables) {
         const ok = await setVariable(auth, projectPath, v.key, v.value, v.masked);
         if (ok) {
@@ -643,7 +741,7 @@ const runAwsInit = async (auth, projectPath) => {
             console.log((0, kleur_1.red)('  Failed: ' + v.key));
         }
     }
-    console.log((0, kleur_1.gray)('\nStep 9: CI Configuration'));
+    console.log((0, kleur_1.gray)('\nStep 10: CI Configuration'));
     console.log((0, kleur_1.gray)('─'.repeat(40)));
     let runnerTagForCi = runnerTag;
     if (!runnerTagForCi) {
@@ -904,6 +1002,7 @@ apt-get install -y curl ca-certificates python3
 # Install Node.js 20.x
 curl -fsSL https://deb.nodesource.com/setup_20.x | bash -
 apt-get install -y nodejs
+npm install -g gitgreen@latest
 
 # Install GitLab Runner
 curl -L https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh | bash
@@ -1110,7 +1209,7 @@ systemctl start gitlab-runner
         }
     } // end else (use existing runner)
     // Step 7: Service Account
-    console.log((0, kleur_1.gray)('\nStep 5: Service Account'));
+    console.log((0, kleur_1.gray)('\nStep 7: Service Account'));
     console.log((0, kleur_1.gray)('─'.repeat(40)));
     let saKeyBase64;
     const { keyMethod } = await (0, prompts_1.default)({
@@ -1177,8 +1276,8 @@ systemctl start gitlab-runner
         }
         saKeyBase64 = fs_1.default.readFileSync(saKeyPath).toString('base64');
     }
-    // Step 6: Electricity Maps API
-    console.log((0, kleur_1.gray)('\nStep 6: Electricity Maps API'));
+    // Step 8: Electricity Maps API
+    console.log((0, kleur_1.gray)('\nStep 8: Electricity Maps API'));
     console.log((0, kleur_1.gray)('─'.repeat(40)));
     console.log((0, kleur_1.gray)('Get free key: https://api-portal.electricitymaps.com'));
     const { electricityMapsKey } = await (0, prompts_1.default)({
@@ -1190,8 +1289,8 @@ systemctl start gitlab-runner
         console.log((0, kleur_1.red)('API key required'));
         process.exit(1);
     }
-    // Step 7: Optional
-    console.log((0, kleur_1.gray)('\nStep 7: Optional Settings'));
+    // Step 9: Optional
+    console.log((0, kleur_1.gray)('\nStep 9: Optional Settings'));
     console.log((0, kleur_1.gray)('─'.repeat(40)));
     console.log((0, kleur_1.gray)('Set a carbon budget to track emissions against a limit.'));
     console.log((0, kleur_1.gray)('Example: 10 grams CO2e per job. Leave empty to skip.\n'));
@@ -1211,8 +1310,9 @@ systemctl start gitlab-runner
         });
         failOnBudget = shouldFail;
     }
-    // Step 8: Set Variables
-    console.log((0, kleur_1.gray)('\nStep 8: Setting CI/CD Variables'));
+    // Step 10: Output integrations
+    const gcpIntegrationVariables = await promptOutputIntegrations(10);
+    console.log((0, kleur_1.gray)('\nStep 11: Setting CI/CD Variables'));
     console.log((0, kleur_1.gray)('─'.repeat(40)));
     const variables = [
         { key: 'GCP_PROJECT_ID', value: gcpProjectId, masked: false },
@@ -1228,6 +1328,9 @@ systemctl start gitlab-runner
     if (failOnBudget) {
         variables.push({ key: 'FAIL_ON_BUDGET', value: 'true', masked: false });
     }
+    if (gcpIntegrationVariables.length) {
+        variables.push(...gcpIntegrationVariables);
+    }
     for (const v of variables) {
         const ok = await setVariable(auth, projectPath, v.key, v.value, v.masked);
         if (ok) {
@@ -1237,8 +1340,8 @@ systemctl start gitlab-runner
             console.log((0, kleur_1.red)('  Failed: ' + v.key));
         }
     }
-    // Step 9: Generate CI job
-    console.log((0, kleur_1.gray)('\nStep 9: CI Configuration'));
+    // Step 12: Generate CI job
+    console.log((0, kleur_1.gray)('\nStep 12: CI Configuration'));
     console.log((0, kleur_1.gray)('─'.repeat(40)));
     // Only prompt for runner tag if not already set from provisioning
     if (!runnerTag) {

package/dist/lib/integrations/db-setup.js

@@ -0,0 +1,122 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ensurePostgresResources = exports.ensureMysqlResources = void 0;
+const promise_1 = __importDefault(require("mysql2/promise"));
+const pg_1 = require("pg");
+const kleur_1 = require("kleur");
+const escapeMysqlIdentifier = (value) => {
+    return '`' + value.replace(/`/g, '``') + '`';
+};
+const escapePgIdentifier = (value) => {
+    return '"' + value.replace(/"/g, '""') + '"';
+};
+const getTableDefinition = (_target) => {
+    return `(
+    id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
+    ingested_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    payload JSON NOT NULL,
+    PRIMARY KEY (id)
+  ) ENGINE=InnoDB`;
+};
+const getPgTableDefinition = (_target) => {
+    return `(
+    id BIGSERIAL PRIMARY KEY,
+    ingested_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    payload JSONB NOT NULL
+  )`;
+};
+const normalizeHost = (input) => {
+    if (!input)
+        return input;
+    const trimmed = input.trim();
+    const withoutProtocol = trimmed.replace(/^[a-zA-Z]+:\/\//, '');
+    return withoutProtocol.split(/[/?#]/)[0];
+};
+const buildPgSslConfig = (mode) => {
+    if (!mode || mode === 'disable') {
+        return undefined;
+    }
+    if (mode === 'verify-full') {
+        return { rejectUnauthorized: true };
+    }
+    // Modes like "require", "allow", "prefer" fall back to TLS without cert validation
+    return { rejectUnauthorized: false };
+};
+const ensureMysqlResources = async (params) => {
+    const host = normalizeHost(params.host);
+    console.log((0, kleur_1.gray)(`Verifying MySQL database "${params.database}"...`));
+    const serverConnection = await promise_1.default.createConnection({
+        host,
+        port: params.port,
+        user: params.username,
+        password: params.password
+    });
+    try {
+        await serverConnection.query(`CREATE DATABASE IF NOT EXISTS ${escapeMysqlIdentifier(params.database)}`);
+    }
+    finally {
+        await serverConnection.end();
+    }
+    const dbConnection = await promise_1.default.createConnection({
+        host,
+        port: params.port,
+        user: params.username,
+        password: params.password,
+        database: params.database
+    });
+    try {
+        const tableDefinition = getTableDefinition(params.target);
+        await dbConnection.query(`CREATE TABLE IF NOT EXISTS ${escapeMysqlIdentifier(params.table)} ${tableDefinition}`);
+    }
+    finally {
+        await dbConnection.end();
+    }
+    console.log((0, kleur_1.green)(`MySQL ready: ${params.database}.${params.table}`));
+};
+exports.ensureMysqlResources = ensureMysqlResources;
+const ensurePostgresResources = async (params) => {
+    const schemaName = params.schema || 'public';
+    const host = normalizeHost(params.host);
+    console.log((0, kleur_1.gray)(`Verifying PostgreSQL database "${params.database}"...`));
+    const adminClient = new pg_1.Client({
+        host,
+        port: params.port,
+        user: params.username,
+        password: params.password,
+        database: 'postgres',
+        ssl: buildPgSslConfig(params.sslMode)
+    });
+    await adminClient.connect();
+    try {
+        const dbExists = await adminClient.query('SELECT 1 FROM pg_database WHERE datname = $1', [params.database]);
+        if (!dbExists.rowCount) {
+            await adminClient.query(`CREATE DATABASE ${escapePgIdentifier(params.database)}`);
+        }
+    }
+    finally {
+        await adminClient.end();
+    }
+    const dbClient = new pg_1.Client({
+        host,
+        port: params.port,
+        user: params.username,
+        password: params.password,
+        database: params.database,
+        ssl: buildPgSslConfig(params.sslMode)
+    });
+    await dbClient.connect();
+    try {
+        await dbClient.query(`CREATE SCHEMA IF NOT EXISTS ${escapePgIdentifier(schemaName)}`);
+        const fullTable = `${escapePgIdentifier(schemaName)}.${escapePgIdentifier(params.table)}`;
+        const tableDefinition = getPgTableDefinition(params.target);
+        await dbClient.query(`CREATE TABLE IF NOT EXISTS ${fullTable} ${tableDefinition}`);
+    }
+    finally {
+        await dbClient.end();
+    }
+    console.log((0, kleur_1.green)(`PostgreSQL ready: ${params.database}.${schemaName}.${params.table}`));
+};
+exports.ensurePostgresResources = ensurePostgresResources;

package/dist/lib/integrations/output-integrations.js

@@ -0,0 +1,163 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.outputIntegrations = void 0;
+const db_setup_1 = require("./db-setup");
+const buildMysqlIntegration = (target) => {
+    const scope = target === 'job' ? 'job' : 'runner';
+    const prefix = target === 'job' ? 'GITGREEN_JOB' : 'GITGREEN_RUNNER';
+    const tableDefault = target === 'job' ? 'gitgreen_carbon_jobs' : 'gitgreen_runner_inventory';
+    return {
+        id: `mysql-${scope}`,
+        name: `MySQL (${scope === 'job' ? 'per-job emissions' : 'runner inventory'})`,
+        description: target === 'job'
+            ? 'Store each CI job calculation (emissions, runtime, runner tags) inside a MySQL table.'
+            : 'Store runner metadata (machine type, scope 3 estimates, tags) alongside usage metrics.',
+        target,
+        fields: [
+            {
+                id: 'host',
+                label: 'MySQL host',
+                envKey: `${prefix}_MYSQL_HOST`,
+                initial: '127.0.0.1',
+                required: true
+            },
+            {
+                id: 'port',
+                label: 'MySQL port',
+                envKey: `${prefix}_MYSQL_PORT`,
+                type: 'number',
+                initial: '3306',
+                required: true
+            },
+            {
+                id: 'username',
+                label: 'MySQL username',
+                envKey: `${prefix}_MYSQL_USERNAME`,
+                required: true
+            },
+            {
+                id: 'password',
+                label: 'MySQL password',
+                envKey: `${prefix}_MYSQL_PASSWORD`,
+                type: 'password',
+                required: true,
+                masked: true
+            },
+            {
+                id: 'database',
+                label: 'Database name',
+                envKey: `${prefix}_MYSQL_DATABASE`,
+                required: true
+            },
+            {
+                id: 'table',
+                label: 'Target table',
+                envKey: `${prefix}_MYSQL_TABLE`,
+                initial: tableDefault,
+                required: true,
+                helpText: 'Specify the table that will receive inserts from the GitGreen component.'
+            }
+        ],
+        ensure: async (values) => {
+            await (0, db_setup_1.ensureMysqlResources)({
+                host: values.host,
+                port: values.port ? Number(values.port) : 3306,
+                username: values.username,
+                password: values.password,
+                database: values.database,
+                table: values.table,
+                target
+            });
+        }
+    };
+};
+const buildPostgresIntegration = (target) => {
+    const scope = target === 'job' ? 'job' : 'runner';
+    const prefix = target === 'job' ? 'GITGREEN_JOB' : 'GITGREEN_RUNNER';
+    const tableDefault = target === 'job' ? 'gitgreen_carbon_jobs' : 'gitgreen_runner_inventory';
+    return {
+        id: `postgres-${scope}`,
+        name: `PostgreSQL (${scope === 'job' ? 'per-job emissions' : 'runner inventory'})`,
+        description: target === 'job'
+            ? 'Insert each CI job calculation into a PostgreSQL table for downstream analytics.'
+            : 'Persist runner metadata in PostgreSQL so you can track each machine in your fleet.',
+        target,
+        fields: [
+            {
+                id: 'host',
+                label: 'PostgreSQL host',
+                envKey: `${prefix}_POSTGRES_HOST`,
+                initial: '127.0.0.1',
+                required: true
+            },
+            {
+                id: 'port',
+                label: 'PostgreSQL port',
+                envKey: `${prefix}_POSTGRES_PORT`,
+                type: 'number',
+                initial: '5432',
+                required: true
+            },
+            {
+                id: 'username',
+                label: 'PostgreSQL username',
+                envKey: `${prefix}_POSTGRES_USERNAME`,
+                required: true
+            },
+            {
+                id: 'password',
+                label: 'PostgreSQL password',
+                envKey: `${prefix}_POSTGRES_PASSWORD`,
+                type: 'password',
+                required: true,
+                masked: true
+            },
+            {
+                id: 'database',
+                label: 'Database name',
+                envKey: `${prefix}_POSTGRES_DATABASE`,
+                required: true
+            },
+            {
+                id: 'schema',
+                label: 'Schema',
+                envKey: `${prefix}_POSTGRES_SCHEMA`,
+                initial: 'public',
+                helpText: 'Leave as "public" unless you organize tables under a different schema.'
+            },
+            {
+                id: 'table',
+                label: 'Target table',
+                envKey: `${prefix}_POSTGRES_TABLE`,
+                initial: tableDefault,
+                required: true
+            },
+            {
+                id: 'sslmode',
+                label: 'SSL mode',
+                envKey: `${prefix}_POSTGRES_SSLMODE`,
+                initial: 'require',
+                helpText: 'Example: require, verify-full, disable'
+            }
+        ],
+        ensure: async (values) => {
+            await (0, db_setup_1.ensurePostgresResources)({
+                host: values.host,
+                port: values.port ? Number(values.port) : 5432,
+                username: values.username,
+                password: values.password,
+                database: values.database,
+                schema: values.schema,
+                sslMode: values.sslmode,
+                table: values.table,
+                target
+            });
+        }
+    };
+};
+exports.outputIntegrations = [
+    buildMysqlIntegration('job'),
+    buildMysqlIntegration('runner'),
+    buildPostgresIntegration('job'),
+    buildPostgresIntegration('runner')
+];

package/dist/lib/integrations/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gitgreen",
-  "version": "1.0.3",
+  "version": "1.0.4",
   "description": "GitGreen CLI for carbon reporting in GitLab pipelines (GCP/AWS)",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -41,10 +41,13 @@
     "cubic-spline": "^3.0.3",
     "dotenv": "^16.5.0",
     "kleur": "^4.1.5",
+    "mysql2": "^3.11.3",
+    "pg": "^8.12.0",
     "prompts": "^2.4.2"
   },
   "devDependencies": {
     "@types/node": "^20.11.30",
+    "@types/pg": "^8.11.6",
     "@types/prompts": "^2.4.9",
     "ts-node": "^10.9.2",
     "typescript": "^5.4.0",