gitgreen 1.0.4 → 1.1.0

package/README.md

@@ -151,7 +151,10 @@ Built-in connectors today:
 - **MySQL** – populates `GITGREEN_JOB_MYSQL_*` / `GITGREEN_RUNNER_MYSQL_*` and inserts rows through a standard MySQL client.
 - **PostgreSQL** – captures host, port, credentials, schema, table, and SSL mode (`GITGREEN_JOB_POSTGRES_*` / `GITGREEN_RUNNER_POSTGRES_*`) for storage in Postgres.
 
-When you select either connector, the wizard captures host, port, username, password, database, and target table names and stores them in CI/CD variables. It immediately connects with those credentials to ensure the database, schema, and table exist (creating them with a generic `payload JSON/JSONB` + `ingested_at` structure when missing). The GitLab component then uses those variables to insert rows via the corresponding database driver.
+When you select either connector, the wizard captures host, port, username, password, database, and target table names and stores them in CI/CD variables. It immediately connects with those credentials to ensure the database, schema, and table exist (job sinks also create a `<table>_timeseries` table linked via foreign key). During CI, the GitGreen CLI automatically detects those env vars and:
+
+- runs `gitgreen migrate --scope job|runner` to apply any pending migrations (tracked per DB via `gitgreen_migrations`);
+- writes each carbon calculation (typed summary columns plus CPU/RAM timeseries rows) and optional runner inventory snapshot into the configured sink.
 
 ### Extending the interface
 
@@ -163,6 +166,18 @@ Additional connectors can be added without touching the wizard logic. Each desti
 
 To add another sink (for example PostgreSQL or a webhook), create a new entry in that file with the fields your integration needs. Re-run `gitgreen init` and the option will automatically appear in the integration step.
 
+### Database migrations
+
+Structured sinks rely on migrations tracked in `gitgreen_migrations`. Run them whenever you update GitGreen or change table names:
+
+```bash
+gitgreen migrate --scope job      # apply job sink migrations (summary + timeseries)
+gitgreen migrate --scope runner   # apply runner inventory migrations
+gitgreen migrate --scope all      # convenience wrapper (used by the GitLab component)
+```
+
+The GitLab component automatically runs `gitgreen migrate --scope job` and `--scope runner` before calculating emissions, so pipelines stay in sync even when you change versions.
+
 ## Adding a provider
 1. Extend `CloudProvider` and the provider guard in `src/index.ts` so the calculator accepts the new key.
 2. Add machine power data (`<provider>_machine_power_profiles.json`) and, if needed, CPU profiles to `data/`, then update `PowerProfileRepository.loadMachineData` to load it.

package/dist/cli.js

@@ -14,6 +14,9 @@ const index_1 = require("./index");
 const init_1 = require("./init");
 const cloudwatch_1 = require("./lib/aws/cloudwatch");
 const power_profile_repository_1 = require("./lib/carbon/power-profile-repository");
+const sink_writer_1 = require("./lib/integrations/sink-writer");
+const migration_runner_1 = require("./lib/integrations/migration-runner");
+const data_sink_1 = require("./lib/integrations/data-sink");
 const program = new commander_1.Command();
 const toIsoTimestamp = (input) => {
     if (input instanceof Date) {
@@ -209,6 +212,14 @@ const runCalculate = async (opts) => {
         ramSizeTimeseries = parseTimeseriesFile(opts.ramSizeTimeseries);
     }
     console.log((0, kleur_1.gray)(`Loaded ${cpuTimeseries.length} CPU points, ${ramUsedTimeseries.length} RAM points`));
+    const jobInput = {
+        provider,
+        machineType: opts.machine,
+        region: calculationRegion,
+        cpuTimeseries,
+        ramUsedTimeseries,
+        ramSizeTimeseries
+    };
     // Sort by timestamp for charts
     const cpuSorted = [...cpuTimeseries].sort((a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime());
     const ramSorted = [...ramUsedTimeseries].sort((a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime());
@@ -232,13 +243,13 @@ const runCalculate = async (opts) => {
         console.log(asciichart_1.default.plot(ramPercent, { height: 8, format: (x) => x.toFixed(1).padStart(6) }));
     }
     try {
-        const { result, budget, markdown } = await (0, index_1.calculate)({
-            provider,
-            machineType: opts.machine,
-            region: calculationRegion,
-            cpuTimeseries,
-            ramUsedTimeseries,
-            ramSizeTimeseries,
+        const { result, budget, markdown, jsonReport } = await (0, index_1.calculate)({
+            provider: jobInput.provider,
+            machineType: jobInput.machineType,
+            region: jobInput.region,
+            cpuTimeseries: jobInput.cpuTimeseries,
+            ramUsedTimeseries: jobInput.ramUsedTimeseries,
+            ramSizeTimeseries: jobInput.ramSizeTimeseries,
             carbonBudgetGrams: opts.budget,
             failOnBreach: opts.failOnBudget,
             runnerTags: opts.runnerTags ? String(opts.runnerTags).split(/[,\s]+/).filter(Boolean) : (0, index_1.readRunnerTagsFromEnv)(),
@@ -269,6 +280,8 @@ const runCalculate = async (opts) => {
         if (!opts.outMd && !opts.outJson) {
             console.log('\n' + markdown);
         }
+        await (0, sink_writer_1.persistJobPayload)({ jobInput, result, budget, jsonReport });
+        await (0, sink_writer_1.persistRunnerPayload)({ jobInput, result });
         if (budget.overBudget && opts.failOnBudget) {
             process.exitCode = 1;
         }
@@ -312,4 +325,41 @@ program
     .action(async (cmdOpts) => {
     await (0, init_1.runInit)({ assumeYes: Boolean(cmdOpts.yes) });
 });
+program
+    .command('migrate')
+    .description('Run output-integration database migrations')
+    .option('--scope <scope>', 'Scope to migrate (job|runner|all)', 'all')
+    .action(async (opts) => {
+    const rawScope = String(opts.scope || 'all').toLowerCase();
+    const scopes = rawScope === 'all' ? ['job', 'runner'] : (rawScope === 'job' || rawScope === 'runner') ? [rawScope] : [];
+    if (!scopes.length) {
+        console.error((0, kleur_1.red)('Invalid scope. Use job, runner, or all.'));
+        process.exitCode = 1;
+        return;
+    }
+    for (const scope of scopes) {
+        await runMigrationsForScope(scope);
+    }
+});
 program.parseAsync(process.argv);
+const buildMigrationInput = (config) => ({
+    driver: config.driver,
+    host: config.host,
+    port: config.port,
+    username: config.username,
+    password: config.password,
+    database: config.database,
+    schema: config.driver === 'postgres' ? config.schema : undefined,
+    sslMode: config.driver === 'postgres' ? config.sslMode : undefined,
+    table: config.table,
+    timeseriesTable: config.timeseriesTable,
+    scope: config.scope
+});
+const runMigrationsForScope = async (scope) => {
+    const configs = (0, data_sink_1.resolveAllSinkConfigs)(scope);
+    if (!configs.length)
+        return;
+    for (const cfg of configs) {
+        await (0, migration_runner_1.runMigrations)(buildMigrationInput(cfg));
+    }
+};

package/dist/lib/integrations/data-sink.js

@@ -0,0 +1,250 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.insertRunnerRow = exports.insertJobRow = exports.resolveAllSinkConfigs = exports.resolveSinkConfig = void 0;
+const promise_1 = __importDefault(require("mysql2/promise"));
+const pg_1 = require("pg");
+const kleur_1 = require("kleur");
+const normalizeHost = (input) => {
+    if (!input)
+        return input;
+    const trimmed = input.trim();
+    const withoutProtocol = trimmed.replace(/^[a-zA-Z]+:\/\//, '');
+    return withoutProtocol.split(/[/?#]/)[0];
+};
+const buildPgSslConfig = (mode) => {
+    if (!mode || mode === 'disable')
+        return undefined;
+    if (mode === 'verify-full')
+        return { rejectUnauthorized: true };
+    return { rejectUnauthorized: false };
+};
+const loadEnv = (key) => {
+    const value = process.env[key];
+    if (!value)
+        return undefined;
+    return value;
+};
+const buildMysqlConfig = (scope) => {
+    const prefix = scope === 'job' ? 'GITGREEN_JOB' : 'GITGREEN_RUNNER';
+    const host = loadEnv(`${prefix}_MYSQL_HOST`);
+    const username = loadEnv(`${prefix}_MYSQL_USERNAME`);
+    const password = loadEnv(`${prefix}_MYSQL_PASSWORD`);
+    const database = loadEnv(`${prefix}_MYSQL_DATABASE`);
+    const table = loadEnv(`${prefix}_MYSQL_TABLE`);
+    if (!host || !username || !password || !database || !table)
+        return undefined;
+    const port = Number(loadEnv(`${prefix}_MYSQL_PORT`) || 3306);
+    return {
+        driver: 'mysql',
+        host: normalizeHost(host),
+        port,
+        username,
+        password,
+        database,
+        table,
+        timeseriesTable: scope === 'job' ? `${table}_timeseries` : undefined,
+        scope
+    };
+};
+const buildPostgresConfig = (scope) => {
+    const prefix = scope === 'job' ? 'GITGREEN_JOB' : 'GITGREEN_RUNNER';
+    const host = loadEnv(`${prefix}_POSTGRES_HOST`);
+    const username = loadEnv(`${prefix}_POSTGRES_USERNAME`);
+    const password = loadEnv(`${prefix}_POSTGRES_PASSWORD`);
+    const database = loadEnv(`${prefix}_POSTGRES_DATABASE`);
+    const table = loadEnv(`${prefix}_POSTGRES_TABLE`);
+    if (!host || !username || !password || !database || !table)
+        return undefined;
+    const port = Number(loadEnv(`${prefix}_POSTGRES_PORT`) || 5432);
+    const schema = loadEnv(`${prefix}_POSTGRES_SCHEMA`) || 'public';
+    const sslMode = loadEnv(`${prefix}_POSTGRES_SSLMODE`);
+    return {
+        driver: 'postgres',
+        host: normalizeHost(host),
+        port,
+        username,
+        password,
+        database,
+        table,
+        timeseriesTable: scope === 'job' ? `${table}_timeseries` : undefined,
+        schema,
+        sslMode,
+        scope
+    };
+};
+const resolveSinkConfig = (scope, driver) => {
+    if (driver === 'postgres')
+        return buildPostgresConfig(scope);
+    if (driver === 'mysql')
+        return buildMysqlConfig(scope);
+    return buildPostgresConfig(scope) || buildMysqlConfig(scope);
+};
+exports.resolveSinkConfig = resolveSinkConfig;
+const resolveAllSinkConfigs = (scope) => {
+    const configs = [];
+    const pg = buildPostgresConfig(scope);
+    if (pg)
+        configs.push(pg);
+    const mysqlCfg = buildMysqlConfig(scope);
+    if (mysqlCfg)
+        configs.push(mysqlCfg);
+    return configs;
+};
+exports.resolveAllSinkConfigs = resolveAllSinkConfigs;
+const insertWithRetry = async (action, attempts = 3, delayMs = 2000) => {
+    let lastError;
+    for (let attempt = 1; attempt <= attempts; attempt++) {
+        try {
+            return await action();
+        }
+        catch (err) {
+            lastError = err;
+            if (attempt < attempts) {
+                await new Promise(resolve => setTimeout(resolve, delayMs));
+            }
+        }
+    }
+    throw lastError;
+};
+const jobColumns = [
+    ['provider', (row) => row.provider ?? null],
+    ['region', (row) => row.region ?? null],
+    ['machine_type', (row) => row.machineType ?? null],
+    ['cpu_points', (row) => row.cpuPoints ?? null],
+    ['ram_points', (row) => row.ramPoints ?? null],
+    ['runtime_seconds', (row) => row.runtimeSeconds ?? null],
+    ['total_emissions', (row) => row.totalEmissions ?? null],
+    ['cpu_emissions', (row) => row.cpuEmissions ?? null],
+    ['ram_emissions', (row) => row.ramEmissions ?? null],
+    ['scope3_emissions', (row) => row.scope3Emissions ?? null],
+    ['carbon_intensity', (row) => row.carbonIntensity ?? null],
+    ['pue', (row) => row.pue ?? null],
+    ['carbon_budget', (row) => row.carbonBudget ?? null],
+    ['over_budget', (row) => typeof row.overBudget === 'boolean' ? row.overBudget : null],
+    ['gitlab_project_id', (row) => row.gitlabProjectId ?? null],
+    ['gitlab_pipeline_id', (row) => row.gitlabPipelineId ?? null],
+    ['gitlab_job_id', (row) => row.gitlabJobId ?? null],
+    ['gitlab_job_name', (row) => row.gitlabJobName ?? null],
+    ['runner_id', (row) => row.runnerId ?? null],
+    ['runner_description', (row) => row.runnerDescription ?? null],
+    ['runner_tags', (row) => row.runnerTags ?? null],
+    ['runner_version', (row) => row.runnerVersion ?? null],
+    ['runner_revision', (row) => row.runnerRevision ?? null],
+    ['payload', (row) => JSON.stringify(row.payload)]
+];
+const runnerColumns = [
+    ['runner_id', (row) => row.runnerId ?? null],
+    ['runner_description', (row) => row.runnerDescription ?? null],
+    ['runner_version', (row) => row.runnerVersion ?? null],
+    ['runner_revision', (row) => row.runnerRevision ?? null],
+    ['runner_platform', (row) => row.runnerPlatform ?? null],
+    ['runner_architecture', (row) => row.runnerArchitecture ?? null],
+    ['runner_executor', (row) => row.runnerExecutor ?? null],
+    ['runner_tags', (row) => row.runnerTags ?? null],
+    ['machine_type', (row) => row.machineType ?? null],
+    ['provider', (row) => row.provider ?? null],
+    ['region', (row) => row.region ?? null],
+    ['gcp_project_id', (row) => row.gcpProjectId ?? null],
+    ['gcp_instance_id', (row) => row.gcpInstanceId ?? null],
+    ['gcp_zone', (row) => row.gcpZone ?? null],
+    ['aws_region', (row) => row.awsRegion ?? null],
+    ['aws_instance_id', (row) => row.awsInstanceId ?? null],
+    ['last_job_machine_type', (row) => row.lastJobMachineType ?? null],
+    ['last_job_region', (row) => row.lastJobRegion ?? null],
+    ['last_job_provider', (row) => row.lastJobProvider ?? null],
+    ['last_job_runtime_seconds', (row) => row.lastJobRuntimeSeconds ?? null],
+    ['last_job_total_emissions', (row) => row.lastJobTotalEmissions ?? null],
+    ['last_job_recorded_at', (row) => row.lastJobRecordedAt ?? null],
+    ['payload', (row) => JSON.stringify(row.payload)]
+];
+const runMysqlInsert = async (config, columns, row, timeseries) => {
+    const connection = await promise_1.default.createConnection({
+        host: config.host,
+        port: config.port,
+        user: config.username,
+        password: config.password,
+        database: config.database
+    });
+    try {
+        await connection.beginTransaction();
+        const colNames = columns.map(([name]) => `\`${name}\``).join(', ');
+        const placeholders = columns.map(() => '?').join(', ');
+        const values = columns.map(([, getter]) => getter(row));
+        const [result] = await connection.query(`INSERT INTO \`${config.table}\` (${colNames}) VALUES (${placeholders})`, values);
+        const insertedId = result.insertId;
+        if (timeseries && timeseries.length > 0) {
+            const tsTable = config.timeseriesTable || `${config.table}_timeseries`;
+            const tsValues = timeseries.map(point => [insertedId, point.metric, point.timestamp, point.value]);
+            await connection.query(`INSERT INTO \`${tsTable}\` (job_id, metric, ts, value) VALUES ?`, [tsValues]);
+        }
+        await connection.commit();
+    }
+    finally {
+        await connection.end();
+    }
+};
+const runPostgresInsert = async (config, columns, row, timeseries) => {
+    const schema = config.schema || 'public';
+    const client = new pg_1.Client({
+        host: config.host,
+        port: config.port,
+        user: config.username,
+        password: config.password,
+        database: config.database,
+        ssl: buildPgSslConfig(config.sslMode)
+    });
+    await client.connect();
+    try {
+        await client.query('BEGIN');
+        const colNames = columns.map(([name]) => `"${name}"`).join(', ');
+        const placeholders = columns.map((_, idx) => `$${idx + 1}`).join(', ');
+        const values = columns.map(([, getter]) => getter(row));
+        const insertResult = await client.query(`INSERT INTO "${schema}"."${config.table}" (${colNames}) VALUES (${placeholders}) RETURNING id`, values);
+        const jobId = insertResult.rows[0]?.id;
+        if (jobId && timeseries && timeseries.length > 0) {
+            const tsTableName = config.timeseriesTable || `${config.table}_timeseries`;
+            const tsTable = `"${schema}"."${tsTableName}"`;
+            const valueFragments = [];
+            const params = [];
+            timeseries.forEach((point, index) => {
+                const base = index * 4;
+                valueFragments.push(`($${base + 1}, $${base + 2}, $${base + 3}, $${base + 4})`);
+                params.push(jobId, point.metric, point.timestamp, point.value);
+            });
+            await client.query(`INSERT INTO ${tsTable} (job_id, metric, ts, value) VALUES ${valueFragments.join(', ')}`, params);
+        }
+        await client.query('COMMIT');
+    }
+    catch (err) {
+        await client.query('ROLLBACK').catch(() => { });
+        throw err;
+    }
+    finally {
+        await client.end();
+    }
+};
+const writeStructuredRow = async (config, columns, row, timeseries) => {
+    try {
+        if (config.driver === 'mysql') {
+            await insertWithRetry(() => runMysqlInsert(config, columns, row, timeseries));
+        }
+        else {
+            await insertWithRetry(() => runPostgresInsert(config, columns, row, timeseries));
+        }
+        console.log((0, kleur_1.green)(`Saved ${config.scope} payload to ${config.driver} sink (${config.database}.${config.table})`));
+    }
+    catch (err) {
+        console.log((0, kleur_1.red)(`Failed to write ${config.scope} payload: ${err?.message || err}`));
+    }
+};
+const insertJobRow = (config, row, timeseries) => {
+    return writeStructuredRow(config, jobColumns, row, timeseries);
+};
+exports.insertJobRow = insertJobRow;
+const insertRunnerRow = (config, row) => {
+    return writeStructuredRow(config, runnerColumns, row);
+};
+exports.insertRunnerRow = insertRunnerRow;

package/dist/lib/integrations/migration-runner.js

@@ -0,0 +1,214 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runMigrations = void 0;
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const promise_1 = __importDefault(require("mysql2/promise"));
+const pg_1 = require("pg");
+const kleur_1 = require("kleur");
+const normalizeHost = (input) => {
+    if (!input)
+        return input;
+    const trimmed = input.trim();
+    const withoutProtocol = trimmed.replace(/^[a-zA-Z]+:\/\//, '');
+    return withoutProtocol.split(/[/?#]/)[0];
+};
+const buildPgSslConfig = (mode) => {
+    if (!mode || mode === 'disable')
+        return undefined;
+    if (mode === 'verify-full')
+        return { rejectUnauthorized: true };
+    return { rejectUnauthorized: false };
+};
+const migrationsDir = path_1.default.join(__dirname, '..', '..', '..', 'migrations');
+const loadMigrations = (scope, driver) => {
+    const prefix = `${scope}_${driver}_`;
+    const files = fs_1.default.readdirSync(migrationsDir)
+        .filter(file => file.endsWith('.sql') && file.startsWith(prefix))
+        .sort();
+    return files.map(file => {
+        const id = Number(file.split('_')[0]);
+        return {
+            id,
+            name: file,
+            sql: fs_1.default.readFileSync(path_1.default.join(migrationsDir, file), 'utf8')
+        };
+    });
+};
+const ensureMetadataTable = async (config) => {
+    if (config.driver === 'mysql') {
+        const connection = await promise_1.default.createConnection({
+            host: config.host,
+            port: config.port,
+            user: config.username,
+            password: config.password,
+            database: config.database
+        });
+        try {
+            await connection.query(`CREATE TABLE IF NOT EXISTS gitgreen_migrations (
+        scope VARCHAR(32) NOT NULL,
+        id INT NOT NULL,
+        name VARCHAR(255) NOT NULL,
+        applied_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+        PRIMARY KEY (scope, id)
+      ) ENGINE=InnoDB`);
+        }
+        finally {
+            await connection.end();
+        }
+    }
+    else {
+        const client = new pg_1.Client({
+            host: config.host,
+            port: config.port,
+            user: config.username,
+            password: config.password,
+            database: config.database,
+            ssl: buildPgSslConfig(config.sslMode)
+        });
+        await client.connect();
+        try {
+            await client.query('CREATE TABLE IF NOT EXISTS gitgreen_migrations (scope TEXT NOT NULL, id INT NOT NULL, name TEXT NOT NULL, applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), PRIMARY KEY (scope, id))');
+        }
+        finally {
+            await client.end();
+        }
+    }
+};
+const fetchAppliedMigrations = async (config) => {
+    const applied = new Set();
+    if (config.driver === 'mysql') {
+        const connection = await promise_1.default.createConnection({
+            host: config.host,
+            port: config.port,
+            user: config.username,
+            password: config.password,
+            database: config.database
+        });
+        try {
+            const [rows] = await connection.query('SELECT id FROM gitgreen_migrations WHERE scope = ?', [config.scope]);
+            for (const row of rows) {
+                applied.add(row.id);
+            }
+        }
+        finally {
+            await connection.end();
+        }
+    }
+    else {
+        const client = new pg_1.Client({
+            host: config.host,
+            port: config.port,
+            user: config.username,
+            password: config.password,
+            database: config.database,
+            ssl: buildPgSslConfig(config.sslMode)
+        });
+        await client.connect();
+        try {
+            const result = await client.query('SELECT id FROM gitgreen_migrations WHERE scope = $1', [config.scope]);
+            result.rows.forEach(row => applied.add(Number(row.id)));
+        }
+        finally {
+            await client.end();
+        }
+    }
+    return applied;
+};
+const renderSql = (migration, config) => {
+    const tableName = config.table;
+    const timeseriesName = config.timeseriesTable || `${config.table}_timeseries`;
+    const qualifiedTable = config.driver === 'postgres' && config.schema
+        ? `"${config.schema}"."${tableName}"`
+        : config.driver === 'postgres'
+            ? `"${tableName}"`
+            : `\`${tableName}\``;
+    const qualifiedTimeseries = config.driver === 'postgres' && config.schema
+        ? `"${config.schema}"."${timeseriesName}"`
+        : config.driver === 'postgres'
+            ? `"${timeseriesName}"`
+            : `\`${timeseriesName}\``;
+    return migration.sql
+        .replace(/\{\{TABLE\}\}/g, qualifiedTable)
+        .replace(/\{\{TIMESERIES_TABLE\}\}/g, qualifiedTimeseries);
+};
+const runMysqlMigration = async (config, migration) => {
+    const connection = await promise_1.default.createConnection({
+        host: config.host,
+        port: config.port,
+        user: config.username,
+        password: config.password,
+        database: config.database,
+        multipleStatements: true
+    });
+    try {
+        await connection.beginTransaction();
+        const sql = renderSql(migration, config);
+        await connection.query(sql);
+        await connection.query('INSERT INTO gitgreen_migrations (scope, id, name) VALUES (?, ?, ?)', [config.scope, migration.id, migration.name]);
+        await connection.commit();
+    }
+    catch (err) {
+        await connection.rollback().catch(() => { });
+        throw err;
+    }
+    finally {
+        await connection.end();
+    }
+};
+const runPostgresMigration = async (config, migration) => {
+    const client = new pg_1.Client({
+        host: config.host,
+        port: config.port,
+        user: config.username,
+        password: config.password,
+        database: config.database,
+        ssl: buildPgSslConfig(config.sslMode)
+    });
+    await client.connect();
+    try {
+        await client.query('BEGIN');
+        const sql = renderSql(migration, config);
+        await client.query(sql);
+        await client.query('INSERT INTO gitgreen_migrations (scope, id, name) VALUES ($1, $2, $3)', [config.scope, migration.id, migration.name]);
+        await client.query('COMMIT');
+    }
+    catch (err) {
+        await client.query('ROLLBACK').catch(() => { });
+        throw err;
+    }
+    finally {
+        await client.end();
+    }
+};
+const runMigrations = async (config) => {
+    const normalizedHost = normalizeHost(config.host);
+    const driver = config.driver;
+    const finalConfig = { ...config, host: normalizedHost };
+    await ensureMetadataTable(finalConfig);
+    const applied = await fetchAppliedMigrations(finalConfig);
+    const migrations = loadMigrations(finalConfig.scope, driver).filter(mig => !applied.has(mig.id));
+    if (!migrations.length) {
+        console.log((0, kleur_1.green)('No database migrations to run.'));
+        return;
+    }
+    for (const migration of migrations) {
+        try {
+            if (driver === 'mysql') {
+                await runMysqlMigration(finalConfig, migration);
+            }
+            else {
+                await runPostgresMigration(finalConfig, migration);
+            }
+            console.log((0, kleur_1.green)(`Applied migration ${migration.name}`));
+        }
+        catch (err) {
+            console.log((0, kleur_1.red)(`Failed to apply migration ${migration.name}: ${err?.message || err}`));
+            throw err;
+        }
+    }
+};
+exports.runMigrations = runMigrations;

package/dist/lib/integrations/sink-writer.js

@@ -0,0 +1,181 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.persistRunnerPayload = exports.persistJobPayload = void 0;
+const data_sink_1 = require("./data-sink");
+const migration_runner_1 = require("./migration-runner");
+const readRunnerTags = () => {
+    const tagsEnv = process.env.CI_RUNNER_TAGS || process.env.RUNNER_TAGS || '';
+    return tagsEnv.split(/[,\s]+/).filter(Boolean);
+};
+const migrationsRan = new Set();
+const keyForConfig = (config) => {
+    const schemaPart = config.driver === 'postgres' ? (config.schema || 'public') : '';
+    return [
+        config.driver,
+        config.host,
+        config.port,
+        config.database,
+        config.table,
+        schemaPart
+    ].join(':');
+};
+const ensureMigrationsForConfig = async (config) => {
+    const key = keyForConfig(config);
+    if (migrationsRan.has(key))
+        return;
+    await (0, migration_runner_1.runMigrations)({
+        driver: config.driver,
+        host: config.host,
+        port: config.port,
+        username: config.username,
+        password: config.password,
+        database: config.database,
+        schema: config.driver === 'postgres' ? config.schema : undefined,
+        sslMode: config.driver === 'postgres' ? config.sslMode : undefined,
+        table: config.table,
+        timeseriesTable: config.timeseriesTable,
+        scope: config.scope
+    });
+    migrationsRan.add(key);
+};
+const persistJobPayload = async (options) => {
+    const config = (0, data_sink_1.resolveSinkConfig)('job');
+    if (!config)
+        return;
+    await ensureMigrationsForConfig(config);
+    const payload = {
+        ingestedAt: new Date().toISOString(),
+        gitlab: {
+            projectId: process.env.CI_PROJECT_ID,
+            pipelineId: process.env.CI_PIPELINE_ID,
+            jobId: process.env.CI_JOB_ID,
+            jobName: process.env.CI_JOB_NAME,
+            runnerId: process.env.CI_RUNNER_ID,
+            runnerDescription: process.env.CI_RUNNER_DESCRIPTION,
+            runnerTags: readRunnerTags(),
+            runnerVersion: process.env.CI_RUNNER_VERSION,
+            runnerRevision: process.env.CI_RUNNER_REVISION
+        },
+        job: {
+            provider: options.jobInput.provider,
+            region: options.jobInput.region,
+            machineType: options.jobInput.machineType,
+            cpuDataPoints: options.jobInput.cpuTimeseries.length,
+            ramDataPoints: options.jobInput.ramUsedTimeseries.length
+        },
+        emissions: options.result,
+        budget: options.budget,
+        report: options.jsonReport
+    };
+    const timeseriesRows = [
+        ...toTimeseries('cpu', options.jobInput.cpuTimeseries),
+        ...toTimeseries('ram_used', options.jobInput.ramUsedTimeseries),
+        ...toTimeseries('ram_size', options.jobInput.ramSizeTimeseries)
+    ];
+    await (0, data_sink_1.insertJobRow)(config, {
+        provider: options.jobInput.provider,
+        region: options.jobInput.region,
+        machineType: options.jobInput.machineType,
+        cpuPoints: options.jobInput.cpuTimeseries.length,
+        ramPoints: options.jobInput.ramUsedTimeseries.length,
+        runtimeSeconds: Math.round(options.result.runtimeHours * 3600),
+        totalEmissions: options.result.totalEmissions,
+        cpuEmissions: options.result.cpuEmissions,
+        ramEmissions: options.result.ramEmissions,
+        scope3Emissions: options.result.scope3Emissions,
+        carbonIntensity: options.result.carbonIntensity,
+        pue: options.result.pue,
+        carbonBudget: options.budget.limitGrams,
+        overBudget: options.budget.overBudget,
+        gitlabProjectId: payload.gitlab.projectId,
+        gitlabPipelineId: payload.gitlab.pipelineId,
+        gitlabJobId: payload.gitlab.jobId,
+        gitlabJobName: payload.gitlab.jobName,
+        runnerId: payload.gitlab.runnerId,
+        runnerDescription: payload.gitlab.runnerDescription,
+        runnerTags: (payload.gitlab.runnerTags || []).join(','),
+        runnerVersion: payload.gitlab.runnerVersion,
+        runnerRevision: payload.gitlab.runnerRevision,
+        payload
+    }, timeseriesRows);
+};
+exports.persistJobPayload = persistJobPayload;
+const persistRunnerPayload = async (options) => {
+    const config = (0, data_sink_1.resolveSinkConfig)('runner');
+    if (!config)
+        return;
+    await ensureMigrationsForConfig(config);
+    const env = process.env;
+    const machineType = env.MACHINE_TYPE || options.jobInput.machineType;
+    const payload = {
+        ingestedAt: new Date().toISOString(),
+        runner: {
+            id: env.CI_RUNNER_ID,
+            description: env.CI_RUNNER_DESCRIPTION,
+            version: env.CI_RUNNER_VERSION,
+            revision: env.CI_RUNNER_REVISION,
+            platform: env.CI_RUNNER_PLATFORM,
+            architecture: env.CI_RUNNER_ARCH,
+            executor: env.CI_RUNNER_EXECUTOR,
+            tags: readRunnerTags(),
+            machineType,
+            provider: options.jobInput.provider,
+            region: options.jobInput.region,
+            gcp: {
+                projectId: env.GCP_PROJECT_ID,
+                instanceId: env.GCP_INSTANCE_ID,
+                zone: env.GCP_ZONE
+            },
+            aws: {
+                region: env.AWS_REGION,
+                instanceId: env.AWS_INSTANCE_ID
+            }
+        },
+        gitlab: {
+            projectId: env.CI_PROJECT_ID,
+            pipelineId: env.CI_PIPELINE_ID,
+            jobId: env.CI_JOB_ID
+        },
+        lastJob: {
+            machineType: options.jobInput.machineType,
+            provider: options.jobInput.provider,
+            region: options.jobInput.region,
+            runtimeSeconds: Math.round(options.result.runtimeHours * 3600),
+            totalEmissions: options.result.totalEmissions,
+            recordedAt: new Date().toISOString()
+        }
+    };
+    await (0, data_sink_1.insertRunnerRow)(config, {
+        runnerId: payload.runner.id,
+        runnerDescription: payload.runner.description,
+        runnerVersion: payload.runner.version,
+        runnerRevision: payload.runner.revision,
+        runnerPlatform: payload.runner.platform,
+        runnerArchitecture: payload.runner.architecture,
+        runnerExecutor: payload.runner.executor,
+        runnerTags: (payload.runner.tags || []).join(','),
+        machineType: payload.runner.machineType,
+        provider: payload.runner.provider,
+        region: payload.runner.region,
+        gcpProjectId: payload.runner.gcp.projectId,
+        gcpInstanceId: payload.runner.gcp.instanceId,
+        gcpZone: payload.runner.gcp.zone,
+        awsRegion: payload.runner.aws.region,
+        awsInstanceId: payload.runner.aws.instanceId,
+        lastJobMachineType: payload.lastJob.machineType,
+        lastJobRegion: payload.lastJob.region,
+        lastJobProvider: payload.lastJob.provider,
+        lastJobRuntimeSeconds: payload.lastJob.runtimeSeconds,
+        lastJobTotalEmissions: payload.lastJob.totalEmissions,
+        lastJobRecordedAt: payload.lastJob.recordedAt,
+        payload
+    });
+};
+exports.persistRunnerPayload = persistRunnerPayload;
+const toTimeseries = (metric, points) => {
+    return points.map(point => ({
+        metric,
+        timestamp: new Date(point.timestamp).toISOString(),
+        value: typeof point.value === 'number' ? point.value : Number(point.value)
+    }));
+};

package/migrations/job_mysql_001_init.sql

@@ -0,0 +1,40 @@
+CREATE TABLE IF NOT EXISTS {{TABLE}} (
+  id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
+  ingested_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  provider VARCHAR(32),
+  region VARCHAR(128),
+  machine_type VARCHAR(128),
+  cpu_points INT,
+  ram_points INT,
+  runtime_seconds INT,
+  total_emissions DOUBLE,
+  cpu_emissions DOUBLE,
+  ram_emissions DOUBLE,
+  scope3_emissions DOUBLE,
+  carbon_intensity DOUBLE,
+  pue DOUBLE,
+  carbon_budget DOUBLE,
+  over_budget BOOLEAN,
+  gitlab_project_id BIGINT,
+  gitlab_pipeline_id BIGINT,
+  gitlab_job_id BIGINT,
+  gitlab_job_name VARCHAR(255),
+  runner_id VARCHAR(255),
+  runner_description VARCHAR(255),
+  runner_tags TEXT,
+  runner_version VARCHAR(128),
+  runner_revision VARCHAR(128),
+  payload JSON NOT NULL,
+  PRIMARY KEY (id)
+) ENGINE=InnoDB;
+
+CREATE TABLE IF NOT EXISTS {{TIMESERIES_TABLE}} (
+  id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
+  job_id BIGINT UNSIGNED NOT NULL,
+  metric VARCHAR(64) NOT NULL,
+  ts DATETIME NOT NULL,
+  value DOUBLE NOT NULL,
+  PRIMARY KEY (id),
+  KEY job_id_idx (job_id),
+  CONSTRAINT {{TIMESERIES_TABLE}}_fk FOREIGN KEY (job_id) REFERENCES {{TABLE}}(id) ON DELETE CASCADE
+) ENGINE=InnoDB;

package/migrations/job_postgres_001_init.sql

@@ -0,0 +1,36 @@
+CREATE TABLE IF NOT EXISTS {{TABLE}} (
+  id BIGSERIAL PRIMARY KEY,
+  ingested_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  provider TEXT,
+  region TEXT,
+  machine_type TEXT,
+  cpu_points INT,
+  ram_points INT,
+  runtime_seconds INT,
+  total_emissions DOUBLE PRECISION,
+  cpu_emissions DOUBLE PRECISION,
+  ram_emissions DOUBLE PRECISION,
+  scope3_emissions DOUBLE PRECISION,
+  carbon_intensity DOUBLE PRECISION,
+  pue DOUBLE PRECISION,
+  carbon_budget DOUBLE PRECISION,
+  over_budget BOOLEAN,
+  gitlab_project_id BIGINT,
+  gitlab_pipeline_id BIGINT,
+  gitlab_job_id BIGINT,
+  gitlab_job_name TEXT,
+  runner_id TEXT,
+  runner_description TEXT,
+  runner_tags TEXT,
+  runner_version TEXT,
+  runner_revision TEXT,
+  payload JSONB NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS {{TIMESERIES_TABLE}} (
+  id BIGSERIAL PRIMARY KEY,
+  job_id BIGINT NOT NULL REFERENCES {{TABLE}}(id) ON DELETE CASCADE,
+  metric TEXT NOT NULL,
+  ts TIMESTAMPTZ NOT NULL,
+  value DOUBLE PRECISION NOT NULL
+);

package/migrations/runner_mysql_001_init.sql

@@ -0,0 +1,28 @@
+CREATE TABLE IF NOT EXISTS {{TABLE}} (
+  id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
+  ingested_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  runner_id VARCHAR(255),
+  runner_description VARCHAR(255),
+  runner_version VARCHAR(128),
+  runner_revision VARCHAR(128),
+  runner_platform VARCHAR(255),
+  runner_architecture VARCHAR(128),
+  runner_executor VARCHAR(128),
+  runner_tags TEXT,
+  machine_type VARCHAR(128),
+  provider VARCHAR(32),
+  region VARCHAR(128),
+  gcp_project_id VARCHAR(128),
+  gcp_instance_id VARCHAR(128),
+  gcp_zone VARCHAR(64),
+  aws_region VARCHAR(64),
+  aws_instance_id VARCHAR(128),
+  last_job_machine_type VARCHAR(128),
+  last_job_region VARCHAR(128),
+  last_job_provider VARCHAR(32),
+  last_job_runtime_seconds INT,
+  last_job_total_emissions DOUBLE,
+  last_job_recorded_at TIMESTAMP NULL,
+  payload JSON NOT NULL,
+  PRIMARY KEY (id)
+) ENGINE=InnoDB;

package/migrations/runner_postgres_001_init.sql

@@ -0,0 +1,27 @@
+CREATE TABLE IF NOT EXISTS {{TABLE}} (
+  id BIGSERIAL PRIMARY KEY,
+  ingested_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  runner_id TEXT,
+  runner_description TEXT,
+  runner_version TEXT,
+  runner_revision TEXT,
+  runner_platform TEXT,
+  runner_architecture TEXT,
+  runner_executor TEXT,
+  runner_tags TEXT,
+  machine_type TEXT,
+  provider TEXT,
+  region TEXT,
+  gcp_project_id TEXT,
+  gcp_instance_id TEXT,
+  gcp_zone TEXT,
+  aws_region TEXT,
+  aws_instance_id TEXT,
+  last_job_machine_type TEXT,
+  last_job_region TEXT,
+  last_job_provider TEXT,
+  last_job_runtime_seconds INT,
+  last_job_total_emissions DOUBLE PRECISION,
+  last_job_recorded_at TIMESTAMPTZ,
+  payload JSONB NOT NULL
+);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gitgreen",
-  "version": "1.0.4",
+  "version": "1.1.0",
   "description": "GitGreen CLI for carbon reporting in GitLab pipelines (GCP/AWS)",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -30,6 +30,7 @@
   "files": [
     "dist",
     "data",
+    "migrations",
     "README.md"
   ],
   "dependencies": {