duoops 0.0.0 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -0
- package/bin/dev.js +0 -0
- package/dist/commands/init.d.ts +3 -0
- package/dist/commands/init.js +432 -16
- package/dist/commands/runner/logs.d.ts +9 -0
- package/dist/commands/runner/logs.js +36 -0
- package/dist/commands/runner/reinstall-duoops.d.ts +8 -0
- package/dist/commands/runner/reinstall-duoops.js +31 -0
- package/dist/lib/config.d.ts +8 -0
- package/dist/portal/assets/MetricsDashboard-DIsoz4Sl.js +71 -0
- package/dist/portal/assets/index-B6bzT1Vv.js +14 -0
- package/dist/portal/assets/index-CKvD-wbH.js +1 -0
- package/dist/portal/assets/index-D8LC1KwM.js +16 -0
- package/dist/portal/assets/index-DkVG3jel.js +70 -0
- package/dist/portal/index.html +1 -1
- package/oclif.manifest.json +161 -92
- package/package.json +12 -7
- package/dist/portal/assets/index-MU6EBerh.js +0 -188
package/README.md
CHANGED
|
@@ -146,6 +146,35 @@ duoops portal
|
|
|
146
146
|
|
|
147
147
|
Open your browser to `http://localhost:3000`.
|
|
148
148
|
|
|
149
|
+
### 6. Runner Utilities
|
|
150
|
+
|
|
151
|
+
Once you've provisioned a runner with `duoops init`, you can inspect and manage it directly from the CLI:
|
|
152
|
+
|
|
153
|
+
```bash
|
|
154
|
+
# Tail the runner's systemd logs (defaults to gitlab-runner service)
|
|
155
|
+
duoops runner:logs --lines 300
|
|
156
|
+
|
|
157
|
+
# Reinstall the DuoOps CLI on the VM (uses the current CLI version by default)
|
|
158
|
+
duoops runner:reinstall-duoops --version 0.1.0
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
### 7. Use the GitLab CI Component
|
|
162
|
+
|
|
163
|
+
Instead of copying `.duoops/measure-component.yml` into every project, you can reference the published component directly:
|
|
164
|
+
|
|
165
|
+
```yaml
|
|
166
|
+
include:
|
|
167
|
+
- component: gitlab.com/youneslaaroussi/duoops/templates/duoops-measure-component@v0.1.0
|
|
168
|
+
inputs:
|
|
169
|
+
gcp_project_id: "my-project"
|
|
170
|
+
gcp_instance_id: "1234567890123456789"
|
|
171
|
+
gcp_zone: "us-central1-a"
|
|
172
|
+
machine_type: "e2-standard-4"
|
|
173
|
+
tags: ["gcp"]
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
Make sure your runner already has DuoOps installed (the provisioning flow handles this) and that `GCP_SA_KEY_BASE64` plus any optional BigQuery variables are set in the project’s CI/CD settings. Pin to a version tag (`@v0.1.0`) and update when you’re ready to adopt new behavior.
|
|
177
|
+
|
|
149
178
|
## 🛠️ Development
|
|
150
179
|
|
|
151
180
|
### Project Structure
|
|
@@ -176,6 +205,13 @@ This compiles the TypeScript CLI and builds the React frontend, copying assets t
|
|
|
176
205
|
3. Inspect the publish payload locally with `pnpm pack` (this runs the `prepack` script, which now builds the CLI, portal, and Oclif manifest automatically). Untar the generated `.tgz` if you want to double-check the contents.
|
|
177
206
|
4. When you're satisfied, publish the package: `pnpm publish --access public`.
|
|
178
207
|
|
|
208
|
+
## 📦 Publishing the CI Component
|
|
209
|
+
|
|
210
|
+
1. Update `templates/duoops-measure-component.yml` and commit the changes.
|
|
211
|
+
2. Let the `.gitlab-ci.yml` validation job pass (runs automatically on MRs, default branch, and tags).
|
|
212
|
+
3. Tag the repository (e.g., `git tag v0.1.0 && git push origin v0.1.0`). Consumers reference the component with the tag via the snippet above.
|
|
213
|
+
4. Update release notes/README with the new component version so teams know what changed.
|
|
214
|
+
|
|
179
215
|
## 📄 License
|
|
180
216
|
|
|
181
217
|
MIT
|
package/bin/dev.js
CHANGED
|
File without changes
|
package/dist/commands/init.d.ts
CHANGED
package/dist/commands/init.js
CHANGED
|
@@ -1,14 +1,201 @@
|
|
|
1
1
|
import { BigQuery } from '@google-cloud/bigquery';
|
|
2
|
-
import { confirm, input, password } from '@inquirer/prompts';
|
|
2
|
+
import { confirm, input, password, select } from '@inquirer/prompts';
|
|
3
3
|
import { Command } from '@oclif/core';
|
|
4
|
+
import axios from 'axios';
|
|
5
|
+
import { bold, gray, green } from 'kleur/colors';
|
|
6
|
+
import { execSync } from 'node:child_process';
|
|
7
|
+
import fs from 'node:fs';
|
|
8
|
+
import os from 'node:os';
|
|
9
|
+
import path from 'node:path';
|
|
4
10
|
import { configManager } from '../lib/config.js';
|
|
11
|
+
const hasBinary = (command) => {
|
|
12
|
+
try {
|
|
13
|
+
execSync(`${command} --version`, { stdio: 'ignore' });
|
|
14
|
+
return true;
|
|
15
|
+
}
|
|
16
|
+
catch {
|
|
17
|
+
return false;
|
|
18
|
+
}
|
|
19
|
+
};
|
|
20
|
+
const detectGitRemotePath = () => {
|
|
21
|
+
try {
|
|
22
|
+
const remote = execSync('git remote get-url origin', { encoding: 'utf8' }).trim();
|
|
23
|
+
if (remote.startsWith('git@')) {
|
|
24
|
+
const [, repoPath] = remote.split(':');
|
|
25
|
+
return repoPath?.replace(/\.git$/, '');
|
|
26
|
+
}
|
|
27
|
+
if (remote.startsWith('http')) {
|
|
28
|
+
const url = new URL(remote);
|
|
29
|
+
return url.pathname.replace(/^\/+/, '').replace(/\.git$/, '');
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
catch { /* ignore */ }
|
|
33
|
+
};
|
|
34
|
+
const detectGcpProject = () => {
|
|
35
|
+
try {
|
|
36
|
+
const value = execSync('gcloud config get-value project', { encoding: 'utf8' }).trim();
|
|
37
|
+
return value && value !== '(unset)' ? value : undefined;
|
|
38
|
+
}
|
|
39
|
+
catch { }
|
|
40
|
+
};
|
|
41
|
+
const normalizeProjectPath = (project) => project.replace(/^\/+/, '');
|
|
42
|
+
const normalizeGitLabUrl = (url) => url.replace(/\/+$/, '');
|
|
43
|
+
const setGitlabVariable = async (auth, projectPath, variable) => {
|
|
44
|
+
const base = normalizeGitLabUrl(auth.baseUrl);
|
|
45
|
+
const project = encodeURIComponent(normalizeProjectPath(projectPath));
|
|
46
|
+
const apiUrl = `${base}/api/v4/projects/${project}/variables`;
|
|
47
|
+
await axios
|
|
48
|
+
.delete(`${apiUrl}/${variable.key}`, {
|
|
49
|
+
headers: { 'PRIVATE-TOKEN': auth.token },
|
|
50
|
+
})
|
|
51
|
+
.catch(() => { });
|
|
52
|
+
await axios.post(apiUrl, {
|
|
53
|
+
key: variable.key,
|
|
54
|
+
masked: Boolean(variable.masked),
|
|
55
|
+
protected: false,
|
|
56
|
+
value: variable.value,
|
|
57
|
+
}, { headers: { 'PRIVATE-TOKEN': auth.token } });
|
|
58
|
+
};
|
|
59
|
+
const ensureServiceAccount = (projectId, serviceAccount) => {
|
|
60
|
+
try {
|
|
61
|
+
execSync(`gcloud iam service-accounts describe ${serviceAccount} --project=${projectId}`, {
|
|
62
|
+
stdio: 'ignore',
|
|
63
|
+
});
|
|
64
|
+
}
|
|
65
|
+
catch {
|
|
66
|
+
const name = serviceAccount.split('@')[0] ?? 'duoops-runner';
|
|
67
|
+
execSync(`gcloud iam service-accounts create ${name} --project=${projectId} --display-name="DuoOps Runner"`, { stdio: 'inherit' });
|
|
68
|
+
}
|
|
69
|
+
try {
|
|
70
|
+
execSync(`gcloud projects add-iam-policy-binding ${projectId} --member="serviceAccount:${serviceAccount}" --role="roles/monitoring.viewer" --quiet`, { stdio: 'ignore' });
|
|
71
|
+
}
|
|
72
|
+
catch { /* ignore */ }
|
|
73
|
+
};
|
|
74
|
+
const createServiceAccountKey = (projectId, serviceAccount) => {
|
|
75
|
+
const tmpPath = path.join(os.tmpdir(), `duoops-sa-${Date.now()}.json`);
|
|
76
|
+
execSync(`gcloud iam service-accounts keys create ${tmpPath} --iam-account=${serviceAccount} --project=${projectId}`, { stdio: 'inherit' });
|
|
77
|
+
const contents = fs.readFileSync(tmpPath);
|
|
78
|
+
fs.unlinkSync(tmpPath);
|
|
79
|
+
return contents.toString('base64');
|
|
80
|
+
};
|
|
81
|
+
const ensureServiceEnabled = (projectId, service, label, log) => {
|
|
82
|
+
try {
|
|
83
|
+
log?.(gray(`Ensuring ${label} API is enabled...`));
|
|
84
|
+
execSync(`gcloud services enable ${service} --project=${projectId} --quiet`, {
|
|
85
|
+
stdio: 'ignore',
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
catch {
|
|
89
|
+
log?.(gray(`${label} API enablement skipped (it may already be enabled).`));
|
|
90
|
+
}
|
|
91
|
+
};
|
|
92
|
+
const ensureComputeApiEnabled = (projectId, log) => {
|
|
93
|
+
ensureServiceEnabled(projectId, 'compute.googleapis.com', 'Compute Engine', log);
|
|
94
|
+
};
|
|
95
|
+
const ensureMonitoringApiEnabled = (projectId, log) => {
|
|
96
|
+
ensureServiceEnabled(projectId, 'monitoring.googleapis.com', 'Cloud Monitoring', log);
|
|
97
|
+
};
|
|
98
|
+
const sleep = (ms) => new Promise((resolve) => {
|
|
99
|
+
setTimeout(resolve, ms);
|
|
100
|
+
});
|
|
101
|
+
const runWithRetries = async (command, options, attempts, log) => {
|
|
102
|
+
let delay = 2000;
|
|
103
|
+
for (let attempt = 1; attempt <= attempts; attempt++) {
|
|
104
|
+
try {
|
|
105
|
+
execSync(command, options);
|
|
106
|
+
return;
|
|
107
|
+
}
|
|
108
|
+
catch (error) {
|
|
109
|
+
if (attempt === attempts) {
|
|
110
|
+
throw error;
|
|
111
|
+
}
|
|
112
|
+
log?.(gray(`Command failed (attempt ${attempt}/${attempts}). Retrying in ${Math.round(delay / 1000)}s...`));
|
|
113
|
+
// eslint-disable-next-line no-await-in-loop
|
|
114
|
+
await sleep(delay);
|
|
115
|
+
delay *= 2;
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
};
|
|
119
|
+
const buildStartupScript = (opts) => `#!/usr/bin/env bash
|
|
120
|
+
set -euo pipefail
|
|
121
|
+
apt-get update
|
|
122
|
+
apt-get install -y curl ca-certificates git python3
|
|
123
|
+
|
|
124
|
+
# Install Node.js 20.x
|
|
125
|
+
curl -fsSL https://deb.nodesource.com/setup_20.x | bash -
|
|
126
|
+
apt-get install -y nodejs
|
|
127
|
+
npm install -g duoops@latest
|
|
128
|
+
|
|
129
|
+
# Install GitLab Runner
|
|
130
|
+
curl -L https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh | bash
|
|
131
|
+
apt-get install -y gitlab-runner
|
|
132
|
+
|
|
133
|
+
INSTANCE_ID=$(curl -s -H "Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/instance/id)
|
|
134
|
+
TOKEN=$(curl -s -H "Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/instance/attributes/runner-token)
|
|
135
|
+
|
|
136
|
+
gitlab-runner register \\
|
|
137
|
+
--non-interactive \\
|
|
138
|
+
--url "${opts.gitlabUrl}" \\
|
|
139
|
+
--token "$TOKEN" \\
|
|
140
|
+
--executor shell \\
|
|
141
|
+
--description "${opts.vmName}"
|
|
142
|
+
|
|
143
|
+
echo "$INSTANCE_ID" > /opt/gitlab-runner-instance-id.txt
|
|
144
|
+
|
|
145
|
+
TAG_VALUE=$(curl -s -f -H "Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/instance/attributes/runner-tag || echo "${opts.runnerTag}")
|
|
146
|
+
CONFIG_FILE=/etc/gitlab-runner/config.toml
|
|
147
|
+
|
|
148
|
+
if [ -f "$CONFIG_FILE" ]; then
|
|
149
|
+
TAG_ESCAPED=$(printf '%s' "$TAG_VALUE" | sed 's/"/\\"/g')
|
|
150
|
+
CONFIG_FILE="$CONFIG_FILE" TAG_VALUE="$TAG_ESCAPED" python3 <<'PY'
|
|
151
|
+
import os
|
|
152
|
+
import re
|
|
153
|
+
from pathlib import Path
|
|
154
|
+
|
|
155
|
+
config_path = Path(os.environ.get('CONFIG_FILE', '/etc/gitlab-runner/config.toml'))
|
|
156
|
+
tag_value = os.environ.get('TAG_VALUE', 'gcp')
|
|
157
|
+
if not config_path.exists():
|
|
158
|
+
raise SystemExit(0)
|
|
159
|
+
|
|
160
|
+
text = config_path.read_text()
|
|
161
|
+
pattern = re.compile('(\\[\\[runners\\]\\][\\s\\S]*?)(?=\\n\\[\\[|$)', re.MULTILINE)
|
|
162
|
+
|
|
163
|
+
def transform(block: str) -> str:
|
|
164
|
+
if 'run_untagged' not in block:
|
|
165
|
+
block = block.replace(' executor = "shell"\n', ' executor = "shell"\n run_untagged = false\n', 1)
|
|
166
|
+
block = re.sub(' tags = \\[[^\\]]*\\]\\n', '', block)
|
|
167
|
+
if 'tags =' not in block:
|
|
168
|
+
block = block.rstrip() + f'\n tags = ["{tag_value}","duoops"]\n'
|
|
169
|
+
return block
|
|
170
|
+
|
|
171
|
+
text = pattern.sub(lambda m: transform(m.group(1)), text, count=1)
|
|
172
|
+
config_path.write_text(text)
|
|
173
|
+
PY
|
|
174
|
+
fi
|
|
175
|
+
|
|
176
|
+
systemctl enable gitlab-runner
|
|
177
|
+
systemctl restart gitlab-runner
|
|
178
|
+
`;
|
|
179
|
+
const tryCreateRunnerToken = (projectPath, runnerDescription, runnerTag) => {
|
|
180
|
+
if (!hasBinary('glab')) {
|
|
181
|
+
return;
|
|
182
|
+
}
|
|
183
|
+
try {
|
|
184
|
+
const encoded = encodeURIComponent(normalizeProjectPath(projectPath));
|
|
185
|
+
const projectInfo = JSON.parse(execSync(`glab api "projects/${encoded}"`, { encoding: 'utf8' }));
|
|
186
|
+
const runner = execSync(`glab api --method POST "user/runners" -f runner_type=project_type -f project_id=${projectInfo.id} -f description="${runnerDescription}" -f tag_list="${runnerTag}" -f run_untagged=true`, { encoding: 'utf8' });
|
|
187
|
+
const payload = JSON.parse(runner);
|
|
188
|
+
return payload.token;
|
|
189
|
+
}
|
|
190
|
+
catch {
|
|
191
|
+
}
|
|
192
|
+
};
|
|
5
193
|
export default class Init extends Command {
|
|
6
|
-
static description = 'Initialize
|
|
194
|
+
static description = 'Initialize DuoOps, optionally wiring a GitLab project and GCP runner';
|
|
7
195
|
async run() {
|
|
8
|
-
this.log('Welcome to DuoOps!
|
|
196
|
+
this.log(bold('Welcome to DuoOps!'));
|
|
9
197
|
this.log('You will need a GitLab Personal Access Token with the "api" scope.');
|
|
10
|
-
this.log('
|
|
11
|
-
this.log('');
|
|
198
|
+
this.log('Create one at https://gitlab.com/-/user_settings/personal_access_tokens\n');
|
|
12
199
|
const gitlabUrl = await input({
|
|
13
200
|
default: 'https://gitlab.com',
|
|
14
201
|
message: 'GitLab URL',
|
|
@@ -24,8 +211,9 @@ export default class Init extends Command {
|
|
|
24
211
|
let bigqueryTable;
|
|
25
212
|
let googleProjectId;
|
|
26
213
|
if (enableMeasure) {
|
|
27
|
-
this.log('Note:
|
|
214
|
+
this.log(gray('Note: ensure the Google Cloud SDK is authenticated (gcloud auth application-default login) or supply a service account.'));
|
|
28
215
|
googleProjectId = await input({
|
|
216
|
+
default: detectGcpProject(),
|
|
29
217
|
message: 'Google Cloud Project ID',
|
|
30
218
|
});
|
|
31
219
|
bigqueryDataset = await input({
|
|
@@ -39,11 +227,10 @@ export default class Init extends Command {
|
|
|
39
227
|
const shouldCreate = await confirm({
|
|
40
228
|
message: `Attempt to create BigQuery Dataset (${bigqueryDataset}) and Table (${bigqueryTable}) if missing?`,
|
|
41
229
|
});
|
|
42
|
-
if (shouldCreate) {
|
|
230
|
+
if (shouldCreate && googleProjectId) {
|
|
43
231
|
try {
|
|
44
232
|
this.log('Checking BigQuery resources...');
|
|
45
233
|
const bigquery = new BigQuery({ projectId: googleProjectId });
|
|
46
|
-
// 1. Create Dataset
|
|
47
234
|
const dataset = bigquery.dataset(bigqueryDataset);
|
|
48
235
|
const [datasetExists] = await dataset.exists();
|
|
49
236
|
if (datasetExists) {
|
|
@@ -51,10 +238,8 @@ export default class Init extends Command {
|
|
|
51
238
|
}
|
|
52
239
|
else {
|
|
53
240
|
this.log(`Creating dataset '${bigqueryDataset}'...`);
|
|
54
|
-
await dataset.create({ location: 'US' });
|
|
55
|
-
this.log('Dataset created.');
|
|
241
|
+
await dataset.create({ location: 'US' });
|
|
56
242
|
}
|
|
57
|
-
// 2. Create Table with Schema
|
|
58
243
|
const table = dataset.table(bigqueryTable);
|
|
59
244
|
const [tableExists] = await table.exists();
|
|
60
245
|
if (tableExists) {
|
|
@@ -74,10 +259,9 @@ export default class Init extends Command {
|
|
|
74
259
|
{ name: 'cpu_utilization_avg', type: 'FLOAT' },
|
|
75
260
|
{ name: 'ram_utilization_avg', type: 'FLOAT' },
|
|
76
261
|
{ name: 'energy_kwh', type: 'FLOAT' },
|
|
77
|
-
{ name: 'total_emissions_g', type: 'FLOAT' }
|
|
262
|
+
{ name: 'total_emissions_g', type: 'FLOAT' },
|
|
78
263
|
];
|
|
79
264
|
await table.create({ schema });
|
|
80
|
-
this.log('Table created successfully.');
|
|
81
265
|
}
|
|
82
266
|
}
|
|
83
267
|
catch (error) {
|
|
@@ -86,12 +270,244 @@ export default class Init extends Command {
|
|
|
86
270
|
}
|
|
87
271
|
}
|
|
88
272
|
}
|
|
89
|
-
|
|
273
|
+
const config = {
|
|
274
|
+
...configManager.get(),
|
|
90
275
|
gitlabToken,
|
|
91
276
|
gitlabUrl,
|
|
92
277
|
measure: enableMeasure ? { bigqueryDataset, bigqueryTable, googleProjectId } : undefined,
|
|
278
|
+
};
|
|
279
|
+
configManager.set(config);
|
|
280
|
+
this.log(green('Configuration saved.'));
|
|
281
|
+
this.log(`Try '${this.config.bin} pipelines:list <project>' to verify GitLab access.\n`);
|
|
282
|
+
const configureProject = await confirm({
|
|
283
|
+
default: true,
|
|
284
|
+
message: 'Configure a GitLab project (CI variables + optional GCP runner) now?',
|
|
285
|
+
});
|
|
286
|
+
if (!configureProject) {
|
|
287
|
+
return;
|
|
288
|
+
}
|
|
289
|
+
const updatedDefaultProject = await this.configureGitLabProject({ baseUrl: gitlabUrl, token: gitlabToken }, config.defaultProjectId);
|
|
290
|
+
if (updatedDefaultProject) {
|
|
291
|
+
const latest = configManager.get();
|
|
292
|
+
configManager.set({ ...latest, defaultProjectId: updatedDefaultProject });
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
async collectExistingRunnerInfo() {
|
|
296
|
+
const instanceName = await input({
|
|
297
|
+
message: 'Runner instance name (Compute Engine VM name)',
|
|
298
|
+
});
|
|
299
|
+
const gcpZone = await input({
|
|
300
|
+
message: 'Runner zone',
|
|
301
|
+
});
|
|
302
|
+
const gcpInstanceId = await input({
|
|
303
|
+
message: 'Runner instance ID (numeric)',
|
|
304
|
+
});
|
|
305
|
+
const machineType = await input({
|
|
306
|
+
default: 'e2-standard-4',
|
|
307
|
+
message: 'Runner machine type',
|
|
308
|
+
});
|
|
309
|
+
const runnerTag = await input({
|
|
310
|
+
default: 'gcp',
|
|
311
|
+
message: 'Runner tag (used in your .gitlab-ci.yml)',
|
|
312
|
+
});
|
|
313
|
+
return {
|
|
314
|
+
gcpInstanceId,
|
|
315
|
+
gcpZone,
|
|
316
|
+
instanceName,
|
|
317
|
+
machineType,
|
|
318
|
+
runnerTag,
|
|
319
|
+
};
|
|
320
|
+
}
|
|
321
|
+
async configureGitLabProject(auth, currentDefault) {
|
|
322
|
+
if (!auth.baseUrl || !auth.token) {
|
|
323
|
+
this.warn('GitLab credentials missing, skipping project configuration.');
|
|
324
|
+
return;
|
|
325
|
+
}
|
|
326
|
+
const projectPath = await input({
|
|
327
|
+
default: currentDefault ?? detectGitRemotePath(),
|
|
328
|
+
message: 'GitLab project path (e.g., group/project)',
|
|
329
|
+
});
|
|
330
|
+
if (!projectPath) {
|
|
331
|
+
this.warn('Project path is required to configure CI variables.');
|
|
332
|
+
}
|
|
333
|
+
const setAsDefault = await confirm({
|
|
334
|
+
default: !currentDefault,
|
|
335
|
+
message: 'Use this project as the default for DuoOps commands?',
|
|
336
|
+
});
|
|
337
|
+
const gcpProjectId = await input({
|
|
338
|
+
default: detectGcpProject(),
|
|
339
|
+
message: 'GCP Project ID',
|
|
340
|
+
});
|
|
341
|
+
if (!gcpProjectId) {
|
|
342
|
+
this.warn('GCP Project ID is required to collect metrics.');
|
|
343
|
+
return setAsDefault ? projectPath : undefined;
|
|
344
|
+
}
|
|
345
|
+
let serviceAccountEmail = await input({
|
|
346
|
+
default: `duoops-runner@${gcpProjectId}.iam.gserviceaccount.com`,
|
|
347
|
+
message: 'Service account email for DuoOps measurements',
|
|
348
|
+
});
|
|
349
|
+
serviceAccountEmail = serviceAccountEmail.trim();
|
|
350
|
+
if (!serviceAccountEmail) {
|
|
351
|
+
this.error('Service account email is required.');
|
|
352
|
+
}
|
|
353
|
+
ensureServiceAccount(gcpProjectId, serviceAccountEmail);
|
|
354
|
+
const runnerChoice = (await select({
|
|
355
|
+
choices: [
|
|
356
|
+
{ name: 'Provision new GCP runner VM', value: 'provision' },
|
|
357
|
+
{ name: 'Use existing runner', value: 'existing' },
|
|
358
|
+
],
|
|
359
|
+
message: 'Runner setup',
|
|
360
|
+
}));
|
|
361
|
+
const runnerInfo = runnerChoice === 'provision'
|
|
362
|
+
? await this.provisionGcpRunner(auth, projectPath, gcpProjectId, serviceAccountEmail)
|
|
363
|
+
: await this.collectExistingRunnerInfo();
|
|
364
|
+
const keySource = (await select({
|
|
365
|
+
choices: [
|
|
366
|
+
{ name: 'Create a new key with gcloud (recommended)', value: 'create' },
|
|
367
|
+
{ name: 'Use an existing JSON key file', value: 'file' },
|
|
368
|
+
],
|
|
369
|
+
message: 'Service account key for CI (used by duoops measure job)',
|
|
370
|
+
}));
|
|
371
|
+
let saKeyBase64;
|
|
372
|
+
if (keySource === 'create') {
|
|
373
|
+
saKeyBase64 = createServiceAccountKey(gcpProjectId, serviceAccountEmail);
|
|
374
|
+
}
|
|
375
|
+
else {
|
|
376
|
+
const keyPath = await input({
|
|
377
|
+
message: 'Path to service account JSON key',
|
|
378
|
+
});
|
|
379
|
+
if (!keyPath) {
|
|
380
|
+
this.error('Key path is required.');
|
|
381
|
+
}
|
|
382
|
+
if (!fs.existsSync(keyPath)) {
|
|
383
|
+
this.error(`File not found: ${keyPath}`);
|
|
384
|
+
}
|
|
385
|
+
saKeyBase64 = fs.readFileSync(keyPath).toString('base64');
|
|
386
|
+
}
|
|
387
|
+
const variables = [
|
|
388
|
+
{ key: 'GCP_PROJECT_ID', value: gcpProjectId },
|
|
389
|
+
{ key: 'GCP_ZONE', value: runnerInfo.gcpZone },
|
|
390
|
+
{ key: 'GCP_INSTANCE_ID', value: runnerInfo.gcpInstanceId },
|
|
391
|
+
{ key: 'MACHINE_TYPE', value: runnerInfo.machineType },
|
|
392
|
+
{ key: 'GCP_SA_KEY_BASE64', masked: true, value: saKeyBase64 },
|
|
393
|
+
];
|
|
394
|
+
this.log('\nUpdating GitLab CI/CD variables...');
|
|
395
|
+
await Promise.all(variables.map(async (variable) => {
|
|
396
|
+
try {
|
|
397
|
+
await setGitlabVariable(auth, projectPath, variable);
|
|
398
|
+
this.log(green(` OK ${variable.key}`));
|
|
399
|
+
}
|
|
400
|
+
catch (error) {
|
|
401
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
402
|
+
this.warn(` WARN Failed to set ${variable.key}: ${message}`);
|
|
403
|
+
}
|
|
404
|
+
}));
|
|
405
|
+
this.log(gray('\nAdd the duoops measure component to your .gitlab-ci.yml (demo/.duoops/measure-component.yml is a good starting point).'));
|
|
406
|
+
this.log(gray(`Use the '${runnerInfo.runnerTag}' tag on jobs that should target this runner.`));
|
|
407
|
+
this.log(green('Project wiring complete!'));
|
|
408
|
+
const currentConfig = configManager.get();
|
|
409
|
+
const existingRunner = currentConfig.runner ?? {};
|
|
410
|
+
const updatedConfig = {
|
|
411
|
+
...currentConfig,
|
|
412
|
+
gitlabProjectPath: projectPath,
|
|
413
|
+
runner: {
|
|
414
|
+
...existingRunner,
|
|
415
|
+
gcpInstanceId: runnerInfo.gcpInstanceId,
|
|
416
|
+
gcpProjectId,
|
|
417
|
+
gcpZone: runnerInfo.gcpZone,
|
|
418
|
+
instanceName: runnerInfo.instanceName,
|
|
419
|
+
machineType: runnerInfo.machineType,
|
|
420
|
+
},
|
|
421
|
+
};
|
|
422
|
+
if (setAsDefault) {
|
|
423
|
+
updatedConfig.defaultProjectId = projectPath;
|
|
424
|
+
}
|
|
425
|
+
configManager.set(updatedConfig);
|
|
426
|
+
return setAsDefault ? projectPath : undefined;
|
|
427
|
+
}
|
|
428
|
+
async provisionGcpRunner(auth, projectPath, gcpProjectId, serviceAccountEmail) {
|
|
429
|
+
if (!hasBinary('gcloud')) {
|
|
430
|
+
this.error('gcloud CLI is required to provision a runner. Install it from https://cloud.google.com/sdk');
|
|
431
|
+
}
|
|
432
|
+
const vmName = await input({
|
|
433
|
+
default: 'duoops-runner',
|
|
434
|
+
message: 'Runner VM name',
|
|
435
|
+
});
|
|
436
|
+
const gcpZone = await input({
|
|
437
|
+
default: 'us-central1-a',
|
|
438
|
+
message: 'GCP zone',
|
|
93
439
|
});
|
|
94
|
-
|
|
95
|
-
|
|
440
|
+
const machineType = await input({
|
|
441
|
+
default: 'e2-standard-4',
|
|
442
|
+
message: 'Machine type',
|
|
443
|
+
});
|
|
444
|
+
const runnerTag = await input({
|
|
445
|
+
default: 'gcp',
|
|
446
|
+
message: 'Runner tag (GitLab jobs will use this)',
|
|
447
|
+
});
|
|
448
|
+
ensureComputeApiEnabled(gcpProjectId, (message) => this.log(message));
|
|
449
|
+
ensureMonitoringApiEnabled(gcpProjectId, (message) => this.log(message));
|
|
450
|
+
let runnerToken = tryCreateRunnerToken(projectPath, vmName, runnerTag);
|
|
451
|
+
if (!runnerToken) {
|
|
452
|
+
this.log(gray('Unable to create a runner token automatically. Generate one in your GitLab project (Settings → CI/CD → Runners) and paste it below.'));
|
|
453
|
+
runnerToken = await password({
|
|
454
|
+
mask: '*',
|
|
455
|
+
message: 'Runner token (starts with glrt-)',
|
|
456
|
+
});
|
|
457
|
+
}
|
|
458
|
+
if (!runnerToken) {
|
|
459
|
+
this.error('Runner token required to register the VM.');
|
|
460
|
+
}
|
|
461
|
+
const startupScript = buildStartupScript({
|
|
462
|
+
gitlabUrl: normalizeGitLabUrl(auth.baseUrl),
|
|
463
|
+
runnerTag,
|
|
464
|
+
vmName,
|
|
465
|
+
});
|
|
466
|
+
const scriptPath = path.join(os.tmpdir(), `duoops-runner-${Date.now()}.sh`);
|
|
467
|
+
fs.writeFileSync(scriptPath, startupScript);
|
|
468
|
+
try {
|
|
469
|
+
const metadataArg = `--metadata=runner-token=${JSON.stringify(runnerToken)},runner-tag=${JSON.stringify(runnerTag)}`;
|
|
470
|
+
const metadataFileArg = `--metadata-from-file=startup-script=${JSON.stringify(scriptPath)}`;
|
|
471
|
+
const command = [
|
|
472
|
+
'gcloud',
|
|
473
|
+
'compute',
|
|
474
|
+
'instances',
|
|
475
|
+
'create',
|
|
476
|
+
vmName,
|
|
477
|
+
`--project=${gcpProjectId}`,
|
|
478
|
+
`--zone=${gcpZone}`,
|
|
479
|
+
`--machine-type=${machineType}`,
|
|
480
|
+
`--service-account=${serviceAccountEmail}`,
|
|
481
|
+
'--scopes=https://www.googleapis.com/auth/cloud-platform',
|
|
482
|
+
metadataArg,
|
|
483
|
+
metadataFileArg,
|
|
484
|
+
'--image-family=debian-12',
|
|
485
|
+
'--image-project=debian-cloud',
|
|
486
|
+
].join(' ');
|
|
487
|
+
this.log(gray('\nProvisioning VM (this may take a minute)...'));
|
|
488
|
+
await runWithRetries(command, { stdio: 'inherit' }, 3, (message) => {
|
|
489
|
+
this.log(message);
|
|
490
|
+
});
|
|
491
|
+
this.log(green('VM created. Waiting for instance ID...'));
|
|
492
|
+
}
|
|
493
|
+
finally {
|
|
494
|
+
fs.unlinkSync(scriptPath);
|
|
495
|
+
}
|
|
496
|
+
let gcpInstanceId = '';
|
|
497
|
+
try {
|
|
498
|
+
gcpInstanceId = execSync(`gcloud compute instances describe ${vmName} --project=${gcpProjectId} --zone=${gcpZone} --format="value(id)"`, { encoding: 'utf8' })
|
|
499
|
+
.toString()
|
|
500
|
+
.trim();
|
|
501
|
+
}
|
|
502
|
+
catch {
|
|
503
|
+
this.warn('Unable to read instance ID automatically. Provide it manually in CI variables.');
|
|
504
|
+
}
|
|
505
|
+
return {
|
|
506
|
+
gcpInstanceId,
|
|
507
|
+
gcpZone,
|
|
508
|
+
instanceName: vmName,
|
|
509
|
+
machineType,
|
|
510
|
+
runnerTag,
|
|
511
|
+
};
|
|
96
512
|
}
|
|
97
513
|
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { Command } from '@oclif/core';
|
|
2
|
+
export default class RunnerLogs extends Command {
|
|
3
|
+
static description: string;
|
|
4
|
+
static flags: {
|
|
5
|
+
lines: import("@oclif/core/interfaces").OptionFlag<number, import("@oclif/core/interfaces").CustomOptions>;
|
|
6
|
+
unit: import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
|
|
7
|
+
};
|
|
8
|
+
run(): Promise<void>;
|
|
9
|
+
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { Command, Flags } from '@oclif/core';
|
|
2
|
+
import { execSync } from 'node:child_process';
|
|
3
|
+
import { configManager } from '../../lib/config.js';
|
|
4
|
+
export default class RunnerLogs extends Command {
|
|
5
|
+
static description = 'Show systemd logs from the DuoOps runner VM';
|
|
6
|
+
static flags = {
|
|
7
|
+
lines: Flags.integer({
|
|
8
|
+
char: 'n',
|
|
9
|
+
default: 200,
|
|
10
|
+
description: 'Number of log lines to show',
|
|
11
|
+
}),
|
|
12
|
+
unit: Flags.string({
|
|
13
|
+
char: 'u',
|
|
14
|
+
default: 'gitlab-runner',
|
|
15
|
+
description: 'systemd unit to inspect',
|
|
16
|
+
}),
|
|
17
|
+
};
|
|
18
|
+
async run() {
|
|
19
|
+
const { flags } = await this.parse(RunnerLogs);
|
|
20
|
+
const { runner } = configManager.get();
|
|
21
|
+
if (!runner?.instanceName || !runner?.gcpProjectId || !runner?.gcpZone) {
|
|
22
|
+
this.error('Runner details are missing. Re-run "duoops init" and configure a project so the runner metadata is stored locally.');
|
|
23
|
+
}
|
|
24
|
+
const command = [
|
|
25
|
+
'gcloud',
|
|
26
|
+
'compute',
|
|
27
|
+
'ssh',
|
|
28
|
+
runner.instanceName,
|
|
29
|
+
`--project=${runner.gcpProjectId}`,
|
|
30
|
+
`--zone=${runner.gcpZone}`,
|
|
31
|
+
`--command=${JSON.stringify(`sudo journalctl -u ${flags.unit} -n ${flags.lines} --no-pager`)}`,
|
|
32
|
+
].join(' ');
|
|
33
|
+
this.log(`Executing: ${command}`);
|
|
34
|
+
execSync(command, { stdio: 'inherit' });
|
|
35
|
+
}
|
|
36
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { Command } from '@oclif/core';
|
|
2
|
+
export default class RunnerReinstallDuoops extends Command {
|
|
3
|
+
static description: string;
|
|
4
|
+
static flags: {
|
|
5
|
+
version: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
6
|
+
};
|
|
7
|
+
run(): Promise<void>;
|
|
8
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { Command, Flags } from '@oclif/core';
|
|
2
|
+
import { execSync } from 'node:child_process';
|
|
3
|
+
import { configManager } from '../../lib/config.js';
|
|
4
|
+
export default class RunnerReinstallDuoops extends Command {
|
|
5
|
+
static description = 'Reinstall the DuoOps CLI on the runner VM';
|
|
6
|
+
static flags = {
|
|
7
|
+
version: Flags.string({
|
|
8
|
+
description: 'DuoOps version to install on the runner (defaults to current CLI version)',
|
|
9
|
+
}),
|
|
10
|
+
};
|
|
11
|
+
async run() {
|
|
12
|
+
const { flags } = await this.parse(RunnerReinstallDuoops);
|
|
13
|
+
const { runner } = configManager.get();
|
|
14
|
+
if (!runner?.instanceName || !runner?.gcpProjectId || !runner?.gcpZone) {
|
|
15
|
+
this.error('Runner details are missing. Re-run "duoops init" and configure a project so the runner metadata is stored locally.');
|
|
16
|
+
}
|
|
17
|
+
const targetVersion = flags.version ?? this.config.version ?? 'latest';
|
|
18
|
+
const remoteScript = `sudo npm install -g duoops@${targetVersion} && duoops --version`;
|
|
19
|
+
const command = [
|
|
20
|
+
'gcloud',
|
|
21
|
+
'compute',
|
|
22
|
+
'ssh',
|
|
23
|
+
runner.instanceName,
|
|
24
|
+
`--project=${runner.gcpProjectId}`,
|
|
25
|
+
`--zone=${runner.gcpZone}`,
|
|
26
|
+
`--command=${JSON.stringify(remoteScript)}`,
|
|
27
|
+
].join(' ');
|
|
28
|
+
this.log(`Installing duoops@${targetVersion} on ${runner.instanceName}...`);
|
|
29
|
+
execSync(command, { stdio: 'inherit' });
|
|
30
|
+
}
|
|
31
|
+
}
|
package/dist/lib/config.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
export interface DuoOpsConfig {
|
|
2
2
|
defaultProjectId?: string;
|
|
3
|
+
gitlabProjectPath?: string;
|
|
3
4
|
gitlabToken?: string;
|
|
4
5
|
gitlabUrl?: string;
|
|
5
6
|
measure?: {
|
|
@@ -7,6 +8,13 @@ export interface DuoOpsConfig {
|
|
|
7
8
|
bigqueryTable?: string;
|
|
8
9
|
googleProjectId?: string;
|
|
9
10
|
};
|
|
11
|
+
runner?: {
|
|
12
|
+
gcpInstanceId?: string;
|
|
13
|
+
gcpProjectId?: string;
|
|
14
|
+
gcpZone?: string;
|
|
15
|
+
instanceName?: string;
|
|
16
|
+
machineType?: string;
|
|
17
|
+
};
|
|
10
18
|
}
|
|
11
19
|
export declare class ConfigManager {
|
|
12
20
|
private configPath;
|