@cloud-cli/on 1.2.2 → 1.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -20
- package/dist/db-client.d.ts +14 -0
- package/dist/drivers/index.d.ts +2 -0
- package/dist/drivers/standard-process.driver.d.ts +10 -0
- package/dist/drivers/systemd.driver.d.ts +6 -0
- package/dist/index.d.ts +4 -0
- package/dist/log-redactor.d.ts +6 -0
- package/dist/on.js +9240 -0
- package/dist/parser/include-resolver.d.ts +9 -0
- package/dist/parser/matrix-expander.d.ts +5 -0
- package/dist/parser/yaml-loader.d.ts +6 -0
- package/dist/plugins/github-status.plugin.d.ts +7 -0
- package/dist/plugins/manager.d.ts +7 -0
- package/dist/queue.d.ts +38 -0
- package/dist/reporters/html.reporter.d.ts +14 -0
- package/dist/reporters/json-file.reporter.d.ts +9 -0
- package/dist/reporters/slack.reporter.d.ts +15 -0
- package/dist/safe-eval.d.ts +28 -0
- package/dist/secrets.d.ts +15 -0
- package/dist/server/preprocessors/github.d.ts +5 -0
- package/dist/server/server.d.ts +32 -0
- package/dist/types.d.ts +181 -0
- package/dist/worker.d.ts +8 -0
- package/package.json +24 -20
- package/dist/config.js +0 -15
- package/dist/db-client.js +0 -38
- package/dist/drivers/index.js +0 -11
- package/dist/drivers/standard-process.driver.js +0 -156
- package/dist/drivers/systemd.driver.js +0 -151
- package/dist/evaluator/safe-eval.js +0 -213
- package/dist/index.js +0 -145
- package/dist/ingress/preprocessors/github.js +0 -30
- package/dist/ingress/server.js +0 -241
- package/dist/logging/redactor.js +0 -20
- package/dist/parser/include-resolver.js +0 -47
- package/dist/parser/matrix-expander.js +0 -43
- package/dist/parser/yaml-loader.js +0 -45
- package/dist/plugins/github-status.plugin.js +0 -19
- package/dist/plugins/manager.js +0 -21
- package/dist/plugins/types.js +0 -1
- package/dist/queue/dispatcher.js +0 -112
- package/dist/reporters/html.reporter.js +0 -108
- package/dist/reporters/json-file.reporter.js +0 -16
- package/dist/reporters/slack.reporter.js +0 -25
- package/dist/reporters/types.js +0 -1
- package/dist/runner/step-runner.js +0 -42
- package/dist/secrets/store.js +0 -40
- package/dist/types.js +0 -1
- package/dist/worker.js +0 -249
- /package/dist/{ingress/types.js → runner/step-runner.d.ts} +0 -0
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
import fs from 'node:fs';
|
|
2
|
-
import path from 'node:path';
|
|
3
|
-
export class JsonFileReporter {
|
|
4
|
-
name = 'json-file-reporter';
|
|
5
|
-
outputDir;
|
|
6
|
-
constructor(options) {
|
|
7
|
-
this.outputDir = options.outputDir;
|
|
8
|
-
}
|
|
9
|
-
async report(execReport) {
|
|
10
|
-
fs.mkdirSync(this.outputDir, { recursive: true });
|
|
11
|
-
const filePath = path.join(this.outputDir, `run-${execReport.jobId}.json`);
|
|
12
|
-
// Save pretty-printed execution report
|
|
13
|
-
fs.writeFileSync(filePath, JSON.stringify(execReport, null, 2), 'utf-8');
|
|
14
|
-
console.log(`📊 Execution report saved to: ${filePath}`);
|
|
15
|
-
}
|
|
16
|
-
}
|
|
@@ -1,25 +0,0 @@
|
|
|
1
|
-
export class SlackReporter {
|
|
2
|
-
name = 'slack-reporter';
|
|
3
|
-
token;
|
|
4
|
-
channel;
|
|
5
|
-
notifyOn;
|
|
6
|
-
constructor(options) {
|
|
7
|
-
this.token = options.token;
|
|
8
|
-
this.channel = options.channel;
|
|
9
|
-
this.notifyOn = options.notifyOn || ['failed']; // Default: notify on failure only
|
|
10
|
-
}
|
|
11
|
-
async report(execReport) {
|
|
12
|
-
if (!this.notifyOn.includes(execReport.status))
|
|
13
|
-
return;
|
|
14
|
-
const emoji = execReport.status === 'success' ? '✅' : '❌';
|
|
15
|
-
const text = `${emoji} *Workflow ${execReport.workflowName} (#${execReport.jobId})* finished with status: *${execReport.status.toUpperCase()}* (${execReport.durationMs}ms)`;
|
|
16
|
-
await fetch('https://slack.com/api/chat.postMessage', {
|
|
17
|
-
method: 'POST',
|
|
18
|
-
headers: {
|
|
19
|
-
'Authorization': `Bearer ${this.token}`,
|
|
20
|
-
'Content-Type': 'application/json'
|
|
21
|
-
},
|
|
22
|
-
body: JSON.stringify({ channel: this.channel, text })
|
|
23
|
-
});
|
|
24
|
-
}
|
|
25
|
-
}
|
package/dist/reporters/types.js
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export {};
|
|
@@ -1,42 +0,0 @@
|
|
|
1
|
-
import fs from 'node:fs';
|
|
2
|
-
import path from 'node:path';
|
|
3
|
-
import { parseEnv } from 'node:util';
|
|
4
|
-
export async function executeStepAndCollectState(stepCtx, driver, currentWorkflowEnv) {
|
|
5
|
-
// 1. Create temporary state files for this step
|
|
6
|
-
const envFilePath = path.join(stepCtx.workspacePath, `.step-${stepCtx.stepId}.env`);
|
|
7
|
-
const outputFilePath = path.join(stepCtx.workspacePath, `.step-${stepCtx.stepId}.out`);
|
|
8
|
-
fs.writeFileSync(envFilePath, '');
|
|
9
|
-
fs.writeFileSync(outputFilePath, '');
|
|
10
|
-
let newEnv = {};
|
|
11
|
-
let outputs = {};
|
|
12
|
-
try {
|
|
13
|
-
// 2. Inject environment file paths into step execution context
|
|
14
|
-
const stepEnv = {
|
|
15
|
-
...currentWorkflowEnv,
|
|
16
|
-
...stepCtx.env,
|
|
17
|
-
WORKFLOW_ENV: envFilePath,
|
|
18
|
-
WORKFLOW_OUTPUT: outputFilePath,
|
|
19
|
-
};
|
|
20
|
-
// 3. Execute step
|
|
21
|
-
const handle = await driver.execute({ ...stepCtx, env: stepEnv });
|
|
22
|
-
const result = await handle.done;
|
|
23
|
-
// 4. Parse step-exported environment variables using Node's built-in parseEnv
|
|
24
|
-
if (fs.existsSync(envFilePath)) {
|
|
25
|
-
newEnv = parseEnv(fs.readFileSync(envFilePath, 'utf-8'));
|
|
26
|
-
}
|
|
27
|
-
if (fs.existsSync(outputFilePath)) {
|
|
28
|
-
outputs = parseEnv(fs.readFileSync(outputFilePath, 'utf-8'));
|
|
29
|
-
}
|
|
30
|
-
if (result.exitCode !== 0) {
|
|
31
|
-
return { result, newEnv: {}, outputs: {} };
|
|
32
|
-
}
|
|
33
|
-
return { result, newEnv, outputs };
|
|
34
|
-
}
|
|
35
|
-
finally {
|
|
36
|
-
// ALWAYS cleans up temp state files, regardless of success or failure
|
|
37
|
-
if (fs.existsSync(envFilePath))
|
|
38
|
-
fs.unlinkSync(envFilePath);
|
|
39
|
-
if (fs.existsSync(outputFilePath))
|
|
40
|
-
fs.unlinkSync(outputFilePath);
|
|
41
|
-
}
|
|
42
|
-
}
|
package/dist/secrets/store.js
DELETED
|
@@ -1,40 +0,0 @@
|
|
|
1
|
-
import dotenv from 'dotenv';
|
|
2
|
-
import fs from 'node:fs';
|
|
3
|
-
export class SecretStore {
|
|
4
|
-
envFilePath;
|
|
5
|
-
secrets = new Map();
|
|
6
|
-
/**
|
|
7
|
-
* Initialize secrets from host environment or a specified .env file
|
|
8
|
-
*/
|
|
9
|
-
constructor(envFilePath) {
|
|
10
|
-
this.envFilePath = envFilePath;
|
|
11
|
-
}
|
|
12
|
-
reload() {
|
|
13
|
-
// 1. Load host process.env variables prefixed with SECRET_
|
|
14
|
-
for (const [key, val] of Object.entries(process.env)) {
|
|
15
|
-
if (key.startsWith('SECRET_') && val) {
|
|
16
|
-
// Strip prefix: SECRET_SLACK_TOKEN -> SLACK_TOKEN
|
|
17
|
-
this.secrets.set(key.replace('SECRET_', ''), val);
|
|
18
|
-
}
|
|
19
|
-
}
|
|
20
|
-
// 2. Override/add from .env file if present
|
|
21
|
-
if (this.envFilePath && fs.existsSync(this.envFilePath)) {
|
|
22
|
-
const parsed = dotenv.parse(fs.readFileSync(this.envFilePath));
|
|
23
|
-
for (const [key, val] of Object.entries(parsed)) {
|
|
24
|
-
this.secrets.set(key, val);
|
|
25
|
-
}
|
|
26
|
-
}
|
|
27
|
-
}
|
|
28
|
-
get(key) {
|
|
29
|
-
return this.secrets.get(key);
|
|
30
|
-
}
|
|
31
|
-
getAll() {
|
|
32
|
-
return Object.fromEntries(this.secrets);
|
|
33
|
-
}
|
|
34
|
-
/**
|
|
35
|
-
* Returns a list of secret values to be redacted from logs
|
|
36
|
-
*/
|
|
37
|
-
getSecretValuesForRedaction() {
|
|
38
|
-
return Array.from(this.secrets.values()).filter((v) => v.length > 3); // Avoid masking tiny strings
|
|
39
|
-
}
|
|
40
|
-
}
|
package/dist/types.js
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export {};
|
package/dist/worker.js
DELETED
|
@@ -1,249 +0,0 @@
|
|
|
1
|
-
import { resolveDriver } from './drivers/index.js';
|
|
2
|
-
import { SafeExpressionEvaluator } from './evaluator/safe-eval.js';
|
|
3
|
-
export function startWorkers(count, queue, secrets, config) {
|
|
4
|
-
return Array.from({ length: count }, (_, i) => startWorkerLoop(`worker-${i + 1}`, queue, secrets, config));
|
|
5
|
-
}
|
|
6
|
-
/**
|
|
7
|
-
* Main worker loop: continuously polls the SQLite queue for pending jobs.
|
|
8
|
-
*/
|
|
9
|
-
export async function startWorkerLoop(workerId, queue, secrets, config) {
|
|
10
|
-
const driver = await resolveDriver();
|
|
11
|
-
console.log(`[${workerId}] 🚀 Worker started. Driver: ${driver.name}`);
|
|
12
|
-
while (true) {
|
|
13
|
-
try {
|
|
14
|
-
const job = await queue.claimNextJob();
|
|
15
|
-
if (!job) {
|
|
16
|
-
await new Promise((r) => setTimeout(r, 2000));
|
|
17
|
-
continue;
|
|
18
|
-
}
|
|
19
|
-
await processJob(workerId, job, queue, secrets, config, driver);
|
|
20
|
-
}
|
|
21
|
-
catch (error) {
|
|
22
|
-
console.error(`[${workerId}] ⚠️ Worker execution loop error:`, error);
|
|
23
|
-
await new Promise((r) => setTimeout(r, 5000));
|
|
24
|
-
}
|
|
25
|
-
}
|
|
26
|
-
}
|
|
27
|
-
/**
|
|
28
|
-
* Orchestrates the complete execution lifecycle for a claimed job.
|
|
29
|
-
*/
|
|
30
|
-
async function processJob(workerId, job, queue, secrets, config, driver) {
|
|
31
|
-
console.log(`\n[${workerId}] 📦 Claimed Job #${job.id} (Workflow: ${job.workflow_id})`);
|
|
32
|
-
const payload = typeof job.payload === 'string' ? JSON.parse(job.payload) : job.payload;
|
|
33
|
-
const steps = payload.steps || [];
|
|
34
|
-
const inputs = payload.inputs || {};
|
|
35
|
-
const jobStartTime = Date.now();
|
|
36
|
-
const executionContext = {
|
|
37
|
-
inputs,
|
|
38
|
-
env: { ...config.env },
|
|
39
|
-
secrets: secrets.getAll(),
|
|
40
|
-
steps: {},
|
|
41
|
-
};
|
|
42
|
-
const stepReports = [];
|
|
43
|
-
let jobFailed = false;
|
|
44
|
-
let isCancelled = false;
|
|
45
|
-
// Execute steps sequentially
|
|
46
|
-
for (let i = 0; i < steps.length; i++) {
|
|
47
|
-
const step = steps[i];
|
|
48
|
-
const stepResult = await executeSingleStep({
|
|
49
|
-
workerId,
|
|
50
|
-
jobId: job.id,
|
|
51
|
-
step,
|
|
52
|
-
stepIndex: i,
|
|
53
|
-
totalSteps: steps.length,
|
|
54
|
-
executionContext,
|
|
55
|
-
driver,
|
|
56
|
-
queue,
|
|
57
|
-
config,
|
|
58
|
-
});
|
|
59
|
-
stepReports.push(stepResult.report);
|
|
60
|
-
if (stepResult.isCancelled) {
|
|
61
|
-
isCancelled = true;
|
|
62
|
-
jobFailed = true;
|
|
63
|
-
break;
|
|
64
|
-
}
|
|
65
|
-
if (stepResult.failed) {
|
|
66
|
-
jobFailed = true;
|
|
67
|
-
break;
|
|
68
|
-
}
|
|
69
|
-
}
|
|
70
|
-
// Mark unexecuted steps as skipped if execution halted early
|
|
71
|
-
if (jobFailed && stepReports.length < steps.length) {
|
|
72
|
-
fillSkippedSteps(steps, stepReports.length, stepReports);
|
|
73
|
-
}
|
|
74
|
-
const finalStatus = isCancelled ? 'cancelled' : jobFailed ? 'failed' : 'success';
|
|
75
|
-
// Persist status and execution report to SQLite Queue
|
|
76
|
-
await queue.finishJob(job.id, finalStatus);
|
|
77
|
-
console.log(`[${workerId}] ✅ Job #${job.id} completed as: ${finalStatus}`);
|
|
78
|
-
const executionReport = buildExecutionReport(job, finalStatus, jobStartTime, inputs, executionContext.env, stepReports, payload);
|
|
79
|
-
await queue.saveReport(job.id, executionReport);
|
|
80
|
-
// Dispatch reports to all registered plugins/reporters
|
|
81
|
-
await dispatchReporters(workerId, config.reporters, executionReport);
|
|
82
|
-
}
|
|
83
|
-
/**
|
|
84
|
-
* Routes step execution to either JS Eval or Process Driver handler.
|
|
85
|
-
*/
|
|
86
|
-
async function executeSingleStep(params) {
|
|
87
|
-
const { workerId, step, stepIndex, totalSteps, executionContext } = params;
|
|
88
|
-
const stepId = step.id || `step-${stepIndex}`;
|
|
89
|
-
const stepName = step.name || stepId;
|
|
90
|
-
console.log(`[${workerId}] ▶️ Running step ${stepIndex + 1}/${totalSteps}: ${stepName}`);
|
|
91
|
-
if (step.eval) {
|
|
92
|
-
return executeEvalStep(stepId, stepName, step.eval, executionContext);
|
|
93
|
-
}
|
|
94
|
-
else {
|
|
95
|
-
return executeRunStep({ ...params, stepId, stepName });
|
|
96
|
-
}
|
|
97
|
-
}
|
|
98
|
-
/**
|
|
99
|
-
* Executes an in-process JS 'eval:' step.
|
|
100
|
-
*/
|
|
101
|
-
async function executeEvalStep(stepId, stepName, evalExpr, executionContext) {
|
|
102
|
-
const startTime = Date.now();
|
|
103
|
-
try {
|
|
104
|
-
const evalResult = await SafeExpressionEvaluator.evaluateExpression(evalExpr, executionContext);
|
|
105
|
-
executionContext.steps[stepId] = { status: 'success', outputs: evalResult };
|
|
106
|
-
console.log(`[${stepId}] ✅ JS Eval step complete.`);
|
|
107
|
-
return {
|
|
108
|
-
failed: false,
|
|
109
|
-
isCancelled: false,
|
|
110
|
-
report: {
|
|
111
|
-
id: stepId,
|
|
112
|
-
name: stepName,
|
|
113
|
-
status: 'success',
|
|
114
|
-
durationMs: Date.now() - startTime,
|
|
115
|
-
exitCode: 0,
|
|
116
|
-
outputs: evalResult || {},
|
|
117
|
-
logFilePath: '',
|
|
118
|
-
},
|
|
119
|
-
};
|
|
120
|
-
}
|
|
121
|
-
catch (err) {
|
|
122
|
-
console.error(`[${stepId}] ❌ JS Eval step failed:`, err.message);
|
|
123
|
-
executionContext.steps[stepId] = { status: 'failed', error: err.message };
|
|
124
|
-
return {
|
|
125
|
-
failed: true,
|
|
126
|
-
isCancelled: false,
|
|
127
|
-
report: {
|
|
128
|
-
id: stepId,
|
|
129
|
-
name: stepName,
|
|
130
|
-
status: 'failed',
|
|
131
|
-
durationMs: Date.now() - startTime,
|
|
132
|
-
exitCode: 1,
|
|
133
|
-
error: err.message,
|
|
134
|
-
outputs: {},
|
|
135
|
-
logFilePath: '',
|
|
136
|
-
},
|
|
137
|
-
};
|
|
138
|
-
}
|
|
139
|
-
}
|
|
140
|
-
/**
|
|
141
|
-
* Executes an out-of-process 'run:' step via the system execution driver.
|
|
142
|
-
*/
|
|
143
|
-
async function executeRunStep(params) {
|
|
144
|
-
const { workerId, jobId, step, stepId, stepName, executionContext, driver, queue, config } = params;
|
|
145
|
-
// Evaluate step environment variables using deterministic evaluateValue rule
|
|
146
|
-
const evaluatedStepEnv = {};
|
|
147
|
-
if (step.env) {
|
|
148
|
-
for (const [key, val] of Object.entries(step.env)) {
|
|
149
|
-
evaluatedStepEnv[key] = String(await SafeExpressionEvaluator.evaluateValue(val, executionContext));
|
|
150
|
-
}
|
|
151
|
-
}
|
|
152
|
-
const stepCtx = {
|
|
153
|
-
jobId: jobId.toString(),
|
|
154
|
-
stepId,
|
|
155
|
-
workspacePath: `${config.storagePath || '/tmp/workspaces'}/job-${jobId}`,
|
|
156
|
-
command: step.run,
|
|
157
|
-
image: step.image,
|
|
158
|
-
env: {
|
|
159
|
-
...executionContext.env,
|
|
160
|
-
...evaluatedStepEnv,
|
|
161
|
-
},
|
|
162
|
-
timeoutMs: step.timeoutMs,
|
|
163
|
-
};
|
|
164
|
-
const handle = await driver.execute(stepCtx);
|
|
165
|
-
let isCancelled = false;
|
|
166
|
-
// Periodically poll SQLite for mid-run job cancellation signals
|
|
167
|
-
const cancelCheckInterval = setInterval(async () => {
|
|
168
|
-
if (await queue.isCancelled(+jobId)) {
|
|
169
|
-
console.log(`[${workerId}] 🛑 Job #${jobId} was cancelled! Halting step execution.`);
|
|
170
|
-
isCancelled = true;
|
|
171
|
-
clearInterval(cancelCheckInterval);
|
|
172
|
-
await handle.cancel();
|
|
173
|
-
}
|
|
174
|
-
}, 3000);
|
|
175
|
-
const result = await handle.done;
|
|
176
|
-
clearInterval(cancelCheckInterval);
|
|
177
|
-
const failed = result.exitCode !== 0 || isCancelled;
|
|
178
|
-
const stepStatus = result.exitCode === 0 ? 'success' : isCancelled ? 'cancelled' : 'failed';
|
|
179
|
-
if (failed) {
|
|
180
|
-
console.error(`[${workerId}] ❌ Step [${stepId}] failed with status: ${stepStatus}`);
|
|
181
|
-
}
|
|
182
|
-
executionContext.steps[stepId] = { status: stepStatus, exitCode: result.exitCode };
|
|
183
|
-
return {
|
|
184
|
-
failed,
|
|
185
|
-
isCancelled,
|
|
186
|
-
report: {
|
|
187
|
-
id: stepId,
|
|
188
|
-
name: stepName,
|
|
189
|
-
status: stepStatus,
|
|
190
|
-
durationMs: result.durationMs,
|
|
191
|
-
exitCode: result.exitCode,
|
|
192
|
-
error: result.error?.message,
|
|
193
|
-
outputs: executionContext.steps[stepId]?.outputs || {},
|
|
194
|
-
logFilePath: handle.logFilePath,
|
|
195
|
-
},
|
|
196
|
-
};
|
|
197
|
-
}
|
|
198
|
-
/**
|
|
199
|
-
* Fills skipped status for remaining unexecuted steps.
|
|
200
|
-
*/
|
|
201
|
-
function fillSkippedSteps(steps, startIndex, stepReports) {
|
|
202
|
-
for (let j = startIndex; j < steps.length; j++) {
|
|
203
|
-
const skippedStep = steps[j];
|
|
204
|
-
const stepId = skippedStep.id || `step-${j}`;
|
|
205
|
-
stepReports.push({
|
|
206
|
-
id: stepId,
|
|
207
|
-
name: skippedStep.name || stepId,
|
|
208
|
-
status: 'skipped',
|
|
209
|
-
durationMs: 0,
|
|
210
|
-
exitCode: 0,
|
|
211
|
-
outputs: {},
|
|
212
|
-
logFilePath: '',
|
|
213
|
-
});
|
|
214
|
-
}
|
|
215
|
-
}
|
|
216
|
-
/**
|
|
217
|
-
* Assembles the frozen WorkflowExecutionReport JSON payload.
|
|
218
|
-
*/
|
|
219
|
-
function buildExecutionReport(job, status, startTime, inputs, environment, stepReports, payload) {
|
|
220
|
-
return {
|
|
221
|
-
jobId: job.id.toString(),
|
|
222
|
-
workflowName: job.workflow_id,
|
|
223
|
-
status: status,
|
|
224
|
-
durationMs: Date.now() - startTime,
|
|
225
|
-
startedAt: new Date(startTime).toISOString(),
|
|
226
|
-
finishedAt: new Date().toISOString(),
|
|
227
|
-
inputs,
|
|
228
|
-
environment,
|
|
229
|
-
steps: stepReports,
|
|
230
|
-
artifacts: [],
|
|
231
|
-
rerunToken: JSON.stringify({ jobId: job.id, payload }),
|
|
232
|
-
};
|
|
233
|
-
}
|
|
234
|
-
/**
|
|
235
|
-
* Dispatches the execution report concurrently to all registered reporters.
|
|
236
|
-
*/
|
|
237
|
-
async function dispatchReporters(workerId, reporters = [], report) {
|
|
238
|
-
if (!Array.isArray(reporters) || reporters.length === 0)
|
|
239
|
-
return;
|
|
240
|
-
console.log(`[${workerId}] 📢 Dispatching execution report to ${reporters.length} reporter(s)...`);
|
|
241
|
-
await Promise.allSettled(reporters.map(async (reporter) => {
|
|
242
|
-
try {
|
|
243
|
-
await reporter.report(report);
|
|
244
|
-
}
|
|
245
|
-
catch (err) {
|
|
246
|
-
console.error(`[${workerId}] ⚠️ Reporter '${reporter.name || 'unknown'}' failed:`, err.message);
|
|
247
|
-
}
|
|
248
|
-
}));
|
|
249
|
-
}
|
|
File without changes
|