@ryanfw/prompt-orchestration-pipeline 0.15.0 → 0.15.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/core/orchestrator.js +0 -18
- package/src/core/pipeline-runner.js +0 -37
- package/src/core/status-writer.js +0 -18
- package/src/core/symlink-utils.js +0 -12
- package/src/core/task-runner.js +0 -23
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ryanfw/prompt-orchestration-pipeline",
|
|
3
|
-
"version": "0.15.
|
|
3
|
+
"version": "0.15.1",
|
|
4
4
|
"description": "A Prompt-orchestration pipeline (POP) is a framework for building, running, and experimenting with complex chains of LLM tasks.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/ui/server.js",
|
package/src/core/orchestrator.js
CHANGED
|
@@ -123,10 +123,8 @@ export async function startOrchestrator(opts) {
|
|
|
123
123
|
} catch {}
|
|
124
124
|
|
|
125
125
|
// Move seed to current/{jobId}/seed.json
|
|
126
|
-
logger.log("Moving file", { from: filePath, to: dest });
|
|
127
126
|
try {
|
|
128
127
|
await moveFile(filePath, dest);
|
|
129
|
-
logger.log("Successfully moved file", { destination: dest });
|
|
130
128
|
} catch (error) {
|
|
131
129
|
logger.error("Failed to move file", {
|
|
132
130
|
from: filePath,
|
|
@@ -173,12 +171,6 @@ export async function startOrchestrator(opts) {
|
|
|
173
171
|
// Apply artifact initialization to the status
|
|
174
172
|
const updatedStatus = applyArtifacts(status);
|
|
175
173
|
await fs.writeFile(statusPath, JSON.stringify(updatedStatus, null, 2));
|
|
176
|
-
|
|
177
|
-
logger.log("Initialized status from upload artifacts", {
|
|
178
|
-
jobId,
|
|
179
|
-
pipeline: seed?.pipeline,
|
|
180
|
-
artifactsCount: updatedStatus.files?.artifacts?.length || 0,
|
|
181
|
-
});
|
|
182
174
|
} catch (artifactError) {
|
|
183
175
|
// Don't fail job startup if artifact initialization fails, just log
|
|
184
176
|
logger.warn("Failed to initialize status from artifacts", {
|
|
@@ -233,7 +225,6 @@ export async function startOrchestrator(opts) {
|
|
|
233
225
|
|
|
234
226
|
// Watch pending directory for seeds
|
|
235
227
|
const watchPattern = path.join(dirs.pending, "*.json");
|
|
236
|
-
logger.log("Watching pattern", { pattern: watchPattern });
|
|
237
228
|
const watcher = watcherFactory(watchPattern, {
|
|
238
229
|
ignoreInitial: false,
|
|
239
230
|
awaitWriteFinish: false, // Disable awaitWriteFinish for faster detection
|
|
@@ -243,7 +234,6 @@ export async function startOrchestrator(opts) {
|
|
|
243
234
|
// Wait for watcher to be ready before resolving
|
|
244
235
|
await new Promise((resolve, reject) => {
|
|
245
236
|
watcher.on("ready", () => {
|
|
246
|
-
logger.log("Watcher is ready");
|
|
247
237
|
resolve();
|
|
248
238
|
});
|
|
249
239
|
|
|
@@ -254,7 +244,6 @@ export async function startOrchestrator(opts) {
|
|
|
254
244
|
});
|
|
255
245
|
|
|
256
246
|
watcher.on("add", (file) => {
|
|
257
|
-
logger.log("Detected file add", { file });
|
|
258
247
|
// Return promise so tests awaiting the add handler block until processing completes
|
|
259
248
|
return handleSeedAdd(file);
|
|
260
249
|
});
|
|
@@ -330,13 +319,6 @@ function spawnRunner(
|
|
|
330
319
|
const availablePipelines = Object.keys(configSnapshot?.pipelines ?? {});
|
|
331
320
|
const pipelineSlug = seed?.pipeline;
|
|
332
321
|
|
|
333
|
-
logger.log("spawnRunner invoked", {
|
|
334
|
-
jobId,
|
|
335
|
-
pipelineSlug: pipelineSlug ?? null,
|
|
336
|
-
availablePipelines,
|
|
337
|
-
seedKeys: seed ? Object.keys(seed) : null,
|
|
338
|
-
});
|
|
339
|
-
|
|
340
322
|
if (!availablePipelines.length) {
|
|
341
323
|
logger.warn(
|
|
342
324
|
"No pipelines registered in config() when spawnRunner invoked"
|
|
@@ -143,10 +143,6 @@ try {
|
|
|
143
143
|
startFromTask &&
|
|
144
144
|
taskNames.indexOf(taskName) < taskNames.indexOf(startFromTask)
|
|
145
145
|
) {
|
|
146
|
-
logger.log("Skipping task before restart point", {
|
|
147
|
-
taskName,
|
|
148
|
-
startFromTask,
|
|
149
|
-
});
|
|
150
146
|
continue;
|
|
151
147
|
}
|
|
152
148
|
|
|
@@ -155,7 +151,6 @@ try {
|
|
|
155
151
|
const outputPath = path.join(workDir, "tasks", taskName, "output.json");
|
|
156
152
|
const output = JSON.parse(await fs.readFile(outputPath, "utf8"));
|
|
157
153
|
pipelineArtifacts[taskName] = output;
|
|
158
|
-
logger.log("Task already completed", { taskName });
|
|
159
154
|
} catch {
|
|
160
155
|
logger.warn("Failed to read completed task output", { taskName });
|
|
161
156
|
}
|
|
@@ -190,7 +185,6 @@ try {
|
|
|
190
185
|
}
|
|
191
186
|
}
|
|
192
187
|
|
|
193
|
-
logger.log("Starting task", { taskName });
|
|
194
188
|
await updateStatus(taskName, {
|
|
195
189
|
state: TaskState.RUNNING,
|
|
196
190
|
startedAt: now(),
|
|
@@ -265,13 +259,6 @@ try {
|
|
|
265
259
|
process.exitCode = 1;
|
|
266
260
|
process.exit(1);
|
|
267
261
|
}
|
|
268
|
-
|
|
269
|
-
logger.log("Task symlinks repaired successfully", {
|
|
270
|
-
taskName,
|
|
271
|
-
taskDir,
|
|
272
|
-
repairDuration: repairResult.duration,
|
|
273
|
-
relocatedEntry: repairResult.relocatedEntry,
|
|
274
|
-
});
|
|
275
262
|
} else {
|
|
276
263
|
logger.debug("Task symlinks validation passed", {
|
|
277
264
|
taskName,
|
|
@@ -295,7 +282,6 @@ try {
|
|
|
295
282
|
statusPath: tasksStatusPath,
|
|
296
283
|
});
|
|
297
284
|
|
|
298
|
-
logger.log("Running task", { taskName, modulePath: absoluteModulePath });
|
|
299
285
|
const result = await runPipeline(relocatedEntry, ctx);
|
|
300
286
|
|
|
301
287
|
if (!result.ok) {
|
|
@@ -363,13 +349,6 @@ try {
|
|
|
363
349
|
process.exit(1);
|
|
364
350
|
}
|
|
365
351
|
|
|
366
|
-
logger.log("Task completed successfully", {
|
|
367
|
-
taskName,
|
|
368
|
-
executionTimeMs:
|
|
369
|
-
result.logs?.reduce((total, log) => total + (log.ms || 0), 0) || 0,
|
|
370
|
-
refinementAttempts: result.refinementAttempts || 0,
|
|
371
|
-
});
|
|
372
|
-
|
|
373
352
|
// The file I/O system automatically handles writing outputs and updating tasks-status.json
|
|
374
353
|
// No need to manually write output.json or enumerate artifacts
|
|
375
354
|
|
|
@@ -396,7 +375,6 @@ try {
|
|
|
396
375
|
|
|
397
376
|
// Check if this is a single task run and we've completed the target task
|
|
398
377
|
if (runSingleTask && taskName === startFromTask) {
|
|
399
|
-
logger.log("Stopping after single task execution", { taskName });
|
|
400
378
|
break;
|
|
401
379
|
}
|
|
402
380
|
} catch (err) {
|
|
@@ -415,19 +393,6 @@ try {
|
|
|
415
393
|
await fs.mkdir(COMPLETE_DIR, { recursive: true });
|
|
416
394
|
const dest = path.join(COMPLETE_DIR, jobId);
|
|
417
395
|
|
|
418
|
-
logger.log("Pipeline completed", {
|
|
419
|
-
jobId,
|
|
420
|
-
totalExecutionTime: Object.values(status.tasks).reduce(
|
|
421
|
-
(total, t) => total + (t.executionTimeMs || 0),
|
|
422
|
-
0
|
|
423
|
-
),
|
|
424
|
-
totalRefinementAttempts: Object.values(status.tasks).reduce(
|
|
425
|
-
(total, t) => total + (t.refinementAttempts || 0),
|
|
426
|
-
0
|
|
427
|
-
),
|
|
428
|
-
finalArtifacts: Object.keys(pipelineArtifacts),
|
|
429
|
-
});
|
|
430
|
-
|
|
431
396
|
await fs.rename(workDir, dest);
|
|
432
397
|
await appendLine(
|
|
433
398
|
path.join(COMPLETE_DIR, "runs.jsonl"),
|
|
@@ -449,8 +414,6 @@ try {
|
|
|
449
414
|
|
|
450
415
|
// Clean up task symlinks to avoid dangling links in archives
|
|
451
416
|
await cleanupTaskSymlinks(dest);
|
|
452
|
-
} else {
|
|
453
|
-
logger.log("Single task run completed, job remains in current", { jobId });
|
|
454
417
|
}
|
|
455
418
|
} catch (error) {
|
|
456
419
|
throw error;
|
|
@@ -164,13 +164,8 @@ export async function writeJobStatus(jobDir, updateFn) {
|
|
|
164
164
|
|
|
165
165
|
const next = prev
|
|
166
166
|
.then(async () => {
|
|
167
|
-
logger.group("Status Write Operation");
|
|
168
|
-
logger.log(`Updating status for job: ${jobId}`);
|
|
169
|
-
logger.log(`Status file path: ${statusPath}`);
|
|
170
|
-
|
|
171
167
|
// Read existing status or create default
|
|
172
168
|
const current = await readStatusFile(statusPath, jobId);
|
|
173
|
-
logger.log("Current status snapshot:", current);
|
|
174
169
|
|
|
175
170
|
// Validate basic structure
|
|
176
171
|
const validated = validateStatusSnapshot(current);
|
|
@@ -188,11 +183,9 @@ export async function writeJobStatus(jobDir, updateFn) {
|
|
|
188
183
|
);
|
|
189
184
|
|
|
190
185
|
snapshot.lastUpdated = new Date().toISOString();
|
|
191
|
-
logger.log("Status after update function:", snapshot);
|
|
192
186
|
|
|
193
187
|
// Atomic write
|
|
194
188
|
await atomicWrite(statusPath, snapshot);
|
|
195
|
-
logger.log("Status file written successfully");
|
|
196
189
|
|
|
197
190
|
// Emit SSE event for tasks-status.json change using logger
|
|
198
191
|
try {
|
|
@@ -202,7 +195,6 @@ export async function writeJobStatus(jobDir, updateFn) {
|
|
|
202
195
|
jobId,
|
|
203
196
|
};
|
|
204
197
|
await logger.sse("state:change", eventData);
|
|
205
|
-
logger.log("SSE event broadcasted successfully");
|
|
206
198
|
} catch (error) {
|
|
207
199
|
// Don't fail the write if SSE emission fails
|
|
208
200
|
logger.error("Failed to emit SSE event:", error);
|
|
@@ -218,10 +210,6 @@ export async function writeJobStatus(jobDir, updateFn) {
|
|
|
218
210
|
reason: snapshot.lifecycleBlockReason,
|
|
219
211
|
};
|
|
220
212
|
await logger.sse("lifecycle_block", lifecycleEventData);
|
|
221
|
-
logger.log(
|
|
222
|
-
"lifecycle_block SSE event broadcasted successfully",
|
|
223
|
-
lifecycleEventData
|
|
224
|
-
);
|
|
225
213
|
} catch (error) {
|
|
226
214
|
// Don't fail the write if SSE emission fails
|
|
227
215
|
logger.error("Failed to emit lifecycle_block SSE event:", error);
|
|
@@ -310,9 +298,6 @@ export async function updateTaskStatus(jobDir, taskId, taskUpdateFn) {
|
|
|
310
298
|
|
|
311
299
|
const next = prev
|
|
312
300
|
.then(async () => {
|
|
313
|
-
logger.group("Task Status Update Operation");
|
|
314
|
-
logger.log(`Updating task ${taskId} for job: ${jobId}`);
|
|
315
|
-
|
|
316
301
|
const statusPath = path.join(jobDir, "tasks-status.json");
|
|
317
302
|
|
|
318
303
|
// Read existing status or create default
|
|
@@ -336,7 +321,6 @@ export async function updateTaskStatus(jobDir, taskId, taskUpdateFn) {
|
|
|
336
321
|
|
|
337
322
|
// Atomic write
|
|
338
323
|
await atomicWrite(statusPath, validated);
|
|
339
|
-
logger.log("Task status file written successfully");
|
|
340
324
|
|
|
341
325
|
// Emit task:updated SSE event after successful write
|
|
342
326
|
try {
|
|
@@ -346,13 +330,11 @@ export async function updateTaskStatus(jobDir, taskId, taskUpdateFn) {
|
|
|
346
330
|
task: validated.tasks[taskId],
|
|
347
331
|
};
|
|
348
332
|
await logger.sse("task:updated", eventData);
|
|
349
|
-
logger.log("task:updated SSE event broadcasted successfully");
|
|
350
333
|
} catch (error) {
|
|
351
334
|
// Don't fail the write if SSE emission fails
|
|
352
335
|
logger.error("Failed to emit task:updated SSE event:", error);
|
|
353
336
|
}
|
|
354
337
|
|
|
355
|
-
logger.groupEnd();
|
|
356
338
|
resultSnapshot = validated;
|
|
357
339
|
})
|
|
358
340
|
.catch((e) => {
|
|
@@ -196,12 +196,6 @@ export async function repairTaskSymlinks(taskDir, poRoot, taskModulePath) {
|
|
|
196
196
|
const startTime = Date.now();
|
|
197
197
|
|
|
198
198
|
try {
|
|
199
|
-
logger.log("Repairing task symlinks", {
|
|
200
|
-
taskDir,
|
|
201
|
-
poRoot,
|
|
202
|
-
taskModulePath,
|
|
203
|
-
});
|
|
204
|
-
|
|
205
199
|
// Use existing ensureTaskSymlinkBridge for repairs
|
|
206
200
|
const relocatedEntry = await ensureTaskSymlinkBridge({
|
|
207
201
|
taskDir,
|
|
@@ -211,12 +205,6 @@ export async function repairTaskSymlinks(taskDir, poRoot, taskModulePath) {
|
|
|
211
205
|
|
|
212
206
|
const duration = Date.now() - startTime;
|
|
213
207
|
|
|
214
|
-
logger.log("Task symlinks repaired successfully", {
|
|
215
|
-
taskDir,
|
|
216
|
-
duration,
|
|
217
|
-
relocatedEntry,
|
|
218
|
-
});
|
|
219
|
-
|
|
220
208
|
return {
|
|
221
209
|
success: true,
|
|
222
210
|
relocatedEntry,
|
package/src/core/task-runner.js
CHANGED
|
@@ -484,10 +484,6 @@ export async function runPipeline(modulePath, initialContext = {}) {
|
|
|
484
484
|
|
|
485
485
|
// Skip stages when skipIf predicate returns true
|
|
486
486
|
if (stageConfig.skipIf && stageConfig.skipIf(context.flags)) {
|
|
487
|
-
logger.log("Skipping stage", {
|
|
488
|
-
stage: stageName,
|
|
489
|
-
reason: "skipIf predicate returned true",
|
|
490
|
-
});
|
|
491
487
|
context.logs.push({
|
|
492
488
|
stage: stageName,
|
|
493
489
|
action: "skipped",
|
|
@@ -499,7 +495,6 @@ export async function runPipeline(modulePath, initialContext = {}) {
|
|
|
499
495
|
|
|
500
496
|
// Skip if handler is not available (not implemented)
|
|
501
497
|
if (typeof stageHandler !== "function") {
|
|
502
|
-
logger.log("Stage not available, skipping", { stage: stageName });
|
|
503
498
|
logs.push({
|
|
504
499
|
stage: stageName,
|
|
505
500
|
skipped: true,
|
|
@@ -526,11 +521,6 @@ export async function runPipeline(modulePath, initialContext = {}) {
|
|
|
526
521
|
// Set current stage before execution
|
|
527
522
|
context.currentStage = stageName;
|
|
528
523
|
|
|
529
|
-
logger.log("Starting stage execution", {
|
|
530
|
-
stage: stageName,
|
|
531
|
-
taskName: context.meta.taskName,
|
|
532
|
-
});
|
|
533
|
-
|
|
534
524
|
// Write stage start status using writeJobStatus
|
|
535
525
|
if (context.meta.workDir && context.meta.taskName) {
|
|
536
526
|
try {
|
|
@@ -719,12 +709,6 @@ export async function runPipeline(modulePath, initialContext = {}) {
|
|
|
719
709
|
);
|
|
720
710
|
|
|
721
711
|
const ms = +(performance.now() - start).toFixed(2);
|
|
722
|
-
logger.log("Stage completed successfully", {
|
|
723
|
-
stage: stageName,
|
|
724
|
-
executionTimeMs: ms,
|
|
725
|
-
outputType: typeof stageResult.output,
|
|
726
|
-
flagKeys: Object.keys(stageResult.flags),
|
|
727
|
-
});
|
|
728
712
|
logs.push({
|
|
729
713
|
stage: stageName,
|
|
730
714
|
ok: true,
|
|
@@ -819,13 +803,6 @@ export async function runPipeline(modulePath, initialContext = {}) {
|
|
|
819
803
|
|
|
820
804
|
llmEvents.off("llm:request:complete", onLLMComplete);
|
|
821
805
|
|
|
822
|
-
logger.log("Pipeline completed successfully", {
|
|
823
|
-
taskName: context.meta.taskName,
|
|
824
|
-
totalStages: PIPELINE_STAGES.length,
|
|
825
|
-
executedStages: logs.filter((l) => l.ok).length,
|
|
826
|
-
llmMetricsCount: llmMetrics.length,
|
|
827
|
-
});
|
|
828
|
-
|
|
829
806
|
// Write final status with currentStage: null to indicate completion
|
|
830
807
|
if (context.meta.workDir && context.meta.taskName) {
|
|
831
808
|
try {
|