@pikku/core 0.12.13 → 0.12.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +46 -0
- package/dist/errors/errors.d.ts +18 -0
- package/dist/errors/errors.js +33 -0
- package/dist/function/function-runner.d.ts +1 -1
- package/dist/function/function-runner.js +10 -5
- package/dist/function/functions.types.js +1 -0
- package/dist/function/index.d.ts +2 -1
- package/dist/function/index.js +1 -0
- package/dist/handle-error.d.ts +2 -2
- package/dist/handle-error.js +8 -8
- package/dist/index.d.ts +1 -2
- package/dist/index.js +1 -2
- package/dist/middleware/index.d.ts +2 -0
- package/dist/middleware/index.js +2 -0
- package/dist/middleware/remote-auth.js +5 -2
- package/dist/permissions.d.ts +13 -1
- package/dist/permissions.js +51 -0
- package/dist/pikku-state.d.ts +0 -1
- package/dist/pikku-state.js +2 -4
- package/dist/remote.d.ts +10 -0
- package/dist/remote.js +32 -0
- package/dist/schema.js +2 -0
- package/dist/services/credential-service.d.ts +11 -0
- package/dist/services/credential-wire-service.d.ts +14 -3
- package/dist/services/credential-wire-service.js +49 -6
- package/dist/services/deployment-service.d.ts +12 -1
- package/dist/services/in-memory-queue-service.d.ts +7 -0
- package/dist/services/in-memory-queue-service.js +28 -0
- package/dist/services/index.d.ts +4 -1
- package/dist/services/index.js +2 -1
- package/dist/services/local-credential-service.d.ts +2 -0
- package/dist/services/local-credential-service.js +20 -0
- package/dist/services/logger-console.d.ts +26 -40
- package/dist/services/logger-console.js +102 -46
- package/dist/services/logger.d.ts +5 -0
- package/dist/services/meta-service.d.ts +175 -0
- package/dist/services/meta-service.js +310 -0
- package/dist/services/pikku-user-id.d.ts +3 -0
- package/dist/services/pikku-user-id.js +16 -0
- package/dist/services/typed-credential-service.d.ts +2 -0
- package/dist/services/typed-credential-service.js +6 -0
- package/dist/services/workflow-service.d.ts +6 -1
- package/dist/testing/service-tests.js +0 -8
- package/dist/types/core.types.d.ts +14 -2
- package/dist/types/state.types.d.ts +9 -2
- package/dist/wirings/ai-agent/ai-agent-prepare.d.ts +21 -0
- package/dist/wirings/ai-agent/ai-agent-prepare.js +90 -21
- package/dist/wirings/ai-agent/ai-agent-registry.js +2 -2
- package/dist/wirings/ai-agent/ai-agent-runner.js +24 -2
- package/dist/wirings/ai-agent/ai-agent-stream.d.ts +2 -1
- package/dist/wirings/ai-agent/ai-agent-stream.js +61 -3
- package/dist/wirings/ai-agent/ai-agent.types.d.ts +25 -4
- package/dist/wirings/ai-agent/index.d.ts +1 -1
- package/dist/wirings/ai-agent/index.js +1 -1
- package/dist/wirings/channel/channel-runner.js +3 -3
- package/dist/wirings/cli/cli-runner.js +4 -3
- package/dist/wirings/cli/command-parser.js +7 -3
- package/dist/wirings/http/http-runner.d.ts +1 -1
- package/dist/wirings/http/http-runner.js +23 -15
- package/dist/wirings/http/http.types.d.ts +2 -0
- package/dist/wirings/mcp/mcp-runner.js +5 -3
- package/dist/wirings/queue/queue-runner.d.ts +3 -1
- package/dist/wirings/queue/queue-runner.js +8 -3
- package/dist/wirings/queue/queue.types.d.ts +6 -0
- package/dist/wirings/rpc/rpc-runner.js +57 -90
- package/dist/wirings/scheduler/scheduler-runner.d.ts +3 -1
- package/dist/wirings/scheduler/scheduler-runner.js +9 -5
- package/dist/wirings/trigger/trigger-runner.js +4 -2
- package/dist/wirings/workflow/dsl/workflow-dsl.types.d.ts +2 -0
- package/dist/wirings/workflow/dsl/workflow-runner.d.ts +1 -1
- package/dist/wirings/workflow/dsl/workflow-runner.js +6 -9
- package/dist/wirings/workflow/graph/graph-runner.d.ts +1 -1
- package/dist/wirings/workflow/graph/graph-runner.js +18 -4
- package/dist/wirings/workflow/index.d.ts +4 -2
- package/dist/wirings/workflow/index.js +4 -2
- package/dist/wirings/workflow/pikku-workflow-service.d.ts +16 -4
- package/dist/wirings/workflow/pikku-workflow-service.js +218 -24
- package/dist/wirings/workflow/workflow-queue-workers.d.ts +40 -0
- package/dist/wirings/workflow/workflow-queue-workers.js +41 -0
- package/dist/wirings/workflow/workflow.types.d.ts +19 -1
- package/package.json +4 -1
- package/src/errors/errors.ts +44 -0
- package/src/function/function-runner.ts +24 -13
- package/src/function/functions.types.ts +1 -0
- package/src/function/index.ts +10 -0
- package/src/handle-error.test.ts +2 -2
- package/src/handle-error.ts +8 -8
- package/src/index.ts +1 -2
- package/src/middleware/index.ts +2 -0
- package/src/middleware/remote-auth.test.ts +4 -4
- package/src/middleware/remote-auth.ts +7 -2
- package/src/permissions.ts +70 -0
- package/src/pikku-state.test.ts +0 -26
- package/src/pikku-state.ts +2 -5
- package/src/remote.ts +46 -0
- package/src/schema.ts +1 -0
- package/src/services/credential-service.ts +13 -0
- package/src/services/credential-wire-service.test.ts +174 -0
- package/src/services/credential-wire-service.ts +52 -8
- package/src/services/deployment-service.ts +17 -1
- package/src/services/in-memory-queue-service.ts +46 -0
- package/src/services/index.ts +21 -1
- package/src/services/local-credential-service.ts +22 -0
- package/src/services/logger-console.ts +127 -47
- package/src/services/logger.ts +6 -0
- package/src/services/meta-service.ts +523 -0
- package/src/services/pikku-user-id.test.ts +62 -0
- package/src/services/pikku-user-id.ts +17 -0
- package/src/services/typed-credential-service.ts +8 -0
- package/src/services/workflow-service.ts +8 -0
- package/src/testing/service-tests.ts +0 -10
- package/src/types/core.types.ts +16 -2
- package/src/types/state.types.ts +7 -2
- package/src/wirings/ai-agent/ai-agent-prepare.ts +122 -41
- package/src/wirings/ai-agent/ai-agent-registry.ts +3 -3
- package/src/wirings/ai-agent/ai-agent-runner.ts +22 -3
- package/src/wirings/ai-agent/ai-agent-stream.ts +115 -3
- package/src/wirings/ai-agent/ai-agent.types.ts +27 -4
- package/src/wirings/ai-agent/index.ts +1 -0
- package/src/wirings/channel/channel-handler.ts +0 -1
- package/src/wirings/channel/channel-runner.ts +4 -5
- package/src/wirings/cli/cli-runner.test.ts +8 -7
- package/src/wirings/cli/cli-runner.ts +5 -4
- package/src/wirings/cli/command-parser.ts +8 -3
- package/src/wirings/http/http-runner.ts +27 -18
- package/src/wirings/http/http.types.ts +2 -0
- package/src/wirings/mcp/mcp-runner.ts +7 -10
- package/src/wirings/queue/queue-runner.test.ts +5 -11
- package/src/wirings/queue/queue-runner.ts +12 -4
- package/src/wirings/queue/queue.types.ts +6 -0
- package/src/wirings/rpc/rpc-runner.ts +91 -118
- package/src/wirings/scheduler/scheduler-runner.test.ts +5 -11
- package/src/wirings/scheduler/scheduler-runner.ts +13 -5
- package/src/wirings/trigger/trigger-runner.test.ts +10 -11
- package/src/wirings/trigger/trigger-runner.ts +6 -4
- package/src/wirings/workflow/dsl/workflow-dsl.types.ts +2 -0
- package/src/wirings/workflow/dsl/workflow-runner.ts +11 -10
- package/src/wirings/workflow/graph/graph-runner.ts +19 -4
- package/src/wirings/workflow/index.ts +17 -6
- package/src/wirings/workflow/pikku-workflow-service.ts +286 -24
- package/src/wirings/workflow/workflow-queue-workers.ts +85 -0
- package/src/wirings/workflow/workflow.types.ts +18 -1
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/wirings/ai-agent/agent-dynamic-workflow.d.ts +0 -6
- package/dist/wirings/ai-agent/agent-dynamic-workflow.js +0 -468
- package/dist/wirings/workflow/workflow-helpers.d.ts +0 -36
- package/dist/wirings/workflow/workflow-helpers.js +0 -38
- package/src/wirings/ai-agent/agent-dynamic-workflow.ts +0 -610
- package/src/wirings/workflow/workflow-helpers.test.ts +0 -129
- package/src/wirings/workflow/workflow-helpers.ts +0 -79
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { SerializedError } from '../../types/core.types.js';
|
|
2
|
-
import type { PikkuWorkflowWire, StepState, WorkflowRun, WorkflowRunWire, WorkflowStatus, WorkflowVersionStatus, WorkflowStepOptions } from './workflow.types.js';
|
|
2
|
+
import type { PikkuWorkflowWire, StepState, WorkflowRun, WorkflowRunStatus, WorkflowRunWire, WorkflowStatus, WorkflowVersionStatus, WorkflowStepOptions } from './workflow.types.js';
|
|
3
3
|
import type { WorkflowService } from '../../services/workflow-service.js';
|
|
4
4
|
import { PikkuError } from '../../errors/error-handler.js';
|
|
5
5
|
/**
|
|
@@ -69,6 +69,11 @@ export declare abstract class PikkuWorkflowService implements WorkflowService {
|
|
|
69
69
|
* @returns Workflow run or null if not found
|
|
70
70
|
*/
|
|
71
71
|
abstract getRun(id: string): Promise<WorkflowRun | null>;
|
|
72
|
+
/**
|
|
73
|
+
* Get minimal workflow run status with step summaries.
|
|
74
|
+
* Used by the public API — the console addon provides the full verbose view.
|
|
75
|
+
*/
|
|
76
|
+
getRunStatus(id: string): Promise<WorkflowRunStatus | null>;
|
|
72
77
|
/**
|
|
73
78
|
* Get workflow run history (all step attempts in chronological order)
|
|
74
79
|
* @param runId - Run ID
|
|
@@ -219,7 +224,7 @@ export declare abstract class PikkuWorkflowService implements WorkflowService {
|
|
|
219
224
|
* Resume a paused workflow by triggering the orchestrator
|
|
220
225
|
* @param runId - Run ID
|
|
221
226
|
*/
|
|
222
|
-
resumeWorkflow(runId: string): Promise<void>;
|
|
227
|
+
resumeWorkflow(runId: string, workflowName?: string): Promise<void>;
|
|
223
228
|
queueStepWorker(runId: string, stepName: string, rpcName: string, data: any): Promise<void>;
|
|
224
229
|
/**
|
|
225
230
|
* Execute a workflow sleep step completion
|
|
@@ -232,7 +237,7 @@ export declare abstract class PikkuWorkflowService implements WorkflowService {
|
|
|
232
237
|
* @param runId - Run ID
|
|
233
238
|
* @param retryDelay - Delay in milliseconds or duration string (optional)
|
|
234
239
|
*/
|
|
235
|
-
protected scheduleOrchestratorRetry(runId: string, retryDelay?: number | string): Promise<void>;
|
|
240
|
+
protected scheduleOrchestratorRetry(runId: string, retryDelay?: number | string, workflowName?: string): Promise<void>;
|
|
236
241
|
/**
|
|
237
242
|
* Start a new workflow run
|
|
238
243
|
* Automatically detects workflow type (DSL or graph) from meta and executes accordingly
|
|
@@ -269,7 +274,14 @@ export declare abstract class PikkuWorkflowService implements WorkflowService {
|
|
|
269
274
|
private sleepStep;
|
|
270
275
|
private getSuspendStepName;
|
|
271
276
|
private suspendStep;
|
|
272
|
-
createWorkflowWire(name: string, runId: string, rpcService: any): PikkuWorkflowWire;
|
|
277
|
+
createWorkflowWire(name: string, runId: string, rpcService: any, addonNamespace?: string | null): PikkuWorkflowWire;
|
|
273
278
|
private verifyStepName;
|
|
274
279
|
private getConfig;
|
|
280
|
+
/**
|
|
281
|
+
* Get the orchestrator queue name for a specific workflow.
|
|
282
|
+
* Checks queue meta for a per-workflow queue first (e.g. wf-orchestrator-{name}),
|
|
283
|
+
* falls back to the shared orchestrator queue.
|
|
284
|
+
*/
|
|
285
|
+
protected getOrchestratorQueueName(workflowName?: string): string;
|
|
286
|
+
protected getStepWorkerQueueName(rpcName?: string): string;
|
|
275
287
|
}
|
|
@@ -1,6 +1,33 @@
|
|
|
1
|
-
import { runPikkuFunc } from '../../function/function-runner.js';
|
|
1
|
+
import { runPikkuFunc, addFunction } from '../../function/function-runner.js';
|
|
2
|
+
import { pikkuWorkflowWorkerFunc, pikkuWorkflowOrchestratorFunc, pikkuWorkflowSleeperFunc, } from './workflow-queue-workers.js';
|
|
3
|
+
import { wireQueueWorker } from '../queue/queue-runner.js';
|
|
2
4
|
import { getSingletonServices, getCreateWireServices, pikkuState, } from '../../pikku-state.js';
|
|
3
5
|
import { getDurationInMilliseconds } from '../../time-utils.js';
|
|
6
|
+
const resolveWorkflowMeta = (name) => {
|
|
7
|
+
const rootMeta = pikkuState(null, 'workflows', 'meta');
|
|
8
|
+
if (rootMeta[name]) {
|
|
9
|
+
return { meta: rootMeta[name], packageName: null, resolvedName: name };
|
|
10
|
+
}
|
|
11
|
+
const colonIndex = name.indexOf(':');
|
|
12
|
+
if (colonIndex !== -1) {
|
|
13
|
+
const namespace = name.substring(0, colonIndex);
|
|
14
|
+
const localName = name.substring(colonIndex + 1);
|
|
15
|
+
const addons = pikkuState(null, 'addons', 'packages');
|
|
16
|
+
const pkgConfig = addons?.get(namespace);
|
|
17
|
+
if (pkgConfig) {
|
|
18
|
+
const addonMeta = pikkuState(pkgConfig.package, 'workflows', 'meta');
|
|
19
|
+
if (addonMeta?.[localName]) {
|
|
20
|
+
return {
|
|
21
|
+
meta: addonMeta[localName],
|
|
22
|
+
packageName: pkgConfig.package,
|
|
23
|
+
resolvedName: localName,
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
return null;
|
|
29
|
+
};
|
|
30
|
+
const toKebab = (s) => s.replace(/([a-z0-9])([A-Z])/g, '$1-$2').toLowerCase();
|
|
4
31
|
import { continueGraph, executeGraphStep, runWorkflowGraph, runFromMeta, } from './graph/graph-runner.js';
|
|
5
32
|
import { PikkuError, addError } from '../../errors/error-handler.js';
|
|
6
33
|
import { RPCNotFoundError } from '../rpc/rpc-runner.js';
|
|
@@ -88,7 +115,62 @@ export class PikkuWorkflowService {
|
|
|
88
115
|
get logger() {
|
|
89
116
|
return getSingletonServices()?.logger;
|
|
90
117
|
}
|
|
91
|
-
constructor() {
|
|
118
|
+
constructor() {
|
|
119
|
+
const functions = pikkuState(null, 'function', 'functions');
|
|
120
|
+
const functionsMeta = pikkuState(null, 'function', 'meta');
|
|
121
|
+
// Minimal meta for internal workflow functions (satisfies FunctionMeta)
|
|
122
|
+
const mkMeta = (funcId) => ({
|
|
123
|
+
pikkuFuncId: funcId,
|
|
124
|
+
sessionless: true,
|
|
125
|
+
functionType: 'helper',
|
|
126
|
+
inputSchemaName: null,
|
|
127
|
+
outputSchemaName: null,
|
|
128
|
+
});
|
|
129
|
+
const queueMeta = pikkuState(null, 'queue', 'meta');
|
|
130
|
+
const registerWorkflowFunc = (funcId, func, queueName) => {
|
|
131
|
+
if (functions.has(funcId))
|
|
132
|
+
return;
|
|
133
|
+
addFunction(funcId, func);
|
|
134
|
+
if (!queueMeta[queueName]) {
|
|
135
|
+
queueMeta[queueName] = { pikkuFuncId: funcId, name: queueName };
|
|
136
|
+
}
|
|
137
|
+
wireQueueWorker({ name: queueName, func });
|
|
138
|
+
if (!functionsMeta[funcId]) {
|
|
139
|
+
functionsMeta[funcId] = mkMeta(funcId);
|
|
140
|
+
}
|
|
141
|
+
};
|
|
142
|
+
// Register shared queue workers for monolith deployments
|
|
143
|
+
registerWorkflowFunc('pikkuWorkflowOrchestrator', { func: pikkuWorkflowOrchestratorFunc }, 'pikku-workflow-orchestrator');
|
|
144
|
+
registerWorkflowFunc('pikkuWorkflowStepWorker', { func: pikkuWorkflowWorkerFunc }, 'pikku-workflow-step-worker');
|
|
145
|
+
// Register per-workflow queue workers (root + addon packages)
|
|
146
|
+
const registerQueueWorkers = (queueMeta) => {
|
|
147
|
+
for (const [queueName, meta] of Object.entries(queueMeta)) {
|
|
148
|
+
if (functions.has(meta.pikkuFuncId))
|
|
149
|
+
continue;
|
|
150
|
+
if (queueName.startsWith('wf-orchestrator-')) {
|
|
151
|
+
registerWorkflowFunc(meta.pikkuFuncId, { func: pikkuWorkflowOrchestratorFunc }, queueName);
|
|
152
|
+
}
|
|
153
|
+
else if (queueName.startsWith('wf-step-')) {
|
|
154
|
+
registerWorkflowFunc(meta.pikkuFuncId, { func: pikkuWorkflowWorkerFunc }, queueName);
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
};
|
|
158
|
+
registerQueueWorkers(pikkuState(null, 'queue', 'meta'));
|
|
159
|
+
const addons = pikkuState(null, 'addons', 'packages');
|
|
160
|
+
if (addons) {
|
|
161
|
+
for (const [, addon] of addons) {
|
|
162
|
+
const addonQueueMeta = pikkuState(addon.package, 'queue', 'meta');
|
|
163
|
+
if (addonQueueMeta) {
|
|
164
|
+
registerQueueWorkers(addonQueueMeta);
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
if (!functions.has('pikkuWorkflowSleeper')) {
|
|
169
|
+
addFunction('pikkuWorkflowSleeper', {
|
|
170
|
+
func: pikkuWorkflowSleeperFunc,
|
|
171
|
+
});
|
|
172
|
+
}
|
|
173
|
+
}
|
|
92
174
|
/**
|
|
93
175
|
* Check if a run is executing inline (without queues)
|
|
94
176
|
*/
|
|
@@ -115,6 +197,47 @@ export class PikkuWorkflowService {
|
|
|
115
197
|
await this.upsertWorkflowVersion(name, meta.graphHash, meta, meta.source);
|
|
116
198
|
}
|
|
117
199
|
}
|
|
200
|
+
/**
|
|
201
|
+
* Get minimal workflow run status with step summaries.
|
|
202
|
+
* Used by the public API — the console addon provides the full verbose view.
|
|
203
|
+
*/
|
|
204
|
+
async getRunStatus(id) {
|
|
205
|
+
const run = await this.getRun(id);
|
|
206
|
+
if (!run)
|
|
207
|
+
return null;
|
|
208
|
+
const history = await this.getRunHistory(id);
|
|
209
|
+
const terminalStatuses = new Set(['completed', 'failed', 'cancelled']);
|
|
210
|
+
// Build step summaries from history (latest attempt per step)
|
|
211
|
+
const stepMap = new Map();
|
|
212
|
+
for (const step of history) {
|
|
213
|
+
const existing = stepMap.get(step.stepName);
|
|
214
|
+
if (!existing || step.updatedAt > existing.completedAt) {
|
|
215
|
+
stepMap.set(step.stepName, {
|
|
216
|
+
status: step.status,
|
|
217
|
+
startedAt: step.runningAt ?? step.createdAt,
|
|
218
|
+
completedAt: step.succeededAt ?? step.failedAt,
|
|
219
|
+
});
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
const steps = [...stepMap.entries()].map(([name, s]) => ({
|
|
223
|
+
name,
|
|
224
|
+
status: s.status,
|
|
225
|
+
duration: s.startedAt && s.completedAt
|
|
226
|
+
? s.completedAt.getTime() - s.startedAt.getTime()
|
|
227
|
+
: undefined,
|
|
228
|
+
}));
|
|
229
|
+
return {
|
|
230
|
+
id: run.id,
|
|
231
|
+
status: run.status,
|
|
232
|
+
startedAt: run.createdAt,
|
|
233
|
+
completedAt: terminalStatuses.has(run.status) ? run.updatedAt : undefined,
|
|
234
|
+
steps,
|
|
235
|
+
output: run.status === 'completed' ? run.output : undefined,
|
|
236
|
+
error: run.error
|
|
237
|
+
? { message: run.error.message ?? 'Unknown error' }
|
|
238
|
+
: undefined,
|
|
239
|
+
};
|
|
240
|
+
}
|
|
118
241
|
// ============================================================================
|
|
119
242
|
// Workflow Lifecycle Methods
|
|
120
243
|
// ============================================================================
|
|
@@ -122,13 +245,19 @@ export class PikkuWorkflowService {
|
|
|
122
245
|
* Resume a paused workflow by triggering the orchestrator
|
|
123
246
|
* @param runId - Run ID
|
|
124
247
|
*/
|
|
125
|
-
async resumeWorkflow(runId) {
|
|
248
|
+
async resumeWorkflow(runId, workflowName) {
|
|
126
249
|
const queueService = this.verifyQueueService();
|
|
127
|
-
|
|
250
|
+
if (!workflowName) {
|
|
251
|
+
const run = await this.getRun(runId);
|
|
252
|
+
workflowName = run?.workflow;
|
|
253
|
+
}
|
|
254
|
+
await queueService.add(this.getOrchestratorQueueName(workflowName), {
|
|
255
|
+
runId,
|
|
256
|
+
});
|
|
128
257
|
}
|
|
129
258
|
async queueStepWorker(runId, stepName, rpcName, data) {
|
|
130
259
|
const queueService = this.verifyQueueService();
|
|
131
|
-
await queueService.add(this.
|
|
260
|
+
await queueService.add(this.getStepWorkerQueueName(rpcName), JSON.parse(JSON.stringify({
|
|
132
261
|
runId,
|
|
133
262
|
stepName,
|
|
134
263
|
rpcName,
|
|
@@ -149,9 +278,13 @@ export class PikkuWorkflowService {
|
|
|
149
278
|
* @param runId - Run ID
|
|
150
279
|
* @param retryDelay - Delay in milliseconds or duration string (optional)
|
|
151
280
|
*/
|
|
152
|
-
async scheduleOrchestratorRetry(runId, retryDelay) {
|
|
281
|
+
async scheduleOrchestratorRetry(runId, retryDelay, workflowName) {
|
|
153
282
|
const queueService = this.verifyQueueService();
|
|
154
|
-
|
|
283
|
+
if (!workflowName) {
|
|
284
|
+
const run = await this.getRun(runId);
|
|
285
|
+
workflowName = run?.workflow;
|
|
286
|
+
}
|
|
287
|
+
await queueService.add(this.getOrchestratorQueueName(workflowName), { runId }, retryDelay ? { delay: getDurationInMilliseconds(retryDelay) } : undefined);
|
|
155
288
|
}
|
|
156
289
|
/**
|
|
157
290
|
* Start a new workflow run
|
|
@@ -160,21 +293,30 @@ export class PikkuWorkflowService {
|
|
|
160
293
|
* @param options.startNode - Starting node ID for graph workflows (from wire config)
|
|
161
294
|
*/
|
|
162
295
|
async startWorkflow(name, input, wire, rpcService, options) {
|
|
163
|
-
//
|
|
164
|
-
const
|
|
165
|
-
|
|
296
|
+
// Resolve workflow from static meta (root or addon namespace), then dynamic DB
|
|
297
|
+
const resolved = resolveWorkflowMeta(name);
|
|
298
|
+
let workflowMeta = resolved?.meta;
|
|
299
|
+
const packageName = resolved?.packageName ?? null;
|
|
300
|
+
if (!workflowMeta) {
|
|
301
|
+
const dynamicWorkflows = await this.getAIGeneratedWorkflows();
|
|
302
|
+
const match = dynamicWorkflows.find((w) => w.workflowName === name);
|
|
303
|
+
if (match?.graph) {
|
|
304
|
+
workflowMeta = match.graph;
|
|
305
|
+
}
|
|
306
|
+
}
|
|
166
307
|
if (!workflowMeta) {
|
|
167
308
|
throw new WorkflowNotFoundError(name);
|
|
168
309
|
}
|
|
169
|
-
if (workflowMeta.source === 'graph' ||
|
|
310
|
+
if (workflowMeta.source === 'graph' ||
|
|
311
|
+
workflowMeta.source === 'dynamic-workflow') {
|
|
170
312
|
const shouldInline = options?.inline ||
|
|
171
313
|
workflowMeta.inline ||
|
|
172
314
|
!getSingletonServices()?.queueService;
|
|
173
|
-
return runWorkflowGraph(this, name, input, rpcService, shouldInline, options?.startNode, wire);
|
|
315
|
+
return runWorkflowGraph(this, name, input, rpcService, shouldInline, options?.startNode, wire, workflowMeta);
|
|
174
316
|
}
|
|
175
317
|
// DSL workflow - check registration exists
|
|
176
|
-
const registrations = pikkuState(
|
|
177
|
-
const workflow = registrations.get(name);
|
|
318
|
+
const registrations = pikkuState(packageName, 'workflows', 'registrations');
|
|
319
|
+
const workflow = registrations.get(resolved?.resolvedName ?? name);
|
|
178
320
|
if (!workflow) {
|
|
179
321
|
throw new WorkflowNotFoundError(name);
|
|
180
322
|
}
|
|
@@ -231,15 +373,17 @@ export class PikkuWorkflowService {
|
|
|
231
373
|
if (!run) {
|
|
232
374
|
throw new WorkflowRunNotFoundError(runId);
|
|
233
375
|
}
|
|
234
|
-
const
|
|
235
|
-
const workflowMeta = meta
|
|
376
|
+
const resolved = resolveWorkflowMeta(run.workflow);
|
|
377
|
+
const workflowMeta = resolved?.meta;
|
|
378
|
+
const pkgName = resolved?.packageName ?? null;
|
|
236
379
|
if (run.graphHash &&
|
|
237
380
|
workflowMeta?.graphHash &&
|
|
238
381
|
run.graphHash !== workflowMeta.graphHash) {
|
|
239
382
|
await this.runVersionMismatchFallback(run, workflowMeta, rpcService);
|
|
240
383
|
return;
|
|
241
384
|
}
|
|
242
|
-
if (workflowMeta?.source === 'graph'
|
|
385
|
+
if (workflowMeta?.source === 'graph' ||
|
|
386
|
+
workflowMeta?.source === 'dynamic-workflow') {
|
|
243
387
|
await continueGraph(this, runId, run.workflow);
|
|
244
388
|
const updatedRun = await this.getRun(runId);
|
|
245
389
|
if (updatedRun?.status === 'completed') {
|
|
@@ -251,15 +395,36 @@ export class PikkuWorkflowService {
|
|
|
251
395
|
}
|
|
252
396
|
return;
|
|
253
397
|
}
|
|
254
|
-
|
|
255
|
-
|
|
398
|
+
if (!workflowMeta) {
|
|
399
|
+
const dynamicWorkflows = await this.getAIGeneratedWorkflows();
|
|
400
|
+
const match = dynamicWorkflows.find((w) => w.workflowName === run.workflow);
|
|
401
|
+
if (match?.graph) {
|
|
402
|
+
await continueGraph(this, runId, run.workflow, match.graph);
|
|
403
|
+
const updatedRun = await this.getRun(runId);
|
|
404
|
+
if (updatedRun?.status === 'completed') {
|
|
405
|
+
await this.onChildWorkflowCompleted(updatedRun, updatedRun.output);
|
|
406
|
+
}
|
|
407
|
+
else if (updatedRun?.status === 'failed' ||
|
|
408
|
+
updatedRun?.status === 'cancelled') {
|
|
409
|
+
await this.onChildWorkflowFailed(updatedRun, new Error(updatedRun.error?.message || 'Child workflow failed'));
|
|
410
|
+
}
|
|
411
|
+
return;
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
const registrations = pikkuState(pkgName, 'workflows', 'registrations');
|
|
415
|
+
const workflow = registrations.get(resolved?.resolvedName ?? run.workflow);
|
|
256
416
|
if (!workflow) {
|
|
257
417
|
throw new WorkflowNotFoundError(run.workflow);
|
|
258
418
|
}
|
|
259
419
|
await this.withRunLock(runId, async () => {
|
|
260
|
-
const
|
|
420
|
+
const addonNs = run.workflow.includes(':')
|
|
421
|
+
? run.workflow.substring(0, run.workflow.indexOf(':'))
|
|
422
|
+
: null;
|
|
423
|
+
const workflowWire = this.createWorkflowWire(run.workflow, runId, rpcService, addonNs);
|
|
424
|
+
workflowWire.pikkuUserId = run.wire?.pikkuUserId;
|
|
261
425
|
const wire = {
|
|
262
426
|
workflow: workflowWire,
|
|
427
|
+
pikkuUserId: run.wire?.pikkuUserId,
|
|
263
428
|
session: rpcService.wire?.session,
|
|
264
429
|
rpc: rpcService.wire?.rpc,
|
|
265
430
|
};
|
|
@@ -269,6 +434,7 @@ export class PikkuWorkflowService {
|
|
|
269
434
|
wire,
|
|
270
435
|
createWireServices: getCreateWireServices(),
|
|
271
436
|
data: () => run.input,
|
|
437
|
+
packageName: pkgName ?? undefined,
|
|
272
438
|
});
|
|
273
439
|
await this.updateRunStatus(runId, 'completed', result);
|
|
274
440
|
await this.onChildWorkflowCompleted(run, result);
|
|
@@ -374,7 +540,7 @@ export class PikkuWorkflowService {
|
|
|
374
540
|
const meta = pikkuState(null, 'workflows', 'meta');
|
|
375
541
|
const workflowMeta = meta[run.workflow];
|
|
376
542
|
const isGraphWorkflow = workflowMeta?.source === 'graph' ||
|
|
377
|
-
workflowMeta?.source === '
|
|
543
|
+
workflowMeta?.source === 'dynamic-workflow';
|
|
378
544
|
if (isGraphWorkflow &&
|
|
379
545
|
workflowMeta?.nodes &&
|
|
380
546
|
stepName in workflowMeta.nodes) {
|
|
@@ -512,7 +678,7 @@ export class PikkuWorkflowService {
|
|
|
512
678
|
// Map step retry options to queue job options
|
|
513
679
|
const retries = stepOptions?.retries ?? 0;
|
|
514
680
|
const retryDelay = stepOptions?.retryDelay;
|
|
515
|
-
await getSingletonServices().queueService.add(this.
|
|
681
|
+
await getSingletonServices().queueService.add(this.getStepWorkerQueueName(rpcName), JSON.parse(JSON.stringify({
|
|
516
682
|
runId,
|
|
517
683
|
stepName,
|
|
518
684
|
rpcName,
|
|
@@ -756,7 +922,7 @@ export class PikkuWorkflowService {
|
|
|
756
922
|
throw new WorkflowSuspendedException(runId, reason);
|
|
757
923
|
});
|
|
758
924
|
}
|
|
759
|
-
createWorkflowWire(name, runId, rpcService) {
|
|
925
|
+
createWorkflowWire(name, runId, rpcService, addonNamespace) {
|
|
760
926
|
const workflowWire = {
|
|
761
927
|
name,
|
|
762
928
|
runId,
|
|
@@ -765,7 +931,10 @@ export class PikkuWorkflowService {
|
|
|
765
931
|
do: async (stepName, rpcNameOrFn, dataOrOptions, options) => {
|
|
766
932
|
this.verifyStepName(stepName);
|
|
767
933
|
if (typeof rpcNameOrFn === 'string') {
|
|
768
|
-
|
|
934
|
+
const resolvedRpcName = addonNamespace && !rpcNameOrFn.includes(':')
|
|
935
|
+
? `${addonNamespace}:${rpcNameOrFn}`
|
|
936
|
+
: rpcNameOrFn;
|
|
937
|
+
return await this.rpcStep(runId, stepName, resolvedRpcName, dataOrOptions, rpcService, options);
|
|
769
938
|
}
|
|
770
939
|
else {
|
|
771
940
|
return await this.inlineStep(runId, stepName, rpcNameOrFn, dataOrOptions);
|
|
@@ -798,4 +967,29 @@ export class PikkuWorkflowService {
|
|
|
798
967
|
sleeperRPCName: workflow?.sleeperRPCName ?? 'pikkuWorkflowSleeper',
|
|
799
968
|
};
|
|
800
969
|
}
|
|
970
|
+
/**
|
|
971
|
+
* Get the orchestrator queue name for a specific workflow.
|
|
972
|
+
* Checks queue meta for a per-workflow queue first (e.g. wf-orchestrator-{name}),
|
|
973
|
+
* falls back to the shared orchestrator queue.
|
|
974
|
+
*/
|
|
975
|
+
getOrchestratorQueueName(workflowName) {
|
|
976
|
+
if (workflowName) {
|
|
977
|
+
const perWorkflow = `wf-orchestrator-${toKebab(workflowName)}`;
|
|
978
|
+
const registrations = pikkuState(null, 'queue', 'registrations');
|
|
979
|
+
if (registrations.has(perWorkflow)) {
|
|
980
|
+
return perWorkflow;
|
|
981
|
+
}
|
|
982
|
+
}
|
|
983
|
+
return this.getConfig().orchestratorQueueName;
|
|
984
|
+
}
|
|
985
|
+
getStepWorkerQueueName(rpcName) {
|
|
986
|
+
if (rpcName) {
|
|
987
|
+
const perStep = `wf-step-${toKebab(rpcName)}`;
|
|
988
|
+
const registrations = pikkuState(null, 'queue', 'registrations');
|
|
989
|
+
if (registrations.has(perStep)) {
|
|
990
|
+
return perStep;
|
|
991
|
+
}
|
|
992
|
+
}
|
|
993
|
+
return this.getConfig().stepWorkerQueueName;
|
|
994
|
+
}
|
|
801
995
|
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Queue worker functions for workflow orchestration.
|
|
3
|
+
*
|
|
4
|
+
* These are registered as queue consumers by the codegen pipeline
|
|
5
|
+
* (injected into queue meta when workflows exist). They provide
|
|
6
|
+
* the runtime implementations that process queued workflow jobs.
|
|
7
|
+
*/
|
|
8
|
+
import type { PikkuRPC } from '../rpc/rpc-types.js';
|
|
9
|
+
export interface WorkflowStepInput {
|
|
10
|
+
runId: string;
|
|
11
|
+
stepName: string;
|
|
12
|
+
rpcName: string;
|
|
13
|
+
data: unknown;
|
|
14
|
+
}
|
|
15
|
+
export interface PikkuWorkflowOrchestratorInput {
|
|
16
|
+
runId: string;
|
|
17
|
+
}
|
|
18
|
+
export interface PikkuWorkflowSleeperInput {
|
|
19
|
+
runId: string;
|
|
20
|
+
stepId: string;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Step worker — executes individual workflow steps dispatched via queue.
|
|
24
|
+
* The orchestrator queues steps here when they're async (not inline).
|
|
25
|
+
*/
|
|
26
|
+
export declare function pikkuWorkflowWorkerFunc(_services: Record<string, unknown>, { runId, stepName, rpcName, data }: WorkflowStepInput, { rpc }: {
|
|
27
|
+
rpc: PikkuRPC;
|
|
28
|
+
}): Promise<void>;
|
|
29
|
+
/**
|
|
30
|
+
* Orchestrator — resumes workflow execution after an async step completes.
|
|
31
|
+
* Called when a step worker finishes and the workflow needs to continue.
|
|
32
|
+
*/
|
|
33
|
+
export declare function pikkuWorkflowOrchestratorFunc(_services: Record<string, unknown>, { runId }: PikkuWorkflowOrchestratorInput, { rpc }: {
|
|
34
|
+
rpc: PikkuRPC;
|
|
35
|
+
}): Promise<void>;
|
|
36
|
+
/**
|
|
37
|
+
* Sleeper — wakes a workflow after a workflow.sleep() duration expires.
|
|
38
|
+
* Triggered by a delayed queue message or scheduler callback.
|
|
39
|
+
*/
|
|
40
|
+
export declare function pikkuWorkflowSleeperFunc(_services: Record<string, unknown>, { runId, stepId }: PikkuWorkflowSleeperInput): Promise<void>;
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Queue worker functions for workflow orchestration.
|
|
3
|
+
*
|
|
4
|
+
* These are registered as queue consumers by the codegen pipeline
|
|
5
|
+
* (injected into queue meta when workflows exist). They provide
|
|
6
|
+
* the runtime implementations that process queued workflow jobs.
|
|
7
|
+
*/
|
|
8
|
+
import { getSingletonServices } from '../../pikku-state.js';
|
|
9
|
+
/**
|
|
10
|
+
* Step worker — executes individual workflow steps dispatched via queue.
|
|
11
|
+
* The orchestrator queues steps here when they're async (not inline).
|
|
12
|
+
*/
|
|
13
|
+
export async function pikkuWorkflowWorkerFunc(_services, { runId, stepName, rpcName, data }, { rpc }) {
|
|
14
|
+
const services = getSingletonServices();
|
|
15
|
+
if (!services?.workflowService) {
|
|
16
|
+
throw new Error(`Workflow service not initialized: cannot execute workflow step for runId ${runId}, stepName ${stepName}`);
|
|
17
|
+
}
|
|
18
|
+
await services.workflowService.executeWorkflowStep(runId, stepName, rpcName, data, rpc);
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Orchestrator — resumes workflow execution after an async step completes.
|
|
22
|
+
* Called when a step worker finishes and the workflow needs to continue.
|
|
23
|
+
*/
|
|
24
|
+
export async function pikkuWorkflowOrchestratorFunc(_services, { runId }, { rpc }) {
|
|
25
|
+
const services = getSingletonServices();
|
|
26
|
+
if (!services?.workflowService) {
|
|
27
|
+
throw new Error(`Workflow service not initialized: cannot orchestrate workflow for runId ${runId}`);
|
|
28
|
+
}
|
|
29
|
+
await services.workflowService.orchestrateWorkflow(runId, rpc);
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Sleeper — wakes a workflow after a workflow.sleep() duration expires.
|
|
33
|
+
* Triggered by a delayed queue message or scheduler callback.
|
|
34
|
+
*/
|
|
35
|
+
export async function pikkuWorkflowSleeperFunc(_services, { runId, stepId }) {
|
|
36
|
+
const services = getSingletonServices();
|
|
37
|
+
if (!services?.workflowService) {
|
|
38
|
+
throw new Error(`Workflow service not initialized: cannot execute workflow sleep completed for runId ${runId}, stepId ${stepId}`);
|
|
39
|
+
}
|
|
40
|
+
await services.workflowService.executeWorkflowSleepCompleted(runId, stepId);
|
|
41
|
+
}
|
|
@@ -8,6 +8,8 @@ export interface WorkflowRunWire {
|
|
|
8
8
|
id?: string;
|
|
9
9
|
parentRunId?: string;
|
|
10
10
|
parentStepId?: string;
|
|
11
|
+
/** Pikku user ID propagated from the originating request for credential resolution */
|
|
12
|
+
pikkuUserId?: string;
|
|
11
13
|
}
|
|
12
14
|
export interface WorkflowServiceConfig {
|
|
13
15
|
retries: number;
|
|
@@ -88,6 +90,21 @@ export interface StepState {
|
|
|
88
90
|
/** Timestamp when step failed */
|
|
89
91
|
failedAt?: Date;
|
|
90
92
|
}
|
|
93
|
+
export interface WorkflowRunStatus {
|
|
94
|
+
id: string;
|
|
95
|
+
status: WorkflowStatus;
|
|
96
|
+
startedAt: Date;
|
|
97
|
+
completedAt?: Date;
|
|
98
|
+
steps: Array<{
|
|
99
|
+
name: string;
|
|
100
|
+
status: StepStatus;
|
|
101
|
+
duration?: number;
|
|
102
|
+
}>;
|
|
103
|
+
output?: unknown;
|
|
104
|
+
error?: {
|
|
105
|
+
message: string;
|
|
106
|
+
};
|
|
107
|
+
}
|
|
91
108
|
export interface WorkflowRunService {
|
|
92
109
|
listRuns(options?: {
|
|
93
110
|
workflowName?: string;
|
|
@@ -168,6 +185,7 @@ export type WorkflowsMeta = Record<string, CommonWireMeta & {
|
|
|
168
185
|
context?: WorkflowContext;
|
|
169
186
|
dsl?: boolean;
|
|
170
187
|
inline?: boolean;
|
|
188
|
+
expose?: boolean;
|
|
171
189
|
}>;
|
|
172
190
|
/**
|
|
173
191
|
* Unified workflow runtime meta (used by runtime to execute workflows)
|
|
@@ -180,7 +198,7 @@ export interface WorkflowRuntimeMeta {
|
|
|
180
198
|
/** Pikku function name (for execution) */
|
|
181
199
|
pikkuFuncId: string;
|
|
182
200
|
/** Source type: 'dsl' (serializable), 'complex' (has inline steps), 'graph' */
|
|
183
|
-
source: 'dsl' | 'complex' | 'graph' | '
|
|
201
|
+
source: 'dsl' | 'complex' | 'graph' | 'dynamic-workflow';
|
|
184
202
|
/** Optional description */
|
|
185
203
|
description?: string;
|
|
186
204
|
/** Tags for organization */
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@pikku/core",
|
|
3
|
-
"version": "0.12.
|
|
3
|
+
"version": "0.12.15",
|
|
4
4
|
"author": "yasser.fadl@gmail.com",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"module": "dist/index.js",
|
|
@@ -25,6 +25,7 @@
|
|
|
25
25
|
"./channel/local": "./dist/wirings/channel/local/index.js",
|
|
26
26
|
"./channel/serverless": "./dist/wirings/channel/serverless/index.js",
|
|
27
27
|
"./http": "./dist/wirings/http/index.js",
|
|
28
|
+
"./remote": "./dist/remote.js",
|
|
28
29
|
"./queue": "./dist/wirings/queue/index.js",
|
|
29
30
|
"./scheduler": "./dist/wirings/scheduler/index.js",
|
|
30
31
|
"./trigger": "./dist/wirings/trigger/index.js",
|
|
@@ -41,6 +42,7 @@
|
|
|
41
42
|
"./oauth2": "./dist/wirings/oauth2/index.js",
|
|
42
43
|
"./errors": "./dist/errors/index.js",
|
|
43
44
|
"./services": "./dist/services/index.js",
|
|
45
|
+
"./services/local-meta": "./dist/services/meta-service.js",
|
|
44
46
|
"./services/local-content": "./dist/services/local-content.js",
|
|
45
47
|
"./services/gopass-secrets": "./dist/services/gopass-secrets.js",
|
|
46
48
|
"./crypto-utils": "./dist/crypto-utils.js",
|
|
@@ -52,6 +54,7 @@
|
|
|
52
54
|
"dependencies": {
|
|
53
55
|
"@standard-schema/spec": "^1.1.0",
|
|
54
56
|
"cookie": "^1.1.1",
|
|
57
|
+
"json-schema": "^0.4.0",
|
|
55
58
|
"path-to-regexp": "^8.3.0",
|
|
56
59
|
"picoquery": "^2.5.0"
|
|
57
60
|
},
|
package/src/errors/errors.ts
CHANGED
|
@@ -15,6 +15,18 @@ addError(PikkuMissingMetaError, {
|
|
|
15
15
|
message: 'Required metadata is missing',
|
|
16
16
|
})
|
|
17
17
|
|
|
18
|
+
export class MissingServiceError extends PikkuError {}
|
|
19
|
+
addError(MissingServiceError, {
|
|
20
|
+
status: 500,
|
|
21
|
+
message: 'A required service is not configured',
|
|
22
|
+
})
|
|
23
|
+
|
|
24
|
+
export class LocalEnvironmentOnlyError extends PikkuError {}
|
|
25
|
+
addError(LocalEnvironmentOnlyError, {
|
|
26
|
+
status: 403,
|
|
27
|
+
message: 'This operation is only available in local development mode',
|
|
28
|
+
})
|
|
29
|
+
|
|
18
30
|
/**
|
|
19
31
|
* The server cannot or will not process the request due to client error (e.g., malformed request syntax).
|
|
20
32
|
* @group Error
|
|
@@ -76,6 +88,38 @@ addError(ForbiddenError, {
|
|
|
76
88
|
'The client does not have permission to access the requested resource.',
|
|
77
89
|
})
|
|
78
90
|
|
|
91
|
+
/**
|
|
92
|
+
* A required credential is missing. The payload contains metadata
|
|
93
|
+
* about the credential so the client can initiate a connect flow.
|
|
94
|
+
* @group Error
|
|
95
|
+
*/
|
|
96
|
+
export class MissingCredentialError extends PikkuError {
|
|
97
|
+
public payload: {
|
|
98
|
+
error: 'missing_credential'
|
|
99
|
+
credentialName: string
|
|
100
|
+
credentialType: 'oauth2' | 'apikey'
|
|
101
|
+
connectUrl?: string
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
constructor(
|
|
105
|
+
credentialName: string,
|
|
106
|
+
credentialType: 'oauth2' | 'apikey',
|
|
107
|
+
connectUrl?: string
|
|
108
|
+
) {
|
|
109
|
+
super(`Missing credential: ${credentialName}`)
|
|
110
|
+
this.payload = {
|
|
111
|
+
error: 'missing_credential',
|
|
112
|
+
credentialName,
|
|
113
|
+
credentialType,
|
|
114
|
+
connectUrl,
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
addError(MissingCredentialError, {
|
|
119
|
+
status: 403,
|
|
120
|
+
message: 'A required credential is not configured.',
|
|
121
|
+
})
|
|
122
|
+
|
|
79
123
|
/**
|
|
80
124
|
* The session is readonly and cannot access a non-readonly function.
|
|
81
125
|
* @group Error
|