@rigkit/engine 0.2.3 → 0.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/authoring.ts +28 -13
- package/src/authoring.typecheck.ts +21 -1
- package/src/engine.test.ts +224 -28
- package/src/engine.ts +294 -31
- package/src/host-storage.ts +128 -0
- package/src/index.ts +5 -0
- package/src/provider/types.ts +2 -0
- package/src/state.ts +40 -0
- package/src/types.ts +139 -46
- package/src/version.ts +1 -1
package/src/engine.ts
CHANGED
|
@@ -4,7 +4,12 @@ import { pathToFileURL } from "node:url";
|
|
|
4
4
|
import { isRigkitConfig, isProviderDefinition, isWorkflowNode } from "./authoring.ts";
|
|
5
5
|
import { loadDotEnv } from "./env-file.ts";
|
|
6
6
|
import { hash } from "./hash.ts";
|
|
7
|
-
import {
|
|
7
|
+
import {
|
|
8
|
+
createFileProviderHostStorage,
|
|
9
|
+
defaultProviderHostStorageDir,
|
|
10
|
+
type ProviderHostStorageFactory,
|
|
11
|
+
} from "./host-storage.ts";
|
|
12
|
+
import { RESERVED_WORKFLOW_OPERATION_IDS, STEP_INVALIDATION_KIND } from "./types.ts";
|
|
8
13
|
import type {
|
|
9
14
|
BaseProviderPlugin,
|
|
10
15
|
InteractionPresenter,
|
|
@@ -32,11 +37,14 @@ import type {
|
|
|
32
37
|
WorkflowInputFieldDefinition,
|
|
33
38
|
WorkflowDefinition,
|
|
34
39
|
WorkflowEvent,
|
|
40
|
+
WorkflowLogStream,
|
|
35
41
|
WorkflowNodeDefinition,
|
|
36
42
|
WorkflowOperationDefinition,
|
|
37
43
|
WorkflowPlan,
|
|
38
44
|
WorkflowPlanNode,
|
|
39
45
|
WorkflowProviderMap,
|
|
46
|
+
WorkflowStepInvalidation,
|
|
47
|
+
WorkflowTaskCacheTTL,
|
|
40
48
|
WorkflowTaskNode,
|
|
41
49
|
WorkspaceRecord,
|
|
42
50
|
WorkspaceRuntimeRecord,
|
|
@@ -51,6 +59,8 @@ export type CreateDevMachineEngineOptions = {
|
|
|
51
59
|
providers?: BaseProviderPlugin[];
|
|
52
60
|
providerFactory?: ProviderFactory;
|
|
53
61
|
stateFactory?: StateServiceFactory;
|
|
62
|
+
hostStorageDir?: string;
|
|
63
|
+
hostStorageFactory?: ProviderHostStorageFactory;
|
|
54
64
|
interaction?: {
|
|
55
65
|
present?: InteractionPresenter;
|
|
56
66
|
};
|
|
@@ -147,10 +157,16 @@ type EvaluationMode = "plan" | "apply";
|
|
|
147
157
|
type EvaluationState = {
|
|
148
158
|
context: Record<string, JsonValue>;
|
|
149
159
|
upstreamRunIds: string[];
|
|
160
|
+
previousTasks: EvaluationPreviousTask[];
|
|
150
161
|
known: boolean;
|
|
151
162
|
blockedReason?: string;
|
|
152
163
|
};
|
|
153
164
|
|
|
165
|
+
type EvaluationPreviousTask = {
|
|
166
|
+
name: string;
|
|
167
|
+
path: string;
|
|
168
|
+
};
|
|
169
|
+
|
|
154
170
|
type EvaluationResult = EvaluationState & {
|
|
155
171
|
planNodes: WorkflowPlanNode[];
|
|
156
172
|
};
|
|
@@ -179,6 +195,30 @@ type RuntimeWorkspaceOperationEntry = {
|
|
|
179
195
|
readonly run: (input: { workspace: string; workflow?: string; input?: unknown }) => Promise<unknown>;
|
|
180
196
|
};
|
|
181
197
|
|
|
198
|
+
class StepInvalidationRestart extends Error {
|
|
199
|
+
readonly workflow: string;
|
|
200
|
+
readonly target: string;
|
|
201
|
+
readonly targetNodePath: string;
|
|
202
|
+
readonly currentNodePath: string;
|
|
203
|
+
readonly invalidatedRunIds: string[];
|
|
204
|
+
|
|
205
|
+
constructor(input: {
|
|
206
|
+
workflow: string;
|
|
207
|
+
target: string;
|
|
208
|
+
targetNodePath: string;
|
|
209
|
+
currentNodePath: string;
|
|
210
|
+
invalidatedRunIds: string[];
|
|
211
|
+
}) {
|
|
212
|
+
super(`Task ${input.currentNodePath} invalidated ${input.targetNodePath}`);
|
|
213
|
+
this.name = "StepInvalidationRestart";
|
|
214
|
+
this.workflow = input.workflow;
|
|
215
|
+
this.target = input.target;
|
|
216
|
+
this.targetNodePath = input.targetNodePath;
|
|
217
|
+
this.currentNodePath = input.currentNodePath;
|
|
218
|
+
this.invalidatedRunIds = input.invalidatedRunIds;
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
|
|
182
222
|
let configImportCounter = 0;
|
|
183
223
|
|
|
184
224
|
export class DevMachineEngine {
|
|
@@ -189,6 +229,9 @@ export class DevMachineEngine {
|
|
|
189
229
|
private providers: BaseProviderPlugin[];
|
|
190
230
|
private readonly providerFactory: ProviderFactory;
|
|
191
231
|
private readonly stateFactory: StateServiceFactory;
|
|
232
|
+
private readonly hostStorageDir: string;
|
|
233
|
+
private readonly hostStorageFactory: ProviderHostStorageFactory;
|
|
234
|
+
private readonly providerHostStorage = new Map<string, ReturnType<ProviderHostStorageFactory>>();
|
|
192
235
|
private readonly interactionPresenter: InteractionPresenter;
|
|
193
236
|
private readonly local: LocalWorkspaceRuntime;
|
|
194
237
|
private readonly handlers = new Set<EventHandler>();
|
|
@@ -204,6 +247,8 @@ export class DevMachineEngine {
|
|
|
204
247
|
this.providers = options.providers ?? [];
|
|
205
248
|
this.providerFactory = options.providerFactory ?? ((input) => this.createProviderFromPlugin(input));
|
|
206
249
|
this.stateFactory = options.stateFactory ?? createStateStore;
|
|
250
|
+
this.hostStorageDir = options.hostStorageDir ? resolve(options.hostStorageDir) : defaultProviderHostStorageDir();
|
|
251
|
+
this.hostStorageFactory = options.hostStorageFactory ?? createFileProviderHostStorage;
|
|
207
252
|
this.interactionPresenter = options.interaction?.present ?? defaultInteractionPresenter;
|
|
208
253
|
this.local = {
|
|
209
254
|
open: options.local?.open ?? openLocalTarget,
|
|
@@ -581,6 +626,7 @@ export class DevMachineEngine {
|
|
|
581
626
|
providers: runtime,
|
|
582
627
|
local: this.local,
|
|
583
628
|
workflow: workflow.name,
|
|
629
|
+
step: this.createStepRuntime(workflow.name, `operation.${operation.id}`, metadata),
|
|
584
630
|
});
|
|
585
631
|
if (result !== undefined) assertJsonValue(result, `Operation ${operation.id} result`);
|
|
586
632
|
return result ?? null;
|
|
@@ -648,11 +694,27 @@ export class DevMachineEngine {
|
|
|
648
694
|
}> {
|
|
649
695
|
const workflow = this.getWorkflow(input.workflow ?? input.machine);
|
|
650
696
|
const providers = await this.createProviders(workflow);
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
697
|
+
let result: { context: Record<string, JsonValue>; plan: WorkflowPlan } | undefined;
|
|
698
|
+
const maxRestarts = 8;
|
|
699
|
+
for (let attempt = 0; attempt <= maxRestarts; attempt++) {
|
|
700
|
+
try {
|
|
701
|
+
result = await this.evaluate({
|
|
702
|
+
workflow,
|
|
703
|
+
providers,
|
|
704
|
+
mode: "apply",
|
|
705
|
+
});
|
|
706
|
+
break;
|
|
707
|
+
} catch (error) {
|
|
708
|
+
if (!(error instanceof StepInvalidationRestart)) throw error;
|
|
709
|
+
if (attempt === maxRestarts) {
|
|
710
|
+
throw new Error(
|
|
711
|
+
`Task ${error.currentNodePath} repeatedly invalidated ${error.targetNodePath}; stopping after ${maxRestarts + 1} attempts`,
|
|
712
|
+
{ cause: error },
|
|
713
|
+
);
|
|
714
|
+
}
|
|
715
|
+
}
|
|
716
|
+
}
|
|
717
|
+
if (!result) throw new Error(`Workflow ${workflow.name} did not produce an apply result`);
|
|
656
718
|
|
|
657
719
|
return {
|
|
658
720
|
context: result.context,
|
|
@@ -665,7 +727,7 @@ export class DevMachineEngine {
|
|
|
665
727
|
}
|
|
666
728
|
|
|
667
729
|
async createWorkspace(input: { workflow?: string; machine?: string; name: string }): Promise<WorkspaceRecord> {
|
|
668
|
-
|
|
730
|
+
assertValidWorkspaceName(input.name);
|
|
669
731
|
if (this.getStateService().getWorkspace(input.name)) {
|
|
670
732
|
throw new Error(`Workspace ${input.name} already exists`);
|
|
671
733
|
}
|
|
@@ -749,6 +811,7 @@ export class DevMachineEngine {
|
|
|
749
811
|
workspace: workspaceRuntime,
|
|
750
812
|
providers: runtime,
|
|
751
813
|
local: this.local,
|
|
814
|
+
step: this.createStepRuntime(workflow.name, `workspace.${workspace.name}.remove`, metadata),
|
|
752
815
|
});
|
|
753
816
|
|
|
754
817
|
this.getStateService().deleteWorkspace(input.workspace);
|
|
@@ -771,6 +834,7 @@ export class DevMachineEngine {
|
|
|
771
834
|
state: {
|
|
772
835
|
context: {},
|
|
773
836
|
upstreamRunIds: [],
|
|
837
|
+
previousTasks: [],
|
|
774
838
|
known: true,
|
|
775
839
|
},
|
|
776
840
|
prefix: [],
|
|
@@ -819,6 +883,7 @@ export class DevMachineEngine {
|
|
|
819
883
|
state = {
|
|
820
884
|
context: result.context,
|
|
821
885
|
upstreamRunIds: result.upstreamRunIds,
|
|
886
|
+
previousTasks: result.previousTasks,
|
|
822
887
|
known: result.known,
|
|
823
888
|
blockedReason: result.blockedReason,
|
|
824
889
|
};
|
|
@@ -831,6 +896,7 @@ export class DevMachineEngine {
|
|
|
831
896
|
const branches = parallelBranches(input.node);
|
|
832
897
|
const branchOutputs: Record<string, JsonValue> = {};
|
|
833
898
|
const joinedRunIds: string[] = [];
|
|
899
|
+
let joinedPreviousTasks = [...input.state.previousTasks];
|
|
834
900
|
let known = input.state.known;
|
|
835
901
|
let blockedReason = input.state.blockedReason;
|
|
836
902
|
|
|
@@ -845,6 +911,7 @@ export class DevMachineEngine {
|
|
|
845
911
|
state: {
|
|
846
912
|
context: { ...input.state.context },
|
|
847
913
|
upstreamRunIds: [...input.state.upstreamRunIds],
|
|
914
|
+
previousTasks: [...input.state.previousTasks],
|
|
848
915
|
known: input.state.known,
|
|
849
916
|
blockedReason: input.state.blockedReason,
|
|
850
917
|
},
|
|
@@ -856,6 +923,7 @@ export class DevMachineEngine {
|
|
|
856
923
|
if (branchState.known) {
|
|
857
924
|
branchOutputs[branchName] = branchState.context;
|
|
858
925
|
joinedRunIds.push(...branchState.upstreamRunIds);
|
|
926
|
+
joinedPreviousTasks = mergePreviousTasks(joinedPreviousTasks, branchState.previousTasks);
|
|
859
927
|
} else {
|
|
860
928
|
known = false;
|
|
861
929
|
blockedReason ??= branchState.blockedReason ?? `depends on ${branchName}`;
|
|
@@ -865,6 +933,7 @@ export class DevMachineEngine {
|
|
|
865
933
|
return {
|
|
866
934
|
context: known ? { ...input.state.context, ...branchOutputs } : { ...input.state.context },
|
|
867
935
|
upstreamRunIds: known ? joinedRunIds.sort() : [],
|
|
936
|
+
previousTasks: known ? joinedPreviousTasks : input.state.previousTasks,
|
|
868
937
|
known,
|
|
869
938
|
blockedReason,
|
|
870
939
|
planNodes: input.planNodes,
|
|
@@ -875,7 +944,7 @@ export class DevMachineEngine {
|
|
|
875
944
|
const nodePath = [...input.prefix, input.node.name].join(".");
|
|
876
945
|
const upstreamRunIds = [...input.state.upstreamRunIds];
|
|
877
946
|
const nodeKey = hash({
|
|
878
|
-
cache: "task-
|
|
947
|
+
cache: "task-v3",
|
|
879
948
|
kind: "task",
|
|
880
949
|
path: nodePath,
|
|
881
950
|
name: input.node.name,
|
|
@@ -897,6 +966,7 @@ export class DevMachineEngine {
|
|
|
897
966
|
return {
|
|
898
967
|
context: input.state.context,
|
|
899
968
|
upstreamRunIds: [],
|
|
969
|
+
previousTasks: input.state.previousTasks,
|
|
900
970
|
known: false,
|
|
901
971
|
blockedReason: input.state.blockedReason ?? `depends on ${nodePath}`,
|
|
902
972
|
planNodes: input.planNodes,
|
|
@@ -911,9 +981,14 @@ export class DevMachineEngine {
|
|
|
911
981
|
upstreamRunIds,
|
|
912
982
|
providers: input.providers,
|
|
913
983
|
outputSchema: input.node.options?.output,
|
|
984
|
+
cacheTTL: input.node.options?.cacheTTL,
|
|
914
985
|
});
|
|
915
986
|
|
|
916
987
|
if (cached) {
|
|
988
|
+
const previousTasks = appendPreviousTask(input.state.previousTasks, {
|
|
989
|
+
name: input.node.name,
|
|
990
|
+
path: nodePath,
|
|
991
|
+
});
|
|
917
992
|
this.emit({ type: "node.cached", nodePath, runId: cached.id });
|
|
918
993
|
input.planNodes.push({
|
|
919
994
|
index: planIndex,
|
|
@@ -924,8 +999,9 @@ export class DevMachineEngine {
|
|
|
924
999
|
upstreamRunIds,
|
|
925
1000
|
});
|
|
926
1001
|
return {
|
|
927
|
-
context:
|
|
1002
|
+
context: cached.output,
|
|
928
1003
|
upstreamRunIds: [cached.id],
|
|
1004
|
+
previousTasks,
|
|
929
1005
|
known: true,
|
|
930
1006
|
planNodes: input.planNodes,
|
|
931
1007
|
};
|
|
@@ -944,6 +1020,7 @@ export class DevMachineEngine {
|
|
|
944
1020
|
return {
|
|
945
1021
|
context: input.state.context,
|
|
946
1022
|
upstreamRunIds: [],
|
|
1023
|
+
previousTasks: input.state.previousTasks,
|
|
947
1024
|
known: false,
|
|
948
1025
|
blockedReason: `depends on ${nodePath}`,
|
|
949
1026
|
planNodes: input.planNodes,
|
|
@@ -958,28 +1035,38 @@ export class DevMachineEngine {
|
|
|
958
1035
|
nodePath,
|
|
959
1036
|
metadata,
|
|
960
1037
|
});
|
|
1038
|
+
const step = this.createStepRuntime(
|
|
1039
|
+
input.workflow.name,
|
|
1040
|
+
nodePath,
|
|
1041
|
+
metadata,
|
|
1042
|
+
input.state.context,
|
|
1043
|
+
input.state.previousTasks,
|
|
1044
|
+
);
|
|
961
1045
|
const result = await input.node.handler({
|
|
962
1046
|
...runtime,
|
|
963
1047
|
providers: runtime,
|
|
964
|
-
|
|
965
|
-
runtime: {
|
|
966
|
-
workflow: input.workflow.name,
|
|
967
|
-
nodePath,
|
|
968
|
-
metadata: (value) => {
|
|
969
|
-
Object.assign(metadata, value);
|
|
970
|
-
},
|
|
971
|
-
log: (data, options = {}) => {
|
|
972
|
-
this.emit({
|
|
973
|
-
type: "log.output",
|
|
974
|
-
nodePath,
|
|
975
|
-
stream: options.stream ?? "info",
|
|
976
|
-
label: options.label,
|
|
977
|
-
data,
|
|
978
|
-
});
|
|
979
|
-
},
|
|
980
|
-
},
|
|
1048
|
+
step,
|
|
981
1049
|
});
|
|
982
|
-
|
|
1050
|
+
if (isStepInvalidation(result)) {
|
|
1051
|
+
const invalidatedRunIds = this.getStateService().invalidateNodeRuns({
|
|
1052
|
+
workflow: input.workflow.name,
|
|
1053
|
+
nodePaths: [result.targetNodePath, nodePath],
|
|
1054
|
+
});
|
|
1055
|
+
throw new StepInvalidationRestart({
|
|
1056
|
+
workflow: input.workflow.name,
|
|
1057
|
+
target: result.target,
|
|
1058
|
+
targetNodePath: result.targetNodePath,
|
|
1059
|
+
currentNodePath: nodePath,
|
|
1060
|
+
invalidatedRunIds,
|
|
1061
|
+
});
|
|
1062
|
+
}
|
|
1063
|
+
const output = normalizeTaskOutput(
|
|
1064
|
+
nodePath,
|
|
1065
|
+
result,
|
|
1066
|
+
input.node.options?.output,
|
|
1067
|
+
"fresh",
|
|
1068
|
+
input.state.context,
|
|
1069
|
+
);
|
|
983
1070
|
if (!output) {
|
|
984
1071
|
throw new Error(`Task ${nodePath} output failed schema validation`);
|
|
985
1072
|
}
|
|
@@ -1013,9 +1100,15 @@ export class DevMachineEngine {
|
|
|
1013
1100
|
}
|
|
1014
1101
|
this.emit({ type: "node.completed", nodePath, runId: record.id });
|
|
1015
1102
|
|
|
1103
|
+
const previousTasks = appendPreviousTask(input.state.previousTasks, {
|
|
1104
|
+
name: input.node.name,
|
|
1105
|
+
path: nodePath,
|
|
1106
|
+
});
|
|
1107
|
+
|
|
1016
1108
|
return {
|
|
1017
|
-
context:
|
|
1109
|
+
context: output,
|
|
1018
1110
|
upstreamRunIds: [record.id],
|
|
1111
|
+
previousTasks,
|
|
1019
1112
|
known: true,
|
|
1020
1113
|
planNodes: input.planNodes,
|
|
1021
1114
|
};
|
|
@@ -1029,9 +1122,11 @@ export class DevMachineEngine {
|
|
|
1029
1122
|
upstreamRunIds: readonly string[];
|
|
1030
1123
|
providers: ProviderControllers;
|
|
1031
1124
|
outputSchema?: OutputSchema;
|
|
1125
|
+
cacheTTL?: WorkflowTaskCacheTTL;
|
|
1032
1126
|
}): Promise<WorkflowNodeRunRecord | undefined> {
|
|
1033
1127
|
const cached = this.getStateService().findReusableNodeRun(input);
|
|
1034
1128
|
if (!cached) return undefined;
|
|
1129
|
+
if (!isCacheFresh(cached.createdAt, input.cacheTTL)) return undefined;
|
|
1035
1130
|
|
|
1036
1131
|
const parsed = normalizeTaskOutput(input.nodePath, cached.output, input.outputSchema, "cached");
|
|
1037
1132
|
if (!parsed) return undefined;
|
|
@@ -1140,6 +1235,7 @@ export class DevMachineEngine {
|
|
|
1140
1235
|
},
|
|
1141
1236
|
providers,
|
|
1142
1237
|
local: this.local,
|
|
1238
|
+
step: this.createStepRuntime(input.workflow.name, `workspace.${input.name}.create`, metadata),
|
|
1143
1239
|
});
|
|
1144
1240
|
assertJsonValue(data, `Workflow ${input.workflow.name} workspace create result`);
|
|
1145
1241
|
if (!isPlainObject(data)) {
|
|
@@ -1180,6 +1276,11 @@ export class DevMachineEngine {
|
|
|
1180
1276
|
workspace,
|
|
1181
1277
|
providers,
|
|
1182
1278
|
local: this.local,
|
|
1279
|
+
step: this.createStepRuntime(
|
|
1280
|
+
input.workflow.name,
|
|
1281
|
+
`workspace.${input.workspace.name}.${input.operation.id}`,
|
|
1282
|
+
metadata,
|
|
1283
|
+
),
|
|
1183
1284
|
});
|
|
1184
1285
|
if (result !== undefined) assertJsonValue(result, `Workspace operation ${input.operation.id} result`);
|
|
1185
1286
|
return result ?? null;
|
|
@@ -1192,6 +1293,46 @@ export class DevMachineEngine {
|
|
|
1192
1293
|
}) as WorkspaceRuntimeRecord<Data>;
|
|
1193
1294
|
}
|
|
1194
1295
|
|
|
1296
|
+
private createStepRuntime<Context extends JsonObject = JsonObject>(
|
|
1297
|
+
workflow: string,
|
|
1298
|
+
nodePath: string,
|
|
1299
|
+
metadata: JsonObject,
|
|
1300
|
+
context: Context = {} as Context,
|
|
1301
|
+
previousTasks: readonly EvaluationPreviousTask[] = [],
|
|
1302
|
+
) {
|
|
1303
|
+
return {
|
|
1304
|
+
workflow,
|
|
1305
|
+
nodePath,
|
|
1306
|
+
ctx: Object.freeze({ ...context }) as Readonly<Context>,
|
|
1307
|
+
metadata: (value: JsonObject) => {
|
|
1308
|
+
Object.assign(metadata, value);
|
|
1309
|
+
},
|
|
1310
|
+
log: (data: string, options: { stream?: WorkflowLogStream; label?: string } = {}) => {
|
|
1311
|
+
this.emit({
|
|
1312
|
+
type: "log.output",
|
|
1313
|
+
nodePath,
|
|
1314
|
+
stream: options.stream ?? "info",
|
|
1315
|
+
label: options.label,
|
|
1316
|
+
data,
|
|
1317
|
+
});
|
|
1318
|
+
},
|
|
1319
|
+
invalidate: <Target extends string>(target: Target) => {
|
|
1320
|
+
const matches = previousTasks.filter((task) => task.name === target || task.path === target);
|
|
1321
|
+
if (matches.length === 0) {
|
|
1322
|
+
throw new Error(`Task ${nodePath} cannot invalidate ${target} because it has not run earlier in this workflow`);
|
|
1323
|
+
}
|
|
1324
|
+
if (matches.length > 1) {
|
|
1325
|
+
throw new Error(`Task ${nodePath} cannot invalidate ${target} because it matches multiple earlier tasks`);
|
|
1326
|
+
}
|
|
1327
|
+
return {
|
|
1328
|
+
kind: STEP_INVALIDATION_KIND,
|
|
1329
|
+
target,
|
|
1330
|
+
targetNodePath: matches[0]!.path,
|
|
1331
|
+
};
|
|
1332
|
+
},
|
|
1333
|
+
};
|
|
1334
|
+
}
|
|
1335
|
+
|
|
1195
1336
|
private getWorkflow(name: string | undefined): LoadedWorkflow {
|
|
1196
1337
|
if (this.workflows.size === 0) {
|
|
1197
1338
|
throw new Error(`No workflows loaded. Call engine.load() first.`);
|
|
@@ -1324,6 +1465,8 @@ export class DevMachineEngine {
|
|
|
1324
1465
|
const controller = await this.providerFactory({
|
|
1325
1466
|
provider,
|
|
1326
1467
|
storage: this.getStateService().providerStorage(provider.providerId),
|
|
1468
|
+
hostStorage: this.getProviderHostStorage(provider.providerId),
|
|
1469
|
+
local: this.local,
|
|
1327
1470
|
});
|
|
1328
1471
|
return [name, controller] as const;
|
|
1329
1472
|
}),
|
|
@@ -1342,6 +1485,18 @@ export class DevMachineEngine {
|
|
|
1342
1485
|
return await plugin.createProvider(input);
|
|
1343
1486
|
}
|
|
1344
1487
|
|
|
1488
|
+
private getProviderHostStorage(providerId: string): ReturnType<ProviderHostStorageFactory> {
|
|
1489
|
+
let storage = this.providerHostStorage.get(providerId);
|
|
1490
|
+
if (!storage) {
|
|
1491
|
+
storage = this.hostStorageFactory({
|
|
1492
|
+
providerId,
|
|
1493
|
+
rootDir: this.hostStorageDir,
|
|
1494
|
+
});
|
|
1495
|
+
this.providerHostStorage.set(providerId, storage);
|
|
1496
|
+
}
|
|
1497
|
+
return storage;
|
|
1498
|
+
}
|
|
1499
|
+
|
|
1345
1500
|
private async resolveWorkflow(root: WorkflowNodeDefinition<any, any, any>): Promise<LoadedWorkflow> {
|
|
1346
1501
|
const providers: Record<string, LoadedProviderDefinition> = {};
|
|
1347
1502
|
for (const [name, definition] of Object.entries(root.workflow.providers)) {
|
|
@@ -1389,6 +1544,17 @@ function parseWorkspaceOperationId(value: string): { workspace: string; operatio
|
|
|
1389
1544
|
};
|
|
1390
1545
|
}
|
|
1391
1546
|
|
|
1547
|
+
const workspaceNamePattern = /^(?!-)[A-Za-z0-9._-]+$/;
|
|
1548
|
+
|
|
1549
|
+
function assertValidWorkspaceName(value: string): void {
|
|
1550
|
+
if (!value) throw new Error(`create requires a workspace name`);
|
|
1551
|
+
if (!workspaceNamePattern.test(value)) {
|
|
1552
|
+
throw new Error(
|
|
1553
|
+
`Workspace name "${value}" is invalid. Use only letters, numbers, ".", "_", and "-", and do not start with "-".`,
|
|
1554
|
+
);
|
|
1555
|
+
}
|
|
1556
|
+
}
|
|
1557
|
+
|
|
1392
1558
|
async function resolveProviderDefinition(
|
|
1393
1559
|
definition: WorkflowDefinition<any, any>["providers"][string],
|
|
1394
1560
|
): Promise<LoadedProviderDefinition> {
|
|
@@ -1553,26 +1719,123 @@ function parallelBranches(node: WorkflowNodeDefinition<any, any, any>): Record<s
|
|
|
1553
1719
|
return (node as { branches?: Record<string, WorkflowNodeDefinition<any, any, any>> }).branches ?? {};
|
|
1554
1720
|
}
|
|
1555
1721
|
|
|
1722
|
+
function appendPreviousTask(
|
|
1723
|
+
tasks: readonly EvaluationPreviousTask[],
|
|
1724
|
+
task: EvaluationPreviousTask,
|
|
1725
|
+
): EvaluationPreviousTask[] {
|
|
1726
|
+
return mergePreviousTasks([...tasks], [task]);
|
|
1727
|
+
}
|
|
1728
|
+
|
|
1729
|
+
function mergePreviousTasks(
|
|
1730
|
+
left: readonly EvaluationPreviousTask[],
|
|
1731
|
+
right: readonly EvaluationPreviousTask[],
|
|
1732
|
+
): EvaluationPreviousTask[] {
|
|
1733
|
+
const seen = new Set<string>();
|
|
1734
|
+
const result: EvaluationPreviousTask[] = [];
|
|
1735
|
+
for (const task of [...left, ...right]) {
|
|
1736
|
+
if (seen.has(task.path)) continue;
|
|
1737
|
+
seen.add(task.path);
|
|
1738
|
+
result.push(task);
|
|
1739
|
+
}
|
|
1740
|
+
return result;
|
|
1741
|
+
}
|
|
1742
|
+
|
|
1556
1743
|
function normalizeTaskOutput(
|
|
1557
1744
|
nodePath: string,
|
|
1558
1745
|
result: unknown,
|
|
1559
1746
|
schema: OutputSchema | undefined,
|
|
1560
1747
|
source: "fresh" | "cached",
|
|
1748
|
+
currentContext: Record<string, JsonValue> = {},
|
|
1561
1749
|
): Record<string, JsonValue> | undefined {
|
|
1750
|
+
if (source === "fresh") {
|
|
1751
|
+
if (result === undefined) return { ...currentContext };
|
|
1752
|
+
if (!isPlainObject(result) || !("ctx" in result)) {
|
|
1753
|
+
throw new Error(`Task ${nodePath} must return { ctx: { ... } } or step.invalidate(...)`);
|
|
1754
|
+
}
|
|
1755
|
+
const ctx = result.ctx;
|
|
1756
|
+
const value = schema ? parseWithSchema(schema, ctx, source) : ctx;
|
|
1757
|
+
if (!isPlainObject(value)) {
|
|
1758
|
+
throw new Error(`Task ${nodePath} ctx must be a JSON-serializable object`);
|
|
1759
|
+
}
|
|
1760
|
+
|
|
1761
|
+
for (const [key, item] of Object.entries(value)) {
|
|
1762
|
+
assertJsonValue(item, `Task ${nodePath} ctx value ${key}`);
|
|
1763
|
+
}
|
|
1764
|
+
|
|
1765
|
+
return value as Record<string, JsonValue>;
|
|
1766
|
+
}
|
|
1767
|
+
|
|
1562
1768
|
const value = schema ? parseWithSchema(schema, result, source) : result;
|
|
1563
|
-
if (value === undefined) return
|
|
1769
|
+
if (value === undefined) return schema ? undefined : {};
|
|
1564
1770
|
if (!isPlainObject(value)) {
|
|
1565
1771
|
if (source === "cached") return undefined;
|
|
1566
|
-
throw new Error(`Task ${nodePath}
|
|
1772
|
+
throw new Error(`Task ${nodePath} cached ctx must be a JSON-serializable object`);
|
|
1567
1773
|
}
|
|
1568
1774
|
|
|
1569
1775
|
for (const [key, item] of Object.entries(value)) {
|
|
1570
|
-
assertJsonValue(item, `Task ${nodePath}
|
|
1776
|
+
assertJsonValue(item, `Task ${nodePath} ctx value ${key}`);
|
|
1571
1777
|
}
|
|
1572
1778
|
|
|
1573
1779
|
return value as Record<string, JsonValue>;
|
|
1574
1780
|
}
|
|
1575
1781
|
|
|
1782
|
+
function isCacheFresh(createdAt: string, ttl: WorkflowTaskCacheTTL | undefined): boolean {
|
|
1783
|
+
const ttlMs = parseCacheTTL(ttl);
|
|
1784
|
+
if (ttlMs === undefined) return true;
|
|
1785
|
+
if (ttlMs <= 0) return false;
|
|
1786
|
+
const createdTime = Date.parse(createdAt);
|
|
1787
|
+
if (Number.isNaN(createdTime)) return false;
|
|
1788
|
+
return Date.now() - createdTime <= ttlMs;
|
|
1789
|
+
}
|
|
1790
|
+
|
|
1791
|
+
function parseCacheTTL(ttl: WorkflowTaskCacheTTL | undefined): number | undefined {
|
|
1792
|
+
if (ttl === undefined) return undefined;
|
|
1793
|
+
if (typeof ttl === "number") {
|
|
1794
|
+
assertFiniteTTL(ttl, "cacheTTL");
|
|
1795
|
+
return ttl;
|
|
1796
|
+
}
|
|
1797
|
+
if (typeof ttl === "string") return parseCacheTTLString(ttl);
|
|
1798
|
+
|
|
1799
|
+
const total =
|
|
1800
|
+
(ttl.seconds ?? 0) * 1000 +
|
|
1801
|
+
(ttl.minutes ?? 0) * 60 * 1000 +
|
|
1802
|
+
(ttl.hours ?? 0) * 60 * 60 * 1000 +
|
|
1803
|
+
(ttl.days ?? 0) * 24 * 60 * 60 * 1000;
|
|
1804
|
+
assertFiniteTTL(total, "cacheTTL");
|
|
1805
|
+
return total;
|
|
1806
|
+
}
|
|
1807
|
+
|
|
1808
|
+
function parseCacheTTLString(value: string): number {
|
|
1809
|
+
const input = value.trim();
|
|
1810
|
+
const match = input.match(/^(\d+(?:\.\d+)?)\s*(ms|s|m|h|d)$/i);
|
|
1811
|
+
if (!match) {
|
|
1812
|
+
throw new Error(`cacheTTL must be a number, an object, or a string like "30m", "6h", or "1d"`);
|
|
1813
|
+
}
|
|
1814
|
+
const amount = Number(match[1]);
|
|
1815
|
+
assertFiniteTTL(amount, "cacheTTL");
|
|
1816
|
+
const unit = match[2].toLowerCase();
|
|
1817
|
+
const multiplier =
|
|
1818
|
+
unit === "ms" ? 1
|
|
1819
|
+
: unit === "s" ? 1000
|
|
1820
|
+
: unit === "m" ? 60 * 1000
|
|
1821
|
+
: unit === "h" ? 60 * 60 * 1000
|
|
1822
|
+
: 24 * 60 * 60 * 1000;
|
|
1823
|
+
return amount * multiplier;
|
|
1824
|
+
}
|
|
1825
|
+
|
|
1826
|
+
function assertFiniteTTL(value: number, label: string): void {
|
|
1827
|
+
if (!Number.isFinite(value) || value < 0) {
|
|
1828
|
+
throw new Error(`${label} must be a finite non-negative duration`);
|
|
1829
|
+
}
|
|
1830
|
+
}
|
|
1831
|
+
|
|
1832
|
+
function isStepInvalidation(value: unknown): value is WorkflowStepInvalidation<string> {
|
|
1833
|
+
return isPlainObject(value) &&
|
|
1834
|
+
value.kind === STEP_INVALIDATION_KIND &&
|
|
1835
|
+
typeof value.target === "string" &&
|
|
1836
|
+
typeof value.targetNodePath === "string";
|
|
1837
|
+
}
|
|
1838
|
+
|
|
1576
1839
|
function parseWithSchema(
|
|
1577
1840
|
schema: OutputSchema,
|
|
1578
1841
|
value: unknown,
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import { chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { homedir } from "node:os";
|
|
3
|
+
import { dirname, join } from "node:path";
|
|
4
|
+
import { hash } from "./hash.ts";
|
|
5
|
+
import type { ProviderStorage, ProviderStorageRecord } from "./provider/types.ts";
|
|
6
|
+
import type { JsonValue } from "./types.ts";
|
|
7
|
+
|
|
8
|
+
export type ProviderHostStorageOptions = {
|
|
9
|
+
providerId: string;
|
|
10
|
+
rootDir?: string;
|
|
11
|
+
};
|
|
12
|
+
|
|
13
|
+
export type ProviderHostStorageFactory = (options: ProviderHostStorageOptions) => ProviderStorage;
|
|
14
|
+
|
|
15
|
+
type ProviderHostStorageFile = {
|
|
16
|
+
providerId: string;
|
|
17
|
+
records: Record<string, Omit<ProviderStorageRecord, "providerId" | "key">>;
|
|
18
|
+
};
|
|
19
|
+
|
|
20
|
+
export function defaultProviderHostStorageDir(): string {
|
|
21
|
+
return process.env.RIGKIT_HOST_STORAGE_DIR ?? join(homedir(), ".rigkit", "providers");
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export function createFileProviderHostStorage(options: ProviderHostStorageOptions): ProviderStorage {
|
|
25
|
+
return new FileProviderHostStorage(options.providerId, providerHostStoragePath(options));
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function providerHostStoragePath(options: ProviderHostStorageOptions): string {
|
|
29
|
+
const rootDir = options.rootDir ?? defaultProviderHostStorageDir();
|
|
30
|
+
const slug = options.providerId.replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "") || "provider";
|
|
31
|
+
return join(rootDir, `${slug}-${hash(options.providerId).slice(0, 12)}.json`);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
class FileProviderHostStorage implements ProviderStorage {
|
|
35
|
+
constructor(
|
|
36
|
+
private readonly providerId: string,
|
|
37
|
+
private readonly path: string,
|
|
38
|
+
) {}
|
|
39
|
+
|
|
40
|
+
get<Value extends JsonValue = JsonValue>(key: string): ProviderStorageRecord<Value> | undefined {
|
|
41
|
+
const file = this.read();
|
|
42
|
+
const record = file.records[key];
|
|
43
|
+
return record
|
|
44
|
+
? {
|
|
45
|
+
providerId: this.providerId,
|
|
46
|
+
key,
|
|
47
|
+
value: record.value as Value,
|
|
48
|
+
createdAt: record.createdAt,
|
|
49
|
+
updatedAt: record.updatedAt,
|
|
50
|
+
}
|
|
51
|
+
: undefined;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
set<Value extends JsonValue = JsonValue>(key: string, value: Value): ProviderStorageRecord<Value> {
|
|
55
|
+
const file = this.read();
|
|
56
|
+
const now = new Date().toISOString();
|
|
57
|
+
const existing = file.records[key];
|
|
58
|
+
const record: ProviderStorageRecord<Value> = {
|
|
59
|
+
providerId: this.providerId,
|
|
60
|
+
key,
|
|
61
|
+
value,
|
|
62
|
+
createdAt: existing?.createdAt ?? now,
|
|
63
|
+
updatedAt: now,
|
|
64
|
+
};
|
|
65
|
+
file.records[key] = {
|
|
66
|
+
value,
|
|
67
|
+
createdAt: record.createdAt,
|
|
68
|
+
updatedAt: record.updatedAt,
|
|
69
|
+
};
|
|
70
|
+
this.write(file);
|
|
71
|
+
return record;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
delete(key: string): void {
|
|
75
|
+
const file = this.read();
|
|
76
|
+
delete file.records[key];
|
|
77
|
+
this.write(file);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
entries(prefix = ""): ProviderStorageRecord[] {
|
|
81
|
+
const file = this.read();
|
|
82
|
+
return Object.entries(file.records)
|
|
83
|
+
.filter(([key]) => key.startsWith(prefix))
|
|
84
|
+
.sort(([a], [b]) => a.localeCompare(b))
|
|
85
|
+
.map(([key, record]) => ({
|
|
86
|
+
providerId: this.providerId,
|
|
87
|
+
key,
|
|
88
|
+
value: record.value,
|
|
89
|
+
createdAt: record.createdAt,
|
|
90
|
+
updatedAt: record.updatedAt,
|
|
91
|
+
}));
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
private read(): ProviderHostStorageFile {
|
|
95
|
+
if (!existsSync(this.path)) {
|
|
96
|
+
return { providerId: this.providerId, records: {} };
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
const parsed = JSON.parse(readFileSync(this.path, "utf8")) as unknown;
|
|
100
|
+
if (!isHostStorageFile(parsed, this.providerId)) {
|
|
101
|
+
throw new Error(`Invalid Rigkit provider host storage at ${this.path}`);
|
|
102
|
+
}
|
|
103
|
+
return parsed;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
private write(file: ProviderHostStorageFile): void {
|
|
107
|
+
mkdirSync(dirname(this.path), { recursive: true, mode: 0o700 });
|
|
108
|
+
writeFileSync(this.path, `${JSON.stringify(file, null, 2)}\n`, { encoding: "utf8", mode: 0o600 });
|
|
109
|
+
chmodSync(this.path, 0o600);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
function isHostStorageFile(value: unknown, providerId: string): value is ProviderHostStorageFile {
|
|
114
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
|
115
|
+
const record = value as { providerId?: unknown; records?: unknown };
|
|
116
|
+
if (record.providerId !== providerId) return false;
|
|
117
|
+
if (!record.records || typeof record.records !== "object" || Array.isArray(record.records)) return false;
|
|
118
|
+
return Object.values(record.records).every((entry) =>
|
|
119
|
+
Boolean(
|
|
120
|
+
entry &&
|
|
121
|
+
typeof entry === "object" &&
|
|
122
|
+
!Array.isArray(entry) &&
|
|
123
|
+
typeof (entry as { createdAt?: unknown }).createdAt === "string" &&
|
|
124
|
+
typeof (entry as { updatedAt?: unknown }).updatedAt === "string" &&
|
|
125
|
+
"value" in entry
|
|
126
|
+
)
|
|
127
|
+
);
|
|
128
|
+
}
|