@langchain/langgraph 0.2.8 → 0.2.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/constants.cjs +12 -2
- package/dist/constants.d.ts +7 -1
- package/dist/constants.js +11 -1
- package/dist/errors.cjs +5 -4
- package/dist/errors.d.ts +1 -1
- package/dist/errors.js +5 -4
- package/dist/graph/graph.cjs +7 -2
- package/dist/graph/graph.js +8 -3
- package/dist/graph/message.cjs +2 -0
- package/dist/graph/message.js +2 -0
- package/dist/graph/state.cjs +8 -3
- package/dist/graph/state.d.ts +1 -1
- package/dist/graph/state.js +9 -4
- package/dist/managed/shared_value.cjs +1 -1
- package/dist/managed/shared_value.js +1 -1
- package/dist/pregel/algo.cjs +119 -54
- package/dist/pregel/algo.d.ts +5 -2
- package/dist/pregel/algo.js +117 -53
- package/dist/pregel/debug.cjs +15 -18
- package/dist/pregel/debug.d.ts +3 -2
- package/dist/pregel/debug.js +16 -19
- package/dist/pregel/index.cjs +295 -110
- package/dist/pregel/index.d.ts +16 -4
- package/dist/pregel/index.js +292 -110
- package/dist/pregel/io.cjs +15 -8
- package/dist/pregel/io.d.ts +2 -2
- package/dist/pregel/io.js +15 -8
- package/dist/pregel/loop.cjs +256 -111
- package/dist/pregel/loop.d.ts +21 -5
- package/dist/pregel/loop.js +256 -109
- package/dist/pregel/read.cjs +9 -2
- package/dist/pregel/read.d.ts +2 -1
- package/dist/pregel/read.js +9 -2
- package/dist/pregel/retry.d.ts +1 -1
- package/dist/pregel/types.d.ts +5 -1
- package/dist/pregel/utils/config.cjs +72 -0
- package/dist/pregel/utils/config.d.ts +2 -0
- package/dist/pregel/utils/config.js +68 -0
- package/dist/pregel/{utils.cjs → utils/index.cjs} +33 -10
- package/dist/pregel/{utils.d.ts → utils/index.d.ts} +4 -7
- package/dist/pregel/{utils.js → utils/index.js} +30 -8
- package/dist/utils.cjs +5 -3
- package/dist/utils.js +5 -3
- package/dist/web.d.ts +2 -1
- package/package.json +1 -1
package/dist/pregel/io.js
CHANGED
|
@@ -73,12 +73,14 @@ export function* mapOutputValues(outputChannels, pendingWrites, channels
|
|
|
73
73
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
74
74
|
) {
|
|
75
75
|
if (Array.isArray(outputChannels)) {
|
|
76
|
-
if (pendingWrites
|
|
76
|
+
if (pendingWrites === true ||
|
|
77
|
+
pendingWrites.find(([chan, _]) => outputChannels.includes(chan))) {
|
|
77
78
|
yield readChannels(channels, outputChannels);
|
|
78
79
|
}
|
|
79
80
|
}
|
|
80
81
|
else {
|
|
81
|
-
if (pendingWrites
|
|
82
|
+
if (pendingWrites === true ||
|
|
83
|
+
pendingWrites.some(([chan, _]) => chan === outputChannels)) {
|
|
82
84
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
83
85
|
yield readChannel(channels, outputChannels);
|
|
84
86
|
}
|
|
@@ -87,30 +89,32 @@ export function* mapOutputValues(outputChannels, pendingWrites, channels
|
|
|
87
89
|
/**
|
|
88
90
|
* Map pending writes (a sequence of tuples (channel, value)) to output chunk.
|
|
89
91
|
*/
|
|
90
|
-
export function* mapOutputUpdates(outputChannels, tasks
|
|
92
|
+
export function* mapOutputUpdates(outputChannels, tasks, cached
|
|
91
93
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
92
94
|
) {
|
|
93
|
-
const outputTasks = tasks.filter((task) =>
|
|
95
|
+
const outputTasks = tasks.filter(([task]) => {
|
|
96
|
+
return task.config === undefined || !task.config.tags?.includes(TAG_HIDDEN);
|
|
97
|
+
});
|
|
94
98
|
if (!outputTasks.length) {
|
|
95
99
|
return;
|
|
96
100
|
}
|
|
97
101
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
98
102
|
let updated;
|
|
99
103
|
if (!Array.isArray(outputChannels)) {
|
|
100
|
-
updated = outputTasks.flatMap((task) => task.writes
|
|
104
|
+
updated = outputTasks.flatMap(([task]) => task.writes
|
|
101
105
|
.filter(([chan, _]) => chan === outputChannels)
|
|
102
106
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
103
107
|
.map(([_, value]) => [task.name, value]));
|
|
104
108
|
}
|
|
105
109
|
else {
|
|
106
110
|
updated = outputTasks
|
|
107
|
-
.filter((task) => task.writes.some(([chan]) => outputChannels.includes(chan)))
|
|
108
|
-
.map((task) => [
|
|
111
|
+
.filter(([task]) => task.writes.some(([chan]) => outputChannels.includes(chan)))
|
|
112
|
+
.map(([task]) => [
|
|
109
113
|
task.name,
|
|
110
114
|
Object.fromEntries(task.writes.filter(([chan]) => outputChannels.includes(chan))),
|
|
111
115
|
]);
|
|
112
116
|
}
|
|
113
|
-
const grouped = Object.fromEntries(outputTasks.map((t) => [t.name, []])
|
|
117
|
+
const grouped = Object.fromEntries(outputTasks.map(([t]) => [t.name, []])
|
|
114
118
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
115
119
|
);
|
|
116
120
|
for (const [node, value] of updated) {
|
|
@@ -126,6 +130,9 @@ export function* mapOutputUpdates(outputChannels, tasks
|
|
|
126
130
|
grouped[node] = value[0];
|
|
127
131
|
}
|
|
128
132
|
}
|
|
133
|
+
if (cached) {
|
|
134
|
+
grouped["__metadata__"] = { cached };
|
|
135
|
+
}
|
|
129
136
|
yield grouped;
|
|
130
137
|
}
|
|
131
138
|
export function single(iter) {
|
package/dist/pregel/loop.cjs
CHANGED
|
@@ -1,10 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.PregelLoop = void 0;
|
|
7
|
-
const double_ended_queue_1 = __importDefault(require("double-ended-queue"));
|
|
3
|
+
exports.PregelLoop = exports.StreamProtocol = void 0;
|
|
8
4
|
const langgraph_checkpoint_1 = require("@langchain/langgraph-checkpoint");
|
|
9
5
|
const base_js_1 = require("../channels/base.cjs");
|
|
10
6
|
const constants_js_1 = require("../constants.cjs");
|
|
@@ -12,12 +8,41 @@ const algo_js_1 = require("./algo.cjs");
|
|
|
12
8
|
const utils_js_1 = require("../utils.cjs");
|
|
13
9
|
const io_js_1 = require("./io.cjs");
|
|
14
10
|
const errors_js_1 = require("../errors.cjs");
|
|
15
|
-
const
|
|
11
|
+
const index_js_1 = require("./utils/index.cjs");
|
|
16
12
|
const debug_js_1 = require("./debug.cjs");
|
|
17
13
|
const batch_js_1 = require("../store/batch.cjs");
|
|
18
14
|
const INPUT_DONE = Symbol.for("INPUT_DONE");
|
|
19
15
|
const INPUT_RESUMING = Symbol.for("INPUT_RESUMING");
|
|
20
16
|
const DEFAULT_LOOP_LIMIT = 25;
|
|
17
|
+
const SPECIAL_CHANNELS = [constants_js_1.ERROR, constants_js_1.INTERRUPT];
|
|
18
|
+
class StreamProtocol {
|
|
19
|
+
constructor(pushFn, modes) {
|
|
20
|
+
Object.defineProperty(this, "push", {
|
|
21
|
+
enumerable: true,
|
|
22
|
+
configurable: true,
|
|
23
|
+
writable: true,
|
|
24
|
+
value: void 0
|
|
25
|
+
});
|
|
26
|
+
Object.defineProperty(this, "modes", {
|
|
27
|
+
enumerable: true,
|
|
28
|
+
configurable: true,
|
|
29
|
+
writable: true,
|
|
30
|
+
value: void 0
|
|
31
|
+
});
|
|
32
|
+
this.push = pushFn;
|
|
33
|
+
this.modes = modes;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
exports.StreamProtocol = StreamProtocol;
|
|
37
|
+
function createDuplexStream(...streams) {
|
|
38
|
+
return new StreamProtocol((value) => {
|
|
39
|
+
for (const stream of streams) {
|
|
40
|
+
if (stream.modes.has(value[1])) {
|
|
41
|
+
stream.push(value);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
}, new Set(streams.flatMap((s) => Array.from(s.modes))));
|
|
45
|
+
}
|
|
21
46
|
class PregelLoop {
|
|
22
47
|
constructor(params) {
|
|
23
48
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
@@ -27,6 +52,13 @@ class PregelLoop {
|
|
|
27
52
|
writable: true,
|
|
28
53
|
value: void 0
|
|
29
54
|
});
|
|
55
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
56
|
+
Object.defineProperty(this, "output", {
|
|
57
|
+
enumerable: true,
|
|
58
|
+
configurable: true,
|
|
59
|
+
writable: true,
|
|
60
|
+
value: void 0
|
|
61
|
+
});
|
|
30
62
|
Object.defineProperty(this, "config", {
|
|
31
63
|
enumerable: true,
|
|
32
64
|
configurable: true,
|
|
@@ -75,6 +107,12 @@ class PregelLoop {
|
|
|
75
107
|
writable: true,
|
|
76
108
|
value: void 0
|
|
77
109
|
});
|
|
110
|
+
Object.defineProperty(this, "checkpointNamespace", {
|
|
111
|
+
enumerable: true,
|
|
112
|
+
configurable: true,
|
|
113
|
+
writable: true,
|
|
114
|
+
value: void 0
|
|
115
|
+
});
|
|
78
116
|
Object.defineProperty(this, "checkpointPendingWrites", {
|
|
79
117
|
enumerable: true,
|
|
80
118
|
configurable: true,
|
|
@@ -123,6 +161,12 @@ class PregelLoop {
|
|
|
123
161
|
writable: true,
|
|
124
162
|
value: void 0
|
|
125
163
|
});
|
|
164
|
+
Object.defineProperty(this, "taskWritesLeft", {
|
|
165
|
+
enumerable: true,
|
|
166
|
+
configurable: true,
|
|
167
|
+
writable: true,
|
|
168
|
+
value: 0
|
|
169
|
+
});
|
|
126
170
|
Object.defineProperty(this, "status", {
|
|
127
171
|
enumerable: true,
|
|
128
172
|
configurable: true,
|
|
@@ -134,14 +178,14 @@ class PregelLoop {
|
|
|
134
178
|
enumerable: true,
|
|
135
179
|
configurable: true,
|
|
136
180
|
writable: true,
|
|
137
|
-
value:
|
|
181
|
+
value: {}
|
|
138
182
|
});
|
|
139
183
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
140
184
|
Object.defineProperty(this, "stream", {
|
|
141
185
|
enumerable: true,
|
|
142
186
|
configurable: true,
|
|
143
187
|
writable: true,
|
|
144
|
-
value:
|
|
188
|
+
value: void 0
|
|
145
189
|
});
|
|
146
190
|
Object.defineProperty(this, "checkpointerPromises", {
|
|
147
191
|
enumerable: true,
|
|
@@ -168,7 +212,6 @@ class PregelLoop {
|
|
|
168
212
|
value: void 0
|
|
169
213
|
});
|
|
170
214
|
this.input = params.input;
|
|
171
|
-
this.config = params.config;
|
|
172
215
|
this.checkpointer = params.checkpointer;
|
|
173
216
|
// TODO: if managed values no longer needs graph we can replace with
|
|
174
217
|
// managed_specs, channel_specs
|
|
@@ -179,7 +222,6 @@ class PregelLoop {
|
|
|
179
222
|
this.checkpointerGetNextVersion = algo_js_1.increment;
|
|
180
223
|
}
|
|
181
224
|
this.checkpoint = params.checkpoint;
|
|
182
|
-
this.checkpointConfig = params.checkpointConfig;
|
|
183
225
|
this.checkpointMetadata = params.checkpointMetadata;
|
|
184
226
|
this.checkpointPreviousVersions = params.checkpointPreviousVersions;
|
|
185
227
|
this.channels = params.channels;
|
|
@@ -187,29 +229,58 @@ class PregelLoop {
|
|
|
187
229
|
this.checkpointPendingWrites = params.checkpointPendingWrites;
|
|
188
230
|
this.step = params.step;
|
|
189
231
|
this.stop = params.stop;
|
|
190
|
-
this.
|
|
232
|
+
this.config = params.config;
|
|
233
|
+
this.checkpointConfig = params.checkpointConfig;
|
|
234
|
+
this.isNested = params.isNested;
|
|
191
235
|
this.outputKeys = params.outputKeys;
|
|
192
236
|
this.streamKeys = params.streamKeys;
|
|
193
237
|
this.nodes = params.nodes;
|
|
194
|
-
this.skipDoneTasks =
|
|
238
|
+
this.skipDoneTasks = params.skipDoneTasks;
|
|
195
239
|
this.store = params.store;
|
|
240
|
+
this.stream = params.stream;
|
|
241
|
+
this.checkpointNamespace = params.checkpointNamespace;
|
|
196
242
|
}
|
|
197
243
|
static async initialize(params) {
|
|
198
|
-
|
|
199
|
-
|
|
244
|
+
let { config, stream } = params;
|
|
245
|
+
if (stream !== undefined &&
|
|
246
|
+
config.configurable?.[constants_js_1.CONFIG_KEY_STREAM] !== undefined) {
|
|
247
|
+
stream = createDuplexStream(stream, config.configurable[constants_js_1.CONFIG_KEY_STREAM]);
|
|
248
|
+
}
|
|
249
|
+
const skipDoneTasks = config.configurable?.checkpoint_id === undefined;
|
|
250
|
+
const isNested = constants_js_1.CONFIG_KEY_READ in (config.configurable ?? {});
|
|
251
|
+
if (!isNested &&
|
|
252
|
+
config.configurable?.checkpoint_ns !== undefined &&
|
|
253
|
+
config.configurable?.checkpoint_ns !== "") {
|
|
254
|
+
config = (0, index_js_1.patchConfigurable)(config, {
|
|
255
|
+
checkpoint_ns: "",
|
|
256
|
+
checkpoint_id: undefined,
|
|
257
|
+
});
|
|
258
|
+
}
|
|
259
|
+
let checkpointConfig = config;
|
|
260
|
+
if (config.configurable?.[constants_js_1.CONFIG_KEY_CHECKPOINT_MAP] !== undefined &&
|
|
261
|
+
config.configurable?.[constants_js_1.CONFIG_KEY_CHECKPOINT_MAP]?.[config.configurable?.checkpoint_ns]) {
|
|
262
|
+
checkpointConfig = (0, index_js_1.patchConfigurable)(config, {
|
|
263
|
+
checkpoint_id: config.configurable[constants_js_1.CONFIG_KEY_CHECKPOINT_MAP][config.configurable?.checkpoint_ns],
|
|
264
|
+
});
|
|
265
|
+
}
|
|
266
|
+
const checkpointNamespace = config.configurable?.checkpoint_ns?.split(constants_js_1.CHECKPOINT_NAMESPACE_SEPARATOR) ?? [];
|
|
267
|
+
const saved = (await params.checkpointer?.getTuple(checkpointConfig)) ?? {
|
|
268
|
+
config,
|
|
200
269
|
checkpoint: (0, langgraph_checkpoint_1.emptyCheckpoint)(),
|
|
201
270
|
metadata: {
|
|
202
271
|
source: "input",
|
|
203
272
|
step: -2,
|
|
204
273
|
writes: null,
|
|
274
|
+
parents: {},
|
|
205
275
|
},
|
|
206
276
|
pendingWrites: [],
|
|
207
277
|
};
|
|
208
|
-
|
|
209
|
-
...
|
|
278
|
+
checkpointConfig = {
|
|
279
|
+
...config,
|
|
210
280
|
...saved.config,
|
|
211
281
|
configurable: {
|
|
212
|
-
|
|
282
|
+
checkpoint_ns: "",
|
|
283
|
+
...config.configurable,
|
|
213
284
|
...saved.config.configurable,
|
|
214
285
|
},
|
|
215
286
|
};
|
|
@@ -218,7 +289,7 @@ class PregelLoop {
|
|
|
218
289
|
const checkpointPendingWrites = saved.pendingWrites ?? [];
|
|
219
290
|
const channels = (0, base_js_1.emptyChannels)(params.channelSpecs, checkpoint);
|
|
220
291
|
const step = (checkpointMetadata.step ?? 0) + 1;
|
|
221
|
-
const stop = step + (
|
|
292
|
+
const stop = step + (config.recursionLimit ?? DEFAULT_LOOP_LIMIT) + 1;
|
|
222
293
|
const checkpointPreviousVersions = { ...checkpoint.channel_versions };
|
|
223
294
|
const store = params.store
|
|
224
295
|
? new batch_js_1.AsyncBatchedStore(params.store)
|
|
@@ -229,13 +300,16 @@ class PregelLoop {
|
|
|
229
300
|
}
|
|
230
301
|
return new PregelLoop({
|
|
231
302
|
input: params.input,
|
|
232
|
-
config
|
|
303
|
+
config,
|
|
233
304
|
checkpointer: params.checkpointer,
|
|
234
305
|
checkpoint,
|
|
235
306
|
checkpointMetadata,
|
|
236
307
|
checkpointConfig,
|
|
308
|
+
checkpointNamespace,
|
|
237
309
|
channels,
|
|
238
310
|
managed: params.managed,
|
|
311
|
+
isNested,
|
|
312
|
+
skipDoneTasks,
|
|
239
313
|
step,
|
|
240
314
|
stop,
|
|
241
315
|
checkpointPreviousVersions,
|
|
@@ -243,6 +317,7 @@ class PregelLoop {
|
|
|
243
317
|
outputKeys: params.outputKeys ?? [],
|
|
244
318
|
streamKeys: params.streamKeys ?? [],
|
|
245
319
|
nodes: params.nodes,
|
|
320
|
+
stream,
|
|
246
321
|
store,
|
|
247
322
|
});
|
|
248
323
|
}
|
|
@@ -265,6 +340,22 @@ class PregelLoop {
|
|
|
265
340
|
* @param writes
|
|
266
341
|
*/
|
|
267
342
|
putWrites(taskId, writes) {
|
|
343
|
+
if (writes.length === 0) {
|
|
344
|
+
return;
|
|
345
|
+
}
|
|
346
|
+
// adjust taskWritesLeft
|
|
347
|
+
const firstChannel = writes[0][0];
|
|
348
|
+
const anyChannelIsSend = writes.find(([channel]) => channel === constants_js_1.TASKS);
|
|
349
|
+
const alwaysSave = anyChannelIsSend || SPECIAL_CHANNELS.includes(firstChannel);
|
|
350
|
+
if (!alwaysSave && !this.taskWritesLeft) {
|
|
351
|
+
return this._outputWrites(taskId, writes);
|
|
352
|
+
}
|
|
353
|
+
else if (firstChannel !== constants_js_1.INTERRUPT) {
|
|
354
|
+
// INTERRUPT makes us want to save the last task's writes
|
|
355
|
+
// so we don't decrement tasksWritesLeft in that case
|
|
356
|
+
this.taskWritesLeft -= 1;
|
|
357
|
+
}
|
|
358
|
+
// save writes
|
|
268
359
|
const pendingWrites = writes.map(([key, value]) => {
|
|
269
360
|
return [taskId, key, value];
|
|
270
361
|
});
|
|
@@ -280,10 +371,23 @@ class PregelLoop {
|
|
|
280
371
|
if (putWritePromise !== undefined) {
|
|
281
372
|
this.checkpointerPromises.push(putWritePromise);
|
|
282
373
|
}
|
|
283
|
-
|
|
374
|
+
this._outputWrites(taskId, writes);
|
|
375
|
+
}
|
|
376
|
+
_outputWrites(taskId, writes, cached = false) {
|
|
377
|
+
const task = this.tasks[taskId];
|
|
284
378
|
if (task !== undefined) {
|
|
285
|
-
|
|
286
|
-
|
|
379
|
+
if (task.config !== undefined &&
|
|
380
|
+
(task.config.tags ?? []).includes(constants_js_1.TAG_HIDDEN)) {
|
|
381
|
+
return;
|
|
382
|
+
}
|
|
383
|
+
if (writes.length > 0 &&
|
|
384
|
+
writes[0][0] !== constants_js_1.ERROR &&
|
|
385
|
+
writes[0][0] !== constants_js_1.INTERRUPT) {
|
|
386
|
+
this._emit((0, utils_js_1.gatherIteratorSync)((0, utils_js_1.prefixGenerator)((0, io_js_1.mapOutputUpdates)(this.outputKeys, [[task, writes]], cached), "updates")));
|
|
387
|
+
}
|
|
388
|
+
if (!cached) {
|
|
389
|
+
this._emit((0, utils_js_1.gatherIteratorSync)((0, utils_js_1.prefixGenerator)((0, debug_js_1.mapDebugTaskResults)(this.step, [[task, writes]], this.streamKeys), "debug")));
|
|
390
|
+
}
|
|
287
391
|
}
|
|
288
392
|
}
|
|
289
393
|
/**
|
|
@@ -292,35 +396,98 @@ class PregelLoop {
|
|
|
292
396
|
* @param params
|
|
293
397
|
*/
|
|
294
398
|
async tick(params) {
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
if (this.status !== "pending") {
|
|
300
|
-
throw new Error(`Cannot tick when status is no longer "pending". Current status: "${this.status}"`);
|
|
301
|
-
}
|
|
302
|
-
if (![INPUT_DONE, INPUT_RESUMING].includes(this.input)) {
|
|
303
|
-
await this._first(inputKeys);
|
|
304
|
-
}
|
|
305
|
-
else if (this.tasks.every((task) => task.writes.length > 0)) {
|
|
306
|
-
const writes = this.tasks.flatMap((t) => t.writes);
|
|
307
|
-
// All tasks have finished
|
|
308
|
-
const myWrites = (0, algo_js_1._applyWrites)(this.checkpoint, this.channels, this.tasks, this.checkpointerGetNextVersion);
|
|
309
|
-
for (const [key, values] of Object.entries(myWrites)) {
|
|
310
|
-
await this.updateManagedValues(key, values);
|
|
399
|
+
let tickError;
|
|
400
|
+
try {
|
|
401
|
+
if (this.store && !this.store.isRunning) {
|
|
402
|
+
this.store?.start();
|
|
311
403
|
}
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
this.
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
404
|
+
const { inputKeys = [], interruptAfter = [], interruptBefore = [], manager, } = params;
|
|
405
|
+
if (this.status !== "pending") {
|
|
406
|
+
throw new Error(`Cannot tick when status is no longer "pending". Current status: "${this.status}"`);
|
|
407
|
+
}
|
|
408
|
+
if (![INPUT_DONE, INPUT_RESUMING].includes(this.input)) {
|
|
409
|
+
await this._first(inputKeys);
|
|
410
|
+
}
|
|
411
|
+
else if (Object.values(this.tasks).every((task) => task.writes.length > 0)) {
|
|
412
|
+
const writes = Object.values(this.tasks).flatMap((t) => t.writes);
|
|
413
|
+
// All tasks have finished
|
|
414
|
+
const managedValueWrites = (0, algo_js_1._applyWrites)(this.checkpoint, this.channels, Object.values(this.tasks), this.checkpointerGetNextVersion);
|
|
415
|
+
for (const [key, values] of Object.entries(managedValueWrites)) {
|
|
416
|
+
await this.updateManagedValues(key, values);
|
|
417
|
+
}
|
|
418
|
+
// produce values output
|
|
419
|
+
const valuesOutput = await (0, utils_js_1.gatherIterator)((0, utils_js_1.prefixGenerator)((0, io_js_1.mapOutputValues)(this.outputKeys, writes, this.channels), "values"));
|
|
420
|
+
this._emit(valuesOutput);
|
|
421
|
+
// clear pending writes
|
|
422
|
+
this.checkpointPendingWrites = [];
|
|
423
|
+
await this._putCheckpoint({
|
|
424
|
+
source: "loop",
|
|
425
|
+
writes: (0, io_js_1.mapOutputUpdates)(this.outputKeys, Object.values(this.tasks).map((task) => [task, task.writes])).next().value ?? null,
|
|
426
|
+
});
|
|
427
|
+
// after execution, check if we should interrupt
|
|
428
|
+
if ((0, algo_js_1.shouldInterrupt)(this.checkpoint, interruptAfter, Object.values(this.tasks))) {
|
|
429
|
+
this.status = "interrupt_after";
|
|
430
|
+
if (this.isNested) {
|
|
431
|
+
throw new errors_js_1.GraphInterrupt();
|
|
432
|
+
}
|
|
433
|
+
else {
|
|
434
|
+
return false;
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
else {
|
|
439
|
+
return false;
|
|
440
|
+
}
|
|
441
|
+
if (this.step > this.stop) {
|
|
442
|
+
this.status = "out_of_steps";
|
|
443
|
+
return false;
|
|
444
|
+
}
|
|
445
|
+
const nextTasks = (0, algo_js_1._prepareNextTasks)(this.checkpoint, this.nodes, this.channels, this.managed, this.config, true, {
|
|
446
|
+
step: this.step,
|
|
447
|
+
checkpointer: this.checkpointer,
|
|
448
|
+
isResuming: this.input === INPUT_RESUMING,
|
|
449
|
+
manager,
|
|
320
450
|
});
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
451
|
+
this.tasks = nextTasks;
|
|
452
|
+
this.taskWritesLeft = Object.values(this.tasks).length - 1;
|
|
453
|
+
// Produce debug output
|
|
454
|
+
if (this.checkpointer) {
|
|
455
|
+
this._emit(await (0, utils_js_1.gatherIterator)((0, utils_js_1.prefixGenerator)((0, debug_js_1.mapDebugCheckpoint)(this.step - 1, // printing checkpoint for previous step
|
|
456
|
+
this.checkpointConfig, this.channels, this.streamKeys, this.checkpointMetadata, Object.values(this.tasks), this.checkpointPendingWrites), "debug")));
|
|
457
|
+
}
|
|
458
|
+
if (Object.values(this.tasks).length === 0) {
|
|
459
|
+
this.status = "done";
|
|
460
|
+
return false;
|
|
461
|
+
}
|
|
462
|
+
// if there are pending writes from a previous loop, apply them
|
|
463
|
+
if (this.skipDoneTasks && this.checkpointPendingWrites.length > 0) {
|
|
464
|
+
for (const [tid, k, v] of this.checkpointPendingWrites) {
|
|
465
|
+
if (k === constants_js_1.ERROR || k === constants_js_1.INTERRUPT) {
|
|
466
|
+
continue;
|
|
467
|
+
}
|
|
468
|
+
const task = Object.values(this.tasks).find((t) => t.id === tid);
|
|
469
|
+
if (task) {
|
|
470
|
+
task.writes.push([k, v]);
|
|
471
|
+
}
|
|
472
|
+
}
|
|
473
|
+
for (const task of Object.values(this.tasks)) {
|
|
474
|
+
if (task.writes.length > 0) {
|
|
475
|
+
this._outputWrites(task.id, task.writes, true);
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
// if all tasks have finished, re-tick
|
|
480
|
+
if (Object.values(this.tasks).every((task) => task.writes.length > 0)) {
|
|
481
|
+
return this.tick({
|
|
482
|
+
inputKeys,
|
|
483
|
+
interruptAfter,
|
|
484
|
+
interruptBefore,
|
|
485
|
+
manager,
|
|
486
|
+
});
|
|
487
|
+
}
|
|
488
|
+
// Before execution, check if we should interrupt
|
|
489
|
+
if ((0, algo_js_1.shouldInterrupt)(this.checkpoint, interruptBefore, Object.values(this.tasks))) {
|
|
490
|
+
this.status = "interrupt_before";
|
|
324
491
|
if (this.isNested) {
|
|
325
492
|
throw new errors_js_1.GraphInterrupt();
|
|
326
493
|
}
|
|
@@ -328,65 +495,29 @@ class PregelLoop {
|
|
|
328
495
|
return false;
|
|
329
496
|
}
|
|
330
497
|
}
|
|
498
|
+
// Produce debug output
|
|
499
|
+
const debugOutput = await (0, utils_js_1.gatherIterator)((0, utils_js_1.prefixGenerator)((0, debug_js_1.mapDebugTasks)(this.step, Object.values(this.tasks)), "debug"));
|
|
500
|
+
this._emit(debugOutput);
|
|
501
|
+
return true;
|
|
331
502
|
}
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
this.status = "out_of_steps";
|
|
337
|
-
return false;
|
|
338
|
-
}
|
|
339
|
-
const nextTasks = (0, algo_js_1._prepareNextTasks)(this.checkpoint, this.nodes, this.channels, this.managed, this.config, true, {
|
|
340
|
-
step: this.step,
|
|
341
|
-
checkpointer: this.checkpointer,
|
|
342
|
-
isResuming: this.input === INPUT_RESUMING,
|
|
343
|
-
manager,
|
|
344
|
-
});
|
|
345
|
-
this.tasks = nextTasks;
|
|
346
|
-
// Produce debug output
|
|
347
|
-
if (this.checkpointer) {
|
|
348
|
-
this.stream.push(...(await (0, utils_js_1.gatherIterator)((0, utils_js_1.prefixGenerator)((0, debug_js_1.mapDebugCheckpoint)(this.step - 1, // printing checkpoint for previous step
|
|
349
|
-
this.checkpointConfig, this.channels, this.streamKeys, this.checkpointMetadata, this.tasks, this.checkpointPendingWrites), "debug"))));
|
|
350
|
-
}
|
|
351
|
-
if (this.tasks.length === 0) {
|
|
352
|
-
this.status = "done";
|
|
353
|
-
return false;
|
|
354
|
-
}
|
|
355
|
-
// if there are pending writes from a previous loop, apply them
|
|
356
|
-
if (this.checkpointPendingWrites.length > 0 && this.skipDoneTasks) {
|
|
357
|
-
for (const [tid, k, v] of this.checkpointPendingWrites) {
|
|
358
|
-
if (k === constants_js_1.ERROR || k === constants_js_1.INTERRUPT) {
|
|
359
|
-
continue;
|
|
360
|
-
}
|
|
361
|
-
const task = this.tasks.find((t) => t.id === tid);
|
|
362
|
-
if (task) {
|
|
363
|
-
task.writes.push([k, v]);
|
|
364
|
-
}
|
|
365
|
-
}
|
|
366
|
-
}
|
|
367
|
-
// if all tasks have finished, re-tick
|
|
368
|
-
if (this.tasks.every((task) => task.writes.length > 0)) {
|
|
369
|
-
return this.tick({
|
|
370
|
-
inputKeys,
|
|
371
|
-
interruptAfter,
|
|
372
|
-
interruptBefore,
|
|
373
|
-
manager,
|
|
374
|
-
});
|
|
375
|
-
}
|
|
376
|
-
// Before execution, check if we should interrupt
|
|
377
|
-
if ((0, algo_js_1.shouldInterrupt)(this.checkpoint, interruptBefore, this.tasks)) {
|
|
378
|
-
this.status = "interrupt_before";
|
|
379
|
-
if (this.isNested) {
|
|
380
|
-
throw new errors_js_1.GraphInterrupt();
|
|
503
|
+
catch (e) {
|
|
504
|
+
tickError = e;
|
|
505
|
+
if (!this._suppressInterrupt(tickError)) {
|
|
506
|
+
throw tickError;
|
|
381
507
|
}
|
|
382
508
|
else {
|
|
383
|
-
|
|
509
|
+
this.output = (0, io_js_1.readChannels)(this.channels, this.outputKeys);
|
|
510
|
+
}
|
|
511
|
+
return false;
|
|
512
|
+
}
|
|
513
|
+
finally {
|
|
514
|
+
if (tickError === undefined) {
|
|
515
|
+
this.output = (0, io_js_1.readChannels)(this.channels, this.outputKeys);
|
|
384
516
|
}
|
|
385
517
|
}
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
return true;
|
|
518
|
+
}
|
|
519
|
+
_suppressInterrupt(e) {
|
|
520
|
+
return (0, errors_js_1.isGraphInterrupt)(e) && !this.isNested;
|
|
390
521
|
}
|
|
391
522
|
/**
|
|
392
523
|
* Resuming from previous checkpoint requires
|
|
@@ -394,9 +525,9 @@ class PregelLoop {
|
|
|
394
525
|
* - receiving None input (outer graph) or RESUMING flag (subgraph)
|
|
395
526
|
*/
|
|
396
527
|
async _first(inputKeys) {
|
|
397
|
-
const isResuming =
|
|
398
|
-
this.config.configurable?.[constants_js_1.CONFIG_KEY_RESUMING] !== undefined
|
|
399
|
-
|
|
528
|
+
const isResuming = Object.keys(this.checkpoint.channel_versions).length !== 0 &&
|
|
529
|
+
(this.config.configurable?.[constants_js_1.CONFIG_KEY_RESUMING] !== undefined ||
|
|
530
|
+
this.input === null);
|
|
400
531
|
if (isResuming) {
|
|
401
532
|
for (const channelName of Object.keys(this.channels)) {
|
|
402
533
|
if (this.checkpoint.channel_versions[channelName] !== undefined) {
|
|
@@ -407,6 +538,9 @@ class PregelLoop {
|
|
|
407
538
|
};
|
|
408
539
|
}
|
|
409
540
|
}
|
|
541
|
+
// produce values output
|
|
542
|
+
const valuesOutput = await (0, utils_js_1.gatherIterator)((0, utils_js_1.prefixGenerator)((0, io_js_1.mapOutputValues)(this.outputKeys, true, this.channels), "values"));
|
|
543
|
+
this._emit(valuesOutput);
|
|
410
544
|
// map inputs to channel updates
|
|
411
545
|
}
|
|
412
546
|
else {
|
|
@@ -415,7 +549,7 @@ class PregelLoop {
|
|
|
415
549
|
throw new errors_js_1.EmptyInputError(`Received no input writes for ${JSON.stringify(inputKeys, null, 2)}`);
|
|
416
550
|
}
|
|
417
551
|
const discardTasks = (0, algo_js_1._prepareNextTasks)(this.checkpoint, this.nodes, this.channels, this.managed, this.config, true, { step: this.step });
|
|
418
|
-
(0, algo_js_1._applyWrites)(this.checkpoint, this.channels, discardTasks.concat([
|
|
552
|
+
(0, algo_js_1._applyWrites)(this.checkpoint, this.channels, Object.values(discardTasks).concat([
|
|
419
553
|
{
|
|
420
554
|
name: constants_js_1.INPUT,
|
|
421
555
|
writes: inputWrites,
|
|
@@ -430,12 +564,25 @@ class PregelLoop {
|
|
|
430
564
|
}
|
|
431
565
|
// done with input
|
|
432
566
|
this.input = isResuming ? INPUT_RESUMING : INPUT_DONE;
|
|
567
|
+
if (!this.isNested) {
|
|
568
|
+
this.config = (0, index_js_1.patchConfigurable)(this.config, {
|
|
569
|
+
[constants_js_1.CONFIG_KEY_RESUMING]: isResuming,
|
|
570
|
+
});
|
|
571
|
+
}
|
|
572
|
+
}
|
|
573
|
+
_emit(values) {
|
|
574
|
+
for (const chunk of values) {
|
|
575
|
+
if (this.stream.modes.has(chunk[0])) {
|
|
576
|
+
this.stream.push([this.checkpointNamespace, ...chunk]);
|
|
577
|
+
}
|
|
578
|
+
}
|
|
433
579
|
}
|
|
434
580
|
async _putCheckpoint(inputMetadata) {
|
|
435
581
|
// Assign step
|
|
436
582
|
const metadata = {
|
|
437
583
|
...inputMetadata,
|
|
438
584
|
step: this.step,
|
|
585
|
+
parents: this.config.configurable?.[constants_js_1.CONFIG_KEY_CHECKPOINT_MAP] ?? {},
|
|
439
586
|
};
|
|
440
587
|
// Bail if no checkpointer
|
|
441
588
|
if (this.checkpointer !== undefined) {
|
|
@@ -445,9 +592,7 @@ class PregelLoop {
|
|
|
445
592
|
// this is achieved by writing child checkpoints as progress is made
|
|
446
593
|
// (so that error recovery / resuming from interrupt don't lose work)
|
|
447
594
|
// but doing so always with an id equal to that of the parent checkpoint
|
|
448
|
-
this.checkpoint = (0, base_js_1.createCheckpoint)(this.checkpoint, this.channels, this.step
|
|
449
|
-
// id: this.isNested ? this.config.configurable?.checkpoint_id : undefined,
|
|
450
|
-
);
|
|
595
|
+
this.checkpoint = (0, base_js_1.createCheckpoint)(this.checkpoint, this.channels, this.step);
|
|
451
596
|
this.checkpointConfig = {
|
|
452
597
|
...this.checkpointConfig,
|
|
453
598
|
configurable: {
|
|
@@ -456,7 +601,7 @@ class PregelLoop {
|
|
|
456
601
|
},
|
|
457
602
|
};
|
|
458
603
|
const channelVersions = { ...this.checkpoint.channel_versions };
|
|
459
|
-
const newVersions = (0,
|
|
604
|
+
const newVersions = (0, index_js_1.getNewChannelVersions)(this.checkpointPreviousVersions, channelVersions);
|
|
460
605
|
this.checkpointPreviousVersions = channelVersions;
|
|
461
606
|
// save it, without blocking
|
|
462
607
|
// if there's a previous checkpoint save in progress, wait for it
|