@langchain/langgraph 0.2.3 → 0.2.5-rc.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/channels/base.cjs +16 -4
- package/dist/channels/base.d.ts +3 -0
- package/dist/channels/base.js +14 -3
- package/dist/constants.cjs +4 -1
- package/dist/constants.d.ts +3 -0
- package/dist/constants.js +3 -0
- package/dist/graph/annotation.cjs +5 -1
- package/dist/graph/annotation.d.ts +5 -4
- package/dist/graph/annotation.js +5 -1
- package/dist/graph/state.cjs +19 -7
- package/dist/graph/state.d.ts +13 -8
- package/dist/graph/state.js +18 -6
- package/dist/managed/base.cjs +163 -0
- package/dist/managed/base.d.ts +30 -0
- package/dist/managed/base.js +155 -0
- package/dist/managed/index.cjs +19 -0
- package/dist/managed/index.d.ts +3 -0
- package/dist/managed/index.js +3 -0
- package/dist/managed/is_last_step.cjs +11 -0
- package/dist/managed/is_last_step.d.ts +4 -0
- package/dist/managed/is_last_step.js +7 -0
- package/dist/managed/shared_value.cjs +109 -0
- package/dist/managed/shared_value.d.ts +23 -0
- package/dist/managed/shared_value.js +105 -0
- package/dist/prebuilt/agent_executor.d.ts +1 -1
- package/dist/pregel/algo.cjs +111 -50
- package/dist/pregel/algo.d.ts +7 -6
- package/dist/pregel/algo.js +112 -51
- package/dist/pregel/index.cjs +78 -6
- package/dist/pregel/index.d.ts +10 -2
- package/dist/pregel/index.js +80 -8
- package/dist/pregel/loop.cjs +40 -3
- package/dist/pregel/loop.d.ts +10 -0
- package/dist/pregel/loop.js +40 -3
- package/dist/pregel/types.d.ts +9 -2
- package/dist/pregel/validate.d.ts +3 -2
- package/dist/store/base.cjs +12 -0
- package/dist/store/base.d.ts +7 -0
- package/dist/store/base.js +8 -0
- package/dist/store/batch.cjs +126 -0
- package/dist/store/batch.d.ts +55 -0
- package/dist/store/batch.js +122 -0
- package/dist/store/index.cjs +19 -0
- package/dist/store/index.d.ts +3 -0
- package/dist/store/index.js +3 -0
- package/dist/store/memory.cjs +43 -0
- package/dist/store/memory.d.ts +6 -0
- package/dist/store/memory.js +39 -0
- package/dist/utils.cjs +26 -1
- package/dist/utils.d.ts +1 -0
- package/dist/utils.js +24 -0
- package/dist/web.cjs +2 -0
- package/dist/web.d.ts +2 -0
- package/dist/web.js +2 -0
- package/package.json +1 -1
package/dist/pregel/index.cjs
CHANGED
|
@@ -16,6 +16,8 @@ const algo_js_1 = require("./algo.cjs");
|
|
|
16
16
|
const utils_js_1 = require("./utils.cjs");
|
|
17
17
|
const loop_js_1 = require("./loop.cjs");
|
|
18
18
|
const retry_js_1 = require("./retry.cjs");
|
|
19
|
+
const base_js_2 = require("../managed/base.cjs");
|
|
20
|
+
const utils_js_2 = require("../utils.cjs");
|
|
19
21
|
function isString(value) {
|
|
20
22
|
return typeof value === "string";
|
|
21
23
|
}
|
|
@@ -111,6 +113,12 @@ class Pregel extends runnables_1.Runnable {
|
|
|
111
113
|
writable: true,
|
|
112
114
|
value: void 0
|
|
113
115
|
});
|
|
116
|
+
Object.defineProperty(this, "configKeys", {
|
|
117
|
+
enumerable: true,
|
|
118
|
+
configurable: true,
|
|
119
|
+
writable: true,
|
|
120
|
+
value: void 0
|
|
121
|
+
});
|
|
114
122
|
Object.defineProperty(this, "autoValidate", {
|
|
115
123
|
enumerable: true,
|
|
116
124
|
configurable: true,
|
|
@@ -165,6 +173,12 @@ class Pregel extends runnables_1.Runnable {
|
|
|
165
173
|
writable: true,
|
|
166
174
|
value: void 0
|
|
167
175
|
});
|
|
176
|
+
Object.defineProperty(this, "store", {
|
|
177
|
+
enumerable: true,
|
|
178
|
+
configurable: true,
|
|
179
|
+
writable: true,
|
|
180
|
+
value: void 0
|
|
181
|
+
});
|
|
168
182
|
let { streamMode } = fields;
|
|
169
183
|
if (streamMode != null && !Array.isArray(streamMode)) {
|
|
170
184
|
streamMode = [streamMode];
|
|
@@ -175,6 +189,7 @@ class Pregel extends runnables_1.Runnable {
|
|
|
175
189
|
this.streamMode = streamMode ?? this.streamMode;
|
|
176
190
|
this.inputChannels = fields.inputChannels;
|
|
177
191
|
this.outputChannels = fields.outputChannels;
|
|
192
|
+
this.configKeys = fields.configKeys;
|
|
178
193
|
this.streamChannels = fields.streamChannels ?? this.streamChannels;
|
|
179
194
|
this.interruptAfter = fields.interruptAfter;
|
|
180
195
|
this.interruptBefore = fields.interruptBefore;
|
|
@@ -182,6 +197,7 @@ class Pregel extends runnables_1.Runnable {
|
|
|
182
197
|
this.debug = fields.debug ?? this.debug;
|
|
183
198
|
this.checkpointer = fields.checkpointer;
|
|
184
199
|
this.retryPolicy = fields.retryPolicy;
|
|
200
|
+
this.store = fields.store;
|
|
185
201
|
if (this.autoValidate) {
|
|
186
202
|
this.validate();
|
|
187
203
|
}
|
|
@@ -227,7 +243,8 @@ class Pregel extends runnables_1.Runnable {
|
|
|
227
243
|
const saved = await this.checkpointer.getTuple(config);
|
|
228
244
|
const checkpoint = saved ? saved.checkpoint : (0, langgraph_checkpoint_1.emptyCheckpoint)();
|
|
229
245
|
const channels = (0, base_js_1.emptyChannels)(this.channels, checkpoint);
|
|
230
|
-
const
|
|
246
|
+
const { managed } = await this.prepareSpecs(config);
|
|
247
|
+
const nextTasks = (0, algo_js_1._prepareNextTasks)(checkpoint, this.nodes, channels, managed, saved !== undefined ? saved.config : config, false, { step: saved ? (saved.metadata?.step ?? -1) + 1 : -1 });
|
|
231
248
|
return {
|
|
232
249
|
values: (0, io_js_1.readChannels)(channels, this.streamChannelsAsIs),
|
|
233
250
|
next: nextTasks.map((task) => task.name),
|
|
@@ -245,9 +262,10 @@ class Pregel extends runnables_1.Runnable {
|
|
|
245
262
|
if (!this.checkpointer) {
|
|
246
263
|
throw new errors_js_1.GraphValueError("No checkpointer set");
|
|
247
264
|
}
|
|
265
|
+
const { managed } = await this.prepareSpecs(config);
|
|
248
266
|
for await (const saved of this.checkpointer.list(config, options)) {
|
|
249
267
|
const channels = (0, base_js_1.emptyChannels)(this.channels, saved.checkpoint);
|
|
250
|
-
const nextTasks = (0, algo_js_1._prepareNextTasks)(saved.checkpoint, this.nodes, channels, saved.config, false, { step: -1 });
|
|
268
|
+
const nextTasks = (0, algo_js_1._prepareNextTasks)(saved.checkpoint, this.nodes, channels, managed, saved.config, false, { step: -1 });
|
|
251
269
|
yield {
|
|
252
270
|
values: (0, io_js_1.readChannels)(channels, this.streamChannelsAsIs),
|
|
253
271
|
next: nextTasks.map((task) => task.name),
|
|
@@ -336,6 +354,7 @@ class Pregel extends runnables_1.Runnable {
|
|
|
336
354
|
}
|
|
337
355
|
// update channels
|
|
338
356
|
const channels = (0, base_js_1.emptyChannels)(this.channels, checkpoint);
|
|
357
|
+
const { managed } = await this.prepareSpecs(config);
|
|
339
358
|
// run all writers of the chosen node
|
|
340
359
|
const writers = this.nodes[asNode].getWriters();
|
|
341
360
|
if (!writers.length) {
|
|
@@ -356,9 +375,9 @@ class Pregel extends runnables_1.Runnable {
|
|
|
356
375
|
runName: config.runName ?? `${this.getName()}UpdateState`,
|
|
357
376
|
configurable: {
|
|
358
377
|
[constants_js_1.CONFIG_KEY_SEND]: (items) => task.writes.push(...items),
|
|
359
|
-
[constants_js_1.CONFIG_KEY_READ]: algo_js_1._localRead
|
|
378
|
+
[constants_js_1.CONFIG_KEY_READ]: (select_, fresh_ = false) => (0, algo_js_1._localRead)(step, checkpoint, channels, managed,
|
|
360
379
|
// TODO: Why does keyof StrRecord allow number and symbol?
|
|
361
|
-
task),
|
|
380
|
+
task, select_, fresh_),
|
|
362
381
|
},
|
|
363
382
|
}));
|
|
364
383
|
if (saved !== undefined) {
|
|
@@ -413,6 +432,12 @@ class Pregel extends runnables_1.Runnable {
|
|
|
413
432
|
else {
|
|
414
433
|
defaultCheckpointer = this.checkpointer;
|
|
415
434
|
}
|
|
435
|
+
if (this.configKeys !== undefined) {
|
|
436
|
+
const newConfigurable = Object.fromEntries(Object.entries(rest.configurable ?? {}).filter(([key]) => {
|
|
437
|
+
return this.configKeys?.includes(key);
|
|
438
|
+
}));
|
|
439
|
+
rest.configurable = newConfigurable;
|
|
440
|
+
}
|
|
416
441
|
return [
|
|
417
442
|
defaultDebug,
|
|
418
443
|
defaultStreamMode,
|
|
@@ -442,6 +467,43 @@ class Pregel extends runnables_1.Runnable {
|
|
|
442
467
|
async stream(input, options) {
|
|
443
468
|
return super.stream(input, options);
|
|
444
469
|
}
|
|
470
|
+
async prepareSpecs(config) {
|
|
471
|
+
const configForManaged = (0, utils_js_2.patchConfigurable)(config, {
|
|
472
|
+
[constants_js_1.CONFIG_KEY_STORE]: this.store,
|
|
473
|
+
});
|
|
474
|
+
const channelSpecs = {};
|
|
475
|
+
const managedSpecs = {};
|
|
476
|
+
for (const [name, spec] of Object.entries(this.channels)) {
|
|
477
|
+
if ((0, base_js_1.isBaseChannel)(spec)) {
|
|
478
|
+
channelSpecs[name] = spec;
|
|
479
|
+
}
|
|
480
|
+
else {
|
|
481
|
+
managedSpecs[name] = spec;
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
const managed = new base_js_2.ManagedValueMapping(await Object.entries(managedSpecs).reduce(async (accPromise, [key, value]) => {
|
|
485
|
+
const acc = await accPromise;
|
|
486
|
+
let initializedValue;
|
|
487
|
+
if ((0, base_js_2.isConfiguredManagedValue)(value)) {
|
|
488
|
+
if ("key" in value.params &&
|
|
489
|
+
value.params.key === base_js_2.ChannelKeyPlaceholder) {
|
|
490
|
+
value.params.key = key;
|
|
491
|
+
}
|
|
492
|
+
initializedValue = await value.cls.initialize(configForManaged, value.params);
|
|
493
|
+
}
|
|
494
|
+
else {
|
|
495
|
+
initializedValue = await value.initialize(configForManaged);
|
|
496
|
+
}
|
|
497
|
+
if (initializedValue !== undefined) {
|
|
498
|
+
acc.push([key, initializedValue]);
|
|
499
|
+
}
|
|
500
|
+
return acc;
|
|
501
|
+
}, Promise.resolve([])));
|
|
502
|
+
return {
|
|
503
|
+
channelSpecs,
|
|
504
|
+
managed,
|
|
505
|
+
};
|
|
506
|
+
}
|
|
445
507
|
async *_streamIterator(input, options) {
|
|
446
508
|
const inputConfig = (0, runnables_1.ensureConfig)(options);
|
|
447
509
|
if (inputConfig.recursionLimit === undefined ||
|
|
@@ -457,6 +519,7 @@ class Pregel extends runnables_1.Runnable {
|
|
|
457
519
|
delete inputConfig.runId;
|
|
458
520
|
// assign defaults
|
|
459
521
|
const [debug, streamMode, , outputKeys, config, interruptBefore, interruptAfter, checkpointer,] = this._defaults(inputConfig);
|
|
522
|
+
const { channelSpecs, managed } = await this.prepareSpecs(config);
|
|
460
523
|
let loop;
|
|
461
524
|
try {
|
|
462
525
|
loop = await loop_js_1.PregelLoop.initialize({
|
|
@@ -464,9 +527,11 @@ class Pregel extends runnables_1.Runnable {
|
|
|
464
527
|
config,
|
|
465
528
|
checkpointer,
|
|
466
529
|
nodes: this.nodes,
|
|
467
|
-
channelSpecs
|
|
530
|
+
channelSpecs,
|
|
531
|
+
managed,
|
|
468
532
|
outputKeys,
|
|
469
533
|
streamKeys: this.streamChannelsAsIs,
|
|
534
|
+
store: this.store,
|
|
470
535
|
});
|
|
471
536
|
while (await loop.tick({
|
|
472
537
|
inputKeys: this.inputChannels,
|
|
@@ -568,7 +633,14 @@ class Pregel extends runnables_1.Runnable {
|
|
|
568
633
|
throw e;
|
|
569
634
|
}
|
|
570
635
|
finally {
|
|
571
|
-
|
|
636
|
+
// Call `.stop()` again incase it was not called in the loop, e.g due to an error.
|
|
637
|
+
if (loop) {
|
|
638
|
+
loop.store?.stop();
|
|
639
|
+
}
|
|
640
|
+
await Promise.all([
|
|
641
|
+
loop?.checkpointerPromises ?? [],
|
|
642
|
+
...Array.from(managed.values()).map((mv) => mv.promises()),
|
|
643
|
+
]);
|
|
572
644
|
}
|
|
573
645
|
}
|
|
574
646
|
/**
|
package/dist/pregel/index.d.ts
CHANGED
|
@@ -7,6 +7,8 @@ import { ChannelWrite } from "./write.js";
|
|
|
7
7
|
import { PregelInterface, PregelParams, StateSnapshot, StreamMode } from "./types.js";
|
|
8
8
|
import { StrRecord } from "./algo.js";
|
|
9
9
|
import { RetryPolicy } from "./utils.js";
|
|
10
|
+
import { BaseStore } from "../store/base.js";
|
|
11
|
+
import { ManagedValueMapping, type ManagedValueSpec } from "../managed/base.js";
|
|
10
12
|
type WriteValue = Runnable | RunnableFunc<unknown, unknown> | unknown;
|
|
11
13
|
export declare class Channel {
|
|
12
14
|
static subscribeTo(channels: string, options?: {
|
|
@@ -21,7 +23,7 @@ export declare class Channel {
|
|
|
21
23
|
/**
|
|
22
24
|
* Config for executing the graph.
|
|
23
25
|
*/
|
|
24
|
-
export interface PregelOptions<Nn extends StrRecord<string, PregelNode>, Cc extends StrRecord<string, BaseChannel>> extends RunnableConfig {
|
|
26
|
+
export interface PregelOptions<Nn extends StrRecord<string, PregelNode>, Cc extends StrRecord<string, BaseChannel | ManagedValueSpec>> extends RunnableConfig {
|
|
25
27
|
/** The stream mode for the graph run. Default is ["values"]. */
|
|
26
28
|
streamMode?: StreamMode | StreamMode[];
|
|
27
29
|
inputKeys?: keyof Cc | Array<keyof Cc>;
|
|
@@ -36,13 +38,14 @@ export interface PregelOptions<Nn extends StrRecord<string, PregelNode>, Cc exte
|
|
|
36
38
|
}
|
|
37
39
|
export type PregelInputType = any;
|
|
38
40
|
export type PregelOutputType = any;
|
|
39
|
-
export declare class Pregel<Nn extends StrRecord<string, PregelNode>, Cc extends StrRecord<string, BaseChannel>> extends Runnable<PregelInputType, PregelOutputType, PregelOptions<Nn, Cc>> implements PregelInterface<Nn, Cc> {
|
|
41
|
+
export declare class Pregel<Nn extends StrRecord<string, PregelNode>, Cc extends StrRecord<string, BaseChannel | ManagedValueSpec>> extends Runnable<PregelInputType, PregelOutputType, PregelOptions<Nn, Cc>> implements PregelInterface<Nn, Cc> {
|
|
40
42
|
static lc_name(): string;
|
|
41
43
|
lc_namespace: string[];
|
|
42
44
|
nodes: Nn;
|
|
43
45
|
channels: Cc;
|
|
44
46
|
inputChannels: keyof Cc | Array<keyof Cc>;
|
|
45
47
|
outputChannels: keyof Cc | Array<keyof Cc>;
|
|
48
|
+
configKeys?: string[];
|
|
46
49
|
autoValidate: boolean;
|
|
47
50
|
streamMode: StreamMode[];
|
|
48
51
|
streamChannels?: keyof Cc | Array<keyof Cc>;
|
|
@@ -52,6 +55,7 @@ export declare class Pregel<Nn extends StrRecord<string, PregelNode>, Cc extends
|
|
|
52
55
|
debug: boolean;
|
|
53
56
|
checkpointer?: BaseCheckpointSaver;
|
|
54
57
|
retryPolicy?: RetryPolicy;
|
|
58
|
+
store?: BaseStore;
|
|
55
59
|
constructor(fields: PregelParams<Nn, Cc>);
|
|
56
60
|
validate(): this;
|
|
57
61
|
get streamChannelsList(): Array<keyof Cc>;
|
|
@@ -101,6 +105,10 @@ export declare class Pregel<Nn extends StrRecord<string, PregelNode>, Cc extends
|
|
|
101
105
|
* @param options.debug Whether to print debug information during execution.
|
|
102
106
|
*/
|
|
103
107
|
stream(input: PregelInputType, options?: Partial<PregelOptions<Nn, Cc>>): Promise<IterableReadableStream<PregelOutputType>>;
|
|
108
|
+
protected prepareSpecs(config: RunnableConfig): Promise<{
|
|
109
|
+
channelSpecs: Record<string, BaseChannel<unknown, unknown, unknown>>;
|
|
110
|
+
managed: ManagedValueMapping;
|
|
111
|
+
}>;
|
|
104
112
|
_streamIterator(input: PregelInputType, options?: Partial<PregelOptions<Nn, Cc>>): AsyncGenerator<PregelOutputType>;
|
|
105
113
|
/**
|
|
106
114
|
* Run the graph with a single input and config.
|
package/dist/pregel/index.js
CHANGED
|
@@ -1,18 +1,20 @@
|
|
|
1
1
|
/* eslint-disable no-param-reassign */
|
|
2
2
|
import { Runnable, RunnableSequence, _coerceToRunnable, ensureConfig, getCallbackManagerForConfig, patchConfig, } from "@langchain/core/runnables";
|
|
3
3
|
import { compareChannelVersions, copyCheckpoint, emptyCheckpoint, uuid5, } from "@langchain/langgraph-checkpoint";
|
|
4
|
-
import { createCheckpoint, emptyChannels, } from "../channels/base.js";
|
|
4
|
+
import { createCheckpoint, emptyChannels, isBaseChannel, } from "../channels/base.js";
|
|
5
5
|
import { PregelNode } from "./read.js";
|
|
6
6
|
import { validateGraph, validateKeys } from "./validate.js";
|
|
7
7
|
import { readChannels } from "./io.js";
|
|
8
8
|
import { printStepCheckpoint, printStepTasks, printStepWrites, tasksWithWrites, } from "./debug.js";
|
|
9
9
|
import { ChannelWrite, PASSTHROUGH } from "./write.js";
|
|
10
|
-
import { CONFIG_KEY_CHECKPOINTER, CONFIG_KEY_READ, CONFIG_KEY_SEND, ERROR, INTERRUPT, } from "../constants.js";
|
|
10
|
+
import { CONFIG_KEY_CHECKPOINTER, CONFIG_KEY_READ, CONFIG_KEY_SEND, CONFIG_KEY_STORE, ERROR, INTERRUPT, } from "../constants.js";
|
|
11
11
|
import { GraphRecursionError, GraphValueError, InvalidUpdateError, isGraphInterrupt, } from "../errors.js";
|
|
12
12
|
import { _prepareNextTasks, _localRead, _applyWrites, } from "./algo.js";
|
|
13
13
|
import { _coerceToDict, getNewChannelVersions } from "./utils.js";
|
|
14
14
|
import { PregelLoop } from "./loop.js";
|
|
15
15
|
import { executeTasksWithRetry } from "./retry.js";
|
|
16
|
+
import { ChannelKeyPlaceholder, isConfiguredManagedValue, ManagedValueMapping, } from "../managed/base.js";
|
|
17
|
+
import { patchConfigurable } from "../utils.js";
|
|
16
18
|
function isString(value) {
|
|
17
19
|
return typeof value === "string";
|
|
18
20
|
}
|
|
@@ -107,6 +109,12 @@ export class Pregel extends Runnable {
|
|
|
107
109
|
writable: true,
|
|
108
110
|
value: void 0
|
|
109
111
|
});
|
|
112
|
+
Object.defineProperty(this, "configKeys", {
|
|
113
|
+
enumerable: true,
|
|
114
|
+
configurable: true,
|
|
115
|
+
writable: true,
|
|
116
|
+
value: void 0
|
|
117
|
+
});
|
|
110
118
|
Object.defineProperty(this, "autoValidate", {
|
|
111
119
|
enumerable: true,
|
|
112
120
|
configurable: true,
|
|
@@ -161,6 +169,12 @@ export class Pregel extends Runnable {
|
|
|
161
169
|
writable: true,
|
|
162
170
|
value: void 0
|
|
163
171
|
});
|
|
172
|
+
Object.defineProperty(this, "store", {
|
|
173
|
+
enumerable: true,
|
|
174
|
+
configurable: true,
|
|
175
|
+
writable: true,
|
|
176
|
+
value: void 0
|
|
177
|
+
});
|
|
164
178
|
let { streamMode } = fields;
|
|
165
179
|
if (streamMode != null && !Array.isArray(streamMode)) {
|
|
166
180
|
streamMode = [streamMode];
|
|
@@ -171,6 +185,7 @@ export class Pregel extends Runnable {
|
|
|
171
185
|
this.streamMode = streamMode ?? this.streamMode;
|
|
172
186
|
this.inputChannels = fields.inputChannels;
|
|
173
187
|
this.outputChannels = fields.outputChannels;
|
|
188
|
+
this.configKeys = fields.configKeys;
|
|
174
189
|
this.streamChannels = fields.streamChannels ?? this.streamChannels;
|
|
175
190
|
this.interruptAfter = fields.interruptAfter;
|
|
176
191
|
this.interruptBefore = fields.interruptBefore;
|
|
@@ -178,6 +193,7 @@ export class Pregel extends Runnable {
|
|
|
178
193
|
this.debug = fields.debug ?? this.debug;
|
|
179
194
|
this.checkpointer = fields.checkpointer;
|
|
180
195
|
this.retryPolicy = fields.retryPolicy;
|
|
196
|
+
this.store = fields.store;
|
|
181
197
|
if (this.autoValidate) {
|
|
182
198
|
this.validate();
|
|
183
199
|
}
|
|
@@ -223,7 +239,8 @@ export class Pregel extends Runnable {
|
|
|
223
239
|
const saved = await this.checkpointer.getTuple(config);
|
|
224
240
|
const checkpoint = saved ? saved.checkpoint : emptyCheckpoint();
|
|
225
241
|
const channels = emptyChannels(this.channels, checkpoint);
|
|
226
|
-
const
|
|
242
|
+
const { managed } = await this.prepareSpecs(config);
|
|
243
|
+
const nextTasks = _prepareNextTasks(checkpoint, this.nodes, channels, managed, saved !== undefined ? saved.config : config, false, { step: saved ? (saved.metadata?.step ?? -1) + 1 : -1 });
|
|
227
244
|
return {
|
|
228
245
|
values: readChannels(channels, this.streamChannelsAsIs),
|
|
229
246
|
next: nextTasks.map((task) => task.name),
|
|
@@ -241,9 +258,10 @@ export class Pregel extends Runnable {
|
|
|
241
258
|
if (!this.checkpointer) {
|
|
242
259
|
throw new GraphValueError("No checkpointer set");
|
|
243
260
|
}
|
|
261
|
+
const { managed } = await this.prepareSpecs(config);
|
|
244
262
|
for await (const saved of this.checkpointer.list(config, options)) {
|
|
245
263
|
const channels = emptyChannels(this.channels, saved.checkpoint);
|
|
246
|
-
const nextTasks = _prepareNextTasks(saved.checkpoint, this.nodes, channels, saved.config, false, { step: -1 });
|
|
264
|
+
const nextTasks = _prepareNextTasks(saved.checkpoint, this.nodes, channels, managed, saved.config, false, { step: -1 });
|
|
247
265
|
yield {
|
|
248
266
|
values: readChannels(channels, this.streamChannelsAsIs),
|
|
249
267
|
next: nextTasks.map((task) => task.name),
|
|
@@ -332,6 +350,7 @@ export class Pregel extends Runnable {
|
|
|
332
350
|
}
|
|
333
351
|
// update channels
|
|
334
352
|
const channels = emptyChannels(this.channels, checkpoint);
|
|
353
|
+
const { managed } = await this.prepareSpecs(config);
|
|
335
354
|
// run all writers of the chosen node
|
|
336
355
|
const writers = this.nodes[asNode].getWriters();
|
|
337
356
|
if (!writers.length) {
|
|
@@ -352,9 +371,9 @@ export class Pregel extends Runnable {
|
|
|
352
371
|
runName: config.runName ?? `${this.getName()}UpdateState`,
|
|
353
372
|
configurable: {
|
|
354
373
|
[CONFIG_KEY_SEND]: (items) => task.writes.push(...items),
|
|
355
|
-
[CONFIG_KEY_READ]: _localRead
|
|
374
|
+
[CONFIG_KEY_READ]: (select_, fresh_ = false) => _localRead(step, checkpoint, channels, managed,
|
|
356
375
|
// TODO: Why does keyof StrRecord allow number and symbol?
|
|
357
|
-
task),
|
|
376
|
+
task, select_, fresh_),
|
|
358
377
|
},
|
|
359
378
|
}));
|
|
360
379
|
if (saved !== undefined) {
|
|
@@ -409,6 +428,12 @@ export class Pregel extends Runnable {
|
|
|
409
428
|
else {
|
|
410
429
|
defaultCheckpointer = this.checkpointer;
|
|
411
430
|
}
|
|
431
|
+
if (this.configKeys !== undefined) {
|
|
432
|
+
const newConfigurable = Object.fromEntries(Object.entries(rest.configurable ?? {}).filter(([key]) => {
|
|
433
|
+
return this.configKeys?.includes(key);
|
|
434
|
+
}));
|
|
435
|
+
rest.configurable = newConfigurable;
|
|
436
|
+
}
|
|
412
437
|
return [
|
|
413
438
|
defaultDebug,
|
|
414
439
|
defaultStreamMode,
|
|
@@ -438,6 +463,43 @@ export class Pregel extends Runnable {
|
|
|
438
463
|
async stream(input, options) {
|
|
439
464
|
return super.stream(input, options);
|
|
440
465
|
}
|
|
466
|
+
async prepareSpecs(config) {
|
|
467
|
+
const configForManaged = patchConfigurable(config, {
|
|
468
|
+
[CONFIG_KEY_STORE]: this.store,
|
|
469
|
+
});
|
|
470
|
+
const channelSpecs = {};
|
|
471
|
+
const managedSpecs = {};
|
|
472
|
+
for (const [name, spec] of Object.entries(this.channels)) {
|
|
473
|
+
if (isBaseChannel(spec)) {
|
|
474
|
+
channelSpecs[name] = spec;
|
|
475
|
+
}
|
|
476
|
+
else {
|
|
477
|
+
managedSpecs[name] = spec;
|
|
478
|
+
}
|
|
479
|
+
}
|
|
480
|
+
const managed = new ManagedValueMapping(await Object.entries(managedSpecs).reduce(async (accPromise, [key, value]) => {
|
|
481
|
+
const acc = await accPromise;
|
|
482
|
+
let initializedValue;
|
|
483
|
+
if (isConfiguredManagedValue(value)) {
|
|
484
|
+
if ("key" in value.params &&
|
|
485
|
+
value.params.key === ChannelKeyPlaceholder) {
|
|
486
|
+
value.params.key = key;
|
|
487
|
+
}
|
|
488
|
+
initializedValue = await value.cls.initialize(configForManaged, value.params);
|
|
489
|
+
}
|
|
490
|
+
else {
|
|
491
|
+
initializedValue = await value.initialize(configForManaged);
|
|
492
|
+
}
|
|
493
|
+
if (initializedValue !== undefined) {
|
|
494
|
+
acc.push([key, initializedValue]);
|
|
495
|
+
}
|
|
496
|
+
return acc;
|
|
497
|
+
}, Promise.resolve([])));
|
|
498
|
+
return {
|
|
499
|
+
channelSpecs,
|
|
500
|
+
managed,
|
|
501
|
+
};
|
|
502
|
+
}
|
|
441
503
|
async *_streamIterator(input, options) {
|
|
442
504
|
const inputConfig = ensureConfig(options);
|
|
443
505
|
if (inputConfig.recursionLimit === undefined ||
|
|
@@ -453,6 +515,7 @@ export class Pregel extends Runnable {
|
|
|
453
515
|
delete inputConfig.runId;
|
|
454
516
|
// assign defaults
|
|
455
517
|
const [debug, streamMode, , outputKeys, config, interruptBefore, interruptAfter, checkpointer,] = this._defaults(inputConfig);
|
|
518
|
+
const { channelSpecs, managed } = await this.prepareSpecs(config);
|
|
456
519
|
let loop;
|
|
457
520
|
try {
|
|
458
521
|
loop = await PregelLoop.initialize({
|
|
@@ -460,9 +523,11 @@ export class Pregel extends Runnable {
|
|
|
460
523
|
config,
|
|
461
524
|
checkpointer,
|
|
462
525
|
nodes: this.nodes,
|
|
463
|
-
channelSpecs
|
|
526
|
+
channelSpecs,
|
|
527
|
+
managed,
|
|
464
528
|
outputKeys,
|
|
465
529
|
streamKeys: this.streamChannelsAsIs,
|
|
530
|
+
store: this.store,
|
|
466
531
|
});
|
|
467
532
|
while (await loop.tick({
|
|
468
533
|
inputKeys: this.inputChannels,
|
|
@@ -564,7 +629,14 @@ export class Pregel extends Runnable {
|
|
|
564
629
|
throw e;
|
|
565
630
|
}
|
|
566
631
|
finally {
|
|
567
|
-
|
|
632
|
+
// Call `.stop()` again incase it was not called in the loop, e.g due to an error.
|
|
633
|
+
if (loop) {
|
|
634
|
+
loop.store?.stop();
|
|
635
|
+
}
|
|
636
|
+
await Promise.all([
|
|
637
|
+
loop?.checkpointerPromises ?? [],
|
|
638
|
+
...Array.from(managed.values()).map((mv) => mv.promises()),
|
|
639
|
+
]);
|
|
568
640
|
}
|
|
569
641
|
}
|
|
570
642
|
/**
|
package/dist/pregel/loop.cjs
CHANGED
|
@@ -14,6 +14,7 @@ const io_js_1 = require("./io.cjs");
|
|
|
14
14
|
const errors_js_1 = require("../errors.cjs");
|
|
15
15
|
const utils_js_2 = require("./utils.cjs");
|
|
16
16
|
const debug_js_1 = require("./debug.cjs");
|
|
17
|
+
const batch_js_1 = require("../store/batch.cjs");
|
|
17
18
|
const INPUT_DONE = Symbol.for("INPUT_DONE");
|
|
18
19
|
const INPUT_RESUMING = Symbol.for("INPUT_RESUMING");
|
|
19
20
|
const DEFAULT_LOOP_LIMIT = 25;
|
|
@@ -50,6 +51,12 @@ class PregelLoop {
|
|
|
50
51
|
writable: true,
|
|
51
52
|
value: void 0
|
|
52
53
|
});
|
|
54
|
+
Object.defineProperty(this, "managed", {
|
|
55
|
+
enumerable: true,
|
|
56
|
+
configurable: true,
|
|
57
|
+
writable: true,
|
|
58
|
+
value: void 0
|
|
59
|
+
});
|
|
53
60
|
Object.defineProperty(this, "checkpoint", {
|
|
54
61
|
enumerable: true,
|
|
55
62
|
configurable: true,
|
|
@@ -154,6 +161,12 @@ class PregelLoop {
|
|
|
154
161
|
writable: true,
|
|
155
162
|
value: Promise.resolve()
|
|
156
163
|
});
|
|
164
|
+
Object.defineProperty(this, "store", {
|
|
165
|
+
enumerable: true,
|
|
166
|
+
configurable: true,
|
|
167
|
+
writable: true,
|
|
168
|
+
value: void 0
|
|
169
|
+
});
|
|
157
170
|
this.input = params.input;
|
|
158
171
|
this.config = params.config;
|
|
159
172
|
this.checkpointer = params.checkpointer;
|
|
@@ -170,6 +183,7 @@ class PregelLoop {
|
|
|
170
183
|
this.checkpointMetadata = params.checkpointMetadata;
|
|
171
184
|
this.checkpointPreviousVersions = params.checkpointPreviousVersions;
|
|
172
185
|
this.channels = params.channels;
|
|
186
|
+
this.managed = params.managed;
|
|
173
187
|
this.checkpointPendingWrites = params.checkpointPendingWrites;
|
|
174
188
|
this.step = params.step;
|
|
175
189
|
this.stop = params.stop;
|
|
@@ -178,6 +192,7 @@ class PregelLoop {
|
|
|
178
192
|
this.streamKeys = params.streamKeys;
|
|
179
193
|
this.nodes = params.nodes;
|
|
180
194
|
this.skipDoneTasks = this.config.configurable?.checkpoint_id === undefined;
|
|
195
|
+
this.store = params.store;
|
|
181
196
|
}
|
|
182
197
|
static async initialize(params) {
|
|
183
198
|
const saved = (await params.checkpointer?.getTuple(params.config)) ?? {
|
|
@@ -205,6 +220,13 @@ class PregelLoop {
|
|
|
205
220
|
const step = (checkpointMetadata.step ?? 0) + 1;
|
|
206
221
|
const stop = step + (params.config.recursionLimit ?? DEFAULT_LOOP_LIMIT) + 1;
|
|
207
222
|
const checkpointPreviousVersions = { ...checkpoint.channel_versions };
|
|
223
|
+
const store = params.store
|
|
224
|
+
? new batch_js_1.AsyncBatchedStore(params.store)
|
|
225
|
+
: undefined;
|
|
226
|
+
if (store) {
|
|
227
|
+
// Start the store. This is a batch store, so it will run continuously
|
|
228
|
+
store.start();
|
|
229
|
+
}
|
|
208
230
|
return new PregelLoop({
|
|
209
231
|
input: params.input,
|
|
210
232
|
config: params.config,
|
|
@@ -213,6 +235,7 @@ class PregelLoop {
|
|
|
213
235
|
checkpointMetadata,
|
|
214
236
|
checkpointConfig,
|
|
215
237
|
channels,
|
|
238
|
+
managed: params.managed,
|
|
216
239
|
step,
|
|
217
240
|
stop,
|
|
218
241
|
checkpointPreviousVersions,
|
|
@@ -220,6 +243,7 @@ class PregelLoop {
|
|
|
220
243
|
outputKeys: params.outputKeys ?? [],
|
|
221
244
|
streamKeys: params.streamKeys ?? [],
|
|
222
245
|
nodes: params.nodes,
|
|
246
|
+
store,
|
|
223
247
|
});
|
|
224
248
|
}
|
|
225
249
|
_checkpointerPutAfterPrevious(input) {
|
|
@@ -228,6 +252,13 @@ class PregelLoop {
|
|
|
228
252
|
});
|
|
229
253
|
this.checkpointerPromises.push(this._checkpointerChainedPromise);
|
|
230
254
|
}
|
|
255
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
256
|
+
async updateManagedValues(key, values) {
|
|
257
|
+
const mv = this.managed.get(key);
|
|
258
|
+
if (mv && "update" in mv && typeof mv.update === "function") {
|
|
259
|
+
await mv.update(values);
|
|
260
|
+
}
|
|
261
|
+
}
|
|
231
262
|
/**
|
|
232
263
|
* Put writes for a task, to be read by the next tick.
|
|
233
264
|
* @param taskId
|
|
@@ -261,6 +292,9 @@ class PregelLoop {
|
|
|
261
292
|
* @param params
|
|
262
293
|
*/
|
|
263
294
|
async tick(params) {
|
|
295
|
+
if (this.store && !this.store.isRunning) {
|
|
296
|
+
this.store?.start();
|
|
297
|
+
}
|
|
264
298
|
const { inputKeys = [], interruptAfter = [], interruptBefore = [], manager, } = params;
|
|
265
299
|
if (this.status !== "pending") {
|
|
266
300
|
throw new Error(`Cannot tick when status is no longer "pending". Current status: "${this.status}"`);
|
|
@@ -271,7 +305,10 @@ class PregelLoop {
|
|
|
271
305
|
else if (this.tasks.every((task) => task.writes.length > 0)) {
|
|
272
306
|
const writes = this.tasks.flatMap((t) => t.writes);
|
|
273
307
|
// All tasks have finished
|
|
274
|
-
(0, algo_js_1._applyWrites)(this.checkpoint, this.channels, this.tasks, this.checkpointerGetNextVersion);
|
|
308
|
+
const myWrites = (0, algo_js_1._applyWrites)(this.checkpoint, this.channels, this.tasks, this.checkpointerGetNextVersion);
|
|
309
|
+
for (const [key, values] of Object.entries(myWrites)) {
|
|
310
|
+
await this.updateManagedValues(key, values);
|
|
311
|
+
}
|
|
275
312
|
// produce values output
|
|
276
313
|
const valuesOutput = await (0, utils_js_1.gatherIterator)((0, utils_js_1.prefixGenerator)((0, io_js_1.mapOutputValues)(this.outputKeys, writes, this.channels), "values"));
|
|
277
314
|
this.stream.push(...valuesOutput);
|
|
@@ -299,7 +336,7 @@ class PregelLoop {
|
|
|
299
336
|
this.status = "out_of_steps";
|
|
300
337
|
return false;
|
|
301
338
|
}
|
|
302
|
-
const nextTasks = (0, algo_js_1._prepareNextTasks)(this.checkpoint, this.nodes, this.channels, this.config, true, {
|
|
339
|
+
const nextTasks = (0, algo_js_1._prepareNextTasks)(this.checkpoint, this.nodes, this.channels, this.managed, this.config, true, {
|
|
303
340
|
step: this.step,
|
|
304
341
|
checkpointer: this.checkpointer,
|
|
305
342
|
isResuming: this.input === INPUT_RESUMING,
|
|
@@ -377,7 +414,7 @@ class PregelLoop {
|
|
|
377
414
|
if (inputWrites.length === 0) {
|
|
378
415
|
throw new errors_js_1.EmptyInputError(`Received no input writes for ${JSON.stringify(inputKeys, null, 2)}`);
|
|
379
416
|
}
|
|
380
|
-
const discardTasks = (0, algo_js_1._prepareNextTasks)(this.checkpoint, this.nodes, this.channels, this.config, true, { step: this.step });
|
|
417
|
+
const discardTasks = (0, algo_js_1._prepareNextTasks)(this.checkpoint, this.nodes, this.channels, this.managed, this.config, true, { step: this.step });
|
|
381
418
|
(0, algo_js_1._applyWrites)(this.checkpoint, this.channels, discardTasks.concat([
|
|
382
419
|
{
|
|
383
420
|
name: constants_js_1.INPUT,
|
package/dist/pregel/loop.d.ts
CHANGED
|
@@ -5,6 +5,9 @@ import { BaseCheckpointSaver, Checkpoint, PendingWrite, CheckpointPendingWrite,
|
|
|
5
5
|
import { BaseChannel } from "../channels/base.js";
|
|
6
6
|
import { PregelExecutableTask, StreamMode } from "./types.js";
|
|
7
7
|
import { PregelNode } from "./read.js";
|
|
8
|
+
import { BaseStore } from "../store/base.js";
|
|
9
|
+
import { AsyncBatchedStore } from "../store/batch.js";
|
|
10
|
+
import { ManagedValueMapping } from "../managed/base.js";
|
|
8
11
|
export type PregelLoopInitializeParams = {
|
|
9
12
|
input?: any;
|
|
10
13
|
config: RunnableConfig;
|
|
@@ -13,6 +16,8 @@ export type PregelLoopInitializeParams = {
|
|
|
13
16
|
streamKeys: string | string[];
|
|
14
17
|
nodes: Record<string, PregelNode>;
|
|
15
18
|
channelSpecs: Record<string, BaseChannel>;
|
|
19
|
+
managed: ManagedValueMapping;
|
|
20
|
+
store?: BaseStore;
|
|
16
21
|
};
|
|
17
22
|
type PregelLoopParams = {
|
|
18
23
|
input?: any;
|
|
@@ -24,11 +29,13 @@ type PregelLoopParams = {
|
|
|
24
29
|
checkpointPendingWrites: CheckpointPendingWrite[];
|
|
25
30
|
checkpointConfig: RunnableConfig;
|
|
26
31
|
channels: Record<string, BaseChannel>;
|
|
32
|
+
managed: ManagedValueMapping;
|
|
27
33
|
step: number;
|
|
28
34
|
stop: number;
|
|
29
35
|
outputKeys: string | string[];
|
|
30
36
|
streamKeys: string | string[];
|
|
31
37
|
nodes: Record<string, PregelNode>;
|
|
38
|
+
store?: AsyncBatchedStore;
|
|
32
39
|
};
|
|
33
40
|
export declare class PregelLoop {
|
|
34
41
|
protected input?: any;
|
|
@@ -36,6 +43,7 @@ export declare class PregelLoop {
|
|
|
36
43
|
protected checkpointer?: BaseCheckpointSaver;
|
|
37
44
|
protected checkpointerGetNextVersion: (current: number | undefined, channel: BaseChannel) => number;
|
|
38
45
|
channels: Record<string, BaseChannel>;
|
|
46
|
+
managed: ManagedValueMapping;
|
|
39
47
|
protected checkpoint: Checkpoint;
|
|
40
48
|
protected checkpointConfig: RunnableConfig;
|
|
41
49
|
checkpointMetadata: CheckpointMetadata;
|
|
@@ -53,6 +61,7 @@ export declare class PregelLoop {
|
|
|
53
61
|
checkpointerPromises: Promise<unknown>[];
|
|
54
62
|
protected isNested: boolean;
|
|
55
63
|
protected _checkpointerChainedPromise: Promise<unknown>;
|
|
64
|
+
store?: AsyncBatchedStore;
|
|
56
65
|
constructor(params: PregelLoopParams);
|
|
57
66
|
static initialize(params: PregelLoopInitializeParams): Promise<PregelLoop>;
|
|
58
67
|
protected _checkpointerPutAfterPrevious(input: {
|
|
@@ -61,6 +70,7 @@ export declare class PregelLoop {
|
|
|
61
70
|
metadata: CheckpointMetadata;
|
|
62
71
|
newVersions: Record<string, string | number>;
|
|
63
72
|
}): void;
|
|
73
|
+
protected updateManagedValues(key: string, values: any[]): Promise<void>;
|
|
64
74
|
/**
|
|
65
75
|
* Put writes for a task, to be read by the next tick.
|
|
66
76
|
* @param taskId
|