@crewhaus/durable-execution 0.1.4 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +68 -0
- package/dist/index.js +87 -0
- package/package.json +10 -7
- package/src/index.test.ts +0 -249
- package/src/index.ts +0 -117
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Catalog R11 `durable-execution` — exactly-once node-execution wrapper.
|
|
3
|
+
*
|
|
4
|
+
* The graph-engine writes a checkpoint after every successful node, so
|
|
5
|
+
* a crashed run can resume by loading the last checkpoint and replaying
|
|
6
|
+
* the next edge. This package adds two further guarantees:
|
|
7
|
+
*
|
|
8
|
+
* 1. Idempotency keys. Each node-execution attempt computes
|
|
9
|
+
* `idempotencyKey(graphRunId, nodeName, attemptIndex)` and a
|
|
10
|
+
* backing store keeps a record of completed attempts. If the same
|
|
11
|
+
* key is seen twice, the second invocation returns the cached
|
|
12
|
+
* result instead of re-executing — preventing double-spend in
|
|
13
|
+
* side-effectful tools.
|
|
14
|
+
*
|
|
15
|
+
* 2. Resume helpers. `resumeFrom(store, graphRunId)` returns the
|
|
16
|
+
* next-node hint a caller can pass to `graph.run({ resumeFrom })`,
|
|
17
|
+
* computed by walking the checkpoint chain backward to the latest
|
|
18
|
+
* successfully-completed node. The default policy ("re-run the
|
|
19
|
+
* next edge") composes with graph-engine; alternative policies
|
|
20
|
+
* (e.g. retry the failed node) can be plugged in via the optional
|
|
21
|
+
* `policy` callback.
|
|
22
|
+
*
|
|
23
|
+
* Layer R11. Pairs with `checkpoint-store` (R7) and `graph-engine` (R11).
|
|
24
|
+
*/
|
|
25
|
+
import type { CheckpointId, CheckpointStore, GraphRunId } from "@crewhaus/checkpoint-store";
|
|
26
|
+
export type IdempotencyKey = string;
|
|
27
|
+
export declare function idempotencyKey(graphRunId: GraphRunId, nodeName: string, attemptIndex?: number): IdempotencyKey;
|
|
28
|
+
export type IdempotencyRecord = {
|
|
29
|
+
readonly key: IdempotencyKey;
|
|
30
|
+
readonly graphRunId: GraphRunId;
|
|
31
|
+
readonly nodeName: string;
|
|
32
|
+
readonly attempt: number;
|
|
33
|
+
readonly result: unknown;
|
|
34
|
+
readonly completedAt: string;
|
|
35
|
+
};
|
|
36
|
+
export interface IdempotencyStore {
|
|
37
|
+
get(key: IdempotencyKey): Promise<IdempotencyRecord | undefined>;
|
|
38
|
+
put(record: IdempotencyRecord): Promise<void>;
|
|
39
|
+
}
|
|
40
|
+
declare class InMemoryIdempotencyStore implements IdempotencyStore {
|
|
41
|
+
private readonly entries;
|
|
42
|
+
constructor();
|
|
43
|
+
get(key: IdempotencyKey): Promise<IdempotencyRecord | undefined>;
|
|
44
|
+
put(record: IdempotencyRecord): Promise<void>;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Wrap a node fn with idempotency. The first call executes normally and
|
|
48
|
+
* caches the result against `idempotencyKey(graphRunId, nodeName,
|
|
49
|
+
* attempt)`. Subsequent calls with the same key return the cached
|
|
50
|
+
* result without invoking the inner fn — which is exactly what crash
|
|
51
|
+
* recovery needs: if the engine restarts mid-node, the same idempotency
|
|
52
|
+
* key resolves to the prior side-effect's result.
|
|
53
|
+
*/
|
|
54
|
+
export declare function withIdempotency<S>(inner: (graphRunId: GraphRunId, nodeName: string, prev: S) => Promise<S>, opts?: {
|
|
55
|
+
readonly store?: IdempotencyStore;
|
|
56
|
+
readonly attempt?: number;
|
|
57
|
+
}): (graphRunId: GraphRunId, nodeName: string, prev: S) => Promise<S>;
|
|
58
|
+
/**
|
|
59
|
+
* Read the latest checkpoint of `graphRunId` and return the
|
|
60
|
+
* `{ checkpointId, nextNode }` hint for `graph.run({ resumeFrom })`.
|
|
61
|
+
* `nextNode` is the LAST committed node; the engine will re-evaluate
|
|
62
|
+
* its outgoing edge to figure out where to go next.
|
|
63
|
+
*/
|
|
64
|
+
export declare function resumeFrom(store: CheckpointStore, graphRunId: GraphRunId): Promise<{
|
|
65
|
+
checkpointId: CheckpointId;
|
|
66
|
+
nextNode: string;
|
|
67
|
+
} | undefined>;
|
|
68
|
+
export { InMemoryIdempotencyStore };
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Catalog R11 `durable-execution` — exactly-once node-execution wrapper.
|
|
3
|
+
*
|
|
4
|
+
* The graph-engine writes a checkpoint after every successful node, so
|
|
5
|
+
* a crashed run can resume by loading the last checkpoint and replaying
|
|
6
|
+
* the next edge. This package adds two further guarantees:
|
|
7
|
+
*
|
|
8
|
+
* 1. Idempotency keys. Each node-execution attempt computes
|
|
9
|
+
* `idempotencyKey(graphRunId, nodeName, attemptIndex)` and a
|
|
10
|
+
* backing store keeps a record of completed attempts. If the same
|
|
11
|
+
* key is seen twice, the second invocation returns the cached
|
|
12
|
+
* result instead of re-executing — preventing double-spend in
|
|
13
|
+
* side-effectful tools.
|
|
14
|
+
*
|
|
15
|
+
* 2. Resume helpers. `resumeFrom(store, graphRunId)` returns the
|
|
16
|
+
* next-node hint a caller can pass to `graph.run({ resumeFrom })`,
|
|
17
|
+
* computed by walking the checkpoint chain backward to the latest
|
|
18
|
+
* successfully-completed node. The default policy ("re-run the
|
|
19
|
+
* next edge") composes with graph-engine; alternative policies
|
|
20
|
+
* (e.g. retry the failed node) can be plugged in via the optional
|
|
21
|
+
* `policy` callback.
|
|
22
|
+
*
|
|
23
|
+
* Layer R11. Pairs with `checkpoint-store` (R7) and `graph-engine` (R11).
|
|
24
|
+
*/
|
|
25
|
+
import { createHash } from "node:crypto";
|
|
26
|
+
export function idempotencyKey(graphRunId, nodeName, attemptIndex = 0) {
|
|
27
|
+
return createHash("sha256")
|
|
28
|
+
.update(`${graphRunId}|${nodeName}|${attemptIndex}`)
|
|
29
|
+
.digest("hex")
|
|
30
|
+
.slice(0, 24);
|
|
31
|
+
}
|
|
32
|
+
class InMemoryIdempotencyStore {
|
|
33
|
+
entries;
|
|
34
|
+
constructor() {
|
|
35
|
+
this.entries = new Map();
|
|
36
|
+
}
|
|
37
|
+
async get(key) {
|
|
38
|
+
return this.entries.get(key);
|
|
39
|
+
}
|
|
40
|
+
async put(record) {
|
|
41
|
+
this.entries.set(record.key, record);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Wrap a node fn with idempotency. The first call executes normally and
|
|
46
|
+
* caches the result against `idempotencyKey(graphRunId, nodeName,
|
|
47
|
+
* attempt)`. Subsequent calls with the same key return the cached
|
|
48
|
+
* result without invoking the inner fn — which is exactly what crash
|
|
49
|
+
* recovery needs: if the engine restarts mid-node, the same idempotency
|
|
50
|
+
* key resolves to the prior side-effect's result.
|
|
51
|
+
*/
|
|
52
|
+
export function withIdempotency(inner, opts = {}) {
|
|
53
|
+
const store = opts.store ?? new InMemoryIdempotencyStore();
|
|
54
|
+
const attempt = opts.attempt ?? 0;
|
|
55
|
+
return async (graphRunId, nodeName, prev) => {
|
|
56
|
+
const key = idempotencyKey(graphRunId, nodeName, attempt);
|
|
57
|
+
const cached = await store.get(key);
|
|
58
|
+
if (cached !== undefined)
|
|
59
|
+
return cached.result;
|
|
60
|
+
const result = await inner(graphRunId, nodeName, prev);
|
|
61
|
+
await store.put({
|
|
62
|
+
key,
|
|
63
|
+
graphRunId,
|
|
64
|
+
nodeName,
|
|
65
|
+
attempt,
|
|
66
|
+
result,
|
|
67
|
+
completedAt: new Date().toISOString(),
|
|
68
|
+
});
|
|
69
|
+
return result;
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Read the latest checkpoint of `graphRunId` and return the
|
|
74
|
+
* `{ checkpointId, nextNode }` hint for `graph.run({ resumeFrom })`.
|
|
75
|
+
* `nextNode` is the LAST committed node; the engine will re-evaluate
|
|
76
|
+
* its outgoing edge to figure out where to go next.
|
|
77
|
+
*/
|
|
78
|
+
export async function resumeFrom(store, graphRunId) {
|
|
79
|
+
const meta = await store.meta(graphRunId);
|
|
80
|
+
if (meta?.head === undefined)
|
|
81
|
+
return undefined;
|
|
82
|
+
const head = await store.load(graphRunId, meta.head);
|
|
83
|
+
if (head === undefined)
|
|
84
|
+
return undefined;
|
|
85
|
+
return { checkpointId: head.id, nextNode: head.nodeName };
|
|
86
|
+
}
|
|
87
|
+
export { InMemoryIdempotencyStore };
|
package/package.json
CHANGED
|
@@ -1,19 +1,22 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crewhaus/durable-execution",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.6",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Exactly-once node execution wrapper over graph-engine — idempotency keys + crash-replay",
|
|
6
|
-
"main": "
|
|
7
|
-
"types": "
|
|
6
|
+
"main": "dist/index.js",
|
|
7
|
+
"types": "dist/index.d.ts",
|
|
8
8
|
"exports": {
|
|
9
|
-
".":
|
|
9
|
+
".": {
|
|
10
|
+
"types": "./dist/index.d.ts",
|
|
11
|
+
"import": "./dist/index.js"
|
|
12
|
+
}
|
|
10
13
|
},
|
|
11
14
|
"scripts": {
|
|
12
15
|
"test": "bun test src"
|
|
13
16
|
},
|
|
14
17
|
"dependencies": {
|
|
15
|
-
"@crewhaus/checkpoint-store": "0.1.
|
|
16
|
-
"@crewhaus/errors": "0.1.
|
|
18
|
+
"@crewhaus/checkpoint-store": "0.1.6",
|
|
19
|
+
"@crewhaus/errors": "0.1.6"
|
|
17
20
|
},
|
|
18
21
|
"license": "Apache-2.0",
|
|
19
22
|
"author": {
|
|
@@ -33,5 +36,5 @@
|
|
|
33
36
|
"publishConfig": {
|
|
34
37
|
"access": "public"
|
|
35
38
|
},
|
|
36
|
-
"files": ["
|
|
39
|
+
"files": ["dist", "README.md", "LICENSE", "NOTICE"]
|
|
37
40
|
}
|
package/src/index.test.ts
DELETED
|
@@ -1,249 +0,0 @@
|
|
|
1
|
-
import { afterEach, beforeEach, describe, expect, test } from "bun:test";
|
|
2
|
-
import { mkdtempSync, rmSync } from "node:fs";
|
|
3
|
-
import { tmpdir } from "node:os";
|
|
4
|
-
import { join } from "node:path";
|
|
5
|
-
import {
|
|
6
|
-
type CheckpointStore,
|
|
7
|
-
createCheckpointStore,
|
|
8
|
-
newGraphRunId,
|
|
9
|
-
} from "@crewhaus/checkpoint-store";
|
|
10
|
-
import {
|
|
11
|
-
type IdempotencyRecord,
|
|
12
|
-
type IdempotencyStore,
|
|
13
|
-
InMemoryIdempotencyStore,
|
|
14
|
-
idempotencyKey,
|
|
15
|
-
resumeFrom,
|
|
16
|
-
withIdempotency,
|
|
17
|
-
} from "./index";
|
|
18
|
-
|
|
19
|
-
let tmp: string;
|
|
20
|
-
let store: CheckpointStore;
|
|
21
|
-
|
|
22
|
-
beforeEach(() => {
|
|
23
|
-
tmp = mkdtempSync(join(tmpdir(), "durable-exec-"));
|
|
24
|
-
store = createCheckpointStore({ rootDir: tmp });
|
|
25
|
-
});
|
|
26
|
-
|
|
27
|
-
afterEach(() => {
|
|
28
|
-
rmSync(tmp, { recursive: true, force: true });
|
|
29
|
-
});
|
|
30
|
-
|
|
31
|
-
describe("idempotencyKey", () => {
|
|
32
|
-
test("deterministic for the same inputs", () => {
|
|
33
|
-
const grun = newGraphRunId();
|
|
34
|
-
expect(idempotencyKey(grun, "a", 0)).toBe(idempotencyKey(grun, "a", 0));
|
|
35
|
-
});
|
|
36
|
-
test("different attempts produce different keys", () => {
|
|
37
|
-
const grun = newGraphRunId();
|
|
38
|
-
expect(idempotencyKey(grun, "a", 0)).not.toBe(idempotencyKey(grun, "a", 1));
|
|
39
|
-
});
|
|
40
|
-
test("different nodes produce different keys", () => {
|
|
41
|
-
const grun = newGraphRunId();
|
|
42
|
-
expect(idempotencyKey(grun, "a", 0)).not.toBe(idempotencyKey(grun, "b", 0));
|
|
43
|
-
});
|
|
44
|
-
});
|
|
45
|
-
|
|
46
|
-
describe("withIdempotency", () => {
|
|
47
|
-
test("first call invokes inner; second call returns cached value", async () => {
|
|
48
|
-
let invocations = 0;
|
|
49
|
-
const wrapped = withIdempotency(async (_g, _n, _p: { v: number }) => {
|
|
50
|
-
invocations += 1;
|
|
51
|
-
return { v: 42 };
|
|
52
|
-
});
|
|
53
|
-
const grun = newGraphRunId();
|
|
54
|
-
const a = await wrapped(grun, "node-a", { v: 0 });
|
|
55
|
-
const b = await wrapped(grun, "node-a", { v: 999 });
|
|
56
|
-
expect(invocations).toBe(1);
|
|
57
|
-
expect(a).toEqual({ v: 42 });
|
|
58
|
-
expect(b).toEqual({ v: 42 });
|
|
59
|
-
});
|
|
60
|
-
|
|
61
|
-
test("different attempt indexes re-invoke inner", async () => {
|
|
62
|
-
let invocations = 0;
|
|
63
|
-
const grun = newGraphRunId();
|
|
64
|
-
const w0 = withIdempotency<{ x: number }>(async () => {
|
|
65
|
-
invocations += 1;
|
|
66
|
-
return { x: invocations };
|
|
67
|
-
});
|
|
68
|
-
const w1 = withIdempotency<{ x: number }>(
|
|
69
|
-
async () => {
|
|
70
|
-
invocations += 1;
|
|
71
|
-
return { x: invocations };
|
|
72
|
-
},
|
|
73
|
-
{ attempt: 1 },
|
|
74
|
-
);
|
|
75
|
-
await w0(grun, "n", { x: 0 });
|
|
76
|
-
await w1(grun, "n", { x: 0 });
|
|
77
|
-
expect(invocations).toBe(2);
|
|
78
|
-
});
|
|
79
|
-
|
|
80
|
-
test("honours an injected store and persists a well-formed record", async () => {
|
|
81
|
-
const grun = newGraphRunId();
|
|
82
|
-
const store = new InMemoryIdempotencyStore();
|
|
83
|
-
const wrapped = withIdempotency<{ v: number }>(async () => ({ v: 7 }), { store, attempt: 3 });
|
|
84
|
-
const out = await wrapped(grun, "node-z", { v: 0 });
|
|
85
|
-
expect(out).toEqual({ v: 7 });
|
|
86
|
-
// The record must be findable in the *shared* store under the derived key.
|
|
87
|
-
const rec = await store.get(idempotencyKey(grun, "node-z", 3));
|
|
88
|
-
expect(rec).toBeDefined();
|
|
89
|
-
expect(rec?.graphRunId).toBe(grun);
|
|
90
|
-
expect(rec?.nodeName).toBe("node-z");
|
|
91
|
-
expect(rec?.attempt).toBe(3);
|
|
92
|
-
expect(rec?.result).toEqual({ v: 7 });
|
|
93
|
-
// completedAt is a valid ISO-8601 timestamp.
|
|
94
|
-
expect(Number.isNaN(Date.parse(rec?.completedAt ?? ""))).toBe(false);
|
|
95
|
-
});
|
|
96
|
-
|
|
97
|
-
test("a second wrapper sharing the store reads the cached result", async () => {
|
|
98
|
-
const grun = newGraphRunId();
|
|
99
|
-
const store = new InMemoryIdempotencyStore();
|
|
100
|
-
let firstCalls = 0;
|
|
101
|
-
let secondCalls = 0;
|
|
102
|
-
const first = withIdempotency<{ v: number }>(
|
|
103
|
-
async () => {
|
|
104
|
-
firstCalls += 1;
|
|
105
|
-
return { v: 1 };
|
|
106
|
-
},
|
|
107
|
-
{ store },
|
|
108
|
-
);
|
|
109
|
-
const second = withIdempotency<{ v: number }>(
|
|
110
|
-
async () => {
|
|
111
|
-
secondCalls += 1;
|
|
112
|
-
return { v: 2 };
|
|
113
|
-
},
|
|
114
|
-
{ store },
|
|
115
|
-
);
|
|
116
|
-
const a = await first(grun, "shared", { v: 0 });
|
|
117
|
-
const b = await second(grun, "shared", { v: 0 });
|
|
118
|
-
expect(a).toEqual({ v: 1 });
|
|
119
|
-
// Cache hit: the second wrapper never runs its inner fn.
|
|
120
|
-
expect(b).toEqual({ v: 1 });
|
|
121
|
-
expect(firstCalls).toBe(1);
|
|
122
|
-
expect(secondCalls).toBe(0);
|
|
123
|
-
});
|
|
124
|
-
|
|
125
|
-
test("works with a custom IdempotencyStore implementation", async () => {
|
|
126
|
-
const grun = newGraphRunId();
|
|
127
|
-
const backing = new Map<string, IdempotencyRecord>();
|
|
128
|
-
const calls = { get: 0, put: 0 };
|
|
129
|
-
const custom: IdempotencyStore = {
|
|
130
|
-
async get(key) {
|
|
131
|
-
calls.get += 1;
|
|
132
|
-
return backing.get(key);
|
|
133
|
-
},
|
|
134
|
-
async put(record) {
|
|
135
|
-
calls.put += 1;
|
|
136
|
-
backing.set(record.key, record);
|
|
137
|
-
},
|
|
138
|
-
};
|
|
139
|
-
let invocations = 0;
|
|
140
|
-
const wrapped = withIdempotency<{ n: number }>(
|
|
141
|
-
async () => {
|
|
142
|
-
invocations += 1;
|
|
143
|
-
return { n: invocations };
|
|
144
|
-
},
|
|
145
|
-
{ store: custom },
|
|
146
|
-
);
|
|
147
|
-
const a = await wrapped(grun, "node-c", { n: 0 });
|
|
148
|
-
const b = await wrapped(grun, "node-c", { n: 0 });
|
|
149
|
-
expect(a).toEqual({ n: 1 });
|
|
150
|
-
expect(b).toEqual({ n: 1 });
|
|
151
|
-
expect(invocations).toBe(1);
|
|
152
|
-
expect(calls.put).toBe(1);
|
|
153
|
-
expect(calls.get).toBe(2);
|
|
154
|
-
});
|
|
155
|
-
});
|
|
156
|
-
|
|
157
|
-
describe("InMemoryIdempotencyStore", () => {
|
|
158
|
-
test("get returns undefined for an absent key", async () => {
|
|
159
|
-
const store = new InMemoryIdempotencyStore();
|
|
160
|
-
expect(await store.get("does-not-exist")).toBeUndefined();
|
|
161
|
-
});
|
|
162
|
-
|
|
163
|
-
test("put then get round-trips the record", async () => {
|
|
164
|
-
const store = new InMemoryIdempotencyStore();
|
|
165
|
-
const grun = newGraphRunId();
|
|
166
|
-
const record: IdempotencyRecord = {
|
|
167
|
-
key: idempotencyKey(grun, "n", 0),
|
|
168
|
-
graphRunId: grun,
|
|
169
|
-
nodeName: "n",
|
|
170
|
-
attempt: 0,
|
|
171
|
-
result: { ok: true },
|
|
172
|
-
completedAt: new Date(0).toISOString(),
|
|
173
|
-
};
|
|
174
|
-
await store.put(record);
|
|
175
|
-
expect(await store.get(record.key)).toEqual(record);
|
|
176
|
-
});
|
|
177
|
-
|
|
178
|
-
test("put overwrites an existing record for the same key", async () => {
|
|
179
|
-
const store = new InMemoryIdempotencyStore();
|
|
180
|
-
const grun = newGraphRunId();
|
|
181
|
-
const key = idempotencyKey(grun, "n", 0);
|
|
182
|
-
const base = {
|
|
183
|
-
key,
|
|
184
|
-
graphRunId: grun,
|
|
185
|
-
nodeName: "n",
|
|
186
|
-
attempt: 0,
|
|
187
|
-
completedAt: new Date(0).toISOString(),
|
|
188
|
-
};
|
|
189
|
-
await store.put({ ...base, result: { v: 1 } });
|
|
190
|
-
await store.put({ ...base, result: { v: 2 } });
|
|
191
|
-
expect((await store.get(key))?.result).toEqual({ v: 2 });
|
|
192
|
-
});
|
|
193
|
-
});
|
|
194
|
-
|
|
195
|
-
describe("resumeFrom", () => {
|
|
196
|
-
test("returns the head's checkpointId + nodeName", async () => {
|
|
197
|
-
const grun = newGraphRunId();
|
|
198
|
-
await store.save({ graphRunId: grun, nodeName: "plan", state: 1 });
|
|
199
|
-
const cp = await store.save({ graphRunId: grun, nodeName: "execute", state: 2 });
|
|
200
|
-
const r = await resumeFrom(store, grun);
|
|
201
|
-
expect(r?.checkpointId).toBe(cp.id);
|
|
202
|
-
expect(r?.nextNode).toBe("execute");
|
|
203
|
-
});
|
|
204
|
-
|
|
205
|
-
test("returns undefined for an unknown run", async () => {
|
|
206
|
-
const grun = newGraphRunId();
|
|
207
|
-
expect(await resumeFrom(store, grun)).toBeUndefined();
|
|
208
|
-
});
|
|
209
|
-
|
|
210
|
-
test("returns undefined when meta exists but has no head", async () => {
|
|
211
|
-
const grun = newGraphRunId();
|
|
212
|
-
// A freshly-branched run can have meta with `head` absent (see
|
|
213
|
-
// checkpoint-store GraphRunMeta docs). Model it directly via a stub so
|
|
214
|
-
// the test is deterministic and does not depend on branch internals.
|
|
215
|
-
const stub = {
|
|
216
|
-
async meta() {
|
|
217
|
-
return { version: 1 as const, graphRunId: grun, createdAt: new Date(0).toISOString() };
|
|
218
|
-
},
|
|
219
|
-
async load() {
|
|
220
|
-
throw new Error("load must not be called when head is undefined");
|
|
221
|
-
},
|
|
222
|
-
} as unknown as CheckpointStore;
|
|
223
|
-
expect(await resumeFrom(stub, grun)).toBeUndefined();
|
|
224
|
-
});
|
|
225
|
-
|
|
226
|
-
test("returns undefined when head checkpoint is missing", async () => {
|
|
227
|
-
const grun = newGraphRunId();
|
|
228
|
-
// meta names a head, but the underlying checkpoint cannot be loaded
|
|
229
|
-
// (e.g. a corrupted/partially-deleted store). resumeFrom must fail
|
|
230
|
-
// safe rather than dereference an undefined checkpoint.
|
|
231
|
-
let loadCalls = 0;
|
|
232
|
-
const stub = {
|
|
233
|
-
async meta() {
|
|
234
|
-
return {
|
|
235
|
-
version: 1 as const,
|
|
236
|
-
graphRunId: grun,
|
|
237
|
-
head: "ckpt_00000000000000aa",
|
|
238
|
-
createdAt: new Date(0).toISOString(),
|
|
239
|
-
};
|
|
240
|
-
},
|
|
241
|
-
async load() {
|
|
242
|
-
loadCalls += 1;
|
|
243
|
-
return undefined;
|
|
244
|
-
},
|
|
245
|
-
} as unknown as CheckpointStore;
|
|
246
|
-
expect(await resumeFrom(stub, grun)).toBeUndefined();
|
|
247
|
-
expect(loadCalls).toBe(1);
|
|
248
|
-
});
|
|
249
|
-
});
|
package/src/index.ts
DELETED
|
@@ -1,117 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Catalog R11 `durable-execution` — exactly-once node-execution wrapper.
|
|
3
|
-
*
|
|
4
|
-
* The graph-engine writes a checkpoint after every successful node, so
|
|
5
|
-
* a crashed run can resume by loading the last checkpoint and replaying
|
|
6
|
-
* the next edge. This package adds two further guarantees:
|
|
7
|
-
*
|
|
8
|
-
* 1. Idempotency keys. Each node-execution attempt computes
|
|
9
|
-
* `idempotencyKey(graphRunId, nodeName, attemptIndex)` and a
|
|
10
|
-
* backing store keeps a record of completed attempts. If the same
|
|
11
|
-
* key is seen twice, the second invocation returns the cached
|
|
12
|
-
* result instead of re-executing — preventing double-spend in
|
|
13
|
-
* side-effectful tools.
|
|
14
|
-
*
|
|
15
|
-
* 2. Resume helpers. `resumeFrom(store, graphRunId)` returns the
|
|
16
|
-
* next-node hint a caller can pass to `graph.run({ resumeFrom })`,
|
|
17
|
-
* computed by walking the checkpoint chain backward to the latest
|
|
18
|
-
* successfully-completed node. The default policy ("re-run the
|
|
19
|
-
* next edge") composes with graph-engine; alternative policies
|
|
20
|
-
* (e.g. retry the failed node) can be plugged in via the optional
|
|
21
|
-
* `policy` callback.
|
|
22
|
-
*
|
|
23
|
-
* Layer R11. Pairs with `checkpoint-store` (R7) and `graph-engine` (R11).
|
|
24
|
-
*/
|
|
25
|
-
|
|
26
|
-
import { createHash } from "node:crypto";
|
|
27
|
-
import type { CheckpointId, CheckpointStore, GraphRunId } from "@crewhaus/checkpoint-store";
|
|
28
|
-
|
|
29
|
-
export type IdempotencyKey = string;
|
|
30
|
-
|
|
31
|
-
export function idempotencyKey(
|
|
32
|
-
graphRunId: GraphRunId,
|
|
33
|
-
nodeName: string,
|
|
34
|
-
attemptIndex = 0,
|
|
35
|
-
): IdempotencyKey {
|
|
36
|
-
return createHash("sha256")
|
|
37
|
-
.update(`${graphRunId}|${nodeName}|${attemptIndex}`)
|
|
38
|
-
.digest("hex")
|
|
39
|
-
.slice(0, 24);
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
export type IdempotencyRecord = {
|
|
43
|
-
readonly key: IdempotencyKey;
|
|
44
|
-
readonly graphRunId: GraphRunId;
|
|
45
|
-
readonly nodeName: string;
|
|
46
|
-
readonly attempt: number;
|
|
47
|
-
readonly result: unknown;
|
|
48
|
-
readonly completedAt: string;
|
|
49
|
-
};
|
|
50
|
-
|
|
51
|
-
export interface IdempotencyStore {
|
|
52
|
-
get(key: IdempotencyKey): Promise<IdempotencyRecord | undefined>;
|
|
53
|
-
put(record: IdempotencyRecord): Promise<void>;
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
class InMemoryIdempotencyStore implements IdempotencyStore {
|
|
57
|
-
private readonly entries: Map<IdempotencyKey, IdempotencyRecord>;
|
|
58
|
-
constructor() {
|
|
59
|
-
this.entries = new Map<IdempotencyKey, IdempotencyRecord>();
|
|
60
|
-
}
|
|
61
|
-
async get(key: IdempotencyKey): Promise<IdempotencyRecord | undefined> {
|
|
62
|
-
return this.entries.get(key);
|
|
63
|
-
}
|
|
64
|
-
async put(record: IdempotencyRecord): Promise<void> {
|
|
65
|
-
this.entries.set(record.key, record);
|
|
66
|
-
}
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
/**
|
|
70
|
-
* Wrap a node fn with idempotency. The first call executes normally and
|
|
71
|
-
* caches the result against `idempotencyKey(graphRunId, nodeName,
|
|
72
|
-
* attempt)`. Subsequent calls with the same key return the cached
|
|
73
|
-
* result without invoking the inner fn — which is exactly what crash
|
|
74
|
-
* recovery needs: if the engine restarts mid-node, the same idempotency
|
|
75
|
-
* key resolves to the prior side-effect's result.
|
|
76
|
-
*/
|
|
77
|
-
export function withIdempotency<S>(
|
|
78
|
-
inner: (graphRunId: GraphRunId, nodeName: string, prev: S) => Promise<S>,
|
|
79
|
-
opts: { readonly store?: IdempotencyStore; readonly attempt?: number } = {},
|
|
80
|
-
): (graphRunId: GraphRunId, nodeName: string, prev: S) => Promise<S> {
|
|
81
|
-
const store = opts.store ?? new InMemoryIdempotencyStore();
|
|
82
|
-
const attempt = opts.attempt ?? 0;
|
|
83
|
-
return async (graphRunId, nodeName, prev) => {
|
|
84
|
-
const key = idempotencyKey(graphRunId, nodeName, attempt);
|
|
85
|
-
const cached = await store.get(key);
|
|
86
|
-
if (cached !== undefined) return cached.result as S;
|
|
87
|
-
const result = await inner(graphRunId, nodeName, prev);
|
|
88
|
-
await store.put({
|
|
89
|
-
key,
|
|
90
|
-
graphRunId,
|
|
91
|
-
nodeName,
|
|
92
|
-
attempt,
|
|
93
|
-
result,
|
|
94
|
-
completedAt: new Date().toISOString(),
|
|
95
|
-
});
|
|
96
|
-
return result;
|
|
97
|
-
};
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
/**
|
|
101
|
-
* Read the latest checkpoint of `graphRunId` and return the
|
|
102
|
-
* `{ checkpointId, nextNode }` hint for `graph.run({ resumeFrom })`.
|
|
103
|
-
* `nextNode` is the LAST committed node; the engine will re-evaluate
|
|
104
|
-
* its outgoing edge to figure out where to go next.
|
|
105
|
-
*/
|
|
106
|
-
export async function resumeFrom(
|
|
107
|
-
store: CheckpointStore,
|
|
108
|
-
graphRunId: GraphRunId,
|
|
109
|
-
): Promise<{ checkpointId: CheckpointId; nextNode: string } | undefined> {
|
|
110
|
-
const meta = await store.meta(graphRunId);
|
|
111
|
-
if (meta?.head === undefined) return undefined;
|
|
112
|
-
const head = await store.load(graphRunId, meta.head);
|
|
113
|
-
if (head === undefined) return undefined;
|
|
114
|
-
return { checkpointId: head.id, nextNode: head.nodeName };
|
|
115
|
-
}
|
|
116
|
-
|
|
117
|
-
export { InMemoryIdempotencyStore };
|