@pikku/core 0.12.130 → 0.12.134
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +101 -0
- package/dist/middleware-runner.d.ts +4 -0
- package/dist/middleware-runner.js +52 -16
- package/dist/services/in-memory-lease-service.d.ts +13 -0
- package/dist/services/in-memory-lease-service.js +49 -0
- package/dist/services/in-memory-workflow-service.d.ts +11 -3
- package/dist/services/in-memory-workflow-service.js +53 -23
- package/dist/services/index.d.ts +2 -0
- package/dist/services/index.js +2 -0
- package/dist/services/lease-service.d.ts +61 -0
- package/dist/services/lease-service.js +110 -0
- package/dist/services/workflow-service.d.ts +1 -1
- package/dist/testing/index.d.ts +1 -0
- package/dist/testing/service-tests/lease-service-tests.d.ts +3 -0
- package/dist/testing/service-tests/lease-service-tests.js +138 -0
- package/dist/testing/service-tests/workflow-fencing-tests.d.ts +18 -0
- package/dist/testing/service-tests/workflow-fencing-tests.js +216 -0
- package/dist/testing/service-tests.d.ts +5 -0
- package/dist/testing/service-tests.js +8 -0
- package/dist/types/core.types.d.ts +12 -8
- package/dist/utils/hmac.d.ts +14 -10
- package/dist/utils/hmac.js +29 -25
- package/dist/wirings/http/pikku-fetch-http-response.js +13 -0
- package/dist/wirings/secret/derive-oauth2-app-secrets.js +1 -0
- package/dist/wirings/secret/secret.types.d.ts +6 -0
- package/dist/wirings/trigger/index.d.ts +2 -2
- package/dist/wirings/trigger/index.js +1 -1
- package/dist/wirings/trigger/webhook-source-runner.js +52 -1
- package/dist/wirings/trigger/webhook-source.types.d.ts +50 -0
- package/dist/wirings/trigger/webhook-source.types.js +2 -0
- package/dist/wirings/workflow/graph/graph-runner.js +33 -14
- package/dist/wirings/workflow/index.d.ts +4 -3
- package/dist/wirings/workflow/index.js +3 -2
- package/dist/wirings/workflow/pikku-workflow-service.d.ts +57 -18
- package/dist/wirings/workflow/pikku-workflow-service.js +143 -140
- package/dist/wirings/workflow/workflow-constants.d.ts +28 -1
- package/dist/wirings/workflow/workflow-constants.js +28 -1
- package/dist/wirings/workflow/workflow-errors.d.ts +21 -0
- package/dist/wirings/workflow/workflow-errors.js +38 -0
- package/dist/wirings/workflow/workflow-queue-routing.d.ts +4 -2
- package/dist/wirings/workflow/workflow-queue-routing.js +9 -1
- package/dist/wirings/workflow/workflow-run-lease.d.ts +10 -0
- package/dist/wirings/workflow/workflow-run-lease.js +25 -0
- package/dist/wirings/workflow/workflow-run-status.d.ts +4 -0
- package/dist/wirings/workflow/workflow-run-status.js +37 -0
- package/dist/wirings/workflow/workflow-status-stream.d.ts +2 -1
- package/dist/wirings/workflow/workflow-status-stream.js +18 -1
- package/dist/wirings/workflow/workflow-step-claim.d.ts +13 -1
- package/dist/wirings/workflow/workflow-step-claim.js +29 -5
- package/dist/wirings/workflow/workflow-step-lease.d.ts +25 -0
- package/dist/wirings/workflow/workflow-step-lease.js +53 -0
- package/dist/wirings/workflow/workflow-step-retry.d.ts +14 -0
- package/dist/wirings/workflow/workflow-step-retry.js +33 -0
- package/dist/wirings/workflow/workflow-version-fallback.d.ts +11 -0
- package/dist/wirings/workflow/workflow-version-fallback.js +30 -0
- package/dist/wirings/workflow/workflow.types.d.ts +17 -0
- package/knowledge/decisions/internals/a-held-run-is-woken-later-not-retried.md +28 -0
- package/knowledge/decisions/internals/index.md +2 -1
- package/knowledge/decisions/internals/the-in-memory-workflow-service-is-inline-only-and-single-process.md +2 -2
- package/knowledge/decisions/internals/workflow-step-lock-is-held-only-to-claim-the-step.md +12 -5
- package/package.json +1 -1
- package/src/public-surface.json +18 -1
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
import { describe, test } from 'node:test';
|
|
2
|
+
import assert from 'node:assert/strict';
|
|
3
|
+
import { holdLease, } from '../../services/lease-service.js';
|
|
4
|
+
const HOUR = 60 * 60 * 1000;
|
|
5
|
+
/**
|
|
6
|
+
* A lease a test lets lapse, and the lease `holdLease` renews. Both leave room
|
|
7
|
+
* for a real database's round-trips: a lease has to outlive the statement that
|
|
8
|
+
* writes it and the one that reads it back.
|
|
9
|
+
*/
|
|
10
|
+
const SHORT = 500;
|
|
11
|
+
const RENEWED = 1_000;
|
|
12
|
+
const wait = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
13
|
+
/** Conformance suite for `leaseService`. Each test gets a fresh service. */
|
|
14
|
+
export const defineLeaseServiceTests = (name, create) => {
|
|
15
|
+
describe(`LeaseService [${name}]`, () => {
|
|
16
|
+
test('one holder at a time', async () => {
|
|
17
|
+
const leases = await create();
|
|
18
|
+
const a = await leases.acquire('stage', 'a', HOUR);
|
|
19
|
+
assert.equal(a?.holder, 'a');
|
|
20
|
+
assert.equal(a?.token, 1);
|
|
21
|
+
assert.equal(await leases.acquire('stage', 'b', HOUR), null);
|
|
22
|
+
assert.equal((await leases.get('stage'))?.holder, 'a');
|
|
23
|
+
});
|
|
24
|
+
test('keys are independent', async () => {
|
|
25
|
+
const leases = await create();
|
|
26
|
+
await leases.acquire('one', 'a', HOUR);
|
|
27
|
+
assert.equal((await leases.acquire('two', 'b', HOUR))?.holder, 'b');
|
|
28
|
+
});
|
|
29
|
+
test('the holder re-acquires under the same token', async () => {
|
|
30
|
+
const leases = await create();
|
|
31
|
+
const first = await leases.acquire('stage', 'a', SHORT);
|
|
32
|
+
const again = await leases.acquire('stage', 'a', HOUR);
|
|
33
|
+
assert.equal(again?.token, first?.token);
|
|
34
|
+
assert.ok(again.expiresAt > first.expiresAt);
|
|
35
|
+
});
|
|
36
|
+
test('a lapsed lease passes to the next holder with a higher token', async () => {
|
|
37
|
+
const leases = await create();
|
|
38
|
+
const a = await leases.acquire('stage', 'a', SHORT);
|
|
39
|
+
await wait(2 * SHORT);
|
|
40
|
+
const b = await leases.acquire('stage', 'b', HOUR);
|
|
41
|
+
assert.equal(b?.holder, 'b');
|
|
42
|
+
assert.equal(b?.token, a.token + 1);
|
|
43
|
+
assert.equal(await leases.refresh(a, HOUR), null);
|
|
44
|
+
});
|
|
45
|
+
test('release frees the key and the token keeps rising', async () => {
|
|
46
|
+
const leases = await create();
|
|
47
|
+
const a = await leases.acquire('stage', 'a', HOUR);
|
|
48
|
+
await leases.release(a);
|
|
49
|
+
assert.equal(await leases.get('stage'), null);
|
|
50
|
+
const b = await leases.acquire('stage', 'b', HOUR);
|
|
51
|
+
assert.equal(b?.token, a.token + 1);
|
|
52
|
+
});
|
|
53
|
+
test('a stale lease cannot refresh or release the new holder', async () => {
|
|
54
|
+
const leases = await create();
|
|
55
|
+
const a = await leases.acquire('stage', 'a', SHORT);
|
|
56
|
+
await wait(2 * SHORT);
|
|
57
|
+
await leases.acquire('stage', 'b', HOUR);
|
|
58
|
+
await leases.release(a);
|
|
59
|
+
assert.equal((await leases.get('stage'))?.holder, 'b');
|
|
60
|
+
});
|
|
61
|
+
test('refresh extends a live lease', async () => {
|
|
62
|
+
const leases = await create();
|
|
63
|
+
const a = await leases.acquire('stage', 'a', SHORT);
|
|
64
|
+
const refreshed = await leases.refresh(a, HOUR);
|
|
65
|
+
assert.equal(refreshed?.token, a.token);
|
|
66
|
+
await wait(2 * SHORT);
|
|
67
|
+
assert.equal(await leases.acquire('stage', 'b', HOUR), null);
|
|
68
|
+
});
|
|
69
|
+
test('concurrent acquires admit exactly one holder', async () => {
|
|
70
|
+
const leases = await create();
|
|
71
|
+
const results = await Promise.all(['a', 'b', 'c', 'd', 'e'].map((holder) => leases.acquire('stage', holder, HOUR)));
|
|
72
|
+
assert.equal(results.filter(Boolean).length, 1);
|
|
73
|
+
});
|
|
74
|
+
test('holdLease hands the body its lease and frees the key', async () => {
|
|
75
|
+
const leases = await create();
|
|
76
|
+
let seen;
|
|
77
|
+
const result = await holdLease(leases, 'stage', async (lease) => {
|
|
78
|
+
seen = lease;
|
|
79
|
+
return 42;
|
|
80
|
+
});
|
|
81
|
+
assert.equal(result, 42);
|
|
82
|
+
assert.equal(seen?.key, 'stage');
|
|
83
|
+
assert.equal(await leases.get('stage'), null);
|
|
84
|
+
});
|
|
85
|
+
test('holdLease refuses a taken key', async () => {
|
|
86
|
+
const leases = await create();
|
|
87
|
+
await leases.acquire('stage', 'a', HOUR);
|
|
88
|
+
await assert.rejects(holdLease(leases, 'stage', async () => { }), (err) => err.name === 'LeaseTakenError');
|
|
89
|
+
});
|
|
90
|
+
test('holdLease frees the key when the body throws', async () => {
|
|
91
|
+
const leases = await create();
|
|
92
|
+
await assert.rejects(holdLease(leases, 'stage', async () => {
|
|
93
|
+
throw new Error('boom');
|
|
94
|
+
}), /boom/);
|
|
95
|
+
assert.equal(await leases.get('stage'), null);
|
|
96
|
+
});
|
|
97
|
+
test('holdLease refuses to vouch for a body that outlived its lease', async () => {
|
|
98
|
+
const leases = await create();
|
|
99
|
+
await assert.rejects(holdLease(leases, 'stage', async (lease) => {
|
|
100
|
+
await leases.release(lease);
|
|
101
|
+
assert.ok(await leases.acquire('stage', 'other', HOUR));
|
|
102
|
+
await wait(RENEWED);
|
|
103
|
+
return 'unprotected';
|
|
104
|
+
}, RENEWED), (err) => err.name === 'LeaseLostError');
|
|
105
|
+
assert.equal((await leases.get('stage'))?.holder, 'other');
|
|
106
|
+
});
|
|
107
|
+
test('holdLease notices a lease lost after its last renewal', async () => {
|
|
108
|
+
const leases = await create();
|
|
109
|
+
await assert.rejects(holdLease(leases, 'stage', async (lease) => {
|
|
110
|
+
await leases.release(lease);
|
|
111
|
+
assert.ok(await leases.acquire('stage', 'other', HOUR));
|
|
112
|
+
return 'finished before any renewal ran';
|
|
113
|
+
}), (err) => err.name === 'LeaseLostError');
|
|
114
|
+
assert.equal((await leases.get('stage'))?.holder, 'other');
|
|
115
|
+
});
|
|
116
|
+
test('holdLease aborts the body once its lease is lost', async () => {
|
|
117
|
+
const leases = await create();
|
|
118
|
+
let aborted = false;
|
|
119
|
+
await assert.rejects(holdLease(leases, 'stage', async (lease, signal) => {
|
|
120
|
+
await leases.release(lease);
|
|
121
|
+
assert.ok(await leases.acquire('stage', 'other', HOUR));
|
|
122
|
+
await new Promise((resolve) => {
|
|
123
|
+
signal.addEventListener('abort', () => resolve());
|
|
124
|
+
setTimeout(resolve, 5 * RENEWED);
|
|
125
|
+
});
|
|
126
|
+
aborted = signal.aborted;
|
|
127
|
+
}, RENEWED), (err) => err.name === 'LeaseLostError');
|
|
128
|
+
assert.equal(aborted, true, 'the body was never told its lease was gone');
|
|
129
|
+
});
|
|
130
|
+
test('holdLease keeps the lease alive past its ttl', async () => {
|
|
131
|
+
const leases = await create();
|
|
132
|
+
await holdLease(leases, 'stage', async () => {
|
|
133
|
+
await wait(2 * RENEWED);
|
|
134
|
+
assert.ok(await leases.get('stage'), 'the lease lapsed mid-body');
|
|
135
|
+
}, RENEWED);
|
|
136
|
+
});
|
|
137
|
+
});
|
|
138
|
+
};
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { PikkuWorkflowService } from '../../wirings/workflow/pikku-workflow-service.js';
|
|
2
|
+
/** A workflow service under test, and the means to kill the worker holding a step. */
|
|
3
|
+
export type WorkflowFencingHarness = {
|
|
4
|
+
service: PikkuWorkflowService;
|
|
5
|
+
/**
|
|
6
|
+
* Make the step's lease read as lapsed, as it would once its worker died, on
|
|
7
|
+
* whichever clock the store judges leases by.
|
|
8
|
+
*/
|
|
9
|
+
lapseLease: (runId: string, stepName: string) => Promise<void>;
|
|
10
|
+
};
|
|
11
|
+
/**
|
|
12
|
+
* Conformance suite for a workflow service that fences a step's writes to the
|
|
13
|
+
* claim that made them. Each test gets a fresh harness.
|
|
14
|
+
*
|
|
15
|
+
* Every scenario has a stale attempt — claimed, then its lease lapsed and the
|
|
16
|
+
* step claimed again — and the attempt that replaced it.
|
|
17
|
+
*/
|
|
18
|
+
export declare const defineWorkflowFencingTests: (name: string, create: () => Promise<WorkflowFencingHarness>) => void;
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
import { describe, test } from 'node:test';
|
|
2
|
+
import assert from 'node:assert/strict';
|
|
3
|
+
import { RPCNotFoundError } from '../../wirings/rpc/rpc-runner.js';
|
|
4
|
+
import { pikkuState } from '../../pikku-state.js';
|
|
5
|
+
import { ConsoleLogger } from '../../services/logger-console.js';
|
|
6
|
+
import { LogLevel } from '../../services/logger.js';
|
|
7
|
+
const STEP = 'charge';
|
|
8
|
+
const RPC = 'charge:card';
|
|
9
|
+
const LEASE_MS = 60_000;
|
|
10
|
+
const RACES = 25;
|
|
11
|
+
const createSilentLogger = () => {
|
|
12
|
+
const logger = new ConsoleLogger();
|
|
13
|
+
logger.setLevel(LogLevel.critical);
|
|
14
|
+
return logger;
|
|
15
|
+
};
|
|
16
|
+
const superseded = (e) => e.name === 'WorkflowStepSupersededError';
|
|
17
|
+
/**
|
|
18
|
+
* Conformance suite for a workflow service that fences a step's writes to the
|
|
19
|
+
* claim that made them. Each test gets a fresh harness.
|
|
20
|
+
*
|
|
21
|
+
* Every scenario has a stale attempt — claimed, then its lease lapsed and the
|
|
22
|
+
* step claimed again — and the attempt that replaced it.
|
|
23
|
+
*/
|
|
24
|
+
export const defineWorkflowFencingTests = (name, create) => {
|
|
25
|
+
describe(`WorkflowService fencing [${name}]`, () => {
|
|
26
|
+
const claim = ({ service }, runId) => service['claimStepForExecution'](runId, STEP, RPC, LEASE_MS);
|
|
27
|
+
const startRun = async ({ service }) => {
|
|
28
|
+
const runId = await service.createRun('fenced', {}, false, 'hash', {
|
|
29
|
+
type: 'test',
|
|
30
|
+
});
|
|
31
|
+
await service.insertStepState(runId, STEP, RPC, {}, { retries: 5 });
|
|
32
|
+
return runId;
|
|
33
|
+
};
|
|
34
|
+
/** Claim the step, lose its worker, and claim it again. */
|
|
35
|
+
const supersede = async (h, runId) => {
|
|
36
|
+
const stale = await claim(h, runId);
|
|
37
|
+
assert.ok(stale, 'the first dispatch did not get the step');
|
|
38
|
+
await h.lapseLease(runId, STEP);
|
|
39
|
+
const current = await claim(h, runId);
|
|
40
|
+
assert.ok(current, 'the lapsed step was not claimed again');
|
|
41
|
+
assert.equal(current.attemptCount, stale.attemptCount + 1);
|
|
42
|
+
return { stale, current };
|
|
43
|
+
};
|
|
44
|
+
const stepOf = (h, runId) => h.service.getStepState(runId, STEP);
|
|
45
|
+
const leaseOf = async (h, runId) => (await stepOf(h, runId)).leaseExpiresAt?.getTime();
|
|
46
|
+
/** Runs `fn` with the engine's singletons quiet, then puts back what was there. */
|
|
47
|
+
const quietly = async (fn) => {
|
|
48
|
+
const previous = pikkuState(null, 'package', 'singletonServices');
|
|
49
|
+
pikkuState(null, 'package', 'singletonServices', {
|
|
50
|
+
...previous,
|
|
51
|
+
logger: createSilentLogger(),
|
|
52
|
+
});
|
|
53
|
+
try {
|
|
54
|
+
return await fn();
|
|
55
|
+
}
|
|
56
|
+
finally {
|
|
57
|
+
pikkuState(null, 'package', 'singletonServices', previous);
|
|
58
|
+
}
|
|
59
|
+
};
|
|
60
|
+
/**
|
|
61
|
+
* Dispatch the step through the engine, with an RPC that loses the step to
|
|
62
|
+
* a newer claim while it runs and then does `outcome`.
|
|
63
|
+
*/
|
|
64
|
+
const dispatchStale = (h, runId, outcome) => quietly(async () => {
|
|
65
|
+
let current = null;
|
|
66
|
+
const staleWorker = {
|
|
67
|
+
rpcWithWire: async () => {
|
|
68
|
+
await h.lapseLease(runId, STEP);
|
|
69
|
+
current = await claim(h, runId);
|
|
70
|
+
return outcome();
|
|
71
|
+
},
|
|
72
|
+
};
|
|
73
|
+
const surfaced = await h.service
|
|
74
|
+
.executeWorkflowStep(runId, STEP, RPC, {}, staleWorker)
|
|
75
|
+
.then(() => undefined, (e) => e);
|
|
76
|
+
assert.ok(current, 'the step was never claimed from under the worker');
|
|
77
|
+
return { surfaced, current: current };
|
|
78
|
+
});
|
|
79
|
+
test("a stale attempt's result is refused and the current attempt's lands", async () => {
|
|
80
|
+
const h = await create();
|
|
81
|
+
const runId = await startRun(h);
|
|
82
|
+
const { stale, current } = await supersede(h, runId);
|
|
83
|
+
await assert.rejects(h.service.setStepResult(stale.stepId, 'stale', stale.attemptCount), superseded);
|
|
84
|
+
await h.service.setStepResult(current.stepId, 'fresh', current.attemptCount);
|
|
85
|
+
const step = await stepOf(h, runId);
|
|
86
|
+
assert.equal(step.status, 'succeeded');
|
|
87
|
+
assert.equal(step.result, 'fresh');
|
|
88
|
+
assert.equal(step.attemptCount, current.attemptCount);
|
|
89
|
+
const history = (await h.service.getRunHistory(runId)).filter((attempt) => attempt.stepName === STEP);
|
|
90
|
+
assert.equal(history.length, 2, 'each claim keeps its own attempt');
|
|
91
|
+
assert.equal(history[0].status, 'running', 'the stale attempt was recorded with an outcome it was refused');
|
|
92
|
+
assert.notEqual(history[0].result, 'stale');
|
|
93
|
+
assert.equal(history[1].status, 'succeeded');
|
|
94
|
+
assert.equal(history[1].result, 'fresh');
|
|
95
|
+
});
|
|
96
|
+
test("a stale attempt's error is refused and does not fail the step", async () => {
|
|
97
|
+
const h = await create();
|
|
98
|
+
const runId = await startRun(h);
|
|
99
|
+
const { stale, current } = await supersede(h, runId);
|
|
100
|
+
await assert.rejects(h.service.setStepError(stale.stepId, new Error('stale'), stale.attemptCount), superseded);
|
|
101
|
+
const step = await stepOf(h, runId);
|
|
102
|
+
assert.equal(step.status, 'running', 'the stale attempt failed the step');
|
|
103
|
+
assert.equal(step.error, undefined);
|
|
104
|
+
assert.equal(step.attemptCount, current.attemptCount);
|
|
105
|
+
});
|
|
106
|
+
test('a stale attempt cannot renew the lease, nor move the one that replaced it', async () => {
|
|
107
|
+
const h = await create();
|
|
108
|
+
const runId = await startRun(h);
|
|
109
|
+
const { stale, current } = await supersede(h, runId);
|
|
110
|
+
const held = await leaseOf(h, runId);
|
|
111
|
+
assert.equal(await h.service.refreshStepLease(stale.stepId, 10 * LEASE_MS, stale.attemptCount), false, 'the stale attempt was told it still holds the step');
|
|
112
|
+
assert.equal(await leaseOf(h, runId), held, 'the stale renewal landed');
|
|
113
|
+
assert.equal(await h.service.refreshStepLease(stale.stepId, null, stale.attemptCount), false);
|
|
114
|
+
assert.equal(await leaseOf(h, runId), held, 'the stale release landed');
|
|
115
|
+
assert.equal(await h.service.refreshStepLease(current.stepId, 10 * LEASE_MS, current.attemptCount), true);
|
|
116
|
+
});
|
|
117
|
+
test('a stale attempt cannot overwrite a step the current attempt already succeeded', async () => {
|
|
118
|
+
const h = await create();
|
|
119
|
+
const runId = await startRun(h);
|
|
120
|
+
const { stale, current } = await supersede(h, runId);
|
|
121
|
+
await h.service.setStepResult(current.stepId, 'fresh', current.attemptCount);
|
|
122
|
+
await assert.rejects(h.service.setStepResult(stale.stepId, 'stale', stale.attemptCount), superseded);
|
|
123
|
+
await assert.rejects(h.service.setStepError(stale.stepId, new Error('stale'), stale.attemptCount), superseded);
|
|
124
|
+
const step = await stepOf(h, runId);
|
|
125
|
+
assert.equal(step.status, 'succeeded');
|
|
126
|
+
assert.equal(step.result, 'fresh');
|
|
127
|
+
assert.equal(step.error, undefined);
|
|
128
|
+
});
|
|
129
|
+
test('a stale attempt cannot overwrite a step the current attempt already failed', async () => {
|
|
130
|
+
const h = await create();
|
|
131
|
+
const runId = await startRun(h);
|
|
132
|
+
const { stale, current } = await supersede(h, runId);
|
|
133
|
+
await h.service.setStepError(current.stepId, new Error('fresh'), current.attemptCount);
|
|
134
|
+
await assert.rejects(h.service.setStepResult(stale.stepId, 'stale', stale.attemptCount), superseded);
|
|
135
|
+
const step = await stepOf(h, runId);
|
|
136
|
+
assert.equal(step.status, 'failed');
|
|
137
|
+
assert.equal(step.error?.message, 'fresh');
|
|
138
|
+
assert.equal(step.result, undefined);
|
|
139
|
+
});
|
|
140
|
+
test('when both attempts write at once, only the current one ever lands', async () => {
|
|
141
|
+
const h = await create();
|
|
142
|
+
for (let i = 0; i < RACES; i++) {
|
|
143
|
+
const runId = await startRun(h);
|
|
144
|
+
const { stale, current } = await supersede(h, runId);
|
|
145
|
+
const staleWrite = () => i % 2 === 0
|
|
146
|
+
? h.service.setStepResult(stale.stepId, 'stale', stale.attemptCount)
|
|
147
|
+
: h.service.setStepError(stale.stepId, new Error('stale'), stale.attemptCount);
|
|
148
|
+
const currentWrite = () => h.service.setStepResult(current.stepId, 'fresh', current.attemptCount);
|
|
149
|
+
const [staleOutcome, currentOutcome] = i % 3 === 0
|
|
150
|
+
? (await Promise.allSettled([currentWrite(), staleWrite()])).reverse()
|
|
151
|
+
: await Promise.allSettled([staleWrite(), currentWrite()]);
|
|
152
|
+
assert.equal(staleOutcome.status, 'rejected', `race ${i}`);
|
|
153
|
+
assert.ok(superseded(staleOutcome.reason), `race ${i}: ${staleOutcome.reason}`);
|
|
154
|
+
assert.equal(currentOutcome.status, 'fulfilled', `race ${i}`);
|
|
155
|
+
const step = await stepOf(h, runId);
|
|
156
|
+
assert.equal(step.status, 'succeeded', `race ${i}`);
|
|
157
|
+
assert.equal(step.result, 'fresh', `race ${i}`);
|
|
158
|
+
}
|
|
159
|
+
});
|
|
160
|
+
test('a third claim fences out both attempts before it', async () => {
|
|
161
|
+
const h = await create();
|
|
162
|
+
const runId = await startRun(h);
|
|
163
|
+
const { stale: first, current: second } = await supersede(h, runId);
|
|
164
|
+
await h.lapseLease(runId, STEP);
|
|
165
|
+
const third = await claim(h, runId);
|
|
166
|
+
assert.ok(third);
|
|
167
|
+
assert.equal(third.attemptCount, second.attemptCount + 1);
|
|
168
|
+
const held = await leaseOf(h, runId);
|
|
169
|
+
for (const stale of [first, second]) {
|
|
170
|
+
await assert.rejects(h.service.setStepResult(stale.stepId, 'stale', stale.attemptCount), superseded, `attempt ${stale.attemptCount} recorded a result`);
|
|
171
|
+
await assert.rejects(h.service.setStepError(stale.stepId, new Error('stale'), stale.attemptCount), superseded, `attempt ${stale.attemptCount} recorded an error`);
|
|
172
|
+
assert.equal(await h.service.refreshStepLease(stale.stepId, 10 * LEASE_MS, stale.attemptCount), false, `attempt ${stale.attemptCount} renewed the lease`);
|
|
173
|
+
}
|
|
174
|
+
assert.equal(await leaseOf(h, runId), held);
|
|
175
|
+
await h.service.setStepResult(third.stepId, 'third', third.attemptCount);
|
|
176
|
+
const step = await stepOf(h, runId);
|
|
177
|
+
assert.equal(step.status, 'succeeded');
|
|
178
|
+
assert.equal(step.result, 'third');
|
|
179
|
+
});
|
|
180
|
+
test('a superseded dispatch whose RPC is missing neither fails the step nor suspends the run', async () => {
|
|
181
|
+
const h = await create();
|
|
182
|
+
const runId = await startRun(h);
|
|
183
|
+
const { surfaced, current } = await dispatchStale(h, runId, () => {
|
|
184
|
+
throw new RPCNotFoundError(RPC);
|
|
185
|
+
});
|
|
186
|
+
const step = await stepOf(h, runId);
|
|
187
|
+
assert.equal(step.status, 'running', 'the stale attempt failed the step');
|
|
188
|
+
assert.equal(step.attemptCount, current.attemptCount);
|
|
189
|
+
assert.equal((await h.service.getRun(runId))?.status, 'running', 'the stale attempt suspended the run');
|
|
190
|
+
assert.equal(surfaced, undefined, 'a superseded dispatch ends quietly');
|
|
191
|
+
});
|
|
192
|
+
test('a superseded dispatch whose RPC throws ends quietly and leaves the step to its new owner', async () => {
|
|
193
|
+
const h = await create();
|
|
194
|
+
const runId = await startRun(h);
|
|
195
|
+
const { surfaced, current } = await dispatchStale(h, runId, () => {
|
|
196
|
+
throw new Error('the card was declined');
|
|
197
|
+
});
|
|
198
|
+
assert.equal(surfaced, undefined, 'the stale failure surfaced');
|
|
199
|
+
const step = await stepOf(h, runId);
|
|
200
|
+
assert.equal(step.status, 'running', 'the stale attempt failed the step');
|
|
201
|
+
assert.equal(step.error, undefined);
|
|
202
|
+
assert.equal(step.attemptCount, current.attemptCount);
|
|
203
|
+
assert.equal((await h.service.getRun(runId))?.status, 'running');
|
|
204
|
+
});
|
|
205
|
+
test('a superseded dispatch whose RPC succeeds does not record its result', async () => {
|
|
206
|
+
const h = await create();
|
|
207
|
+
const runId = await startRun(h);
|
|
208
|
+
const { surfaced, current } = await dispatchStale(h, runId, () => 'stale');
|
|
209
|
+
assert.equal(surfaced, undefined);
|
|
210
|
+
const step = await stepOf(h, runId);
|
|
211
|
+
assert.equal(step.status, 'running', 'the stale result landed');
|
|
212
|
+
assert.equal(step.result, undefined);
|
|
213
|
+
assert.equal(step.attemptCount, current.attemptCount);
|
|
214
|
+
});
|
|
215
|
+
});
|
|
216
|
+
};
|
|
@@ -9,6 +9,8 @@ import type { SecretService } from '../services/secret-service.js';
|
|
|
9
9
|
import type { CredentialService } from '../services/credential-service.js';
|
|
10
10
|
import type { AgentRunService } from '../wirings/agent/agent.types.js';
|
|
11
11
|
import type { SessionStore } from '../services/session-store.js';
|
|
12
|
+
import type { LeaseService } from '../services/lease-service.js';
|
|
13
|
+
import type { WorkflowFencingHarness } from './service-tests/workflow-fencing-tests.js';
|
|
12
14
|
export interface ServiceTestConfig {
|
|
13
15
|
name: string;
|
|
14
16
|
services: {
|
|
@@ -36,6 +38,9 @@ export interface ServiceTestConfig {
|
|
|
36
38
|
rotateKEK?(): Promise<number>;
|
|
37
39
|
}>;
|
|
38
40
|
sessionStore?: () => Promise<SessionStore>;
|
|
41
|
+
leaseService?: () => Promise<LeaseService>;
|
|
42
|
+
/** A workflow service that fences each step write to the claim that made it. */
|
|
43
|
+
workflowFencing?: () => Promise<WorkflowFencingHarness>;
|
|
39
44
|
};
|
|
40
45
|
}
|
|
41
46
|
/**
|
|
@@ -8,6 +8,8 @@ import { defineSecretServiceTests } from './service-tests/secret-service-tests.j
|
|
|
8
8
|
import { defineCredentialServiceTests } from './service-tests/credential-service-tests.js';
|
|
9
9
|
import { defineAgentRunServiceTests } from './service-tests/agent-run-service-tests.js';
|
|
10
10
|
import { defineSessionStoreTests } from './service-tests/session-store-tests.js';
|
|
11
|
+
import { defineLeaseServiceTests } from './service-tests/lease-service-tests.js';
|
|
12
|
+
import { defineWorkflowFencingTests } from './service-tests/workflow-fencing-tests.js';
|
|
11
13
|
/**
|
|
12
14
|
* The shared conformance suite every storage backend runs.
|
|
13
15
|
*
|
|
@@ -46,4 +48,10 @@ export function defineServiceTests(config) {
|
|
|
46
48
|
if (services.sessionStore) {
|
|
47
49
|
defineSessionStoreTests(name, services.sessionStore);
|
|
48
50
|
}
|
|
51
|
+
if (services.leaseService) {
|
|
52
|
+
defineLeaseServiceTests(name, services.leaseService);
|
|
53
|
+
}
|
|
54
|
+
if (services.workflowFencing) {
|
|
55
|
+
defineWorkflowFencingTests(name, services.workflowFencing);
|
|
56
|
+
}
|
|
49
57
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { IncomingWebhookService } from '../services/incoming-webhook-service.js';
|
|
2
2
|
import type { TriggerSourceStore } from '../services/trigger-source-store.js';
|
|
3
|
+
import type { LeaseService } from '../services/lease-service.js';
|
|
3
4
|
import type { Logger, LogLevel } from '../services/logger.js';
|
|
4
5
|
import type { VariablesService } from '../services/variables-service.js';
|
|
5
6
|
import type { SecretService } from '../services/secret-service.js';
|
|
@@ -139,6 +140,7 @@ export interface CoreSingletonServices<Config extends CoreConfig = CoreConfig> {
|
|
|
139
140
|
incomingWebhookService?: IncomingWebhookService;
|
|
140
141
|
/** Which trigger sources an operator enabled, and what enabling registered. */
|
|
141
142
|
triggerSourceStore?: TriggerSourceStore;
|
|
143
|
+
leaseService?: LeaseService;
|
|
142
144
|
metaService?: MetaService;
|
|
143
145
|
/**
|
|
144
146
|
* Where virtual-user runs are recorded. A run is dispatched and answered for
|
|
@@ -199,19 +201,25 @@ export interface CoreSingletonServices<Config extends CoreConfig = CoreConfig> {
|
|
|
199
201
|
/**
|
|
200
202
|
* Reads a single credential. The first signature resolves the value type from
|
|
201
203
|
* the project's generated `CredentialsMap`; the second keeps a name the map
|
|
202
|
-
* does not know callable with an explicit type.
|
|
204
|
+
* does not know callable with an explicit type. Without a map, every name is a
|
|
205
|
+
* key, so the first signature would swallow an explicit type argument; only
|
|
206
|
+
* the second is offered then.
|
|
203
207
|
*
|
|
204
208
|
* `TCredentials` is unconstrained because the generated map is an interface,
|
|
205
209
|
* which has no implicit index signature and so cannot satisfy
|
|
206
210
|
* `Record<string, unknown>`.
|
|
207
211
|
*/
|
|
208
|
-
export type GetCredential<TCredentials = Record<string, unknown>> = {
|
|
212
|
+
export type GetCredential<TCredentials = Record<string, unknown>> = string extends keyof TCredentials ? <T = unknown>(name: string) => NoInfer<T> | null | Promise<NoInfer<T> | null> : {
|
|
209
213
|
<K extends keyof TCredentials & string>(name: K): TCredentials[K] | null | Promise<TCredentials[K] | null>;
|
|
210
214
|
<T = unknown>(name: string): T | null | Promise<T | null>;
|
|
211
215
|
};
|
|
212
216
|
export type PikkuWire<In = unknown, Out = unknown, HasInitialSession extends boolean = false, UserSession extends CoreUserSession = CoreUserSession, TypedRPC extends PikkuRPC = PikkuRPC, IsChannel extends true | null = null, MCPTools extends string | never = never, TypedWorkflow extends PikkuWorkflowWire | never = PikkuWorkflowWire, TriggerOutput = unknown, TypedScenario extends PikkuScenarioWire<any> | never = PikkuScenarioWire<any>, TypedActors extends ScenarioPersonas = ScenarioPersonas, TypedCredentials = Record<string, unknown>> = {
|
|
213
217
|
/** Always present — lazily initialised on first access for every function invocation */
|
|
214
218
|
rpc: TypedRPC;
|
|
219
|
+
/** Get a single credential by name — lazy-loads from CredentialService on first call, sync thereafter */
|
|
220
|
+
getCredential: GetCredential<TypedCredentials>;
|
|
221
|
+
/** Get all resolved credentials — lazy-loads from CredentialService on first call, sync thereafter */
|
|
222
|
+
getCredentials: () => Record<string, unknown> | Promise<Record<string, unknown>>;
|
|
215
223
|
} & Partial<{
|
|
216
224
|
wireType: PikkuWiringTypes;
|
|
217
225
|
wireId: string;
|
|
@@ -260,10 +268,6 @@ export type PikkuWire<In = unknown, Out = unknown, HasInitialSession extends boo
|
|
|
260
268
|
pikkuUserId: string;
|
|
261
269
|
/** Set a credential value (available in middleware) */
|
|
262
270
|
setCredential: (name: string, value: unknown) => void;
|
|
263
|
-
/** Get a single credential by name — lazy-loads from CredentialService on first call, sync thereafter */
|
|
264
|
-
getCredential: GetCredential<TypedCredentials>;
|
|
265
|
-
/** Get all resolved credentials — lazy-loads from CredentialService on first call, sync thereafter */
|
|
266
|
-
getCredentials: () => Record<string, unknown> | Promise<Record<string, unknown>>;
|
|
267
271
|
audit: {
|
|
268
272
|
durability: AuditDurability;
|
|
269
273
|
};
|
|
@@ -282,8 +286,8 @@ export type PikkuWire<In = unknown, Out = unknown, HasInitialSession extends boo
|
|
|
282
286
|
*/
|
|
283
287
|
beginChanges: () => Promise<void>;
|
|
284
288
|
}>;
|
|
285
|
-
/** Wire as constructed by runners, before the function runner
|
|
286
|
-
export type PikkuRawWire = Omit<PikkuWire, 'rpc'
|
|
289
|
+
/** Wire as constructed by runners, before the function runner adds `rpc` and the credential readers. */
|
|
290
|
+
export type PikkuRawWire = Omit<PikkuWire, 'rpc' | 'getCredential' | 'getCredentials'> & Partial<Pick<PikkuWire, 'getCredential' | 'getCredentials'>>;
|
|
287
291
|
export type CoreServices<SingletonServices = CoreSingletonServices> = SingletonServices;
|
|
288
292
|
/** Strips `secrets` from a services type. */
|
|
289
293
|
export type SecretlessServices<Services> = Omit<Services, 'secrets'>;
|
package/dist/utils/hmac.d.ts
CHANGED
|
@@ -9,18 +9,22 @@ export declare function hmacSha256Hex(secret: string, payload: string): string;
|
|
|
9
9
|
* where `timingSafeEqual` itself throws.
|
|
10
10
|
*/
|
|
11
11
|
export declare function timingSafeStringEqual(a: string, b: string): boolean;
|
|
12
|
-
type WebhookPayload = string | Uint8Array;
|
|
13
|
-
type HmacAlgorithm = 'sha1' | 'sha256' | 'sha512';
|
|
14
|
-
type SecretEncoding = 'utf8' | 'hex' | 'base64';
|
|
12
|
+
export type WebhookPayload = string | Uint8Array;
|
|
13
|
+
export type HmacAlgorithm = 'sha1' | 'sha256' | 'sha512';
|
|
14
|
+
export type SecretEncoding = 'utf8' | 'hex' | 'base64';
|
|
15
15
|
type SecretSource = string | null | (() => Promise<string | null>);
|
|
16
|
+
/** The HMAC of `payload` under `secret`, for handshakes that answer with a digest. */
|
|
17
|
+
export declare function hmacDigest(secret: string, algorithm: HmacAlgorithm, payload: WebhookPayload, encoding: 'hex' | 'base64', secretEncoding?: SecretEncoding): string;
|
|
18
|
+
/** Whether `signature` is the HMAC of `payload` under `secret`, compared in constant time. */
|
|
19
|
+
export declare function verifyHmacSignature(secret: string, signature: string | undefined, algorithm: HmacAlgorithm, payload: WebhookPayload, encoding: 'hex' | 'base64', secretEncoding?: SecretEncoding): boolean;
|
|
20
|
+
/** Whether `signature` (base64) is `payload` signed by the private half of `publicKey` (PEM). */
|
|
21
|
+
export declare function verifyPublicKeySignature(publicKey: string, signature: string | undefined, payload: WebhookPayload, options?: {
|
|
22
|
+
algorithm?: string;
|
|
23
|
+
dsaEncoding?: 'der' | 'ieee-p1363';
|
|
24
|
+
}): boolean;
|
|
16
25
|
/**
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
* null when the app never provisioned the secret: every check then refuses, so
|
|
20
|
-
* an unconfigured receiver accepts nothing.
|
|
21
|
-
*
|
|
22
|
-
* Built with a lookup, as `fromCredential` does, it checks nothing until
|
|
23
|
-
* `load()` reads the current value.
|
|
26
|
+
* @deprecated Declare `credential` and `verify` on `wireTriggerWebhookSource`,
|
|
27
|
+
* which checks every request before `receive` runs, or use the functions above.
|
|
24
28
|
*/
|
|
25
29
|
export declare class WebhookSigningSecret {
|
|
26
30
|
private readonly provider;
|
package/dist/utils/hmac.js
CHANGED
|
@@ -19,14 +19,33 @@ export function timingSafeStringEqual(a, b) {
|
|
|
19
19
|
return false;
|
|
20
20
|
}
|
|
21
21
|
}
|
|
22
|
+
/** The HMAC of `payload` under `secret`, for handshakes that answer with a digest. */
|
|
23
|
+
export function hmacDigest(secret, algorithm, payload, encoding, secretEncoding = 'utf8') {
|
|
24
|
+
return createHmac(algorithm, Buffer.from(secret, secretEncoding))
|
|
25
|
+
.update(payload)
|
|
26
|
+
.digest(encoding);
|
|
27
|
+
}
|
|
28
|
+
/** Whether `signature` is the HMAC of `payload` under `secret`, compared in constant time. */
|
|
29
|
+
export function verifyHmacSignature(secret, signature, algorithm, payload, encoding, secretEncoding = 'utf8') {
|
|
30
|
+
return (!!signature &&
|
|
31
|
+
timingSafeStringEqual(signature, hmacDigest(secret, algorithm, payload, encoding, secretEncoding)));
|
|
32
|
+
}
|
|
33
|
+
/** Whether `signature` (base64) is `payload` signed by the private half of `publicKey` (PEM). */
|
|
34
|
+
export function verifyPublicKeySignature(publicKey, signature, payload, options = {}) {
|
|
35
|
+
if (!signature)
|
|
36
|
+
return false;
|
|
37
|
+
try {
|
|
38
|
+
return createVerify(options.algorithm ?? 'sha256')
|
|
39
|
+
.update(payload)
|
|
40
|
+
.verify({ key: publicKey, dsaEncoding: options.dsaEncoding }, signature, 'base64');
|
|
41
|
+
}
|
|
42
|
+
catch {
|
|
43
|
+
return false;
|
|
44
|
+
}
|
|
45
|
+
}
|
|
22
46
|
/**
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
* null when the app never provisioned the secret: every check then refuses, so
|
|
26
|
-
* an unconfigured receiver accepts nothing.
|
|
27
|
-
*
|
|
28
|
-
* Built with a lookup, as `fromCredential` does, it checks nothing until
|
|
29
|
-
* `load()` reads the current value.
|
|
47
|
+
* @deprecated Declare `credential` and `verify` on `wireTriggerWebhookSource`,
|
|
48
|
+
* which checks every request before `receive` runs, or use the functions above.
|
|
30
49
|
*/
|
|
31
50
|
export class WebhookSigningSecret {
|
|
32
51
|
provider;
|
|
@@ -54,14 +73,11 @@ export class WebhookSigningSecret {
|
|
|
54
73
|
}
|
|
55
74
|
/** For handshakes that answer with a digest, such as Zoom's URL validation. */
|
|
56
75
|
hmac(algorithm, payload, encoding, secretEncoding = 'utf8') {
|
|
57
|
-
return
|
|
58
|
-
.update(payload)
|
|
59
|
-
.digest(encoding);
|
|
76
|
+
return hmacDigest(this.require(), algorithm, payload, encoding, secretEncoding);
|
|
60
77
|
}
|
|
61
78
|
/** Throws unless `signature` is the HMAC of `payload` under the secret. */
|
|
62
79
|
verifyHmac(signature, algorithm, payload, encoding, secretEncoding = 'utf8') {
|
|
63
|
-
|
|
64
|
-
if (!signature || !timingSafeStringEqual(signature, expected)) {
|
|
80
|
+
if (!verifyHmacSignature(this.require(), signature, algorithm, payload, encoding, secretEncoding)) {
|
|
65
81
|
throw this.rejected();
|
|
66
82
|
}
|
|
67
83
|
}
|
|
@@ -73,19 +89,7 @@ export class WebhookSigningSecret {
|
|
|
73
89
|
}
|
|
74
90
|
/** For providers that sign with a private key: the secret is their public key, as PEM. */
|
|
75
91
|
verifyPublicKey(signature, payload, options = {}) {
|
|
76
|
-
|
|
77
|
-
let valid = false;
|
|
78
|
-
try {
|
|
79
|
-
valid =
|
|
80
|
-
!!signature &&
|
|
81
|
-
createVerify(options.algorithm ?? 'sha256')
|
|
82
|
-
.update(payload)
|
|
83
|
-
.verify({ key, dsaEncoding: options.dsaEncoding }, signature, 'base64');
|
|
84
|
-
}
|
|
85
|
-
catch {
|
|
86
|
-
valid = false;
|
|
87
|
-
}
|
|
88
|
-
if (!valid) {
|
|
92
|
+
if (!verifyPublicKeySignature(this.require(), signature, payload, options)) {
|
|
89
93
|
throw this.rejected();
|
|
90
94
|
}
|
|
91
95
|
}
|
|
@@ -7,6 +7,7 @@ export class PikkuFetchHTTPResponse {
|
|
|
7
7
|
#responseMode = null;
|
|
8
8
|
#send = null;
|
|
9
9
|
#close = null;
|
|
10
|
+
#cancelStream = null;
|
|
10
11
|
setMode(mode) {
|
|
11
12
|
this.#responseMode = 'stream';
|
|
12
13
|
if (mode === 'stream') {
|
|
@@ -107,16 +108,28 @@ export class PikkuFetchHTTPResponse {
|
|
|
107
108
|
const encoder = new TextEncoder();
|
|
108
109
|
return new ReadableStream({
|
|
109
110
|
start: (controller) => {
|
|
111
|
+
let closed = false;
|
|
110
112
|
const send = (data) => {
|
|
113
|
+
if (closed)
|
|
114
|
+
return;
|
|
111
115
|
controller.enqueue(encoder.encode(`data: ${data}\n\n`));
|
|
112
116
|
};
|
|
113
117
|
const close = () => {
|
|
118
|
+
if (closed)
|
|
119
|
+
return;
|
|
120
|
+
closed = true;
|
|
114
121
|
controller.close();
|
|
115
122
|
};
|
|
123
|
+
this.#cancelStream = () => {
|
|
124
|
+
closed = true;
|
|
125
|
+
};
|
|
116
126
|
this.#send = send;
|
|
117
127
|
this.#close = close;
|
|
118
128
|
controller.enqueue(encoder.encode(':\n\n'));
|
|
119
129
|
},
|
|
130
|
+
cancel: () => {
|
|
131
|
+
this.#cancelStream?.();
|
|
132
|
+
},
|
|
120
133
|
});
|
|
121
134
|
}
|
|
122
135
|
}
|
|
@@ -37,6 +37,7 @@ export const deriveOAuth2AppSecrets = (credentials, declared) => {
|
|
|
37
37
|
authorizationUrl: oauth2.authorizationUrl,
|
|
38
38
|
tokenUrl: oauth2.tokenUrl,
|
|
39
39
|
scopes: oauth2.scopes,
|
|
40
|
+
scopeSeparator: oauth2.scopeSeparator,
|
|
40
41
|
pkce: oauth2.pkce,
|
|
41
42
|
additionalParams: oauth2.additionalParams,
|
|
42
43
|
},
|
|
@@ -31,6 +31,12 @@ export type OAuth2CredentialConfig = {
|
|
|
31
31
|
authorizationUrl: string;
|
|
32
32
|
tokenUrl: string;
|
|
33
33
|
scopes: string[];
|
|
34
|
+
/**
|
|
35
|
+
* What joins `scopes` in the authorization URL. RFC 6749 says a space, which
|
|
36
|
+
* is the default; a few providers (Twist) document a comma. An empty `scopes`
|
|
37
|
+
* omits the parameter altogether.
|
|
38
|
+
*/
|
|
39
|
+
scopeSeparator?: string;
|
|
34
40
|
pkce?: boolean;
|
|
35
41
|
/** Appended to the authorization URL's query string. */
|
|
36
42
|
additionalParams?: Record<string, string>;
|