@loomwork/middleware 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +8 -0
- package/README.md +82 -2
- package/dist/claimCheck.d.ts +179 -0
- package/dist/claimCheck.d.ts.map +1 -0
- package/dist/claimCheck.js +241 -0
- package/dist/claimCheck.js.map +1 -0
- package/dist/index.d.ts +13 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +48 -0
- package/dist/index.js.map +1 -0
- package/dist/killSwitchRegistry.d.ts +74 -0
- package/dist/killSwitchRegistry.d.ts.map +1 -0
- package/dist/killSwitchRegistry.js +108 -0
- package/dist/killSwitchRegistry.js.map +1 -0
- package/dist/metering.d.ts +81 -0
- package/dist/metering.d.ts.map +1 -0
- package/dist/metering.js +153 -0
- package/dist/metering.js.map +1 -0
- package/dist/nestedBudget.d.ts +60 -0
- package/dist/nestedBudget.d.ts.map +1 -0
- package/dist/nestedBudget.js +134 -0
- package/dist/nestedBudget.js.map +1 -0
- package/dist/profile.d.ts +38 -0
- package/dist/profile.d.ts.map +1 -0
- package/dist/profile.js +70 -0
- package/dist/profile.js.map +1 -0
- package/dist/rateLimit.d.ts +40 -0
- package/dist/rateLimit.d.ts.map +1 -0
- package/dist/rateLimit.js +83 -0
- package/dist/rateLimit.js.map +1 -0
- package/dist/redaction.d.ts +66 -0
- package/dist/redaction.d.ts.map +1 -0
- package/dist/redaction.js +141 -0
- package/dist/redaction.js.map +1 -0
- package/dist/retry.d.ts +50 -0
- package/dist/retry.d.ts.map +1 -0
- package/dist/retry.js +98 -0
- package/dist/retry.js.map +1 -0
- package/dist/safety.d.ts +93 -0
- package/dist/safety.d.ts.map +1 -0
- package/dist/safety.js +152 -0
- package/dist/safety.js.map +1 -0
- package/dist/streamTokenBudget.d.ts +58 -0
- package/dist/streamTokenBudget.d.ts.map +1 -0
- package/dist/streamTokenBudget.js +47 -0
- package/dist/streamTokenBudget.js.map +1 -0
- package/package.json +55 -3
- package/src/claimCheck.ts +410 -0
- package/src/index.ts +79 -0
- package/src/killSwitchRegistry.ts +168 -0
- package/src/metering.ts +226 -0
- package/src/nestedBudget.ts +200 -0
- package/src/profile.ts +86 -0
- package/src/rateLimit.ts +126 -0
- package/src/redaction.ts +184 -0
- package/src/retry.ts +133 -0
- package/src/safety.ts +235 -0
- package/src/streamTokenBudget.ts +98 -0
package/src/rateLimit.ts
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
import type { EffectContext, EffectMiddleware } from "@loomwork/effect-runtime";
|
|
2
|
+
|
|
3
|
+
// ── Effect rate limiting (R9 — autonomy envelope) ────────────────────────────
|
|
4
|
+
//
|
|
5
|
+
// A runaway agent is not only expensive, it is FAST: a tool loop can issue thousands of
|
|
6
|
+
// effects a second long before a token budget notices (a cheap tool meters ~0 tokens).
|
|
7
|
+
// A token bucket bounds the RATE of effects per scope, which the cost budgets cannot.
|
|
8
|
+
//
|
|
9
|
+
// It lives at the EFFECT layer, not in the fold, so it is replay-safe: middleware runs only
|
|
10
|
+
// when an effect actually executes (completed steps are served from the log and never reach
|
|
11
|
+
// it), and a refusal is a thrown error that fails the step through the ordinary path. The
|
|
12
|
+
// bucket state is process-local and wall-clock driven by design — it is admission control,
|
|
13
|
+
// not a fact about the execution — so it never enters the log and two nodes each enforce
|
|
14
|
+
// their own bucket (the limit is per process; cluster-wide limits are a store-backed
|
|
15
|
+
// concern this middleware deliberately does not pretend to solve).
|
|
16
|
+
|
|
17
|
+
/** Which requests share one bucket. */
|
|
18
|
+
export type RateLimitScope = "execution" | "tenant" | "workflow" | "global";
|
|
19
|
+
|
|
20
|
+
export interface RateLimitOptions {
|
|
21
|
+
/** Bucket size: the burst a scope may issue at once. Must be a positive integer. */
|
|
22
|
+
capacity: number;
|
|
23
|
+
/** Sustained rate: tokens added per second. Must be positive. */
|
|
24
|
+
refillPerSecond: number;
|
|
25
|
+
/**
|
|
26
|
+
* Who shares a bucket. Default `"execution"` (one runaway execution cannot starve the
|
|
27
|
+
* rest). `"tenant"` keys on `ctx.principal.tenant` — a request with no tenant shares ONE
|
|
28
|
+
* `"(no-tenant)"` bucket rather than escaping the limit. `"workflow"` keys on the flow id
|
|
29
|
+
* (the per-agent case); `"global"` is one bucket for the process.
|
|
30
|
+
*/
|
|
31
|
+
scope?: RateLimitScope;
|
|
32
|
+
/** Injectable clock in ms (tests); defaults to `Date.now`. */
|
|
33
|
+
clock?: () => number;
|
|
34
|
+
/**
|
|
35
|
+
* Upper bound on live buckets. A fresh bucket is created per execution/tenant key, so an
|
|
36
|
+
* unbounded map would be a leak; the least-recently-used bucket is evicted past this size
|
|
37
|
+
* (an evicted key simply starts full again — it can only err toward permitting). Default 10_000.
|
|
38
|
+
*/
|
|
39
|
+
maxBuckets?: number;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* The typed, NON-retryable refusal for a rate-limit trip. It is the control plane speaking,
|
|
44
|
+
* not a transient fault: retrying immediately repeats it, so callers that classify errors by
|
|
45
|
+
* `name` (loom-ui's `isControlPlaneError`) must treat `"RateLimitExceededError"` as a refusal.
|
|
46
|
+
*/
|
|
47
|
+
export class RateLimitExceededError extends Error {
|
|
48
|
+
readonly scope: RateLimitScope;
|
|
49
|
+
readonly key: string;
|
|
50
|
+
/** Milliseconds until one token is available again (a hint, never a promise). */
|
|
51
|
+
readonly retryAfterMs: number;
|
|
52
|
+
/** Honoured by `retryMiddleware`'s default classifier: an immediate retry repeats the refusal. */
|
|
53
|
+
readonly retryable = false;
|
|
54
|
+
constructor(scope: RateLimitScope, key: string, retryAfterMs: number) {
|
|
55
|
+
super(`Rate limit exceeded for ${scope} "${key}" — retry after ~${retryAfterMs}ms`);
|
|
56
|
+
this.name = "RateLimitExceededError";
|
|
57
|
+
this.scope = scope;
|
|
58
|
+
this.key = key;
|
|
59
|
+
this.retryAfterMs = retryAfterMs;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
interface Bucket {
|
|
64
|
+
tokens: number;
|
|
65
|
+
at: number;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function keyOf(scope: RateLimitScope, ctx: EffectContext): string {
|
|
69
|
+
switch (scope) {
|
|
70
|
+
case "execution":
|
|
71
|
+
return ctx.executionId;
|
|
72
|
+
case "workflow":
|
|
73
|
+
return ctx.workflowId;
|
|
74
|
+
case "tenant":
|
|
75
|
+
return ctx.principal?.tenant ?? "(no-tenant)";
|
|
76
|
+
case "global":
|
|
77
|
+
return "*";
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
export function rateLimitMiddleware(options: RateLimitOptions): EffectMiddleware {
|
|
82
|
+
const { capacity, refillPerSecond } = options;
|
|
83
|
+
if (!Number.isInteger(capacity) || capacity < 1) {
|
|
84
|
+
throw new Error(`rateLimitMiddleware: capacity must be a positive integer (got ${capacity})`);
|
|
85
|
+
}
|
|
86
|
+
if (!(refillPerSecond > 0) || !Number.isFinite(refillPerSecond)) {
|
|
87
|
+
throw new Error(
|
|
88
|
+
`rateLimitMiddleware: refillPerSecond must be a positive number (got ${refillPerSecond})`
|
|
89
|
+
);
|
|
90
|
+
}
|
|
91
|
+
const scope = options.scope ?? "execution";
|
|
92
|
+
const now = options.clock ?? (() => Date.now());
|
|
93
|
+
const maxBuckets = options.maxBuckets ?? 10_000;
|
|
94
|
+
// Map iteration order is insertion order, so delete+set on every touch keeps the OLDEST
|
|
95
|
+
// key first — an O(1) LRU without a second structure.
|
|
96
|
+
const buckets = new Map<string, Bucket>();
|
|
97
|
+
|
|
98
|
+
return async (ctx, _input, next): Promise<unknown> => {
|
|
99
|
+
const key = keyOf(scope, ctx);
|
|
100
|
+
const t = now();
|
|
101
|
+
const bucket = buckets.get(key) ?? { tokens: capacity, at: t };
|
|
102
|
+
buckets.delete(key);
|
|
103
|
+
const elapsedSec = Math.max(0, t - bucket.at) / 1000;
|
|
104
|
+
bucket.tokens = Math.min(capacity, bucket.tokens + elapsedSec * refillPerSecond);
|
|
105
|
+
bucket.at = t;
|
|
106
|
+
const admitted = bucket.tokens >= 1;
|
|
107
|
+
if (admitted) bucket.tokens -= 1;
|
|
108
|
+
buckets.set(key, bucket);
|
|
109
|
+
if (buckets.size > maxBuckets) {
|
|
110
|
+
const oldest = buckets.keys().next().value;
|
|
111
|
+
if (oldest !== undefined) buckets.delete(oldest);
|
|
112
|
+
}
|
|
113
|
+
if (!admitted) {
|
|
114
|
+
const retryAfterMs = Math.ceil(((1 - bucket.tokens) / refillPerSecond) * 1000);
|
|
115
|
+
ctx.emit("SAFETY_BLOCKED", {
|
|
116
|
+
stepId: ctx.stepId,
|
|
117
|
+
validatorId: "rate-limit",
|
|
118
|
+
phase: "input",
|
|
119
|
+
reason: `rate limit exceeded for ${scope} "${key}"`,
|
|
120
|
+
category: "rate-limit",
|
|
121
|
+
});
|
|
122
|
+
throw new RateLimitExceededError(scope, key, retryAfterMs);
|
|
123
|
+
}
|
|
124
|
+
return next();
|
|
125
|
+
};
|
|
126
|
+
}
|
package/src/redaction.ts
ADDED
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
import { pathsOfClass } from "@loomwork/classify";
|
|
2
|
+
import type { StepDefinition } from "@loomwork/core";
|
|
3
|
+
import type { EffectContext, EffectMiddleware, EffectNext } from "@loomwork/effect-runtime";
|
|
4
|
+
|
|
5
|
+
// ── K10.3 — redaction markers: keep secrets OUT of the log entirely ───────────
|
|
6
|
+
//
|
|
7
|
+
// K10.2 (crypto-shredding) stores an encrypted secret IN the log and erases it
|
|
8
|
+
// by destroying the key. K10.3 goes further for values that must never be
|
|
9
|
+
// persisted at all: a step's input carries a deterministic REFERENCE
|
|
10
|
+
// (`secretRef("stripe-key")`) instead of the raw secret, so the STEP_STARTED
|
|
11
|
+
// event — and the fold, and every snapshot — only ever contains the reference.
|
|
12
|
+
// A `SecretResolver` resolves the reference to the real value at effect-execution
|
|
13
|
+
// time (this middleware runs kernel-side, before the K7 boundary, so an isolated
|
|
14
|
+
// tool receives the resolved value while the parent's LOG never sees it — A5).
|
|
15
|
+
// The MUST is "secrets never enter the log", and it holds with or without K7.
|
|
16
|
+
|
|
17
|
+
/** The deterministic reference that stands in for a secret in step I/O and the log. */
|
|
18
|
+
export interface SecretRef {
|
|
19
|
+
readonly __loomSecret: string;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export function secretRef(id: string): SecretRef {
|
|
23
|
+
return { __loomSecret: id };
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export function isSecretRef(value: unknown): value is SecretRef {
|
|
27
|
+
return (
|
|
28
|
+
typeof value === "object" &&
|
|
29
|
+
value !== null &&
|
|
30
|
+
typeof (value as { __loomSecret?: unknown }).__loomSecret === "string"
|
|
31
|
+
);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Resolves secret references to their real values (read) and, for output
|
|
36
|
+
* redaction, stores a value under a reference (write). Back it with a vault /
|
|
37
|
+
* KMS / secret manager in production; the resolver is the boundary the plaintext
|
|
38
|
+
* lives behind. `InMemorySecretResolver` is for tests.
|
|
39
|
+
*/
|
|
40
|
+
export interface SecretResolver {
|
|
41
|
+
resolve(id: string): Promise<unknown> | unknown;
|
|
42
|
+
/** Optional — required only when `redactOutput` mints references from results. */
|
|
43
|
+
store?(id: string, value: unknown): Promise<void> | void;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export class InMemorySecretResolver implements SecretResolver {
|
|
47
|
+
constructor(private readonly secrets: Map<string, unknown> = new Map()) {}
|
|
48
|
+
resolve(id: string): unknown {
|
|
49
|
+
if (!this.secrets.has(id)) throw new Error(`no secret registered for reference "${id}"`);
|
|
50
|
+
return this.secrets.get(id);
|
|
51
|
+
}
|
|
52
|
+
store(id: string, value: unknown): void {
|
|
53
|
+
this.secrets.set(id, value);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export interface RedactionOptions {
|
|
58
|
+
resolver: SecretResolver;
|
|
59
|
+
/**
|
|
60
|
+
* Output dot-paths (per effect) whose values must NOT enter the log: the value
|
|
61
|
+
* is stored in the resolver under a deterministic reference
|
|
62
|
+
* (`executionId:stepId:path`) and REPLACED by that reference in the result, so
|
|
63
|
+
* STEP_COMPLETED logs the reference. Requires `resolver.store`. Omit to only
|
|
64
|
+
* resolve input references (the common case).
|
|
65
|
+
*/
|
|
66
|
+
redactOutput?: (info: { stepId: string }) => readonly string[];
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
async function deepResolve(value: unknown, resolver: SecretResolver): Promise<unknown> {
|
|
70
|
+
if (isSecretRef(value)) return resolver.resolve(value.__loomSecret);
|
|
71
|
+
if (Array.isArray(value)) return Promise.all(value.map((v) => deepResolve(v, resolver)));
|
|
72
|
+
if (typeof value === "object" && value !== null) {
|
|
73
|
+
const out: Record<string, unknown> = {};
|
|
74
|
+
for (const [k, v] of Object.entries(value)) out[k] = await deepResolve(v, resolver);
|
|
75
|
+
return out;
|
|
76
|
+
}
|
|
77
|
+
return value;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
function getPath(
|
|
81
|
+
root: unknown,
|
|
82
|
+
path: string
|
|
83
|
+
): { parent: Record<string, unknown>; leaf: string } | undefined {
|
|
84
|
+
const segs = path.split(".");
|
|
85
|
+
let cursor = root as Record<string, unknown> | undefined;
|
|
86
|
+
for (let i = 0; i < segs.length - 1; i++) {
|
|
87
|
+
const next = cursor?.[segs[i] as string];
|
|
88
|
+
cursor =
|
|
89
|
+
typeof next === "object" && next !== null ? (next as Record<string, unknown>) : undefined;
|
|
90
|
+
if (cursor === undefined) return undefined;
|
|
91
|
+
}
|
|
92
|
+
const leaf = segs[segs.length - 1] as string;
|
|
93
|
+
return cursor !== undefined && Object.hasOwn(cursor, leaf) ? { parent: cursor, leaf } : undefined;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* K10.3 — redaction middleware. Runs on the effect path (like metering/safety):
|
|
98
|
+
* - **input**: resolves every `SecretRef` in the effect input to its real value
|
|
99
|
+
* before the tool runs, WITHOUT touching the logged input (STEP_STARTED keeps
|
|
100
|
+
* the reference — the secret never enters the log).
|
|
101
|
+
* - **output** (optional): replaces marked result fields with deterministic
|
|
102
|
+
* references (storing the value in the resolver), so STEP_COMPLETED logs the
|
|
103
|
+
* reference, not the secret.
|
|
104
|
+
*
|
|
105
|
+
* Unwired (no middleware), effects run byte-identically; with it but no refs in
|
|
106
|
+
* play, it is a deep-walk pass-through (G2).
|
|
107
|
+
*/
|
|
108
|
+
export function redactionMiddleware(options: RedactionOptions): EffectMiddleware {
|
|
109
|
+
const mw = async (ctx: EffectContext, input: unknown, next: EffectNext): Promise<unknown> => {
|
|
110
|
+
// Input phase — resolve references into the value the tool actually sees,
|
|
111
|
+
// WITHOUT touching the logged input (STEP_STARTED keeps the reference).
|
|
112
|
+
const resolvedInput = await deepResolve(input, options.resolver);
|
|
113
|
+
const result = await next(resolvedInput);
|
|
114
|
+
|
|
115
|
+
// Output phase — redact marked result fields into references.
|
|
116
|
+
const paths = options.redactOutput?.({ stepId: ctx.stepId }) ?? [];
|
|
117
|
+
if (paths.length === 0) return result;
|
|
118
|
+
if (options.resolver.store === undefined) {
|
|
119
|
+
throw new Error("redactOutput requires a SecretResolver with store()");
|
|
120
|
+
}
|
|
121
|
+
const redacted = structuredClone(result);
|
|
122
|
+
for (const path of paths) {
|
|
123
|
+
const hit = getPath(redacted, path);
|
|
124
|
+
if (hit === undefined) continue;
|
|
125
|
+
// A transient call stamped as an execution shares (executionId, stepId) with its
|
|
126
|
+
// siblings; callKey keeps their secret refs distinct (undefined for durable steps).
|
|
127
|
+
const refId = `${ctx.executionId}:${ctx.stepId}${
|
|
128
|
+
ctx.callKey !== undefined ? `:${ctx.callKey}` : ""
|
|
129
|
+
}:${path}`;
|
|
130
|
+
await options.resolver.store(refId, hit.parent[hit.leaf]);
|
|
131
|
+
hit.parent[hit.leaf] = secretRef(refId); // the LOG gets the reference
|
|
132
|
+
}
|
|
133
|
+
return redacted;
|
|
134
|
+
};
|
|
135
|
+
// ADR 0119 — output redaction REWRITES the result after `next()`; a streamed delta
|
|
136
|
+
// bypasses that masking, so declare it: the runtime then refuses live deltas (fail
|
|
137
|
+
// closed) rather than stream the secret past the mask. Input-only resolution does not
|
|
138
|
+
// touch output, so it stays streamable.
|
|
139
|
+
return (
|
|
140
|
+
options.redactOutput !== undefined ? Object.assign(mw, { inspectsOutput: true as const }) : mw
|
|
141
|
+
) as EffectMiddleware;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
// ── K10-R2 — derive output redaction from `secret()` marks ────────────────────
|
|
145
|
+
//
|
|
146
|
+
// Instead of hand-authoring a `redactOutput: ({stepId}) => paths` callback, mark the
|
|
147
|
+
// field once — `apiToken: secret(z.string())` — and derive it. A step's output fields
|
|
148
|
+
// classified `"secret"` are the ones that must never be persisted; each becomes a
|
|
149
|
+
// redactOutput path (relative to the effect RESULT, which is why there is no `output.`
|
|
150
|
+
// prefix — the crypto map operates on the event payload, redaction on the raw result).
|
|
151
|
+
//
|
|
152
|
+
// This is the low-risk half of K10-R2: it drives the EXISTING effect-path redaction seam
|
|
153
|
+
// off the R1 taxonomy, with NO kernel change. Automatic secret substitution before
|
|
154
|
+
// STEP_STARTED in the hot path remains deferred (its determinism stakes are higher).
|
|
155
|
+
|
|
156
|
+
/**
|
|
157
|
+
* Precompute a `stepId → secret output paths` lookup from step definitions, for O(1) access
|
|
158
|
+
* in the per-effect redactOutput callback (a Map, not a per-call schema walk). Steps with no
|
|
159
|
+
* `secret()`-marked output field are omitted, so an all-unmarked app yields an empty Map and
|
|
160
|
+
* the redaction pass stays a no-op (G2).
|
|
161
|
+
*/
|
|
162
|
+
export function secretOutputPathsByStep(
|
|
163
|
+
// Only id + output schema are read; Pick drops the invariant handler field so concrete
|
|
164
|
+
// `StepDefinition<In, Out>` values pass without a cast.
|
|
165
|
+
steps: readonly Pick<StepDefinition, "id" | "outputSchema">[]
|
|
166
|
+
): Map<string, readonly string[]> {
|
|
167
|
+
const byStep = new Map<string, readonly string[]>();
|
|
168
|
+
for (const step of steps) {
|
|
169
|
+
if (step.outputSchema === undefined) continue;
|
|
170
|
+
const paths = pathsOfClass(step.outputSchema, "secret");
|
|
171
|
+
if (paths.length > 0) byStep.set(step.id, paths);
|
|
172
|
+
}
|
|
173
|
+
return byStep;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* A {@link RedactionOptions.redactOutput} callback derived from `secret()` marks — pass the
|
|
178
|
+
* result of {@link secretOutputPathsByStep} (or let `defineConfig({ secrets })` wire it).
|
|
179
|
+
*/
|
|
180
|
+
export function redactOutputFrom(
|
|
181
|
+
byStep: Map<string, readonly string[]>
|
|
182
|
+
): (info: { stepId: string }) => readonly string[] {
|
|
183
|
+
return ({ stepId }) => byStep.get(stepId) ?? [];
|
|
184
|
+
}
|
package/src/retry.ts
ADDED
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
import { SuspendError, TypedFailure } from "@loomwork/core";
|
|
2
|
+
import type { EffectContext, EffectMiddleware } from "@loomwork/effect-runtime";
|
|
3
|
+
|
|
4
|
+
export interface RetryOptions {
|
|
5
|
+
/** Max retries after the first attempt. Default 2 (⇒ up to 3 tries). */
|
|
6
|
+
maxRetries?: number;
|
|
7
|
+
/** Delay before the first retry, in ms. Default 100. */
|
|
8
|
+
baseDelayMs?: number;
|
|
9
|
+
/** Exponential growth factor between retries. Default 2. */
|
|
10
|
+
factor?: number;
|
|
11
|
+
/** Upper bound on any single delay, in ms. Default 30_000. */
|
|
12
|
+
maxDelayMs?: number;
|
|
13
|
+
/** Spread delays deterministically to avoid thundering herds. Default true. */
|
|
14
|
+
jitter?: boolean;
|
|
15
|
+
/**
|
|
16
|
+
* Decide whether a given error is retryable. Default: all (non-control) errors
|
|
17
|
+
* EXCEPT `TypedFailure` (DE-4) — a declared domain outcome is not transient, so
|
|
18
|
+
* retrying it by default would just burn attempts on a deterministic result — and any
|
|
19
|
+
* error that declares `retryable: false` (R9's `KillSwitchEngagedError`,
|
|
20
|
+
* `RateLimitExceededError`, `SpawnLimitExceededError`, `SafetyViolationError`).
|
|
21
|
+
* An explicit `isRetryable` retains full control (retrying a TypedFailure
|
|
22
|
+
* in-memory is legal: no durable event has been emitted yet).
|
|
23
|
+
*/
|
|
24
|
+
isRetryable?: (err: unknown) => boolean;
|
|
25
|
+
/** Observability hook fired before each backoff wait. */
|
|
26
|
+
onRetry?: (info: { ctx: EffectContext; attempt: number; delayMs: number; err: unknown }) => void;
|
|
27
|
+
/** Injectable sleep (for tests). Must reject if `signal` aborts. */
|
|
28
|
+
sleep?: (ms: number, signal?: AbortSignal) => Promise<void>;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Retry-with-backoff middleware — fills the gap where effect failures are
|
|
33
|
+
* terminal today. Wraps a single `EffectRuntime.execute`; on a retryable error
|
|
34
|
+
* it waits (exponential backoff + deterministic jitter) and re-invokes the
|
|
35
|
+
* effect, up to `maxRetries` times.
|
|
36
|
+
*
|
|
37
|
+
* Correctness rules:
|
|
38
|
+
* - **Never retries `SuspendError`** (a control-flow signal, not a failure) or a
|
|
39
|
+
* **cancelled** execution (`ctx.signal.aborted`) — both rethrow immediately.
|
|
40
|
+
* - Retries happen **in-memory within one execute call** and emit no durable
|
|
41
|
+
* events, so the STEP_STARTED/COMPLETED envelope and the event log stay
|
|
42
|
+
* deterministic; use `onRetry` for logging/metrics. Exhaustion rethrows the
|
|
43
|
+
* last error into the existing `STEP_FAILED` path — behavior-preserving.
|
|
44
|
+
* - Jitter is a deterministic hash of `executionId:stepId:attempt` (no
|
|
45
|
+
* `Math.random`), so replay/behaviour stays stable.
|
|
46
|
+
*/
|
|
47
|
+
export function retryMiddleware(options: RetryOptions = {}): EffectMiddleware {
|
|
48
|
+
const maxRetries = options.maxRetries ?? 2;
|
|
49
|
+
const baseDelayMs = options.baseDelayMs ?? 100;
|
|
50
|
+
const factor = options.factor ?? 2;
|
|
51
|
+
const maxDelayMs = options.maxDelayMs ?? 30_000;
|
|
52
|
+
const useJitter = options.jitter ?? true;
|
|
53
|
+
const isRetryable =
|
|
54
|
+
options.isRetryable ??
|
|
55
|
+
((err: unknown) =>
|
|
56
|
+
!(err instanceof TypedFailure) &&
|
|
57
|
+
// R9 — a control-plane refusal (kill switch, rate limit, spawn cap) marks itself
|
|
58
|
+
// `retryable: false`: it is a decision, and retrying repeats it.
|
|
59
|
+
(err as { retryable?: unknown } | null)?.retryable !== false);
|
|
60
|
+
const sleep = options.sleep ?? defaultSleep;
|
|
61
|
+
|
|
62
|
+
return async (ctx, _input, next) => {
|
|
63
|
+
for (let attemptIndex = 0; ; attemptIndex++) {
|
|
64
|
+
try {
|
|
65
|
+
return await next();
|
|
66
|
+
} catch (err) {
|
|
67
|
+
// Control-flow + cancellation are never "failures" to retry.
|
|
68
|
+
if (err instanceof SuspendError) throw err;
|
|
69
|
+
if (ctx.signal?.aborted === true) throw err;
|
|
70
|
+
if (attemptIndex >= maxRetries || !isRetryable(err)) throw err;
|
|
71
|
+
|
|
72
|
+
const delayMs = computeDelay(
|
|
73
|
+
attemptIndex,
|
|
74
|
+
{ baseDelayMs, factor, maxDelayMs, useJitter },
|
|
75
|
+
ctx
|
|
76
|
+
);
|
|
77
|
+
options.onRetry?.({ ctx, attempt: attemptIndex + 1, delayMs, err });
|
|
78
|
+
await sleep(delayMs, ctx.signal);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function computeDelay(
|
|
85
|
+
attemptIndex: number,
|
|
86
|
+
cfg: { baseDelayMs: number; factor: number; maxDelayMs: number; useJitter: boolean },
|
|
87
|
+
ctx: EffectContext
|
|
88
|
+
): number {
|
|
89
|
+
const base = Math.min(cfg.maxDelayMs, cfg.baseDelayMs * cfg.factor ** attemptIndex);
|
|
90
|
+
if (!cfg.useJitter) return Math.round(base);
|
|
91
|
+
// Deterministic "full jitter" in [base/2, base] — stable across replay.
|
|
92
|
+
const frac = hashFraction(`${ctx.executionId}:${ctx.stepId}:${attemptIndex}`);
|
|
93
|
+
return Math.round(base / 2 + frac * (base / 2));
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** FNV-1a → [0, 1). Deterministic; no Math.random. */
|
|
97
|
+
function hashFraction(key: string): number {
|
|
98
|
+
let h = 0x811c9dc5;
|
|
99
|
+
for (let i = 0; i < key.length; i++) {
|
|
100
|
+
h ^= key.charCodeAt(i);
|
|
101
|
+
h = Math.imul(h, 0x01000193);
|
|
102
|
+
}
|
|
103
|
+
// >>> 0 to unsigned, scale into [0,1)
|
|
104
|
+
return (h >>> 0) / 0xffffffff;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function defaultSleep(ms: number, signal?: AbortSignal): Promise<void> {
|
|
108
|
+
return new Promise<void>((resolve, reject) => {
|
|
109
|
+
if (signal?.aborted === true) {
|
|
110
|
+
reject(abortError(signal));
|
|
111
|
+
return;
|
|
112
|
+
}
|
|
113
|
+
const timer = setTimeout(() => {
|
|
114
|
+
cleanup();
|
|
115
|
+
resolve();
|
|
116
|
+
}, ms);
|
|
117
|
+
const onAbort = (): void => {
|
|
118
|
+
cleanup();
|
|
119
|
+
reject(abortError(signal));
|
|
120
|
+
};
|
|
121
|
+
const cleanup = (): void => {
|
|
122
|
+
clearTimeout(timer);
|
|
123
|
+
signal?.removeEventListener("abort", onAbort);
|
|
124
|
+
};
|
|
125
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
function abortError(signal?: AbortSignal): Error {
|
|
130
|
+
const reason = signal?.reason;
|
|
131
|
+
if (reason instanceof Error) return reason;
|
|
132
|
+
return new Error("aborted");
|
|
133
|
+
}
|
package/src/safety.ts
ADDED
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
import type { EffectContext, EffectMiddleware } from "@loomwork/effect-runtime";
|
|
2
|
+
|
|
3
|
+
// ── Behavioral safety ring (Loom 2.0 K6) ─────────────────────────────────────
|
|
4
|
+
//
|
|
5
|
+
// K2 governs *who may act*; K3 governs *how much it costs*. Neither governs
|
|
6
|
+
// *whether the action or its output is safe* — the guarantee an autonomous
|
|
7
|
+
// intelligent system most needs. K6 adds a governed check on the effect path:
|
|
8
|
+
// pluggable validators inspect an effect's INPUT before it runs (block an unsafe
|
|
9
|
+
// action, like K3 admission control) and its OUTPUT after (stop an unsafe result
|
|
10
|
+
// from propagating). A block is a durable `SAFETY_BLOCKED` fact (folded into a
|
|
11
|
+
// reserved `variables._safety` audit list) and throws `SafetyViolationError`,
|
|
12
|
+
// which fails the step via the normal `STEP_FAILED` path — exactly like a K3 quota
|
|
13
|
+
// trip. Opt-in and replay-safe: unwired, an effect runs identically to today
|
|
14
|
+
// (P10); a block is a logged fact served from the log on replay (P9), so a blocked
|
|
15
|
+
// run reconstructs identically. Validators run only on live execution (replay
|
|
16
|
+
// serves completed/failed steps from the log and never re-invokes them); keep them
|
|
17
|
+
// cheap and pure for latency and auditability, and route any ML/LLM classifier
|
|
18
|
+
// verdict through the decision seam (P9) if it must be reproduced.
|
|
19
|
+
|
|
20
|
+
export type SafetyPhase = "input" | "output" | "delta";
|
|
21
|
+
|
|
22
|
+
export interface SafetyCheckInfo {
|
|
23
|
+
phase: SafetyPhase;
|
|
24
|
+
stepId: string;
|
|
25
|
+
/** The effect input ("input" and "output" phases; `undefined` on "delta"). */
|
|
26
|
+
input: unknown;
|
|
27
|
+
/** The effect result — present only on the "output" phase. */
|
|
28
|
+
result?: unknown;
|
|
29
|
+
/**
|
|
30
|
+
* The RUNNING text of a streamed call — present only on the "delta" phase. It is the
|
|
31
|
+
* last {@link SafetyOptions.deltaWindow} characters received so far INCLUDING the delta
|
|
32
|
+
* under test, so a pattern split across deltas is seen whole (up to the window).
|
|
33
|
+
*/
|
|
34
|
+
text?: string;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export interface SafetyVerdict {
|
|
38
|
+
ok: boolean;
|
|
39
|
+
reason?: string;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export interface SafetyValidator {
|
|
43
|
+
id: string;
|
|
44
|
+
phase: SafetyPhase;
|
|
45
|
+
category?: string;
|
|
46
|
+
/**
|
|
47
|
+
* Return `{ ok: false }` to block. Should be pure/deterministic. A `"delta"`-phase
|
|
48
|
+
* validator receives `info.text` (the running streamed text) and runs ONLY on a
|
|
49
|
+
* streamed call, severing the stream before the offending delta is delivered.
|
|
50
|
+
*/
|
|
51
|
+
check(info: SafetyCheckInfo): SafetyVerdict;
|
|
52
|
+
/**
|
|
53
|
+
* ADR 0119 — the DELTA FORM of an `"output"` validator: the same policy applied to the
|
|
54
|
+
* running text of a streamed call (`info.phase === "delta"`, `info.text`). An output
|
|
55
|
+
* validator WITHOUT one cannot see a stream, so when any is configured the runtime
|
|
56
|
+
* REFUSES streaming to every delta sink — per-call and configured (`EffectMiddleware.inspectsOutput`) rather than stream
|
|
57
|
+
* text past it. The output form still runs on the final result either way.
|
|
58
|
+
*/
|
|
59
|
+
checkDelta?(info: SafetyCheckInfo): SafetyVerdict;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export interface SafetyOptions {
|
|
63
|
+
validators: readonly SafetyValidator[];
|
|
64
|
+
/**
|
|
65
|
+
* ADR 0119 — how many trailing characters of a streamed call's running text a delta
|
|
66
|
+
* check sees (default 4096). It bounds per-delta cost (the check is O(window), not
|
|
67
|
+
* O(text so far)). A pattern that SPANS more than the window across deltas can evade
|
|
68
|
+
* the delta check; the output form still sees the whole result, but by then the
|
|
69
|
+
* caller's live tail has already received it. Text BEFORE a match completes is also
|
|
70
|
+
* delivered (a card number is severed at its last digit, not its first) — a delta
|
|
71
|
+
* form narrows what a live tail can show, it does not make a stream as safe as a
|
|
72
|
+
* buffered result.
|
|
73
|
+
*/
|
|
74
|
+
deltaWindow?: number;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
export class SafetyViolationError extends Error {
|
|
78
|
+
readonly validatorId: string;
|
|
79
|
+
readonly category: string | undefined;
|
|
80
|
+
readonly reason: string;
|
|
81
|
+
/**
|
|
82
|
+
* Honoured by `retryMiddleware`'s default classifier: a violation is a policy decision,
|
|
83
|
+
* not a transient fault. Retrying re-calls the provider — for a severed stream (ADR 0119)
|
|
84
|
+
* that is a second paid call whose deltas a live tail already started showing. An app
|
|
85
|
+
* that wants "regenerate on violation" opts in with an explicit `isRetryable`.
|
|
86
|
+
*/
|
|
87
|
+
readonly retryable = false;
|
|
88
|
+
constructor(validatorId: string, reason: string, category?: string) {
|
|
89
|
+
super(`Safety violation [${validatorId}]: ${reason}`);
|
|
90
|
+
this.name = "SafetyViolationError";
|
|
91
|
+
this.validatorId = validatorId;
|
|
92
|
+
this.category = category;
|
|
93
|
+
this.reason = reason;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
export function safetyMiddleware(options: SafetyOptions): EffectMiddleware {
|
|
98
|
+
const inputValidators = options.validators.filter((v) => v.phase === "input");
|
|
99
|
+
const outputValidators = options.validators.filter((v) => v.phase === "output");
|
|
100
|
+
// The delta checks: a delta-phase validator's `check`, an output validator's `checkDelta`.
|
|
101
|
+
const deltaChecks: Array<{ v: SafetyValidator; run: (i: SafetyCheckInfo) => SafetyVerdict }> = [];
|
|
102
|
+
for (const v of options.validators) {
|
|
103
|
+
if (v.phase === "delta") deltaChecks.push({ v, run: (i) => v.check(i) });
|
|
104
|
+
else if (v.phase === "output" && v.checkDelta !== undefined) {
|
|
105
|
+
const f = v.checkDelta.bind(v);
|
|
106
|
+
deltaChecks.push({ v, run: f });
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
const uncoveredOutput = outputValidators.some((v) => v.checkDelta === undefined);
|
|
110
|
+
const window = Math.max(1, options.deltaWindow ?? 4096);
|
|
111
|
+
|
|
112
|
+
const block = (ctx: EffectContext, v: SafetyValidator, phase: SafetyPhase, reason: string) => {
|
|
113
|
+
ctx.emit("SAFETY_BLOCKED", {
|
|
114
|
+
stepId: ctx.stepId,
|
|
115
|
+
validatorId: v.id,
|
|
116
|
+
phase,
|
|
117
|
+
reason,
|
|
118
|
+
...(v.category !== undefined && { category: v.category }),
|
|
119
|
+
});
|
|
120
|
+
throw new SafetyViolationError(v.id, reason, v.category);
|
|
121
|
+
};
|
|
122
|
+
|
|
123
|
+
const run = (ctx: EffectContext, validators: SafetyValidator[], info: SafetyCheckInfo): void => {
|
|
124
|
+
for (const v of validators) {
|
|
125
|
+
const verdict = v.check(info);
|
|
126
|
+
if (!verdict.ok) block(ctx, v, info.phase, verdict.reason ?? "blocked");
|
|
127
|
+
}
|
|
128
|
+
};
|
|
129
|
+
|
|
130
|
+
const mw = async (ctx: EffectContext, input: unknown, next: () => Promise<unknown>) => {
|
|
131
|
+
run(ctx, inputValidators, { phase: "input", stepId: ctx.stepId, input });
|
|
132
|
+
const result = await next();
|
|
133
|
+
if (outputValidators.length > 0) {
|
|
134
|
+
run(ctx, outputValidators, { phase: "output", stepId: ctx.stepId, input, result });
|
|
135
|
+
}
|
|
136
|
+
return result;
|
|
137
|
+
};
|
|
138
|
+
|
|
139
|
+
return Object.assign(mw, {
|
|
140
|
+
// An output validator with no delta form cannot see a stream: tell the runtime so it
|
|
141
|
+
// refuses a live sink instead of streaming text past it (fail-closed).
|
|
142
|
+
...(uncoveredOutput && { inspectsOutput: true as const }),
|
|
143
|
+
...(deltaChecks.length > 0 && {
|
|
144
|
+
deltaGuard: (ctx: EffectContext) => {
|
|
145
|
+
let text = "";
|
|
146
|
+
return (delta: string): void => {
|
|
147
|
+
text = (text + delta).slice(-window);
|
|
148
|
+
for (const { v, run: check } of deltaChecks) {
|
|
149
|
+
const verdict = check({ phase: "delta", stepId: ctx.stepId, input: undefined, text });
|
|
150
|
+
if (!verdict.ok) block(ctx, v, "delta", verdict.reason ?? "blocked");
|
|
151
|
+
}
|
|
152
|
+
};
|
|
153
|
+
},
|
|
154
|
+
}),
|
|
155
|
+
}) as EffectMiddleware;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
// ── Built-in validators ──────────────────────────────────────────────────────
|
|
159
|
+
|
|
160
|
+
function asText(value: unknown): string {
|
|
161
|
+
return typeof value === "string" ? value : JSON.stringify(value ?? "");
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/** Block if the checked value (input or result) matches any pattern (jailbreak/policy). */
|
|
165
|
+
export function denyPatterns(
|
|
166
|
+
patterns: readonly RegExp[],
|
|
167
|
+
opts: { id?: string; phase?: SafetyPhase; category?: string } = {}
|
|
168
|
+
): SafetyValidator {
|
|
169
|
+
const phase = opts.phase ?? "output";
|
|
170
|
+
const scan = (text: string): SafetyVerdict => {
|
|
171
|
+
for (const p of patterns) {
|
|
172
|
+
if (p.test(text)) return { ok: false, reason: `matched ${p}` };
|
|
173
|
+
}
|
|
174
|
+
return { ok: true };
|
|
175
|
+
};
|
|
176
|
+
return {
|
|
177
|
+
id: opts.id ?? "deny-patterns",
|
|
178
|
+
phase,
|
|
179
|
+
category: opts.category ?? "policy",
|
|
180
|
+
check(info) {
|
|
181
|
+
return scan(
|
|
182
|
+
info.phase === "delta"
|
|
183
|
+
? (info.text ?? "")
|
|
184
|
+
: asText(phase === "output" ? info.result : info.input)
|
|
185
|
+
);
|
|
186
|
+
},
|
|
187
|
+
// The delta form of the default (output) validator: the same patterns over the running
|
|
188
|
+
// text, so a default config protects a stream. An input-phase validator has none.
|
|
189
|
+
...(phase === "output" && { checkDelta: (info: SafetyCheckInfo) => scan(info.text ?? "") }),
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/** Block if the checked value contains obvious PII (email, US SSN, card-like digits). */
|
|
194
|
+
export function piiValidator(opts: { id?: string; phase?: SafetyPhase } = {}): SafetyValidator {
|
|
195
|
+
const PII: RegExp[] = [
|
|
196
|
+
/\b[\w.+-]+@[\w-]+\.[\w.-]+\b/, // email
|
|
197
|
+
/\b\d{3}-\d{2}-\d{4}\b/, // US SSN
|
|
198
|
+
/\b(?:\d[ -]?){13,16}\b/, // card-like
|
|
199
|
+
];
|
|
200
|
+
return {
|
|
201
|
+
...denyPatterns(PII, { id: opts.id ?? "pii", ...(opts.phase && { phase: opts.phase }) }),
|
|
202
|
+
category: "pii",
|
|
203
|
+
};
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
/** A minimal Zod-compatible shape — anything with a `safeParse`. */
|
|
207
|
+
export interface SchemaLike {
|
|
208
|
+
safeParse(value: unknown): { success: boolean };
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
/** Enforce an effect's output schema at runtime (schemas are defined-but-unchecked today). */
|
|
212
|
+
export function schemaValidator(schema: SchemaLike, opts: { id?: string } = {}): SafetyValidator {
|
|
213
|
+
return {
|
|
214
|
+
id: opts.id ?? "schema",
|
|
215
|
+
phase: "output",
|
|
216
|
+
category: "schema",
|
|
217
|
+
check(info) {
|
|
218
|
+
return schema.safeParse(info.result).success
|
|
219
|
+
? { ok: true }
|
|
220
|
+
: { ok: false, reason: "output failed schema validation" };
|
|
221
|
+
},
|
|
222
|
+
};
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
/** A real kill-switch: block every effect while `isTripped()` returns true (distinct from K4). */
|
|
226
|
+
export function killSwitch(isTripped: () => boolean, opts: { id?: string } = {}): SafetyValidator {
|
|
227
|
+
return {
|
|
228
|
+
id: opts.id ?? "kill-switch",
|
|
229
|
+
phase: "input",
|
|
230
|
+
category: "kill-switch",
|
|
231
|
+
check() {
|
|
232
|
+
return isTripped() ? { ok: false, reason: "kill-switch engaged" } : { ok: true };
|
|
233
|
+
},
|
|
234
|
+
};
|
|
235
|
+
}
|