create-windy 0.8.0 → 0.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +14 -8
- package/package.json +1 -1
- package/template/.windy-template.json +2 -2
- package/template/apps/agent-server/Dockerfile +1 -1
- package/template/apps/agent-server/README.md +1 -1
- package/template/apps/agent-server/src/southwind_agent_server/asgi.py +8 -0
- package/template/apps/agent-server/src/southwind_agent_server/attempt_context.py +9 -1
- package/template/apps/agent-server/src/southwind_agent_server/openai_provider.py +5 -2
- package/template/apps/agent-server/src/southwind_agent_server/provider_output_guidance.py +34 -0
- package/template/apps/agent-server/src/southwind_agent_server/runtime_logging.py +57 -0
- package/template/apps/agent-server/src/southwind_agent_server/structured_logging.py +145 -0
- package/template/apps/agent-server/src/southwind_agent_server/telemetry.py +4 -0
- package/template/apps/agent-server/tests/test_run_operational_logging.py +114 -0
- package/template/apps/agent-server/tests/test_structured_logging.py +142 -0
- package/template/apps/agent-server/tests/test_tool_structured_output.py +347 -0
- package/template/apps/server/src/agent/isolated-execution-guard.test.ts +194 -0
- package/template/apps/server/src/agent/isolated-execution-guard.ts +202 -0
- package/template/apps/server/src/agent/run-definition-support.ts +11 -1
- package/template/apps/server/src/agent/run-facade.ts +7 -11
- package/template/apps/server/src/agent/run-operation-policy.ts +11 -0
- package/template/apps/server/src/agent/workflow-durable-runtime-input.ts +8 -5
- package/template/apps/server/src/jobs/legacy-service-runtime.ts +4 -0
- package/template/apps/server/src/jobs/runtime-logging.test.ts +212 -0
- package/template/apps/server/src/jobs/runtime-logging.ts +58 -0
- package/template/apps/server/src/jobs/worker.ts +54 -12
- package/template/apps/server/src/observability/bootstrap.ts +1 -4
- package/template/apps/server/src/observability/http-observability.ts +2 -3
- package/template/apps/server/src/observability/structured-logger.test.ts +31 -0
- package/template/apps/server/src/observability/structured-logger.ts +27 -0
- package/template/apps/server/src/platform-bootstrap.ts +9 -3
- package/template/apps/server/src/startup-finalization.ts +3 -3
- package/template/apps/server/src/system/built-in-roles.test.ts +23 -0
- package/template/apps/server/src/system/built-in-roles.ts +10 -4
- package/template/apps/web/src/pages/system/composables/useResourceEditor.ts +16 -5
- package/template/apps/web/src/pages/system/composables/useResourceEditor.webtest.ts +4 -0
- package/template/docs/architecture/log-governance-implementation.md +96 -0
- package/template/docs/architecture/log-governance-plan.md +39 -0
- package/template/docs/operations/observability.md +36 -4
- package/template/docs/platform/agent-runtime.md +18 -0
- package/template/packages/modules/package.json +1 -1
- package/template/packages/modules/src/ai-isolated-execution-policy.test.ts +70 -0
- package/template/packages/modules/src/ai-isolated-execution-policy.ts +76 -0
- package/template/packages/modules/src/ai-operation-composition.ts +2 -0
- package/template/packages/modules/src/manifest.ts +23 -0
- package/template/packages/modules/src/system-features.ts +11 -10
- package/template/packages/server-sdk/package.json +1 -1
- package/template/packages/shared/index.ts +1 -0
- package/template/packages/shared/package.json +3 -2
- package/template/packages/shared/src/structured-log.test.ts +147 -0
- package/template/packages/shared/src/structured-log.ts +215 -0
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
AiIsolatedExecutionPolicyV1,
|
|
3
|
+
ModuleManifest,
|
|
4
|
+
} from "@southwind-ai/modules";
|
|
5
|
+
import { evaluateAccessGuards } from "../access-guards.js";
|
|
6
|
+
import type { RequestActor } from "../guards.js";
|
|
7
|
+
import { findFeature } from "../route-guards.js";
|
|
8
|
+
import type { createServerRuntime } from "../runtime.js";
|
|
9
|
+
import type { InitializedModuleHosts } from "../module-host.js";
|
|
10
|
+
import type { AgentRunAuthorizationStore } from "./run-authorization.js";
|
|
11
|
+
import { buildAgentRuntimeRelease } from "./run-release.js";
|
|
12
|
+
|
|
13
|
+
type ServerRuntime = ReturnType<typeof createServerRuntime>;
|
|
14
|
+
|
|
15
|
+
/** Must reserve atomically and durably by Run and call ID; absent storage denies admission. */
|
|
16
|
+
export interface IsolatedExecutionBudgetPort {
|
|
17
|
+
reserve(input: {
|
|
18
|
+
readonly runId: string;
|
|
19
|
+
readonly callId: string;
|
|
20
|
+
readonly maxToolCalls: number;
|
|
21
|
+
readonly maxToolRounds: number;
|
|
22
|
+
readonly maxConcurrentProcesses: number;
|
|
23
|
+
}): Promise<boolean>;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export interface IsolatedExecutionRunStatePort {
|
|
27
|
+
status(runId: string): Promise<"accepted" | "running" | "terminal">;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export class IsolatedExecutionAdmissionError extends Error {
|
|
31
|
+
constructor(
|
|
32
|
+
readonly code:
|
|
33
|
+
| "EXECUTION_DENIED"
|
|
34
|
+
| "BUDGET_EXCEEDED"
|
|
35
|
+
| "RUNTIME_UNAVAILABLE",
|
|
36
|
+
message: string,
|
|
37
|
+
) {
|
|
38
|
+
super(message);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** Revalidates the authoritative Run and current governance facts for every process call. */
|
|
43
|
+
export class IsolatedExecutionGuard {
|
|
44
|
+
readonly #operations: Map<
|
|
45
|
+
string,
|
|
46
|
+
{
|
|
47
|
+
definition: NonNullable<ModuleManifest["aiOperations"]>[number];
|
|
48
|
+
releaseId: string;
|
|
49
|
+
}
|
|
50
|
+
>;
|
|
51
|
+
readonly #features: ModuleManifest["features"];
|
|
52
|
+
|
|
53
|
+
constructor(
|
|
54
|
+
modules: readonly ModuleManifest[],
|
|
55
|
+
hosts: InitializedModuleHosts,
|
|
56
|
+
private readonly runtime: ServerRuntime,
|
|
57
|
+
private readonly authorizations: AgentRunAuthorizationStore,
|
|
58
|
+
private readonly resolveActor: (
|
|
59
|
+
actorId: string,
|
|
60
|
+
) => Promise<RequestActor | undefined>,
|
|
61
|
+
private readonly state?: IsolatedExecutionRunStatePort,
|
|
62
|
+
private readonly budget?: IsolatedExecutionBudgetPort,
|
|
63
|
+
private readonly executionRuntimeAvailable = false,
|
|
64
|
+
) {
|
|
65
|
+
this.#features = modules.flatMap((module) => module.features);
|
|
66
|
+
const handlers = new Map(
|
|
67
|
+
hosts.aiOperationHandlers.map(({ registration }) => [
|
|
68
|
+
registration.operationKey,
|
|
69
|
+
registration,
|
|
70
|
+
]),
|
|
71
|
+
);
|
|
72
|
+
this.#operations = new Map(
|
|
73
|
+
modules.flatMap((module) =>
|
|
74
|
+
(module.aiOperations ?? [])
|
|
75
|
+
.filter((definition) => definition.kind === "agent")
|
|
76
|
+
.map(
|
|
77
|
+
(definition) =>
|
|
78
|
+
[
|
|
79
|
+
definition.key,
|
|
80
|
+
{
|
|
81
|
+
definition,
|
|
82
|
+
releaseId: buildAgentRuntimeRelease(
|
|
83
|
+
modules,
|
|
84
|
+
module,
|
|
85
|
+
definition,
|
|
86
|
+
handlers.get(definition.key),
|
|
87
|
+
).releaseId,
|
|
88
|
+
},
|
|
89
|
+
] as const,
|
|
90
|
+
),
|
|
91
|
+
),
|
|
92
|
+
);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
async admit(input: {
|
|
96
|
+
readonly executionGrant: string;
|
|
97
|
+
readonly runId: string;
|
|
98
|
+
readonly callId: string;
|
|
99
|
+
readonly language: "python" | "node";
|
|
100
|
+
}): Promise<AiIsolatedExecutionPolicyV1> {
|
|
101
|
+
if (!input.callId || !input.runId) throw denied("执行调用坐标缺失");
|
|
102
|
+
let binding;
|
|
103
|
+
try {
|
|
104
|
+
binding = await this.authorizations.authorizeTool(
|
|
105
|
+
input.executionGrant,
|
|
106
|
+
input.runId,
|
|
107
|
+
);
|
|
108
|
+
} catch {
|
|
109
|
+
throw denied("Agent Run 执行授权无效");
|
|
110
|
+
}
|
|
111
|
+
const current = this.#operations.get(binding.operationKey);
|
|
112
|
+
const policy = current?.definition.isolatedExecution;
|
|
113
|
+
if (
|
|
114
|
+
!current ||
|
|
115
|
+
!policy ||
|
|
116
|
+
current.releaseId !== binding.releaseId ||
|
|
117
|
+
!policy.languages.includes(input.language) ||
|
|
118
|
+
binding.authorizationSnapshot.toolNetworkMode !== "forbidden" ||
|
|
119
|
+
binding.authorizationSnapshot.delegation !== undefined ||
|
|
120
|
+
binding.authorizationSnapshot.runtimeBudgetDigest !== undefined ||
|
|
121
|
+
current.definition.featureKey !==
|
|
122
|
+
binding.authorizationSnapshot.featureKey ||
|
|
123
|
+
current.definition.permissionKey !==
|
|
124
|
+
binding.authorizationSnapshot.permissionKey
|
|
125
|
+
) {
|
|
126
|
+
throw denied("Operation 未授权隔离执行或版本已变化");
|
|
127
|
+
}
|
|
128
|
+
let actor: RequestActor | undefined;
|
|
129
|
+
try {
|
|
130
|
+
actor = await this.resolveActor(binding.actorId);
|
|
131
|
+
} catch {
|
|
132
|
+
throw denied("Agent Run Actor 当前不可用");
|
|
133
|
+
}
|
|
134
|
+
if (!actor || actor.id !== binding.actorId || actor.type === "anonymous") {
|
|
135
|
+
throw denied("Agent Run Actor 当前不可用");
|
|
136
|
+
}
|
|
137
|
+
let context: Awaited<
|
|
138
|
+
ReturnType<ServerRuntime["createGuardContextForActor"]>
|
|
139
|
+
>;
|
|
140
|
+
try {
|
|
141
|
+
context = await this.runtime.createGuardContextForActor(
|
|
142
|
+
new Request("http://agent-execution.internal"),
|
|
143
|
+
actor,
|
|
144
|
+
);
|
|
145
|
+
} catch {
|
|
146
|
+
throw denied("Agent Run 当前授权无法复核");
|
|
147
|
+
}
|
|
148
|
+
try {
|
|
149
|
+
this.authorizations.assertCurrentScope(binding, context);
|
|
150
|
+
} catch {
|
|
151
|
+
throw denied("Agent Run 当前数据范围已变化");
|
|
152
|
+
}
|
|
153
|
+
let access: Awaited<ReturnType<typeof evaluateAccessGuards>>;
|
|
154
|
+
try {
|
|
155
|
+
access = await evaluateAccessGuards(this.runtime, context, {
|
|
156
|
+
permission: { permissionKey: current.definition.permissionKey },
|
|
157
|
+
feature: findFeature(this.#features, current.definition.featureKey),
|
|
158
|
+
});
|
|
159
|
+
} catch {
|
|
160
|
+
throw denied("Agent Operation 当前授权无法复核");
|
|
161
|
+
}
|
|
162
|
+
if (!access.decision.allowed) throw denied("Agent Operation 当前无权执行");
|
|
163
|
+
if (!this.executionRuntimeAvailable || !this.state || !this.budget) {
|
|
164
|
+
throw new IsolatedExecutionAdmissionError(
|
|
165
|
+
"RUNTIME_UNAVAILABLE",
|
|
166
|
+
"隔离执行 Runtime 或预算存储未装配",
|
|
167
|
+
);
|
|
168
|
+
}
|
|
169
|
+
let status: "accepted" | "running" | "terminal";
|
|
170
|
+
try {
|
|
171
|
+
status = await this.state.status(input.runId);
|
|
172
|
+
} catch {
|
|
173
|
+
throw denied("Agent Run 状态无法复核");
|
|
174
|
+
}
|
|
175
|
+
if (status !== "running") throw denied("Agent Run 当前不可执行");
|
|
176
|
+
let reserved: boolean;
|
|
177
|
+
try {
|
|
178
|
+
reserved = await this.budget.reserve({
|
|
179
|
+
runId: input.runId,
|
|
180
|
+
callId: input.callId,
|
|
181
|
+
maxToolCalls: policy.limits.maxToolCalls,
|
|
182
|
+
maxToolRounds: policy.limits.maxToolRounds,
|
|
183
|
+
maxConcurrentProcesses: policy.limits.maxConcurrentProcesses,
|
|
184
|
+
});
|
|
185
|
+
} catch {
|
|
186
|
+
throw new IsolatedExecutionAdmissionError(
|
|
187
|
+
"RUNTIME_UNAVAILABLE",
|
|
188
|
+
"执行预算无法原子预留",
|
|
189
|
+
);
|
|
190
|
+
}
|
|
191
|
+
if (!reserved)
|
|
192
|
+
throw new IsolatedExecutionAdmissionError(
|
|
193
|
+
"BUDGET_EXCEEDED",
|
|
194
|
+
"隔离执行预算已耗尽",
|
|
195
|
+
);
|
|
196
|
+
return policy;
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
function denied(message: string): IsolatedExecutionAdmissionError {
|
|
201
|
+
return new IsolatedExecutionAdmissionError("EXECUTION_DENIED", message);
|
|
202
|
+
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { ModuleManifest } from "@southwind-ai/modules";
|
|
2
2
|
import type { ModuleAiJsonValue } from "@southwind-ai/server-sdk";
|
|
3
|
-
import
|
|
3
|
+
import { AgentRuntimeError } from "./run-contracts.js";
|
|
4
4
|
import { AgentRunFacadeError } from "./run-errors.js";
|
|
5
5
|
|
|
6
6
|
export function toolDefinitions(
|
|
@@ -56,3 +56,13 @@ export function publicRunPath(
|
|
|
56
56
|
) {
|
|
57
57
|
return `/api/agent/runs/${encodeURIComponent(runId)}/${resource}?operationKey=${encodeURIComponent(operationKey)}`;
|
|
58
58
|
}
|
|
59
|
+
|
|
60
|
+
export function mapAgentRuntimeError(error: unknown) {
|
|
61
|
+
return error instanceof AgentRuntimeError
|
|
62
|
+
? new AgentRunFacadeError(
|
|
63
|
+
error.code,
|
|
64
|
+
error.message,
|
|
65
|
+
runtimeErrorStatus(error.code),
|
|
66
|
+
)
|
|
67
|
+
: error;
|
|
68
|
+
}
|
|
@@ -17,7 +17,6 @@ import type {
|
|
|
17
17
|
AgentRuntimePort,
|
|
18
18
|
AgentRuntimeStartResponse,
|
|
19
19
|
} from "./run-contracts.js";
|
|
20
|
-
import { AgentRuntimeError } from "./run-contracts.js";
|
|
21
20
|
import {
|
|
22
21
|
rotateVisionArtifacts,
|
|
23
22
|
validateVisionArtifacts,
|
|
@@ -33,7 +32,7 @@ import { attachAgentRun } from "./run-attach.js";
|
|
|
33
32
|
import {
|
|
34
33
|
digest,
|
|
35
34
|
publicRunPath,
|
|
36
|
-
|
|
35
|
+
mapAgentRuntimeError,
|
|
37
36
|
} from "./run-definition-support.js";
|
|
38
37
|
import { AgentRunFacadeError } from "./run-errors.js";
|
|
39
38
|
import { scopedRuntimeIdempotencyKey } from "./idempotency-key.js";
|
|
@@ -63,7 +62,10 @@ import {
|
|
|
63
62
|
resolveAgentRunExecutionPolicy,
|
|
64
63
|
type AgentRunDelegationPolicy,
|
|
65
64
|
} from "./agent-run-delegation-policy.js";
|
|
66
|
-
import {
|
|
65
|
+
import {
|
|
66
|
+
assertAgentRuntimeAssembled,
|
|
67
|
+
publicAgentOperationPolicy,
|
|
68
|
+
} from "./run-operation-policy.js";
|
|
67
69
|
export { AgentRunFacadeError } from "./run-errors.js";
|
|
68
70
|
type ServerRuntime = ReturnType<typeof createServerRuntime>;
|
|
69
71
|
export class AgentRunFacade {
|
|
@@ -124,6 +126,7 @@ export class AgentRunFacade {
|
|
|
124
126
|
? { sessionBinding: this.#sessionBindingAuthorization }
|
|
125
127
|
: {}),
|
|
126
128
|
});
|
|
129
|
+
assertAgentRuntimeAssembled(operation);
|
|
127
130
|
const executionPolicy = resolveAgentRunExecutionPolicy(
|
|
128
131
|
operation,
|
|
129
132
|
context,
|
|
@@ -265,14 +268,7 @@ export class AgentRunFacade {
|
|
|
265
268
|
sessionTurn,
|
|
266
269
|
sessionLinked,
|
|
267
270
|
});
|
|
268
|
-
|
|
269
|
-
throw new AgentRunFacadeError(
|
|
270
|
-
error.code,
|
|
271
|
-
error.message,
|
|
272
|
-
runtimeErrorStatus(error.code),
|
|
273
|
-
);
|
|
274
|
-
}
|
|
275
|
-
throw error;
|
|
271
|
+
throw mapAgentRuntimeError(error);
|
|
276
272
|
}
|
|
277
273
|
}
|
|
278
274
|
async authorizeRun(
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { AgentRunFacadeError } from "./run-errors.js";
|
|
1
2
|
import { resolveProviderEgress } from "@southwind-ai/modules";
|
|
2
3
|
import type { RegisteredOperation } from "./run-operation-registry.js";
|
|
3
4
|
|
|
@@ -18,3 +19,13 @@ export function publicAgentOperationPolicy(
|
|
|
18
19
|
release: structuredClone(operation.release),
|
|
19
20
|
};
|
|
20
21
|
}
|
|
22
|
+
|
|
23
|
+
export function assertAgentRuntimeAssembled(operation: RegisteredOperation) {
|
|
24
|
+
if (operation.definition.isolatedExecution) {
|
|
25
|
+
throw new AgentRunFacadeError(
|
|
26
|
+
"RUNTIME_UNAVAILABLE",
|
|
27
|
+
"隔离执行 Runtime 尚未装配",
|
|
28
|
+
503,
|
|
29
|
+
);
|
|
30
|
+
}
|
|
31
|
+
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { StructuredLogger } from "@southwind-ai/shared";
|
|
1
2
|
import type { DurableHandlerRegistry } from "@southwind-ai/jobs";
|
|
2
3
|
import { DurableArtifactService } from "../jobs/artifact-service.js";
|
|
3
4
|
import type { DurableJobService } from "../jobs/service.js";
|
|
@@ -14,18 +15,20 @@ export function workflowDurableInput([durable, storage, logger]: readonly [
|
|
|
14
15
|
{
|
|
15
16
|
readonly blobStore: ConstructorParameters<typeof DurableArtifactService>[1];
|
|
16
17
|
},
|
|
17
|
-
|
|
18
|
-
error(fields: Record<string, unknown>, message: string): void;
|
|
19
|
-
},
|
|
18
|
+
Pick<StructuredLogger, "error">,
|
|
20
19
|
]) {
|
|
21
20
|
return {
|
|
22
21
|
jobs: durable.service,
|
|
23
22
|
jobWorker: durable.worker,
|
|
24
23
|
jobRegistry: durable.registry,
|
|
25
24
|
jobArtifacts: new DurableArtifactService(durable.facts, storage.blobStore),
|
|
26
|
-
onWorkflowContinuationError: (
|
|
25
|
+
onWorkflowContinuationError: (_error: unknown) =>
|
|
27
26
|
logger.error(
|
|
28
|
-
{
|
|
27
|
+
{
|
|
28
|
+
event: "agent.workflow.continuation.failed",
|
|
29
|
+
outcome: "failure",
|
|
30
|
+
errorCode: "AGENT_WORKFLOW_CONTINUATION_FAILED",
|
|
31
|
+
},
|
|
29
32
|
"Agent Workflow continuation 派发失败",
|
|
30
33
|
),
|
|
31
34
|
};
|
|
@@ -10,6 +10,8 @@ import type { BulkDataRepository } from "../bulk-data/repository.js";
|
|
|
10
10
|
import type { BulkResourceHandler } from "../bulk-data/resource-handler.js";
|
|
11
11
|
// @windy-module system.bulk-data end
|
|
12
12
|
import type { createServerRuntime } from "../runtime.js";
|
|
13
|
+
import type { StructuredLogger } from "@southwind-ai/shared";
|
|
14
|
+
import { createServerLogger } from "../observability/structured-logger.js";
|
|
13
15
|
import type { RequestActor } from "../guards.js";
|
|
14
16
|
// @windy-module system.bulk-data begin
|
|
15
17
|
import { evaluateAccessGuards } from "../access-guards.js";
|
|
@@ -48,6 +50,7 @@ interface LegacyServiceRuntimeInput {
|
|
|
48
50
|
// @windy-module system.workflow end
|
|
49
51
|
installModuleDurableHandlers?: (registry: DurableHandlerRegistry) => void;
|
|
50
52
|
durableContextualize?: DurableWorkerOptions["contextualize"];
|
|
53
|
+
logger?: StructuredLogger;
|
|
51
54
|
}
|
|
52
55
|
|
|
53
56
|
export function createLegacyServiceRuntime(input: LegacyServiceRuntimeInput) {
|
|
@@ -61,6 +64,7 @@ export function createLegacyServiceRuntime(input: LegacyServiceRuntimeInput) {
|
|
|
61
64
|
input.installModuleDurableHandlers?.(registry);
|
|
62
65
|
const worker = new DurableJobWorker(execution, registry, {
|
|
63
66
|
contextualize: input.durableContextualize,
|
|
67
|
+
logger: input.logger ?? createServerLogger(),
|
|
64
68
|
});
|
|
65
69
|
// @windy-module system.workflow begin
|
|
66
70
|
const workflowTimeout = new WorkflowTimeoutRuntime(
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
import { describe, expect, test } from "bun:test";
|
|
2
|
+
import { DurableHandlerRegistry } from "@southwind-ai/jobs";
|
|
3
|
+
import {
|
|
4
|
+
createStructuredLogger,
|
|
5
|
+
type StructuredLogger,
|
|
6
|
+
} from "@southwind-ai/shared";
|
|
7
|
+
import { InMemoryDurableExecutionRepository } from "./in-memory-execution.js";
|
|
8
|
+
import { DurableJobService } from "./service.js";
|
|
9
|
+
import { DurableJobWorker } from "./worker.js";
|
|
10
|
+
|
|
11
|
+
const base = new Date("2026-09-30T00:00:00.000Z");
|
|
12
|
+
|
|
13
|
+
function capture() {
|
|
14
|
+
const records: Record<string, unknown>[] = [];
|
|
15
|
+
const write = (record: Record<string, unknown>) => records.push(record);
|
|
16
|
+
const logger = createStructuredLogger(
|
|
17
|
+
{ info: write, warn: write, error: write },
|
|
18
|
+
"windy-server",
|
|
19
|
+
);
|
|
20
|
+
return { logger, records };
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
async function fixture(
|
|
24
|
+
handler?: () => void | Promise<void>,
|
|
25
|
+
logger?: StructuredLogger,
|
|
26
|
+
) {
|
|
27
|
+
const repository = new InMemoryDurableExecutionRepository();
|
|
28
|
+
const service = new DurableJobService(repository, () => base);
|
|
29
|
+
const registry = new DurableHandlerRegistry();
|
|
30
|
+
if (handler)
|
|
31
|
+
registry.register({
|
|
32
|
+
key: "test.handler",
|
|
33
|
+
payload: { parse: (value) => value },
|
|
34
|
+
execute: handler,
|
|
35
|
+
});
|
|
36
|
+
const worker = new DurableJobWorker(repository, registry, {
|
|
37
|
+
logger,
|
|
38
|
+
now: () => base,
|
|
39
|
+
leaseDurationMs: 60_000,
|
|
40
|
+
});
|
|
41
|
+
return { service, worker };
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
describe("Durable Job runtime logging", () => {
|
|
45
|
+
test("idle 无日志;每个实际 attempt 仅一条开始和一条完成", async () => {
|
|
46
|
+
const { logger, records } = capture();
|
|
47
|
+
const { service, worker } = await fixture(() => undefined, logger);
|
|
48
|
+
expect(await worker.runNext("default", "worker-a")).toEqual({
|
|
49
|
+
status: "idle",
|
|
50
|
+
});
|
|
51
|
+
expect(records).toEqual([]);
|
|
52
|
+
const submitted = await service.submit({
|
|
53
|
+
handlerKey: "test.handler",
|
|
54
|
+
idempotencyKey: "job-log-success",
|
|
55
|
+
payload: { secret: "secret-payload" },
|
|
56
|
+
requestedBy: "user-secret",
|
|
57
|
+
stageKeys: ["prepare", "apply"],
|
|
58
|
+
});
|
|
59
|
+
expect(await worker.runNext("default", "worker-a")).toEqual({
|
|
60
|
+
status: "succeeded",
|
|
61
|
+
jobId: submitted.job.id,
|
|
62
|
+
});
|
|
63
|
+
expect(await worker.runNext("default", "worker-a")).toEqual({
|
|
64
|
+
status: "succeeded",
|
|
65
|
+
jobId: submitted.job.id,
|
|
66
|
+
});
|
|
67
|
+
expect(records.map((record) => record.event)).toEqual([
|
|
68
|
+
"job.attempt.started",
|
|
69
|
+
"job.attempt.completed",
|
|
70
|
+
"job.attempt.started",
|
|
71
|
+
"job.attempt.completed",
|
|
72
|
+
]);
|
|
73
|
+
expect(records.map((record) => record.stageKey)).toEqual([
|
|
74
|
+
"prepare",
|
|
75
|
+
"prepare",
|
|
76
|
+
"apply",
|
|
77
|
+
"apply",
|
|
78
|
+
]);
|
|
79
|
+
for (const record of records) {
|
|
80
|
+
expect(record).toMatchObject({
|
|
81
|
+
jobId: submitted.job.id,
|
|
82
|
+
attemptNumber: 1,
|
|
83
|
+
handlerKey: "test.handler",
|
|
84
|
+
queue: "default",
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
const serialized = JSON.stringify(records);
|
|
88
|
+
expect(serialized).not.toContain("secret-payload");
|
|
89
|
+
expect(serialized).not.toContain("user-secret");
|
|
90
|
+
expect(serialized).not.toContain("leaseToken");
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
test("未注册 handler 与执行失败都配对,且只记录稳定错误码", async () => {
|
|
94
|
+
const missing = capture();
|
|
95
|
+
const missingFixture = await fixture(undefined, missing.logger);
|
|
96
|
+
await missingFixture.service.submit({
|
|
97
|
+
handlerKey: "test.handler",
|
|
98
|
+
idempotencyKey: "job-missing",
|
|
99
|
+
payload: { secret: "secret-payload" },
|
|
100
|
+
requestedBy: "user-1",
|
|
101
|
+
});
|
|
102
|
+
expect(
|
|
103
|
+
(await missingFixture.worker.runNext("default", "worker-a")).status,
|
|
104
|
+
).toBe("failed");
|
|
105
|
+
expect(missing.records.map((record) => record.event)).toEqual([
|
|
106
|
+
"job.attempt.started",
|
|
107
|
+
"job.attempt.completed",
|
|
108
|
+
]);
|
|
109
|
+
expect(missing.records[1]?.errorCode).toBe("HANDLER_NOT_REGISTERED");
|
|
110
|
+
|
|
111
|
+
const failed = capture();
|
|
112
|
+
const failedFixture = await fixture(() => {
|
|
113
|
+
throw new Error("secret-error");
|
|
114
|
+
}, failed.logger);
|
|
115
|
+
await failedFixture.service.submit({
|
|
116
|
+
handlerKey: "test.handler",
|
|
117
|
+
idempotencyKey: "job-failed",
|
|
118
|
+
payload: { secret: "secret-payload" },
|
|
119
|
+
requestedBy: "user-1",
|
|
120
|
+
});
|
|
121
|
+
expect(
|
|
122
|
+
(await failedFixture.worker.runNext("default", "worker-a")).status,
|
|
123
|
+
).toBe("failed");
|
|
124
|
+
expect(failed.records.map((record) => record.event)).toEqual([
|
|
125
|
+
"job.attempt.started",
|
|
126
|
+
"job.attempt.completed",
|
|
127
|
+
]);
|
|
128
|
+
expect(failed.records[1]?.errorCode).toBe("JOB_ATTEMPT_FAILED");
|
|
129
|
+
expect(JSON.stringify(failed.records)).not.toContain("secret-error");
|
|
130
|
+
});
|
|
131
|
+
|
|
132
|
+
test("取消与租约失效各写一次安全终态", async () => {
|
|
133
|
+
const cancelled = capture();
|
|
134
|
+
let entered: () => void = () => undefined;
|
|
135
|
+
let release: () => void = () => undefined;
|
|
136
|
+
const enteredPromise = new Promise<void>((resolve) => {
|
|
137
|
+
entered = resolve;
|
|
138
|
+
});
|
|
139
|
+
const releasePromise = new Promise<void>((resolve) => {
|
|
140
|
+
release = resolve;
|
|
141
|
+
});
|
|
142
|
+
const cancelledFixture = await fixture(async () => {
|
|
143
|
+
entered();
|
|
144
|
+
await releasePromise;
|
|
145
|
+
}, cancelled.logger);
|
|
146
|
+
const cancelledJob = await cancelledFixture.service.submit({
|
|
147
|
+
handlerKey: "test.handler",
|
|
148
|
+
idempotencyKey: "job-cancelled",
|
|
149
|
+
payload: {},
|
|
150
|
+
requestedBy: "user-1",
|
|
151
|
+
});
|
|
152
|
+
const run = cancelledFixture.worker.runNext("default", "worker-a");
|
|
153
|
+
await enteredPromise;
|
|
154
|
+
await cancelledFixture.worker.cancel(cancelledJob.job.id);
|
|
155
|
+
release();
|
|
156
|
+
expect((await run).status).toBe("cancelled");
|
|
157
|
+
expect(cancelled.records.map((record) => record.event)).toEqual([
|
|
158
|
+
"job.attempt.started",
|
|
159
|
+
"job.attempt.completed",
|
|
160
|
+
]);
|
|
161
|
+
expect(cancelled.records[1]?.errorCode).toBe("JOB_CANCELLED");
|
|
162
|
+
|
|
163
|
+
const lost = capture();
|
|
164
|
+
const repository = new InMemoryDurableExecutionRepository();
|
|
165
|
+
const service = new DurableJobService(repository, () => base);
|
|
166
|
+
await service.submit({
|
|
167
|
+
handlerKey: "test.handler",
|
|
168
|
+
idempotencyKey: "job-lease-lost",
|
|
169
|
+
payload: {},
|
|
170
|
+
requestedBy: "user-1",
|
|
171
|
+
});
|
|
172
|
+
let now = base;
|
|
173
|
+
const registry = new DurableHandlerRegistry().register({
|
|
174
|
+
key: "test.handler",
|
|
175
|
+
payload: { parse: (value) => value },
|
|
176
|
+
execute: async (_payload, context) => {
|
|
177
|
+
now = new Date(base.getTime() + 100);
|
|
178
|
+
await context.heartbeat();
|
|
179
|
+
},
|
|
180
|
+
});
|
|
181
|
+
const worker = new DurableJobWorker(repository, registry, {
|
|
182
|
+
logger: lost.logger,
|
|
183
|
+
now: () => now,
|
|
184
|
+
leaseDurationMs: 10,
|
|
185
|
+
});
|
|
186
|
+
expect((await worker.runNext("default", "worker-a")).status).toBe(
|
|
187
|
+
"lease-lost",
|
|
188
|
+
);
|
|
189
|
+
expect(lost.records.map((record) => record.event)).toEqual([
|
|
190
|
+
"job.attempt.started",
|
|
191
|
+
"job.attempt.completed",
|
|
192
|
+
]);
|
|
193
|
+
expect(lost.records[1]?.errorCode).toBe("JOB_LEASE_LOST");
|
|
194
|
+
});
|
|
195
|
+
|
|
196
|
+
test("logger 抛错不改变 Job 结果", async () => {
|
|
197
|
+
const fail = () => {
|
|
198
|
+
throw new Error("sink down");
|
|
199
|
+
};
|
|
200
|
+
const logger: StructuredLogger = { info: fail, warn: fail, error: fail };
|
|
201
|
+
const { service, worker } = await fixture(() => undefined, logger);
|
|
202
|
+
await service.submit({
|
|
203
|
+
handlerKey: "test.handler",
|
|
204
|
+
idempotencyKey: "sink-failure",
|
|
205
|
+
payload: {},
|
|
206
|
+
requestedBy: "user-1",
|
|
207
|
+
});
|
|
208
|
+
expect((await worker.runNext("default", "worker-a")).status).toBe(
|
|
209
|
+
"succeeded",
|
|
210
|
+
);
|
|
211
|
+
});
|
|
212
|
+
});
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import type { ClaimedDurableJob } from "@southwind-ai/jobs";
|
|
2
|
+
import type { StructuredLogger } from "@southwind-ai/shared";
|
|
3
|
+
|
|
4
|
+
export type JobAttemptResult =
|
|
5
|
+
| "succeeded"
|
|
6
|
+
| "failed"
|
|
7
|
+
| "cancelled"
|
|
8
|
+
| "lease-lost";
|
|
9
|
+
|
|
10
|
+
/** Only canonical claim coordinates enter operational logs. */
|
|
11
|
+
export function logJobAttemptStarted(
|
|
12
|
+
logger: StructuredLogger | undefined,
|
|
13
|
+
claim: ClaimedDurableJob,
|
|
14
|
+
): void {
|
|
15
|
+
safeLog(() =>
|
|
16
|
+
logger?.info({
|
|
17
|
+
event: "job.attempt.started",
|
|
18
|
+
jobId: claim.jobId,
|
|
19
|
+
stageKey: claim.stage.key,
|
|
20
|
+
attemptNumber: claim.stage.currentAttempt,
|
|
21
|
+
handlerKey: claim.job.handlerKey,
|
|
22
|
+
queue: claim.job.queue,
|
|
23
|
+
}),
|
|
24
|
+
);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export function logJobAttemptCompleted(
|
|
28
|
+
logger: StructuredLogger | undefined,
|
|
29
|
+
claim: ClaimedDurableJob,
|
|
30
|
+
status: JobAttemptResult,
|
|
31
|
+
durationMs: number,
|
|
32
|
+
errorCode?: string,
|
|
33
|
+
): void {
|
|
34
|
+
const fields = {
|
|
35
|
+
event: "job.attempt.completed",
|
|
36
|
+
jobId: claim.jobId,
|
|
37
|
+
stageKey: claim.stage.key,
|
|
38
|
+
attemptNumber: claim.stage.currentAttempt,
|
|
39
|
+
handlerKey: claim.job.handlerKey,
|
|
40
|
+
queue: claim.job.queue,
|
|
41
|
+
durationMs: Number(durationMs.toFixed(3)),
|
|
42
|
+
outcome:
|
|
43
|
+
status === "succeeded" ? ("success" as const) : ("failure" as const),
|
|
44
|
+
...(errorCode ? { errorCode } : {}),
|
|
45
|
+
};
|
|
46
|
+
safeLog(() => {
|
|
47
|
+
if (status === "failed" || status === "lease-lost") logger?.error(fields);
|
|
48
|
+
else logger?.info(fields);
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function safeLog(write: () => void): void {
|
|
53
|
+
try {
|
|
54
|
+
write();
|
|
55
|
+
} catch {
|
|
56
|
+
// Runtime logging is best effort; Job fact writes retain their own contract.
|
|
57
|
+
}
|
|
58
|
+
}
|