@cosmicdrift/kumiko-framework 0.335.0 → 0.336.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/anonymous-live-entities.d.ts +2 -0
- package/dist/api/anonymous-live-entities.js +17 -0
- package/dist/api/index.d.ts +2 -1
- package/dist/api/index.js +1 -0
- package/dist/api/routes.js +1 -1
- package/dist/api/server.js +6 -3
- package/dist/api/sse-route.d.ts +4 -1
- package/dist/api/sse-route.js +38 -11
- package/dist/crypto/event-pii.js +3 -3
- package/dist/db/blind-index-cleanup.js +27 -16
- package/dist/db/event-store-executor-read.js +6 -2
- package/dist/engine/boot-validator/access-declarations.js +2 -1
- package/dist/engine/boot-validator/query-output-columns.js +12 -0
- package/dist/engine/boot-validator/screens.js +22 -13
- package/dist/engine/build-app-schema.js +5 -2
- package/dist/engine/create-app.js +10 -1
- package/dist/engine/define-handler.js +3 -5
- package/dist/engine/extension-names.d.ts +4 -0
- package/dist/engine/extension-names.js +4 -0
- package/dist/engine/feature-ast/extractors/handlers.js +1 -0
- package/dist/engine/feature-entity-handlers.d.ts +1 -0
- package/dist/engine/feature-entity-handlers.js +2 -0
- package/dist/engine/registry-validate.d.ts +1 -0
- package/dist/engine/registry-validate.js +14 -0
- package/dist/engine/registry.js +2 -1
- package/dist/engine/steps/_event-match.js +12 -4
- package/dist/engine/system-user.d.ts +1 -0
- package/dist/engine/system-user.js +3 -0
- package/dist/env/dry-run.js +10 -3
- package/dist/event-store/event-store.js +4 -1
- package/dist/jobs/job-runner.js +29 -7
- package/dist/migrations/pending-rebuilds.js +4 -3
- package/dist/observability/index.d.ts +1 -1
- package/dist/observability/index.js +1 -1
- package/dist/observability/standard-metrics.d.ts +4 -0
- package/dist/observability/standard-metrics.js +16 -0
- package/dist/pipeline/dispatch-batch.js +2 -2
- package/dist/pipeline/dispatcher.js +1 -1
- package/dist/pipeline/event-dispatcher-delivery.js +5 -0
- package/dist/pipeline/event-dispatcher.js +4 -1
- package/dist/pipeline/system-hooks.js +10 -13
- package/dist/pipeline/write-origin.js +13 -6
- package/dist/stack/drain-event-consumers.js +3 -0
- package/dist/stack/drain-jobs.d.ts +3 -1
- package/dist/stack/drain-jobs.js +31 -1
- package/package.json +4 -4
- package/src/changes.json +64 -0
- package/src/scripts/codemod/pii-personal-migration.ts +65 -45
- package/src/scripts/codemod/unprocessable-error-details-reason.ts +61 -24
package/dist/env/dry-run.js
CHANGED
|
@@ -105,13 +105,19 @@ function renderJson(fields, options) {
|
|
|
105
105
|
withDefault: withDefault.map(toEntry),
|
|
106
106
|
}, null, 2)}\n`;
|
|
107
107
|
}
|
|
108
|
+
// A newline in a `.describe()` text would end the `#` comment, so the rest of
|
|
109
|
+
// the description would run as a shell command or break the YAML.
|
|
110
|
+
function oneLineDescription(f) {
|
|
111
|
+
return f.description?.replace(/\s*\r?\n\s*/g, " ");
|
|
112
|
+
}
|
|
108
113
|
function pulumiConfigSetLine(f, options) {
|
|
109
114
|
const meta = readKumikoMeta(f.field);
|
|
110
115
|
const key = pulumiConfigKey(f.name, f.field, options.pulumiPrefix);
|
|
111
116
|
const secretFlag = meta.pulumi?.secret ? " --secret" : "";
|
|
112
117
|
const value = meta.pulumi?.generator ? `"$(${meta.pulumi.generator})"` : `"<set-me>"`;
|
|
113
|
-
const
|
|
114
|
-
|
|
118
|
+
const description = oneLineDescription(f);
|
|
119
|
+
const comment = description
|
|
120
|
+
? ` # ${f.name} (${f.source}): ${description}`
|
|
115
121
|
: ` # ${f.name} (${f.source})`;
|
|
116
122
|
return `pulumi config set${secretFlag} ${key} ${value}${comment}`;
|
|
117
123
|
}
|
|
@@ -150,7 +156,8 @@ function renderK8s(fields, options) {
|
|
|
150
156
|
if (optional.length > 0) {
|
|
151
157
|
lines.push(" # Optional (uncomment and set to enable):");
|
|
152
158
|
for (const f of optional) {
|
|
153
|
-
const
|
|
159
|
+
const description = oneLineDescription(f);
|
|
160
|
+
const comment = description ? ` # (${f.source}): ${description}` : ` # (${f.source})`;
|
|
154
161
|
lines.push(` # ${f.name}: "<set-me>"${comment}`);
|
|
155
162
|
}
|
|
156
163
|
}
|
|
@@ -173,7 +173,10 @@ function buildStoredEvent(event, newVersion, eventVersion, row) {
|
|
|
173
173
|
// aggregateId alone, matching prior behavior. Callers that share id
|
|
174
174
|
// generation across multiple aggregate types on the same id (e.g. a solon-
|
|
175
175
|
// style collision) must pass it to avoid folding a foreign type's events
|
|
176
|
-
// into the loaded stream.
|
|
176
|
+
// into the loaded stream. The aggregateType condition stays an unindexed
|
|
177
|
+
// post-filter on purpose: events_aggregate_version_uq already narrows the scan
|
|
178
|
+
// to one aggregate's rows, so an index including aggregate_type would only add
|
|
179
|
+
// write cost.
|
|
177
180
|
export async function loadAggregate(db, aggregateId, tenantId, options) {
|
|
178
181
|
// "" must not fall through to the unfiltered load: a dynamically computed
|
|
179
182
|
// type that came out empty would silently fold in foreign aggregate types.
|
package/dist/jobs/job-runner.js
CHANGED
|
@@ -228,6 +228,9 @@ const BOOT_REDIS_TIMEOUT_MS = 10_000;
|
|
|
228
228
|
// Shutdown must fit a k8s grace period even when Redis is down, so the
|
|
229
229
|
// readiness wait in stop() is capped well below the boot timeout.
|
|
230
230
|
const STOP_QUEUE_READY_TIMEOUT_MS = 1_000;
|
|
231
|
+
// A gate without an explicit `timeout` may legitimately run long (migration-style
|
|
232
|
+
// checks), so it is never rejected; a hang just must not stay silent.
|
|
233
|
+
const BOOT_GATE_SLOW_WARN_MS = 60_000;
|
|
231
234
|
function timeoutReject(ms, message) {
|
|
232
235
|
let timer;
|
|
233
236
|
const promise = new Promise((_, reject) => {
|
|
@@ -875,14 +878,28 @@ export function createJobRunner(options) {
|
|
|
875
878
|
continue;
|
|
876
879
|
if (!jobDef.bootGate)
|
|
877
880
|
continue;
|
|
881
|
+
const gateTimeout = jobDef.timeout !== undefined
|
|
882
|
+
? timeoutReject(jobDef.timeout, `job-runner: boot gate "${name}" did not finish within ${jobDef.timeout}ms`)
|
|
883
|
+
: undefined;
|
|
884
|
+
const slowWarnTimer = gateTimeout === undefined
|
|
885
|
+
? setTimeout(() => {
|
|
886
|
+
errorLogger.warn("boot gate still running, no timeout configured", {
|
|
887
|
+
gate: name,
|
|
888
|
+
afterMs: BOOT_GATE_SLOW_WARN_MS,
|
|
889
|
+
});
|
|
890
|
+
}, BOOT_GATE_SLOW_WARN_MS)
|
|
891
|
+
: undefined;
|
|
878
892
|
try {
|
|
879
|
-
await
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
893
|
+
await Promise.race([
|
|
894
|
+
handleJob({
|
|
895
|
+
id: bootJobIdForJobName(name),
|
|
896
|
+
name,
|
|
897
|
+
data: {},
|
|
898
|
+
attemptsMade: 0,
|
|
899
|
+
retryable: false,
|
|
900
|
+
}),
|
|
901
|
+
...(gateTimeout ? [gateTimeout.promise] : []),
|
|
902
|
+
]);
|
|
886
903
|
}
|
|
887
904
|
catch (err) {
|
|
888
905
|
// The worker is already consuming here; leaving its Redis connections
|
|
@@ -892,6 +909,11 @@ export function createJobRunner(options) {
|
|
|
892
909
|
worker = null;
|
|
893
910
|
throw err;
|
|
894
911
|
}
|
|
912
|
+
finally {
|
|
913
|
+
gateTimeout?.cancel();
|
|
914
|
+
if (slowWarnTimer !== undefined)
|
|
915
|
+
clearTimeout(slowWarnTimer);
|
|
916
|
+
}
|
|
895
917
|
}
|
|
896
918
|
}
|
|
897
919
|
const runnerApi = {
|
|
@@ -65,9 +65,10 @@ export async function clearPendingRebuilds(db, rows) {
|
|
|
65
65
|
}
|
|
66
66
|
}
|
|
67
67
|
// buildProjectionTableIndex maps both single-stream and multi-stream projection
|
|
68
|
-
// tables to their name, so the rebuild has to pick the matching path.
|
|
69
|
-
// drains
|
|
70
|
-
//
|
|
68
|
+
// tables to their name, so the rebuild has to pick the matching path. Callers
|
|
69
|
+
// (queue drains, inline enqueue, the manual rebuild job) all rethrow the error,
|
|
70
|
+
// so it is recorded there; a failed MSP replay rolls back and must not park
|
|
71
|
+
// the still-intact live consumer as dead.
|
|
71
72
|
export async function rebuildProjectionOrMultiStream(name, deps) {
|
|
72
73
|
const { db, registry, signal, errorPolicy } = deps;
|
|
73
74
|
if (registry.getAllMultiStreamProjections().has(name)) {
|
|
@@ -12,5 +12,5 @@ export { type MetricEvent, type MetricEventHandler, RecordingMeter, } from "./re
|
|
|
12
12
|
export { type RecordedSpan, RecordingTracer, type RecordingTracerOptions, serializeSpanContext, } from "./recording-tracer.js";
|
|
13
13
|
export { wrapRedisClient } from "./redis-wrapper.js";
|
|
14
14
|
export { DEFAULT_SENSITIVE_CONFIG, mergeSensitiveConfig, REDACTED, redactAttributes, redactHeaders, redactQueryString, redactValue, shouldRedactAttribute, } from "./sensitive-filter.js";
|
|
15
|
-
export { emitDbQuery, emitDispatcherError, emitDispatcherHandler, emitEventConsumerLag, emitEventConsumerPassOutcome, emitEventConsumerRearmExhausted, emitEventDispatcherListenConnected, emitHttpRequest, emitJobLastSuccess, emitJobQueueDepth, registerStandardMetrics, STANDARD_METRIC_DEFS, } from "./standard-metrics.js";
|
|
15
|
+
export { emitDbQuery, emitDispatcherError, emitDispatcherHandler, emitEventConsumerLag, emitEventConsumerPassOutcome, emitEventConsumerPassSkipped, emitEventConsumerRearmExhausted, emitEventDispatcherListenConnected, emitHttpRequest, emitJobLastSuccess, emitJobQueueDepth, registerStandardMetrics, STANDARD_METRIC_DEFS, } from "./standard-metrics.js";
|
|
16
16
|
export type { Counter, Gauge, Histogram, Meter, MetricDefinition, MetricLabels, MetricsHandle, MetricType, ObservabilityOptions, ObservabilityProvider, SamplingConfig, SensitiveFilterConfig, SerializedTraceContext, Span, SpanAttributes, SpanAttributeValue, SpanKind, SpanStatus, StartSpanOptions, Tracer, } from "./types/index.js";
|
|
@@ -13,4 +13,4 @@ export { RecordingMeter, } from "./recording-meter.js";
|
|
|
13
13
|
export { RecordingTracer, serializeSpanContext, } from "./recording-tracer.js";
|
|
14
14
|
export { wrapRedisClient } from "./redis-wrapper.js";
|
|
15
15
|
export { DEFAULT_SENSITIVE_CONFIG, mergeSensitiveConfig, REDACTED, redactAttributes, redactHeaders, redactQueryString, redactValue, shouldRedactAttribute, } from "./sensitive-filter.js";
|
|
16
|
-
export { emitDbQuery, emitDispatcherError, emitDispatcherHandler, emitEventConsumerLag, emitEventConsumerPassOutcome, emitEventConsumerRearmExhausted, emitEventDispatcherListenConnected, emitHttpRequest, emitJobLastSuccess, emitJobQueueDepth, registerStandardMetrics, STANDARD_METRIC_DEFS, } from "./standard-metrics.js";
|
|
16
|
+
export { emitDbQuery, emitDispatcherError, emitDispatcherHandler, emitEventConsumerLag, emitEventConsumerPassOutcome, emitEventConsumerPassSkipped, emitEventConsumerRearmExhausted, emitEventDispatcherListenConnected, emitHttpRequest, emitJobLastSuccess, emitJobQueueDepth, registerStandardMetrics, STANDARD_METRIC_DEFS, } from "./standard-metrics.js";
|
|
@@ -32,6 +32,10 @@ export declare function emitEventConsumerRearmExhausted(meter: Meter, labels: {
|
|
|
32
32
|
readonly consumer: string;
|
|
33
33
|
readonly instanceId: string;
|
|
34
34
|
}): void;
|
|
35
|
+
export declare function emitEventConsumerPassSkipped(meter: Meter, labels: {
|
|
36
|
+
readonly consumer: string;
|
|
37
|
+
readonly instanceId: string;
|
|
38
|
+
}, reason: "idle"): void;
|
|
35
39
|
export declare function emitEventDispatcherListenConnected(meter: Meter, connected: boolean): void;
|
|
36
40
|
export declare function emitEventConsumerPassOutcome(meter: Meter, labels: {
|
|
37
41
|
readonly consumer: string;
|
|
@@ -93,6 +93,15 @@ export const STANDARD_METRIC_DEFS = [
|
|
|
93
93
|
description: "Consumer transitioned to dead with its auto-rearm budget exhausted.",
|
|
94
94
|
labels: ["consumer", "instance_id"],
|
|
95
95
|
},
|
|
96
|
+
// An idle-gated consumer turn opens no events.consumer.pass span (the lock-free
|
|
97
|
+
// pre-check proves there is nothing to do), so this counter is the only trace of
|
|
98
|
+
// per-tick pass frequency for consumers that are idle or disabled.
|
|
99
|
+
{
|
|
100
|
+
name: "kumiko_event_consumer_pass_skipped_total",
|
|
101
|
+
type: "counter",
|
|
102
|
+
description: "Consumer turns skipped before opening a pass span because the idle pre-check proved nothing to deliver.",
|
|
103
|
+
labels: ["consumer", "instance_id", "reason"],
|
|
104
|
+
},
|
|
96
105
|
// LISTEN-subscription health. 1 = active, 0 = dropped. Drops to 0 while
|
|
97
106
|
// the dispatcher is running signal that delivery latency has regressed
|
|
98
107
|
// from sub-millisecond (LISTEN) to pollIntervalMs (timer fallback); ops
|
|
@@ -203,6 +212,13 @@ export function emitEventConsumerRearmExhausted(meter, labels) {
|
|
|
203
212
|
instance_id: labels.instanceId,
|
|
204
213
|
});
|
|
205
214
|
}
|
|
215
|
+
export function emitEventConsumerPassSkipped(meter, labels, reason) {
|
|
216
|
+
meter.counter("kumiko_event_consumer_pass_skipped_total").inc(1, {
|
|
217
|
+
consumer: labels.consumer,
|
|
218
|
+
instance_id: labels.instanceId,
|
|
219
|
+
reason,
|
|
220
|
+
});
|
|
221
|
+
}
|
|
206
222
|
export function emitEventDispatcherListenConnected(meter, connected) {
|
|
207
223
|
meter.gauge("kumiko_event_dispatcher_listen_connected").set(connected ? 1 : 0);
|
|
208
224
|
}
|
|
@@ -24,8 +24,8 @@ export async function runBatch(ctx, commands, user, requestId, inheritedOrigin)
|
|
|
24
24
|
if (!current?.signal) {
|
|
25
25
|
return runBatchBody(ctx, commands, user, requestId, inheritedOrigin);
|
|
26
26
|
}
|
|
27
|
-
// Strip the signal: a disconnect
|
|
28
|
-
//
|
|
27
|
+
// Strip the signal: a client disconnect must not roll back a write whose
|
|
28
|
+
// afterCommit effects would then be lost.
|
|
29
29
|
const { signal: _signal, ...withoutSignal } = current;
|
|
30
30
|
return requestContext.run(withoutSignal, () => runBatchBody(ctx, commands, user, requestId, inheritedOrigin));
|
|
31
31
|
}
|
|
@@ -67,7 +67,7 @@ export function createDispatcher(registry, context, options = {}) {
|
|
|
67
67
|
if (isDeprecatedCrossTenantHandler(def.handler)) {
|
|
68
68
|
context.log?.warn(DEPRECATED_CROSS_TENANT_SIGNAL, {
|
|
69
69
|
handler: def.name,
|
|
70
|
-
migration: 'replace crossTenant: true with escapeHatch: { reason: "<why this operator handler reads/writes every tenant>" } (bun scripts/codemod/migrate-cross-tenant.ts)',
|
|
70
|
+
migration: 'replace crossTenant: true with escapeHatch: { reason: "<why this operator handler reads/writes every tenant>" } (bun node_modules/@cosmicdrift/kumiko-framework/src/scripts/codemod/migrate-cross-tenant.ts)',
|
|
71
71
|
});
|
|
72
72
|
}
|
|
73
73
|
}
|
|
@@ -105,6 +105,11 @@ export async function selectConsumerCursorForUpdate(db, name, instanceId) {
|
|
|
105
105
|
// pre-check reads pending_gaps live on every pass, so it observes that gap
|
|
106
106
|
// too. In short: this can under-report idleness for one tick, never
|
|
107
107
|
// over-report it.
|
|
108
|
+
//
|
|
109
|
+
// Observability: an idle turn opens no events.consumer.pass span (so no
|
|
110
|
+
// consumer.skip_reason either, including for a disabled consumer with nothing
|
|
111
|
+
// pending); the dispatcher counts it in
|
|
112
|
+
// kumiko_event_consumer_pass_skipped_total{reason="idle"} instead.
|
|
108
113
|
export async function selectIdleConsumerKeys(db, pairs) {
|
|
109
114
|
if (pairs.length === 0)
|
|
110
115
|
return new Set();
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { selectSnapshotXmax, selectSnapshotXmin } from "../db/queries/event-consumer.js";
|
|
2
2
|
import { SYSTEM_TENANT_ID } from "../engine/types/identifiers.js";
|
|
3
3
|
import { EVENTS_PUBSUB_CHANNEL } from "../event-store/index.js";
|
|
4
|
-
import { emitEventConsumerPassOutcome, emitEventConsumerRearmExhausted, emitEventDispatcherListenConnected, getFallbackMeter, getFallbackTracer, } from "../observability/index.js";
|
|
4
|
+
import { emitEventConsumerPassOutcome, emitEventConsumerPassSkipped, emitEventConsumerRearmExhausted, emitEventDispatcherListenConnected, getFallbackMeter, getFallbackTracer, } from "../observability/index.js";
|
|
5
5
|
import { SHARED_INSTANCE_SENTINEL } from "./event-consumer-state.js";
|
|
6
6
|
import { acquireConsumerState, consumerInstanceId, deliverEvents, emitLagFromTx, fetchPendingEvents, markProcessing, persistConsumerOutcome, persistConsumerPassFailure, preRegisterConsumers, selectIdleConsumerKeys, } from "./event-dispatcher-delivery.js";
|
|
7
7
|
import { partitionBurntGaps, splitRangeExcludingIds, toIdRanges } from "./pending-gap-ranges.js";
|
|
@@ -132,8 +132,11 @@ export function createEventDispatcher(options) {
|
|
|
132
132
|
// backoff-gate above already confirmed retryAtMs has elapsed, and
|
|
133
133
|
// "provably nothing to deliver" is itself proof the consumer isn't
|
|
134
134
|
// presently failing.
|
|
135
|
+
// No events.consumer.pass span is opened here (so no consumer.skip_reason either),
|
|
136
|
+
// which is why the skip is counted instead.
|
|
135
137
|
if (idleKeys.has(key)) {
|
|
136
138
|
consumerBackoff.delete(key);
|
|
139
|
+
emitEventConsumerPassSkipped(meter, { consumer: consumer.name, instanceId: consumerInstanceId(consumer, options.instanceId) }, "idle");
|
|
137
140
|
return { processed: 0, failed: 0 };
|
|
138
141
|
}
|
|
139
142
|
// Registered before awaiting the slot — a turn queued behind the
|
|
@@ -366,19 +366,17 @@ export async function buildSearchDocument(entityName, entityId, state, registry)
|
|
|
366
366
|
}
|
|
367
367
|
// --- SSE Broadcast (async, via event-dispatcher) ---
|
|
368
368
|
//
|
|
369
|
-
// SSE
|
|
370
|
-
//
|
|
371
|
-
// zwei Konsequenzen:
|
|
369
|
+
// The SSE broadcast runs as an async EventConsumer on the event dispatcher,
|
|
370
|
+
// not as a synchronous postSave/postDelete hook. Two consequences:
|
|
372
371
|
//
|
|
373
|
-
// 1. **
|
|
374
|
-
//
|
|
375
|
-
//
|
|
376
|
-
//
|
|
377
|
-
//
|
|
378
|
-
// 2. **Eventual consistency
|
|
379
|
-
//
|
|
380
|
-
//
|
|
381
|
-
// Clients müssten poll-after-write.
|
|
372
|
+
// 1. **Signal only.** The frame mirrors the StoredEvent's identity: `type` is
|
|
373
|
+
// event.type ("user.created"), `data` holds id, aggregateType, version and
|
|
374
|
+
// createdAt, never the payload. The tenant channel fans out to every member
|
|
375
|
+
// and to anonymous connections, so field values must not enter the broker
|
|
376
|
+
// or Redis; the client refetches through a query with its own access check.
|
|
377
|
+
// 2. **Eventual consistency, not read-after-write.** A frame arrives roughly
|
|
378
|
+
// 10 to 100 ms after the HTTP 200 (depends on pollIntervalMs). Clients that
|
|
379
|
+
// must observe their own write poll after writing.
|
|
382
380
|
//
|
|
383
381
|
// Tests drain deterministically via `await stack.eventDispatcher.runOnce()`,
|
|
384
382
|
// or via `drainEventConsumers` for this consumer name specifically.
|
|
@@ -402,7 +400,6 @@ export function createSseBroadcastEventConsumer(sseBroker) {
|
|
|
402
400
|
id: event.aggregateId,
|
|
403
401
|
aggregateType: event.aggregateType,
|
|
404
402
|
version: event.version,
|
|
405
|
-
payload: event.payload,
|
|
406
403
|
createdAt: event.createdAt,
|
|
407
404
|
},
|
|
408
405
|
});
|
|
@@ -49,8 +49,14 @@ export function rootWriteOrigin(registry, type, user) {
|
|
|
49
49
|
// Owner binding is ignored (honorOwnerBinding=false): all anonymous callers share one
|
|
50
50
|
// user id, so from("user:id", ...) vouches for nobody.
|
|
51
51
|
const personalDataTableMaps = new WeakMap();
|
|
52
|
+
const personalColumnsByEntity = new WeakMap();
|
|
52
53
|
function personalColumnNames(entity) {
|
|
53
|
-
|
|
54
|
+
const cached = personalColumnsByEntity.get(entity);
|
|
55
|
+
if (cached)
|
|
56
|
+
return cached;
|
|
57
|
+
const columns = new Set([...personalFieldNames(entity, false)].map(toSnakeCase));
|
|
58
|
+
personalColumnsByEntity.set(entity, columns);
|
|
59
|
+
return columns;
|
|
54
60
|
}
|
|
55
61
|
function buildPersonalDataTableMap(registry) {
|
|
56
62
|
const map = new Map();
|
|
@@ -73,21 +79,22 @@ function personalDataTableMap(registry) {
|
|
|
73
79
|
personalDataTableMaps.set(registry, built);
|
|
74
80
|
return built;
|
|
75
81
|
}
|
|
76
|
-
//
|
|
82
|
+
// The error reaches the anonymous HTTP caller, so table and column names stay out of
|
|
83
|
+
// message/details (they would map the DB schema); they travel in `cause`, which the
|
|
84
|
+
// serializer never sends to clients. The 4xx log line carries only status and code, so
|
|
85
|
+
// `cause` is not logged today; it is only available to in-process error handling.
|
|
77
86
|
export function publicIntakeRequiredError(origin, target, fields) {
|
|
78
87
|
return new AccessDeniedError({
|
|
79
|
-
message: `Anonymous root handler "${origin.rootHandler}" wrote personal
|
|
80
|
-
`${fields.map((f) => `"${f}"`).join(", ")} on "${target}"` +
|
|
88
|
+
message: `Anonymous root handler "${origin.rootHandler}" wrote personal data` +
|
|
81
89
|
(origin.viaJob ? ` via job "${origin.viaJob}"` : "") +
|
|
82
90
|
'. Declare access: { roles: [..., "anonymous"], personalData: "public-intake" } on ' +
|
|
83
91
|
`"${origin.rootHandler}" to allow anonymous callers to write personal data.`,
|
|
84
92
|
details: {
|
|
85
93
|
reason: FrameworkReasons.publicIntakeRequired,
|
|
86
94
|
rootHandler: origin.rootHandler,
|
|
87
|
-
target,
|
|
88
|
-
fields,
|
|
89
95
|
...(origin.viaJob !== undefined && { job: origin.viaJob }),
|
|
90
96
|
},
|
|
97
|
+
cause: new Error(`personal-data field(s) ${fields.map((f) => `"${f}"`).join(", ")} written on "${target}"`),
|
|
91
98
|
});
|
|
92
99
|
}
|
|
93
100
|
// Without an entity the table name is looked up in the registry map; tables outside it
|
|
@@ -17,6 +17,9 @@ export async function drainEventConsumers(stack, consumerNames, options) {
|
|
|
17
17
|
"multiStreamProjection is wired for this stack.");
|
|
18
18
|
}
|
|
19
19
|
const maxPasses = options?.maxPasses ?? DEFAULT_MAX_PASSES;
|
|
20
|
+
if (!Number.isInteger(maxPasses) || maxPasses < 1) {
|
|
21
|
+
throw new Error("drainEventConsumers: maxPasses must be a positive integer");
|
|
22
|
+
}
|
|
20
23
|
const target = await getEventsHighWaterMark(db);
|
|
21
24
|
for (let pass = 0; pass < maxPasses; pass++) {
|
|
22
25
|
await eventDispatcher.runOnce();
|
|
@@ -17,4 +17,6 @@ export type DrainJobsStack = {
|
|
|
17
17
|
readonly eventDispatcher?: EventDispatcher;
|
|
18
18
|
readonly jobRunner?: JobRunner;
|
|
19
19
|
};
|
|
20
|
-
export declare function drainJobs(stack: DrainJobsStack, tracker: JobFailureTracker
|
|
20
|
+
export declare function drainJobs(stack: DrainJobsStack, tracker: JobFailureTracker, options?: {
|
|
21
|
+
readonly timeoutMs?: number;
|
|
22
|
+
}): Promise<void>;
|
package/dist/stack/drain-jobs.js
CHANGED
|
@@ -12,6 +12,9 @@ const DRAIN_IDLE_POLL_MS = 25;
|
|
|
12
12
|
// enqueue that lands just after the first idle read still gets picked up
|
|
13
13
|
// by the second pass instead of racing straight past it.
|
|
14
14
|
const REQUIRED_IDLE_PASSES = 2;
|
|
15
|
+
// Below the 15s per-test budget so a stuck drain reports what is open rather
|
|
16
|
+
// than surfacing as a bare test timeout.
|
|
17
|
+
const DEFAULT_DRAIN_TIMEOUT_MS = 10_000;
|
|
15
18
|
function sleep(ms) {
|
|
16
19
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
17
20
|
}
|
|
@@ -47,6 +50,26 @@ export function createJobFailureTracker() {
|
|
|
47
50
|
},
|
|
48
51
|
};
|
|
49
52
|
}
|
|
53
|
+
// Diagnostic only: lets the deadline error name what was still open.
|
|
54
|
+
async function describeLaggingConsumers(db, consumerNames) {
|
|
55
|
+
// skip: no shared consumers wired — nothing to catch up on.
|
|
56
|
+
if (consumerNames.length === 0)
|
|
57
|
+
return [];
|
|
58
|
+
const target = await getEventsHighWaterMark(db);
|
|
59
|
+
const lagging = [];
|
|
60
|
+
for (const name of consumerNames) {
|
|
61
|
+
const state = await getConsumerState(db, name);
|
|
62
|
+
if (state && state.lastProcessedEventId < target) {
|
|
63
|
+
lagging.push(`${name} (status=${state.status}, lastProcessedEventId=${state.lastProcessedEventId}, target=${target})`);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
return lagging;
|
|
67
|
+
}
|
|
68
|
+
async function drainTimeoutError(db, consumerNames, timeoutMs, pending) {
|
|
69
|
+
const lagging = await describeLaggingConsumers(db, consumerNames);
|
|
70
|
+
return new Error(`drainJobs: not drained after ${timeoutMs}ms — pending jobs: ${pending}, ` +
|
|
71
|
+
`consumers behind: ${lagging.length === 0 ? "none" : lagging.join("; ")}`);
|
|
72
|
+
}
|
|
50
73
|
async function sharedConsumersCaughtUp(db, consumerNames) {
|
|
51
74
|
// skip: no shared consumers wired — nothing to catch up on.
|
|
52
75
|
if (consumerNames.length === 0)
|
|
@@ -66,8 +89,12 @@ async function sharedConsumersCaughtUp(db, consumerNames) {
|
|
|
66
89
|
}
|
|
67
90
|
return allCaughtUp;
|
|
68
91
|
}
|
|
69
|
-
|
|
92
|
+
// Fails a stuck drain with a diagnosis instead of a generic test-runner timeout.
|
|
93
|
+
// It only bounds the loop; it never replaces waiting for real work.
|
|
94
|
+
export async function drainJobs(stack, tracker, options) {
|
|
70
95
|
const { db, eventDispatcher, jobRunner } = stack;
|
|
96
|
+
const timeoutMs = options?.timeoutMs ?? DEFAULT_DRAIN_TIMEOUT_MS;
|
|
97
|
+
const deadline = Date.now() + timeoutMs;
|
|
71
98
|
if (!jobRunner) {
|
|
72
99
|
throw new Error("drainJobs: stack.jobRunner is undefined — pass `jobs: {...}` to setupTestStack and " +
|
|
73
100
|
"register at least one r.job(...) so a runner is built.");
|
|
@@ -84,6 +111,9 @@ export async function drainJobs(stack, tracker) {
|
|
|
84
111
|
const caughtUp = await sharedConsumersCaughtUp(db, sharedConsumerNames);
|
|
85
112
|
consecutiveIdlePasses = pending === 0 && caughtUp ? consecutiveIdlePasses + 1 : 0;
|
|
86
113
|
if (consecutiveIdlePasses < REQUIRED_IDLE_PASSES) {
|
|
114
|
+
if (Date.now() > deadline) {
|
|
115
|
+
throw await drainTimeoutError(db, sharedConsumerNames, timeoutMs, pending);
|
|
116
|
+
}
|
|
87
117
|
await Promise.race([tracker.waitForActivity(), sleep(DRAIN_IDLE_POLL_MS)]);
|
|
88
118
|
}
|
|
89
119
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cosmicdrift/kumiko-framework",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.336.1",
|
|
4
4
|
"description": "Framework core — engine, pipeline, API, DB, and every other bit that makes Kumiko go.",
|
|
5
5
|
"license": "BUSL-1.1",
|
|
6
6
|
"author": "Marc Frost <marc@cosmicdriftgamestudio.com>",
|
|
@@ -206,8 +206,8 @@
|
|
|
206
206
|
"./package.json": "./package.json"
|
|
207
207
|
},
|
|
208
208
|
"dependencies": {
|
|
209
|
-
"@cosmicdrift/kumiko-http": "0.
|
|
210
|
-
"@cosmicdrift/kumiko-types": "0.
|
|
209
|
+
"@cosmicdrift/kumiko-http": "0.336.1",
|
|
210
|
+
"@cosmicdrift/kumiko-types": "0.336.1",
|
|
211
211
|
"bullmq": "^5.76.7",
|
|
212
212
|
"bun-types": "^1.3.13",
|
|
213
213
|
"hono": "^4.13.1",
|
|
@@ -223,7 +223,7 @@
|
|
|
223
223
|
"zod": "^4.4.3"
|
|
224
224
|
},
|
|
225
225
|
"devDependencies": {
|
|
226
|
-
"@cosmicdrift/kumiko-dispatcher-live": "0.
|
|
226
|
+
"@cosmicdrift/kumiko-dispatcher-live": "0.336.1",
|
|
227
227
|
"bun-types": "^1.3.13",
|
|
228
228
|
"pino-pretty": "^13.1.3"
|
|
229
229
|
},
|
package/src/changes.json
CHANGED
|
@@ -1,4 +1,68 @@
|
|
|
1
1
|
[
|
|
2
|
+
{
|
|
3
|
+
"version": "0.336.1",
|
|
4
|
+
"type": "fix",
|
|
5
|
+
"title": "Cross-tenant deprecation warning names a runnable codemod path",
|
|
6
|
+
"detail": "The `deprecation:entity-handler-cross-tenant` boot warning now names the codemod path that runs from a consumer repo. The `reason` strings on the `passwordHash`, PAT `tokenHash` and PAT `prefix` fields no longer carry an issue-number suffix. The `pii-personal-migration` codemod now skips `.d.ts` files in `--report-stance` mode as well."
|
|
7
|
+
},
|
|
8
|
+
{
|
|
9
|
+
"version": "0.336.0",
|
|
10
|
+
"type": "fix",
|
|
11
|
+
"title": "Boot gates with an explicit timeout reject start() naming the gate; gates without one warn after 60s",
|
|
12
|
+
"detail": "A `bootGate` job with an explicit `timeout` now rejects `start()` with a message naming the gate once that timeout elapses. Without `timeout` the gate keeps waiting and logs a warning naming it after 60s instead of blocking boot silently."
|
|
13
|
+
},
|
|
14
|
+
{
|
|
15
|
+
"version": "0.336.0",
|
|
16
|
+
"type": "fix",
|
|
17
|
+
"title": "Boot rejects unknown visibleWhen fields, bootGate with runOnBoot, and unknown money currency kinds",
|
|
18
|
+
"detail": "Boot now rejects three misconfigurations that previously failed silently: a dashboard screen-panel `visibleWhen.field` that is missing from the query's declared `outputSchema`, a job that combines `bootGate` with `runOnBoot` (it ran twice under one job id), and a money field whose `currency` source has an unknown `kind`. `auth-mfa:query:user-mfa:status` now declares its `outputSchema`."
|
|
19
|
+
},
|
|
20
|
+
{
|
|
21
|
+
"version": "0.336.0",
|
|
22
|
+
"type": "fix",
|
|
23
|
+
"title": "Dry-run env output keeps multi-line field descriptions inside the comment",
|
|
24
|
+
"detail": "`KUMIKO_DRY_RUN_ENV` pulumi and k8s output now folds a multi-line `.describe()` text onto one comment line, so the continuation can no longer escape the `#` and run as a shell command or break the YAML stub. The `unprocessable-error-details-reason` codemod now skips a shorthand `reason` whose variable is only used in `details`."
|
|
25
|
+
},
|
|
26
|
+
{
|
|
27
|
+
"version": "0.336.0",
|
|
28
|
+
"type": "fix",
|
|
29
|
+
"title": "waitForEvent ne-match ignores payloads without the referenced path",
|
|
30
|
+
"detail": "`waitForEvent` match atoms no longer match events that lack the referenced payload path, so `ne` no longer wakes a workflow on an unrelated event shape. Explicit `undefined` optionals on `r.nav()` entries are dropped from the client schema."
|
|
31
|
+
},
|
|
32
|
+
{
|
|
33
|
+
"version": "0.336.0",
|
|
34
|
+
"type": "fix",
|
|
35
|
+
"title": "r.nav() explicit undefined optionals no longer reach the client schema"
|
|
36
|
+
},
|
|
37
|
+
{
|
|
38
|
+
"version": "0.336.0",
|
|
39
|
+
"type": "fix",
|
|
40
|
+
"title": "public_intake_required 403 no longer discloses table and column names to anonymous callers",
|
|
41
|
+
"detail": "The `public_intake_required` 403 no longer puts the physical table name and personal-data column names into `message` or `details` (it keeps `reason`, `rootHandler` and `job`); they travel in the error `cause` for the server log. The event dispatcher counts idle-gated consumer turns in the new `kumiko_event_consumer_pass_skipped_total{reason=\"idle\"}` counter, since those turns open no `events.consumer.pass` span. Personal-data column sets for the anonymous write gate are cached per entity."
|
|
42
|
+
},
|
|
43
|
+
{
|
|
44
|
+
"version": "0.336.0",
|
|
45
|
+
"type": "improvement",
|
|
46
|
+
"title": "Idle-gated event consumer turns are counted in kumiko_event_consumer_pass_skipped_total"
|
|
47
|
+
},
|
|
48
|
+
{
|
|
49
|
+
"version": "0.336.0",
|
|
50
|
+
"type": "breaking",
|
|
51
|
+
"title": "/api/sse sends change signals without field values; anonymous connections only for declared entities",
|
|
52
|
+
"detail": "The SSE broadcast consumer no longer puts the event payload (changes, previous) on the tenant channel, because that channel fans out to every tenant member and to anonymous connections. Entity frames carry `{ id, aggregateType, eventType, version, createdAt }`. Anonymous connections get entity signals only for entities named by an anonymously callable query via the new `liveEntities` option; the query name alone grants nothing, because a signal carries the id of every row, including rows the query filters out. Frames without an entity are delivered only when `data.userId` matches the connected user. Boot fails when `liveEntities` names an unregistered entity.",
|
|
53
|
+
"migration": "Code that reads `data.payload` from SSE frames must load the data with a query after the signal instead. Public pages that update anonymously add `liveEntities: [\"<entity>\"]` to the anonymous query they refetch (for example a `page:current` query), otherwise the live update stays off for anonymous visitors. `createSseRoute(broker)` now needs a second argument, `{ anonymousLiveEntities: collectAnonymousLiveEntities(registry) }`."
|
|
54
|
+
},
|
|
55
|
+
{
|
|
56
|
+
"version": "0.336.0",
|
|
57
|
+
"type": "fix",
|
|
58
|
+
"title": "drainEventConsumers rejects an invalid maxPasses and drainJobs fails a stuck drain with a diagnosis",
|
|
59
|
+
"detail": "`drainEventConsumers` rejects a non-positive or non-integer `maxPasses` instead of returning without draining, and `drainJobs` accepts `timeoutMs` (default 10s) and fails a stuck drain with the pending job count and the consumers still behind. The stance report names which hint category (direct, user-owned, user-reference) a near-miss field resembles, and the `UnprocessableError` reason codemod no longer reports spread literals without a `reason` key and lists non-analyzable `details` arguments as unverified."
|
|
60
|
+
},
|
|
61
|
+
{
|
|
62
|
+
"version": "0.336.0",
|
|
63
|
+
"type": "improvement",
|
|
64
|
+
"title": "Codemod reports name the near-miss hint category and separate unverified sites from manual skips"
|
|
65
|
+
},
|
|
2
66
|
{
|
|
3
67
|
"version": "0.335.0",
|
|
4
68
|
"type": "fix",
|