@cosmicdrift/kumiko-framework 0.335.0 → 0.336.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/anonymous-live-entities.d.ts +2 -0
- package/dist/api/anonymous-live-entities.js +17 -0
- package/dist/api/index.d.ts +2 -1
- package/dist/api/index.js +1 -0
- package/dist/api/server.js +6 -3
- package/dist/api/sse-route.d.ts +4 -1
- package/dist/api/sse-route.js +38 -11
- package/dist/db/blind-index-cleanup.js +27 -16
- package/dist/db/event-store-executor-read.js +6 -2
- package/dist/engine/boot-validator/query-output-columns.js +12 -0
- package/dist/engine/boot-validator/screens.js +22 -13
- package/dist/engine/build-app-schema.js +3 -1
- package/dist/engine/create-app.js +10 -1
- package/dist/engine/extension-names.d.ts +4 -0
- package/dist/engine/extension-names.js +4 -0
- package/dist/engine/feature-ast/extractors/handlers.js +1 -0
- package/dist/engine/feature-entity-handlers.d.ts +1 -0
- package/dist/engine/feature-entity-handlers.js +2 -0
- package/dist/engine/registry-validate.d.ts +1 -0
- package/dist/engine/registry-validate.js +14 -0
- package/dist/engine/registry.js +2 -1
- package/dist/engine/steps/_event-match.js +12 -4
- package/dist/engine/system-user.d.ts +1 -0
- package/dist/engine/system-user.js +3 -0
- package/dist/env/dry-run.js +10 -3
- package/dist/event-store/event-store.js +4 -1
- package/dist/jobs/job-runner.js +29 -7
- package/dist/observability/index.d.ts +1 -1
- package/dist/observability/index.js +1 -1
- package/dist/observability/standard-metrics.d.ts +4 -0
- package/dist/observability/standard-metrics.js +16 -0
- package/dist/pipeline/event-dispatcher-delivery.js +5 -0
- package/dist/pipeline/event-dispatcher.js +4 -1
- package/dist/pipeline/system-hooks.js +10 -13
- package/dist/pipeline/write-origin.js +13 -6
- package/dist/stack/drain-event-consumers.js +3 -0
- package/dist/stack/drain-jobs.d.ts +3 -1
- package/dist/stack/drain-jobs.js +31 -1
- package/package.json +4 -4
- package/src/changes.json +58 -0
- package/src/scripts/codemod/pii-personal-migration.ts +56 -26
- package/src/scripts/codemod/unprocessable-error-details-reason.ts +61 -24
package/dist/jobs/job-runner.js
CHANGED
|
@@ -228,6 +228,9 @@ const BOOT_REDIS_TIMEOUT_MS = 10_000;
|
|
|
228
228
|
// Shutdown must fit a k8s grace period even when Redis is down, so the
|
|
229
229
|
// readiness wait in stop() is capped well below the boot timeout.
|
|
230
230
|
const STOP_QUEUE_READY_TIMEOUT_MS = 1_000;
|
|
231
|
+
// A gate without an explicit `timeout` may legitimately run long (migration-style
|
|
232
|
+
// checks), so it is never rejected; a hang just must not stay silent.
|
|
233
|
+
const BOOT_GATE_SLOW_WARN_MS = 60_000;
|
|
231
234
|
function timeoutReject(ms, message) {
|
|
232
235
|
let timer;
|
|
233
236
|
const promise = new Promise((_, reject) => {
|
|
@@ -875,14 +878,28 @@ export function createJobRunner(options) {
|
|
|
875
878
|
continue;
|
|
876
879
|
if (!jobDef.bootGate)
|
|
877
880
|
continue;
|
|
881
|
+
const gateTimeout = jobDef.timeout !== undefined
|
|
882
|
+
? timeoutReject(jobDef.timeout, `job-runner: boot gate "${name}" did not finish within ${jobDef.timeout}ms`)
|
|
883
|
+
: undefined;
|
|
884
|
+
const slowWarnTimer = gateTimeout === undefined
|
|
885
|
+
? setTimeout(() => {
|
|
886
|
+
errorLogger.warn("boot gate still running, no timeout configured", {
|
|
887
|
+
gate: name,
|
|
888
|
+
afterMs: BOOT_GATE_SLOW_WARN_MS,
|
|
889
|
+
});
|
|
890
|
+
}, BOOT_GATE_SLOW_WARN_MS)
|
|
891
|
+
: undefined;
|
|
878
892
|
try {
|
|
879
|
-
await
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
893
|
+
await Promise.race([
|
|
894
|
+
handleJob({
|
|
895
|
+
id: bootJobIdForJobName(name),
|
|
896
|
+
name,
|
|
897
|
+
data: {},
|
|
898
|
+
attemptsMade: 0,
|
|
899
|
+
retryable: false,
|
|
900
|
+
}),
|
|
901
|
+
...(gateTimeout ? [gateTimeout.promise] : []),
|
|
902
|
+
]);
|
|
886
903
|
}
|
|
887
904
|
catch (err) {
|
|
888
905
|
// The worker is already consuming here; leaving its Redis connections
|
|
@@ -892,6 +909,11 @@ export function createJobRunner(options) {
|
|
|
892
909
|
worker = null;
|
|
893
910
|
throw err;
|
|
894
911
|
}
|
|
912
|
+
finally {
|
|
913
|
+
gateTimeout?.cancel();
|
|
914
|
+
if (slowWarnTimer !== undefined)
|
|
915
|
+
clearTimeout(slowWarnTimer);
|
|
916
|
+
}
|
|
895
917
|
}
|
|
896
918
|
}
|
|
897
919
|
const runnerApi = {
|
|
@@ -12,5 +12,5 @@ export { type MetricEvent, type MetricEventHandler, RecordingMeter, } from "./re
|
|
|
12
12
|
export { type RecordedSpan, RecordingTracer, type RecordingTracerOptions, serializeSpanContext, } from "./recording-tracer.js";
|
|
13
13
|
export { wrapRedisClient } from "./redis-wrapper.js";
|
|
14
14
|
export { DEFAULT_SENSITIVE_CONFIG, mergeSensitiveConfig, REDACTED, redactAttributes, redactHeaders, redactQueryString, redactValue, shouldRedactAttribute, } from "./sensitive-filter.js";
|
|
15
|
-
export { emitDbQuery, emitDispatcherError, emitDispatcherHandler, emitEventConsumerLag, emitEventConsumerPassOutcome, emitEventConsumerRearmExhausted, emitEventDispatcherListenConnected, emitHttpRequest, emitJobLastSuccess, emitJobQueueDepth, registerStandardMetrics, STANDARD_METRIC_DEFS, } from "./standard-metrics.js";
|
|
15
|
+
export { emitDbQuery, emitDispatcherError, emitDispatcherHandler, emitEventConsumerLag, emitEventConsumerPassOutcome, emitEventConsumerPassSkipped, emitEventConsumerRearmExhausted, emitEventDispatcherListenConnected, emitHttpRequest, emitJobLastSuccess, emitJobQueueDepth, registerStandardMetrics, STANDARD_METRIC_DEFS, } from "./standard-metrics.js";
|
|
16
16
|
export type { Counter, Gauge, Histogram, Meter, MetricDefinition, MetricLabels, MetricsHandle, MetricType, ObservabilityOptions, ObservabilityProvider, SamplingConfig, SensitiveFilterConfig, SerializedTraceContext, Span, SpanAttributes, SpanAttributeValue, SpanKind, SpanStatus, StartSpanOptions, Tracer, } from "./types/index.js";
|
|
@@ -13,4 +13,4 @@ export { RecordingMeter, } from "./recording-meter.js";
|
|
|
13
13
|
export { RecordingTracer, serializeSpanContext, } from "./recording-tracer.js";
|
|
14
14
|
export { wrapRedisClient } from "./redis-wrapper.js";
|
|
15
15
|
export { DEFAULT_SENSITIVE_CONFIG, mergeSensitiveConfig, REDACTED, redactAttributes, redactHeaders, redactQueryString, redactValue, shouldRedactAttribute, } from "./sensitive-filter.js";
|
|
16
|
-
export { emitDbQuery, emitDispatcherError, emitDispatcherHandler, emitEventConsumerLag, emitEventConsumerPassOutcome, emitEventConsumerRearmExhausted, emitEventDispatcherListenConnected, emitHttpRequest, emitJobLastSuccess, emitJobQueueDepth, registerStandardMetrics, STANDARD_METRIC_DEFS, } from "./standard-metrics.js";
|
|
16
|
+
export { emitDbQuery, emitDispatcherError, emitDispatcherHandler, emitEventConsumerLag, emitEventConsumerPassOutcome, emitEventConsumerPassSkipped, emitEventConsumerRearmExhausted, emitEventDispatcherListenConnected, emitHttpRequest, emitJobLastSuccess, emitJobQueueDepth, registerStandardMetrics, STANDARD_METRIC_DEFS, } from "./standard-metrics.js";
|
|
@@ -32,6 +32,10 @@ export declare function emitEventConsumerRearmExhausted(meter: Meter, labels: {
|
|
|
32
32
|
readonly consumer: string;
|
|
33
33
|
readonly instanceId: string;
|
|
34
34
|
}): void;
|
|
35
|
+
export declare function emitEventConsumerPassSkipped(meter: Meter, labels: {
|
|
36
|
+
readonly consumer: string;
|
|
37
|
+
readonly instanceId: string;
|
|
38
|
+
}, reason: "idle"): void;
|
|
35
39
|
export declare function emitEventDispatcherListenConnected(meter: Meter, connected: boolean): void;
|
|
36
40
|
export declare function emitEventConsumerPassOutcome(meter: Meter, labels: {
|
|
37
41
|
readonly consumer: string;
|
|
@@ -93,6 +93,15 @@ export const STANDARD_METRIC_DEFS = [
|
|
|
93
93
|
description: "Consumer transitioned to dead with its auto-rearm budget exhausted.",
|
|
94
94
|
labels: ["consumer", "instance_id"],
|
|
95
95
|
},
|
|
96
|
+
// An idle-gated consumer turn opens no events.consumer.pass span (the lock-free
|
|
97
|
+
// pre-check proves there is nothing to do), so this counter is the only trace of
|
|
98
|
+
// per-tick pass frequency for consumers that are idle or disabled.
|
|
99
|
+
{
|
|
100
|
+
name: "kumiko_event_consumer_pass_skipped_total",
|
|
101
|
+
type: "counter",
|
|
102
|
+
description: "Consumer turns skipped before opening a pass span because the idle pre-check proved nothing to deliver.",
|
|
103
|
+
labels: ["consumer", "instance_id", "reason"],
|
|
104
|
+
},
|
|
96
105
|
// LISTEN-subscription health. 1 = active, 0 = dropped. Drops to 0 while
|
|
97
106
|
// the dispatcher is running signal that delivery latency has regressed
|
|
98
107
|
// from sub-millisecond (LISTEN) to pollIntervalMs (timer fallback); ops
|
|
@@ -203,6 +212,13 @@ export function emitEventConsumerRearmExhausted(meter, labels) {
|
|
|
203
212
|
instance_id: labels.instanceId,
|
|
204
213
|
});
|
|
205
214
|
}
|
|
215
|
+
export function emitEventConsumerPassSkipped(meter, labels, reason) {
|
|
216
|
+
meter.counter("kumiko_event_consumer_pass_skipped_total").inc(1, {
|
|
217
|
+
consumer: labels.consumer,
|
|
218
|
+
instance_id: labels.instanceId,
|
|
219
|
+
reason,
|
|
220
|
+
});
|
|
221
|
+
}
|
|
206
222
|
export function emitEventDispatcherListenConnected(meter, connected) {
|
|
207
223
|
meter.gauge("kumiko_event_dispatcher_listen_connected").set(connected ? 1 : 0);
|
|
208
224
|
}
|
|
@@ -105,6 +105,11 @@ export async function selectConsumerCursorForUpdate(db, name, instanceId) {
|
|
|
105
105
|
// pre-check reads pending_gaps live on every pass, so it observes that gap
|
|
106
106
|
// too. In short: this can under-report idleness for one tick, never
|
|
107
107
|
// over-report it.
|
|
108
|
+
//
|
|
109
|
+
// Observability: an idle turn opens no events.consumer.pass span (so no
|
|
110
|
+
// consumer.skip_reason either, including for a disabled consumer with nothing
|
|
111
|
+
// pending); the dispatcher counts it in
|
|
112
|
+
// kumiko_event_consumer_pass_skipped_total{reason="idle"} instead.
|
|
108
113
|
export async function selectIdleConsumerKeys(db, pairs) {
|
|
109
114
|
if (pairs.length === 0)
|
|
110
115
|
return new Set();
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { selectSnapshotXmax, selectSnapshotXmin } from "../db/queries/event-consumer.js";
|
|
2
2
|
import { SYSTEM_TENANT_ID } from "../engine/types/identifiers.js";
|
|
3
3
|
import { EVENTS_PUBSUB_CHANNEL } from "../event-store/index.js";
|
|
4
|
-
import { emitEventConsumerPassOutcome, emitEventConsumerRearmExhausted, emitEventDispatcherListenConnected, getFallbackMeter, getFallbackTracer, } from "../observability/index.js";
|
|
4
|
+
import { emitEventConsumerPassOutcome, emitEventConsumerPassSkipped, emitEventConsumerRearmExhausted, emitEventDispatcherListenConnected, getFallbackMeter, getFallbackTracer, } from "../observability/index.js";
|
|
5
5
|
import { SHARED_INSTANCE_SENTINEL } from "./event-consumer-state.js";
|
|
6
6
|
import { acquireConsumerState, consumerInstanceId, deliverEvents, emitLagFromTx, fetchPendingEvents, markProcessing, persistConsumerOutcome, persistConsumerPassFailure, preRegisterConsumers, selectIdleConsumerKeys, } from "./event-dispatcher-delivery.js";
|
|
7
7
|
import { partitionBurntGaps, splitRangeExcludingIds, toIdRanges } from "./pending-gap-ranges.js";
|
|
@@ -132,8 +132,11 @@ export function createEventDispatcher(options) {
|
|
|
132
132
|
// backoff-gate above already confirmed retryAtMs has elapsed, and
|
|
133
133
|
// "provably nothing to deliver" is itself proof the consumer isn't
|
|
134
134
|
// presently failing.
|
|
135
|
+
// No events.consumer.pass span is opened here (so no consumer.skip_reason either),
|
|
136
|
+
// which is why the skip is counted instead.
|
|
135
137
|
if (idleKeys.has(key)) {
|
|
136
138
|
consumerBackoff.delete(key);
|
|
139
|
+
emitEventConsumerPassSkipped(meter, { consumer: consumer.name, instanceId: consumerInstanceId(consumer, options.instanceId) }, "idle");
|
|
137
140
|
return { processed: 0, failed: 0 };
|
|
138
141
|
}
|
|
139
142
|
// Registered before awaiting the slot — a turn queued behind the
|
|
@@ -366,19 +366,17 @@ export async function buildSearchDocument(entityName, entityId, state, registry)
|
|
|
366
366
|
}
|
|
367
367
|
// --- SSE Broadcast (async, via event-dispatcher) ---
|
|
368
368
|
//
|
|
369
|
-
// SSE
|
|
370
|
-
//
|
|
371
|
-
// zwei Konsequenzen:
|
|
369
|
+
// The SSE broadcast runs as an async EventConsumer on the event dispatcher,
|
|
370
|
+
// not as a synchronous postSave/postDelete hook. Two consequences:
|
|
372
371
|
//
|
|
373
|
-
// 1. **
|
|
374
|
-
//
|
|
375
|
-
//
|
|
376
|
-
//
|
|
377
|
-
//
|
|
378
|
-
// 2. **Eventual consistency
|
|
379
|
-
//
|
|
380
|
-
//
|
|
381
|
-
// Clients müssten poll-after-write.
|
|
372
|
+
// 1. **Signal only.** The frame mirrors the StoredEvent's identity: `type` is
|
|
373
|
+
// event.type ("user.created"), `data` holds id, aggregateType, version and
|
|
374
|
+
// createdAt, never the payload. The tenant channel fans out to every member
|
|
375
|
+
// and to anonymous connections, so field values must not enter the broker
|
|
376
|
+
// or Redis; the client refetches through a query with its own access check.
|
|
377
|
+
// 2. **Eventual consistency, not read-after-write.** A frame arrives roughly
|
|
378
|
+
// 10 to 100 ms after the HTTP 200 (depends on pollIntervalMs). Clients that
|
|
379
|
+
// must observe their own write poll after writing.
|
|
382
380
|
//
|
|
383
381
|
// Tests drain deterministically via `await stack.eventDispatcher.runOnce()`,
|
|
384
382
|
// or via `drainEventConsumers` for this consumer name specifically.
|
|
@@ -402,7 +400,6 @@ export function createSseBroadcastEventConsumer(sseBroker) {
|
|
|
402
400
|
id: event.aggregateId,
|
|
403
401
|
aggregateType: event.aggregateType,
|
|
404
402
|
version: event.version,
|
|
405
|
-
payload: event.payload,
|
|
406
403
|
createdAt: event.createdAt,
|
|
407
404
|
},
|
|
408
405
|
});
|
|
@@ -49,8 +49,14 @@ export function rootWriteOrigin(registry, type, user) {
|
|
|
49
49
|
// Owner binding is ignored (honorOwnerBinding=false): all anonymous callers share one
|
|
50
50
|
// user id, so from("user:id", ...) vouches for nobody.
|
|
51
51
|
const personalDataTableMaps = new WeakMap();
|
|
52
|
+
const personalColumnsByEntity = new WeakMap();
|
|
52
53
|
function personalColumnNames(entity) {
|
|
53
|
-
|
|
54
|
+
const cached = personalColumnsByEntity.get(entity);
|
|
55
|
+
if (cached)
|
|
56
|
+
return cached;
|
|
57
|
+
const columns = new Set([...personalFieldNames(entity, false)].map(toSnakeCase));
|
|
58
|
+
personalColumnsByEntity.set(entity, columns);
|
|
59
|
+
return columns;
|
|
54
60
|
}
|
|
55
61
|
function buildPersonalDataTableMap(registry) {
|
|
56
62
|
const map = new Map();
|
|
@@ -73,21 +79,22 @@ function personalDataTableMap(registry) {
|
|
|
73
79
|
personalDataTableMaps.set(registry, built);
|
|
74
80
|
return built;
|
|
75
81
|
}
|
|
76
|
-
//
|
|
82
|
+
// The error reaches the anonymous HTTP caller, so table and column names stay out of
|
|
83
|
+
// message/details (they would map the DB schema); they travel in `cause`, which the
|
|
84
|
+
// serializer never sends to clients. The 4xx log line carries only status and code, so
|
|
85
|
+
// `cause` is not logged today; it is only available to in-process error handling.
|
|
77
86
|
export function publicIntakeRequiredError(origin, target, fields) {
|
|
78
87
|
return new AccessDeniedError({
|
|
79
|
-
message: `Anonymous root handler "${origin.rootHandler}" wrote personal
|
|
80
|
-
`${fields.map((f) => `"${f}"`).join(", ")} on "${target}"` +
|
|
88
|
+
message: `Anonymous root handler "${origin.rootHandler}" wrote personal data` +
|
|
81
89
|
(origin.viaJob ? ` via job "${origin.viaJob}"` : "") +
|
|
82
90
|
'. Declare access: { roles: [..., "anonymous"], personalData: "public-intake" } on ' +
|
|
83
91
|
`"${origin.rootHandler}" to allow anonymous callers to write personal data.`,
|
|
84
92
|
details: {
|
|
85
93
|
reason: FrameworkReasons.publicIntakeRequired,
|
|
86
94
|
rootHandler: origin.rootHandler,
|
|
87
|
-
target,
|
|
88
|
-
fields,
|
|
89
95
|
...(origin.viaJob !== undefined && { job: origin.viaJob }),
|
|
90
96
|
},
|
|
97
|
+
cause: new Error(`personal-data field(s) ${fields.map((f) => `"${f}"`).join(", ")} written on "${target}"`),
|
|
91
98
|
});
|
|
92
99
|
}
|
|
93
100
|
// Without an entity the table name is looked up in the registry map; tables outside it
|
|
@@ -17,6 +17,9 @@ export async function drainEventConsumers(stack, consumerNames, options) {
|
|
|
17
17
|
"multiStreamProjection is wired for this stack.");
|
|
18
18
|
}
|
|
19
19
|
const maxPasses = options?.maxPasses ?? DEFAULT_MAX_PASSES;
|
|
20
|
+
if (!Number.isInteger(maxPasses) || maxPasses < 1) {
|
|
21
|
+
throw new Error("drainEventConsumers: maxPasses must be a positive integer");
|
|
22
|
+
}
|
|
20
23
|
const target = await getEventsHighWaterMark(db);
|
|
21
24
|
for (let pass = 0; pass < maxPasses; pass++) {
|
|
22
25
|
await eventDispatcher.runOnce();
|
|
@@ -17,4 +17,6 @@ export type DrainJobsStack = {
|
|
|
17
17
|
readonly eventDispatcher?: EventDispatcher;
|
|
18
18
|
readonly jobRunner?: JobRunner;
|
|
19
19
|
};
|
|
20
|
-
export declare function drainJobs(stack: DrainJobsStack, tracker: JobFailureTracker
|
|
20
|
+
export declare function drainJobs(stack: DrainJobsStack, tracker: JobFailureTracker, options?: {
|
|
21
|
+
readonly timeoutMs?: number;
|
|
22
|
+
}): Promise<void>;
|
package/dist/stack/drain-jobs.js
CHANGED
|
@@ -12,6 +12,9 @@ const DRAIN_IDLE_POLL_MS = 25;
|
|
|
12
12
|
// enqueue that lands just after the first idle read still gets picked up
|
|
13
13
|
// by the second pass instead of racing straight past it.
|
|
14
14
|
const REQUIRED_IDLE_PASSES = 2;
|
|
15
|
+
// Below the 15s per-test budget so a stuck drain reports what is open rather
|
|
16
|
+
// than surfacing as a bare test timeout.
|
|
17
|
+
const DEFAULT_DRAIN_TIMEOUT_MS = 10_000;
|
|
15
18
|
function sleep(ms) {
|
|
16
19
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
17
20
|
}
|
|
@@ -47,6 +50,26 @@ export function createJobFailureTracker() {
|
|
|
47
50
|
},
|
|
48
51
|
};
|
|
49
52
|
}
|
|
53
|
+
// Diagnostic only: lets the deadline error name what was still open.
|
|
54
|
+
async function describeLaggingConsumers(db, consumerNames) {
|
|
55
|
+
// skip: no shared consumers wired — nothing to catch up on.
|
|
56
|
+
if (consumerNames.length === 0)
|
|
57
|
+
return [];
|
|
58
|
+
const target = await getEventsHighWaterMark(db);
|
|
59
|
+
const lagging = [];
|
|
60
|
+
for (const name of consumerNames) {
|
|
61
|
+
const state = await getConsumerState(db, name);
|
|
62
|
+
if (state && state.lastProcessedEventId < target) {
|
|
63
|
+
lagging.push(`${name} (status=${state.status}, lastProcessedEventId=${state.lastProcessedEventId}, target=${target})`);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
return lagging;
|
|
67
|
+
}
|
|
68
|
+
async function drainTimeoutError(db, consumerNames, timeoutMs, pending) {
|
|
69
|
+
const lagging = await describeLaggingConsumers(db, consumerNames);
|
|
70
|
+
return new Error(`drainJobs: not drained after ${timeoutMs}ms — pending jobs: ${pending}, ` +
|
|
71
|
+
`consumers behind: ${lagging.length === 0 ? "none" : lagging.join("; ")}`);
|
|
72
|
+
}
|
|
50
73
|
async function sharedConsumersCaughtUp(db, consumerNames) {
|
|
51
74
|
// skip: no shared consumers wired — nothing to catch up on.
|
|
52
75
|
if (consumerNames.length === 0)
|
|
@@ -66,8 +89,12 @@ async function sharedConsumersCaughtUp(db, consumerNames) {
|
|
|
66
89
|
}
|
|
67
90
|
return allCaughtUp;
|
|
68
91
|
}
|
|
69
|
-
|
|
92
|
+
// Fails a stuck drain with a diagnosis instead of a generic test-runner timeout.
|
|
93
|
+
// It only bounds the loop; it never replaces waiting for real work.
|
|
94
|
+
export async function drainJobs(stack, tracker, options) {
|
|
70
95
|
const { db, eventDispatcher, jobRunner } = stack;
|
|
96
|
+
const timeoutMs = options?.timeoutMs ?? DEFAULT_DRAIN_TIMEOUT_MS;
|
|
97
|
+
const deadline = Date.now() + timeoutMs;
|
|
71
98
|
if (!jobRunner) {
|
|
72
99
|
throw new Error("drainJobs: stack.jobRunner is undefined — pass `jobs: {...}` to setupTestStack and " +
|
|
73
100
|
"register at least one r.job(...) so a runner is built.");
|
|
@@ -84,6 +111,9 @@ export async function drainJobs(stack, tracker) {
|
|
|
84
111
|
const caughtUp = await sharedConsumersCaughtUp(db, sharedConsumerNames);
|
|
85
112
|
consecutiveIdlePasses = pending === 0 && caughtUp ? consecutiveIdlePasses + 1 : 0;
|
|
86
113
|
if (consecutiveIdlePasses < REQUIRED_IDLE_PASSES) {
|
|
114
|
+
if (Date.now() > deadline) {
|
|
115
|
+
throw await drainTimeoutError(db, sharedConsumerNames, timeoutMs, pending);
|
|
116
|
+
}
|
|
87
117
|
await Promise.race([tracker.waitForActivity(), sleep(DRAIN_IDLE_POLL_MS)]);
|
|
88
118
|
}
|
|
89
119
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cosmicdrift/kumiko-framework",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.336.0",
|
|
4
4
|
"description": "Framework core — engine, pipeline, API, DB, and every other bit that makes Kumiko go.",
|
|
5
5
|
"license": "BUSL-1.1",
|
|
6
6
|
"author": "Marc Frost <marc@cosmicdriftgamestudio.com>",
|
|
@@ -206,8 +206,8 @@
|
|
|
206
206
|
"./package.json": "./package.json"
|
|
207
207
|
},
|
|
208
208
|
"dependencies": {
|
|
209
|
-
"@cosmicdrift/kumiko-http": "0.
|
|
210
|
-
"@cosmicdrift/kumiko-types": "0.
|
|
209
|
+
"@cosmicdrift/kumiko-http": "0.336.0",
|
|
210
|
+
"@cosmicdrift/kumiko-types": "0.336.0",
|
|
211
211
|
"bullmq": "^5.76.7",
|
|
212
212
|
"bun-types": "^1.3.13",
|
|
213
213
|
"hono": "^4.13.1",
|
|
@@ -223,7 +223,7 @@
|
|
|
223
223
|
"zod": "^4.4.3"
|
|
224
224
|
},
|
|
225
225
|
"devDependencies": {
|
|
226
|
-
"@cosmicdrift/kumiko-dispatcher-live": "0.
|
|
226
|
+
"@cosmicdrift/kumiko-dispatcher-live": "0.336.0",
|
|
227
227
|
"bun-types": "^1.3.13",
|
|
228
228
|
"pino-pretty": "^13.1.3"
|
|
229
229
|
},
|
package/src/changes.json
CHANGED
|
@@ -1,4 +1,62 @@
|
|
|
1
1
|
[
|
|
2
|
+
{
|
|
3
|
+
"version": "0.336.0",
|
|
4
|
+
"type": "fix",
|
|
5
|
+
"title": "Boot gates with an explicit timeout reject start() naming the gate; gates without one warn after 60s",
|
|
6
|
+
"detail": "A `bootGate` job with an explicit `timeout` now rejects `start()` with a message naming the gate once that timeout elapses. Without `timeout` the gate keeps waiting and logs a warning naming it after 60s instead of blocking boot silently."
|
|
7
|
+
},
|
|
8
|
+
{
|
|
9
|
+
"version": "0.336.0",
|
|
10
|
+
"type": "fix",
|
|
11
|
+
"title": "Boot rejects unknown visibleWhen fields, bootGate with runOnBoot, and unknown money currency kinds",
|
|
12
|
+
"detail": "Boot now rejects three misconfigurations that previously failed silently: a dashboard screen-panel `visibleWhen.field` that is missing from the query's declared `outputSchema`, a job that combines `bootGate` with `runOnBoot` (it ran twice under one job id), and a money field whose `currency` source has an unknown `kind`. `auth-mfa:query:user-mfa:status` now declares its `outputSchema`."
|
|
13
|
+
},
|
|
14
|
+
{
|
|
15
|
+
"version": "0.336.0",
|
|
16
|
+
"type": "fix",
|
|
17
|
+
"title": "Dry-run env output keeps multi-line field descriptions inside the comment",
|
|
18
|
+
"detail": "`KUMIKO_DRY_RUN_ENV` pulumi and k8s output now folds a multi-line `.describe()` text onto one comment line, so the continuation can no longer escape the `#` and run as a shell command or break the YAML stub. The `unprocessable-error-details-reason` codemod now skips a shorthand `reason` whose variable is only used in `details`."
|
|
19
|
+
},
|
|
20
|
+
{
|
|
21
|
+
"version": "0.336.0",
|
|
22
|
+
"type": "fix",
|
|
23
|
+
"title": "waitForEvent ne-match ignores payloads without the referenced path",
|
|
24
|
+
"detail": "`waitForEvent` match atoms no longer match events that lack the referenced payload path, so `ne` no longer wakes a workflow on an unrelated event shape. Explicit `undefined` optionals on `r.nav()` entries are dropped from the client schema."
|
|
25
|
+
},
|
|
26
|
+
{
|
|
27
|
+
"version": "0.336.0",
|
|
28
|
+
"type": "fix",
|
|
29
|
+
"title": "r.nav() explicit undefined optionals no longer reach the client schema"
|
|
30
|
+
},
|
|
31
|
+
{
|
|
32
|
+
"version": "0.336.0",
|
|
33
|
+
"type": "fix",
|
|
34
|
+
"title": "public_intake_required 403 no longer discloses table and column names to anonymous callers",
|
|
35
|
+
"detail": "The `public_intake_required` 403 no longer puts the physical table name and personal-data column names into `message` or `details` (it keeps `reason`, `rootHandler` and `job`); they travel in the error `cause` for the server log. The event dispatcher counts idle-gated consumer turns in the new `kumiko_event_consumer_pass_skipped_total{reason=\"idle\"}` counter, since those turns open no `events.consumer.pass` span. Personal-data column sets for the anonymous write gate are cached per entity."
|
|
36
|
+
},
|
|
37
|
+
{
|
|
38
|
+
"version": "0.336.0",
|
|
39
|
+
"type": "improvement",
|
|
40
|
+
"title": "Idle-gated event consumer turns are counted in kumiko_event_consumer_pass_skipped_total"
|
|
41
|
+
},
|
|
42
|
+
{
|
|
43
|
+
"version": "0.336.0",
|
|
44
|
+
"type": "breaking",
|
|
45
|
+
"title": "/api/sse sends change signals without field values; anonymous connections only for declared entities",
|
|
46
|
+
"detail": "The SSE broadcast consumer no longer puts the event payload (changes, previous) on the tenant channel, because that channel fans out to every tenant member and to anonymous connections. Entity frames carry `{ id, aggregateType, eventType, version, createdAt }`. Anonymous connections get entity signals only for entities named by an anonymously callable query via the new `liveEntities` option; the query name alone grants nothing, because a signal carries the id of every row, including rows the query filters out. Frames without an entity are delivered only when `data.userId` matches the connected user. Boot fails when `liveEntities` names an unregistered entity.",
|
|
47
|
+
"migration": "Code that reads `data.payload` from SSE frames must load the data with a query after the signal instead. Public pages that update anonymously add `liveEntities: [\"<entity>\"]` to the anonymous query they refetch (for example a `page:current` query), otherwise the live update stays off for anonymous visitors. `createSseRoute(broker)` now needs a second argument, `{ anonymousLiveEntities: collectAnonymousLiveEntities(registry) }`."
|
|
48
|
+
},
|
|
49
|
+
{
|
|
50
|
+
"version": "0.336.0",
|
|
51
|
+
"type": "fix",
|
|
52
|
+
"title": "drainEventConsumers rejects an invalid maxPasses and drainJobs fails a stuck drain with a diagnosis",
|
|
53
|
+
"detail": "`drainEventConsumers` rejects a non-positive or non-integer `maxPasses` instead of returning without draining, and `drainJobs` accepts `timeoutMs` (default 10s) and fails a stuck drain with the pending job count and the consumers still behind. The stance report names which hint category (direct, user-owned, user-reference) a near-miss field resembles, and the `UnprocessableError` reason codemod no longer reports spread literals without a `reason` key and lists non-analyzable `details` arguments as unverified."
|
|
54
|
+
},
|
|
55
|
+
{
|
|
56
|
+
"version": "0.336.0",
|
|
57
|
+
"type": "improvement",
|
|
58
|
+
"title": "Codemod reports name the near-miss hint category and separate unverified sites from manual skips"
|
|
59
|
+
},
|
|
2
60
|
{
|
|
3
61
|
"version": "0.335.0",
|
|
4
62
|
"type": "fix",
|
|
@@ -440,6 +440,8 @@ function findTargetFiles(rootDir: string): string[] {
|
|
|
440
440
|
|
|
441
441
|
export type StanceClass = "direct" | "user-owned" | "user-reference" | "near-miss" | "unclassified";
|
|
442
442
|
|
|
443
|
+
export type HintCategory = "direct" | "user-owned" | "user-reference";
|
|
444
|
+
|
|
443
445
|
export type StanceSite = {
|
|
444
446
|
readonly line: number;
|
|
445
447
|
readonly field: string;
|
|
@@ -447,14 +449,23 @@ export type StanceSite = {
|
|
|
447
449
|
readonly callee: string;
|
|
448
450
|
readonly stance: StanceClass;
|
|
449
451
|
readonly hint: string | undefined;
|
|
452
|
+
readonly nearMissOf: HintCategory | undefined;
|
|
450
453
|
};
|
|
451
454
|
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
...
|
|
455
|
+
type CategorizedHint = readonly [hint: string, category: HintCategory];
|
|
456
|
+
|
|
457
|
+
const ALL_PII_NAME_HINTS: readonly CategorizedHint[] = [
|
|
458
|
+
...[...PII_DIRECT_NAME_HINTS].map((hint): CategorizedHint => [hint, "direct"]),
|
|
459
|
+
...[...PII_USER_OWNED_NAME_HINTS].map((hint): CategorizedHint => [hint, "user-owned"]),
|
|
460
|
+
...[...PII_USER_REFERENCE_NAME_HINTS].map((hint): CategorizedHint => [hint, "user-reference"]),
|
|
456
461
|
];
|
|
457
462
|
|
|
463
|
+
type FieldClassification = {
|
|
464
|
+
readonly stance: StanceClass;
|
|
465
|
+
readonly hint: string | undefined;
|
|
466
|
+
readonly nearMissOf: HintCategory | undefined;
|
|
467
|
+
};
|
|
468
|
+
|
|
458
469
|
const REPORT_STANCE_CALLEES = new Set(["createTextField", "createLongTextField"]);
|
|
459
470
|
|
|
460
471
|
// Mirrors guard-text-field-stance.ts's hasPersonalStance exactly.
|
|
@@ -539,11 +550,15 @@ function hintOccursAtBoundary(fieldLower: string, fieldOriginal: string, hint: s
|
|
|
539
550
|
}
|
|
540
551
|
}
|
|
541
552
|
|
|
542
|
-
function findLongestBoundaryHint(
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
553
|
+
function findLongestBoundaryHint(
|
|
554
|
+
fieldLower: string,
|
|
555
|
+
fieldOriginal: string,
|
|
556
|
+
): CategorizedHint | undefined {
|
|
557
|
+
let best: CategorizedHint | undefined;
|
|
558
|
+
for (const entry of ALL_PII_NAME_HINTS) {
|
|
559
|
+
const [hint] = entry;
|
|
560
|
+
if (best && hint.length <= best[0].length) continue;
|
|
561
|
+
if (hintOccursAtBoundary(fieldLower, fieldOriginal, hint)) best = entry;
|
|
547
562
|
}
|
|
548
563
|
return best;
|
|
549
564
|
}
|
|
@@ -564,29 +579,36 @@ function segmentAlignedSuffixes(fieldOriginal: string): string[] {
|
|
|
564
579
|
// (e.g. "...UserId" of "assigneeUserId") otherwise slips through undetected.
|
|
565
580
|
// The suffix must be the tail of the hint: plain `includes` matched "number"
|
|
566
581
|
// inside "phonenumber" for "orderNumber".
|
|
567
|
-
function findShortestHintContainingSuffix(fieldOriginal: string):
|
|
568
|
-
let best:
|
|
582
|
+
function findShortestHintContainingSuffix(fieldOriginal: string): CategorizedHint | undefined {
|
|
583
|
+
let best: CategorizedHint | undefined;
|
|
569
584
|
for (const suffix of segmentAlignedSuffixes(fieldOriginal)) {
|
|
570
585
|
if (suffix.length < 5) continue;
|
|
571
|
-
for (const
|
|
586
|
+
for (const entry of ALL_PII_NAME_HINTS) {
|
|
587
|
+
const [hint] = entry;
|
|
572
588
|
if (hint === suffix || !hint.endsWith(suffix)) continue;
|
|
573
|
-
if (!best || hint.length < best.length) best =
|
|
589
|
+
if (!best || hint.length < best[0].length) best = entry;
|
|
574
590
|
}
|
|
575
591
|
}
|
|
576
592
|
return best;
|
|
577
593
|
}
|
|
578
594
|
|
|
579
|
-
function classifyFieldStance(field: string):
|
|
595
|
+
function classifyFieldStance(field: string): FieldClassification {
|
|
580
596
|
const fieldLower = field.replaceAll("_", "").toLowerCase();
|
|
581
|
-
if (PII_DIRECT_NAME_HINTS.has(fieldLower))
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
597
|
+
if (PII_DIRECT_NAME_HINTS.has(fieldLower)) {
|
|
598
|
+
return { stance: "direct", hint: fieldLower, nearMissOf: undefined };
|
|
599
|
+
}
|
|
600
|
+
if (PII_USER_OWNED_NAME_HINTS.has(fieldLower)) {
|
|
601
|
+
return { stance: "user-owned", hint: fieldLower, nearMissOf: undefined };
|
|
602
|
+
}
|
|
603
|
+
if (PII_USER_REFERENCE_NAME_HINTS.has(fieldLower)) {
|
|
604
|
+
return { stance: "user-reference", hint: fieldLower, nearMissOf: undefined };
|
|
605
|
+
}
|
|
606
|
+
const nearMiss =
|
|
607
|
+
findLongestBoundaryHint(field.toLowerCase(), field) ?? findShortestHintContainingSuffix(field);
|
|
608
|
+
if (nearMiss) {
|
|
609
|
+
return { stance: "near-miss", hint: nearMiss[0], nearMissOf: nearMiss[1] };
|
|
610
|
+
}
|
|
611
|
+
return { stance: "unclassified", hint: undefined, nearMissOf: undefined };
|
|
590
612
|
}
|
|
591
613
|
|
|
592
614
|
function createReportStanceProject(): Project {
|
|
@@ -627,9 +649,9 @@ function collectStanceSites(sourceFile: SourceFile): StanceSite[] {
|
|
|
627
649
|
// A call with no enclosing field (e.g. a bare createTextField() at top
|
|
628
650
|
// level) has no real name to classify — `createTextField(...)` would
|
|
629
651
|
// otherwise false-positive as a near-miss on its own "Text".
|
|
630
|
-
const { stance, hint } = enclosingField
|
|
652
|
+
const { stance, hint, nearMissOf }: FieldClassification = enclosingField
|
|
631
653
|
? classifyFieldStance(enclosingField)
|
|
632
|
-
: { stance: "unclassified"
|
|
654
|
+
: { stance: "unclassified", hint: undefined, nearMissOf: undefined };
|
|
633
655
|
|
|
634
656
|
sites.push({
|
|
635
657
|
line: call.getStartLineNumber(),
|
|
@@ -638,6 +660,7 @@ function collectStanceSites(sourceFile: SourceFile): StanceSite[] {
|
|
|
638
660
|
callee,
|
|
639
661
|
stance,
|
|
640
662
|
hint,
|
|
663
|
+
nearMissOf,
|
|
641
664
|
});
|
|
642
665
|
}
|
|
643
666
|
return sites;
|
|
@@ -656,6 +679,13 @@ function findReportStanceFiles(rootDir: string): string[] {
|
|
|
656
679
|
return files.sort();
|
|
657
680
|
}
|
|
658
681
|
|
|
682
|
+
function formatHintSuffix(site: StanceSite): string {
|
|
683
|
+
if (!site.hint) return "";
|
|
684
|
+
return site.nearMissOf
|
|
685
|
+
? ` (hint: ${site.hint}, near ${site.nearMissOf})`
|
|
686
|
+
: ` (hint: ${site.hint})`;
|
|
687
|
+
}
|
|
688
|
+
|
|
659
689
|
function reportStance(rootDir: string): void {
|
|
660
690
|
console.log(
|
|
661
691
|
`Scanning every .ts/.tsx under ${rootDir} except node_modules/dist/build/*.d.ts — guard-text-field-stance scans packages/*/src/** only, so counts can differ outside that scope.`,
|
|
@@ -680,7 +710,7 @@ function reportStance(rootDir: string): void {
|
|
|
680
710
|
totals[site.stance]++;
|
|
681
711
|
total++;
|
|
682
712
|
const entity = site.entity ?? "<unresolved>";
|
|
683
|
-
const hintSuffix =
|
|
713
|
+
const hintSuffix = formatHintSuffix(site);
|
|
684
714
|
console.log(
|
|
685
715
|
` ${site.line} ${site.field} entity=${entity} ${site.callee} ${site.stance}${hintSuffix}`,
|
|
686
716
|
);
|