dsh-observe 0.2.11 → 0.2.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -0
- package/{README.es.md → README-es.md} +21 -8
- package/{README.hi.md → README-hi.md} +21 -8
- package/{README.pt.md → README-pt.md} +21 -8
- package/{README.zh.md → README-zh.md} +21 -8
- package/README.md +20 -8
- package/lib/index.js +93 -39
- package/lib/typert.host.js +18 -15
- package/lib/types/collector.d.ts.map +1 -1
- package/lib/types/collector.js +11 -12
- package/lib/types/collector.js.map +1 -1
- package/lib/types/index.d.ts.map +1 -1
- package/lib/types/index.js +83 -42
- package/lib/types/index.js.map +1 -1
- package/lib/types/pipeline.d.ts +2 -2
- package/lib/types/pipeline.d.ts.map +1 -1
- package/lib/types/spool.d.ts +37 -1
- package/lib/types/spool.d.ts.map +1 -1
- package/lib/types/spool.js +25 -0
- package/lib/types/spool.js.map +1 -1
- package/lib/types/typert.host.d.ts +30 -3
- package/lib/types/typert.host.d.ts.map +1 -1
- package/lib/types/version.d.ts +1 -1
- package/lib/types/version.js +1 -1
- package/lib/types/wire.d.ts +60 -6
- package/lib/types/wire.d.ts.map +1 -1
- package/lib/types/wire.js +13 -15
- package/lib/types/wire.js.map +1 -1
- package/package.json +23 -20
- package/src/collector.ts +11 -11
- package/src/index.ts +98 -45
- package/src/pipeline.ts +2 -2
- package/src/spool.ts +43 -1
- package/src/version.ts +1 -1
- package/src/wire.ts +14 -15
package/src/index.ts
CHANGED
|
@@ -15,10 +15,11 @@
|
|
|
15
15
|
*/
|
|
16
16
|
|
|
17
17
|
import type { Context } from '@deepseek-ai/cordis'
|
|
18
|
+
import type { Domain } from '@deepseek-ai/dsh-storage-domain'
|
|
18
19
|
import type { Session, SessionEvent } from '@deepseek-ai/dsh-session'
|
|
19
20
|
import { Config, resolveConfig } from './config.ts'
|
|
20
21
|
import type { ExportRecord, MetricRecord, SpanRecord } from './model.ts'
|
|
21
|
-
import { observeDomainSpec, openSpool } from './spool.ts'
|
|
22
|
+
import { observeDomainSpec, openSpool, DroppingSpool, type ObserveDomainSpec, type SpoolSurface } from './spool.ts'
|
|
22
23
|
import { OtlpSink, LangfuseSink } from './sinks.ts'
|
|
23
24
|
import { Pipeline } from './pipeline.ts'
|
|
24
25
|
import { Collector } from './collector.ts'
|
|
@@ -58,25 +59,68 @@ export async function apply(ctx: Context, config: Config = {}): Promise<void> {
|
|
|
58
59
|
const isEnabled = () => enabled
|
|
59
60
|
|
|
60
61
|
const otlpSink = resolved.otlp === undefined ? undefined : new OtlpSink(resolved.otlp, logger)
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Structural face of the optional (experimental) `inspector` service. Read
|
|
65
|
+
* structurally on purpose: the service is experimental and must never be
|
|
66
|
+
* injected, and it is an extra outlet beside the OTLP sink, never the only
|
|
67
|
+
* one.
|
|
68
|
+
*/
|
|
69
|
+
interface InspectorLike {
|
|
70
|
+
publish(topic: string, payload: unknown, monotonicMs?: number): void
|
|
71
|
+
}
|
|
72
|
+
const inspector = ctx.get('inspector') as InspectorLike | undefined
|
|
73
|
+
const publishInspector = (topic: string, payload: unknown): void => {
|
|
74
|
+
if (inspector === undefined || typeof inspector.publish !== 'function') return
|
|
75
|
+
try {
|
|
76
|
+
inspector.publish(topic, payload)
|
|
77
|
+
} catch {
|
|
78
|
+
// Diagnostics must never break the export hot path.
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
61
82
|
const recordMetric = (metric: MetricRecord): void => {
|
|
62
|
-
if (isEnabled())
|
|
83
|
+
if (!isEnabled()) return
|
|
84
|
+
otlpSink?.recordMetric(metric)
|
|
85
|
+
publishInspector('observe.metric', metric)
|
|
63
86
|
}
|
|
64
87
|
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
88
|
+
// The durable offline buffer. Opening the domain can fail — storageDomain is
|
|
89
|
+
// single-open per name, so a hot reload that races the previous instance's
|
|
90
|
+
// close fails the second open — and failing the whole mount over a
|
|
91
|
+
// best-effort buffer would take the exporter down with it. A failed open
|
|
92
|
+
// degrades to a dropping spool with one warning; a disposal that lands while
|
|
93
|
+
// the open is in flight closes the freshly opened handle immediately, or the
|
|
94
|
+
// single-open reservation leaks into the next mount (A02).
|
|
95
|
+
let domain: Domain<ObserveDomainSpec> | undefined
|
|
96
|
+
let spool: SpoolSurface = new DroppingSpool()
|
|
97
|
+
try {
|
|
98
|
+
const opened = await ctx.storageDomain.open(observeDomainSpec)
|
|
99
|
+
if (ctx.fiber.uid === null) {
|
|
100
|
+
await opened.close()
|
|
101
|
+
return
|
|
102
|
+
}
|
|
103
|
+
domain = opened
|
|
104
|
+
spool = openSpool(
|
|
105
|
+
opened,
|
|
106
|
+
resolved.batch.maxBufferRecords,
|
|
107
|
+
count => recordMetric({
|
|
108
|
+
name: 'observe.dropped',
|
|
109
|
+
kind: 'counter',
|
|
110
|
+
unit: 'records',
|
|
111
|
+
value: count,
|
|
112
|
+
attributes: { reason: 'buffer_overflow' },
|
|
113
|
+
}),
|
|
114
|
+
count => {
|
|
115
|
+
logger.warn(`spool: ${count} stored record(s) failed validation and were dropped`)
|
|
116
|
+
},
|
|
117
|
+
)
|
|
118
|
+
} catch (error) {
|
|
119
|
+
logger.warn(
|
|
120
|
+
`offline buffer unavailable (${error instanceof Error ? error.message : String(error)}); `
|
|
121
|
+
+ 'continuing without the durable spool — spilled records are dropped instead of persisted',
|
|
122
|
+
)
|
|
123
|
+
}
|
|
80
124
|
|
|
81
125
|
const pipelines: Pipeline[] = []
|
|
82
126
|
if (otlpSink !== undefined) {
|
|
@@ -117,9 +161,13 @@ export async function apply(ctx: Context, config: Config = {}): Promise<void> {
|
|
|
117
161
|
logger,
|
|
118
162
|
)
|
|
119
163
|
|
|
120
|
-
// One effect owns every timer
|
|
121
|
-
//
|
|
122
|
-
//
|
|
164
|
+
// One effect owns every timer, the event subscriptions, and the teardown
|
|
165
|
+
// order: stop the timers and unsubscribe first (so no event reaches the
|
|
166
|
+
// collector after disposal), final-flush the pipelines (spilling failures
|
|
167
|
+
// to the durable buffer), then close the domain. Keeping the listeners
|
|
168
|
+
// inside the effect is what makes a mid-apply disposal safe: a listener
|
|
169
|
+
// registered outside it would throw INACTIVE_EFFECT and take the remaining
|
|
170
|
+
// registrations with it (A02).
|
|
123
171
|
ctx.effect(() => {
|
|
124
172
|
const timers: ReturnType<typeof setInterval>[] = [
|
|
125
173
|
setInterval(() => {
|
|
@@ -132,35 +180,40 @@ export async function apply(ctx: Context, config: Config = {}): Promise<void> {
|
|
|
132
180
|
? []
|
|
133
181
|
: [setInterval(() => { void otlpSink.flushMetrics() }, resolved.batch.flushIntervalMs)]),
|
|
134
182
|
]
|
|
135
|
-
return async () => {
|
|
136
|
-
for (const timer of timers) clearInterval(timer)
|
|
137
|
-
await Promise.all(pipelines.map(pipeline => pipeline.dispose()))
|
|
138
|
-
await domain.close()
|
|
139
|
-
}
|
|
140
|
-
})
|
|
141
183
|
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
184
|
+
// Consumer — consumes the harness session/event stream (and the optional
|
|
185
|
+
// tokenMeter) and feeds the export pipeline.
|
|
186
|
+
const offEvent = ctx.on('session/event', (session: Session, event: SessionEvent) => {
|
|
187
|
+
try {
|
|
188
|
+
collector.handleEvent(session, event)
|
|
189
|
+
} catch (error) {
|
|
190
|
+
logger.warn(`session "${session.id}": event handling failed: ${error instanceof Error ? error.message : String(error)}`)
|
|
191
|
+
}
|
|
192
|
+
})
|
|
150
193
|
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
194
|
+
// Best-effort kick: the session-flush durability checkpoint must not wait
|
|
195
|
+
// on a remote observability backend, so exports run in the background.
|
|
196
|
+
const offFlush = ctx.on('session/flush', () => {
|
|
197
|
+
for (const pipeline of pipelines) pipeline.kick()
|
|
198
|
+
})
|
|
156
199
|
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
200
|
+
const offDisposed = ctx.on('session/disposed', (session: Session) => {
|
|
201
|
+
try {
|
|
202
|
+
collector.handleSessionDisposed(session)
|
|
203
|
+
} catch (error) {
|
|
204
|
+
logger.warn(`session "${session.id}": disposal handling failed: ${error instanceof Error ? error.message : String(error)}`)
|
|
205
|
+
}
|
|
206
|
+
for (const pipeline of pipelines) pipeline.kick()
|
|
207
|
+
})
|
|
208
|
+
|
|
209
|
+
return async () => {
|
|
210
|
+
for (const timer of timers) clearInterval(timer)
|
|
211
|
+
offEvent()
|
|
212
|
+
offFlush()
|
|
213
|
+
offDisposed()
|
|
214
|
+
await Promise.all(pipelines.map(pipeline => pipeline.dispose()))
|
|
215
|
+
await domain?.close()
|
|
162
216
|
}
|
|
163
|
-
for (const pipeline of pipelines) pipeline.kick()
|
|
164
217
|
})
|
|
165
218
|
|
|
166
219
|
if (resolved.remote) {
|
package/src/pipeline.ts
CHANGED
|
@@ -10,13 +10,13 @@ import type { ResolvedBatch, ResolvedRetry } from './config.ts'
|
|
|
10
10
|
import type { BackendName, ExportRecord, MetricRecord } from './model.ts'
|
|
11
11
|
import { backoffDelayMs } from './backoff.ts'
|
|
12
12
|
import type { ObserveLogger, SpanSink } from './sinks.ts'
|
|
13
|
-
import type {
|
|
13
|
+
import type { SpoolSurface } from './spool.ts'
|
|
14
14
|
|
|
15
15
|
/** Everything one pipeline needs. */
|
|
16
16
|
export interface PipelineOptions {
|
|
17
17
|
name: BackendName
|
|
18
18
|
sink: SpanSink
|
|
19
|
-
spool:
|
|
19
|
+
spool: SpoolSurface
|
|
20
20
|
batch: ResolvedBatch
|
|
21
21
|
retry: ResolvedRetry
|
|
22
22
|
logger: ObserveLogger
|
package/src/spool.ts
CHANGED
|
@@ -55,11 +55,23 @@ export interface SpoolCounts {
|
|
|
55
55
|
records: number
|
|
56
56
|
}
|
|
57
57
|
|
|
58
|
+
/**
|
|
59
|
+
* The offline-buffer surface the delivery pipeline depends on. Declared
|
|
60
|
+
* structurally so a mount without durable storage can substitute
|
|
61
|
+
* {@link DroppingSpool} instead of pretending to hold the real spool.
|
|
62
|
+
*/
|
|
63
|
+
export interface SpoolSurface {
|
|
64
|
+
push(backend: BackendName, records: ExportRecord[]): Promise<void>
|
|
65
|
+
peek(backend: BackendName, maxRecords: number): { batches: SpoolBatch[]; records: ExportRecord[] }
|
|
66
|
+
remove(batches: readonly SpoolBatch[]): Promise<void>
|
|
67
|
+
counts(): Record<BackendName, SpoolCounts>
|
|
68
|
+
}
|
|
69
|
+
|
|
58
70
|
/**
|
|
59
71
|
* The offline buffer over one storage-domain table. The domain itself is
|
|
60
72
|
* opened and closed by the runtime; the spool only owns the table handle.
|
|
61
73
|
*/
|
|
62
|
-
export class Spool {
|
|
74
|
+
export class Spool implements SpoolSurface {
|
|
63
75
|
private nextSeq = 0
|
|
64
76
|
private bufferedRecords = 0
|
|
65
77
|
|
|
@@ -194,3 +206,33 @@ export function openSpool(
|
|
|
194
206
|
): Spool {
|
|
195
207
|
return new Spool(domain.table('spool'), maxBufferRecords, onDropped, onInvalid)
|
|
196
208
|
}
|
|
209
|
+
|
|
210
|
+
/**
|
|
211
|
+
* The offline buffer's degraded stand-in, used when the storage domain could
|
|
212
|
+
* not be opened at mount (storageDomain is single-open per name, so a hot
|
|
213
|
+
* reload racing the previous instance's close fails the second open).
|
|
214
|
+
* Nothing is persisted and drains are no-ops; the runtime reports the
|
|
215
|
+
* degradation once at mount and keeps exporting in memory.
|
|
216
|
+
*/
|
|
217
|
+
export class DroppingSpool implements SpoolSurface {
|
|
218
|
+
/** Records that would have been spilled into the durable buffer. */
|
|
219
|
+
dropped = 0
|
|
220
|
+
|
|
221
|
+
/** Count the records the durable buffer would have held. */
|
|
222
|
+
async push(_backend: BackendName, records: ExportRecord[]): Promise<void> {
|
|
223
|
+
this.dropped += records.length
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/** Nothing is durable, so a drain always finds an empty buffer. */
|
|
227
|
+
peek(): { batches: SpoolBatch[]; records: ExportRecord[] } {
|
|
228
|
+
return { batches: [], records: [] }
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
/** No stored batches exist to delete. */
|
|
232
|
+
async remove(): Promise<void> {}
|
|
233
|
+
|
|
234
|
+
/** Occupancy of a buffer that holds nothing. */
|
|
235
|
+
counts(): Record<BackendName, SpoolCounts> {
|
|
236
|
+
return { otlp: { batches: 0, records: 0 }, langfuse: { batches: 0, records: 0 } }
|
|
237
|
+
}
|
|
238
|
+
}
|
package/src/version.ts
CHANGED
package/src/wire.ts
CHANGED
|
@@ -49,6 +49,17 @@ const SET_ENABLED_ARGS_CODEC = z.object({
|
|
|
49
49
|
enabled: z.boolean(),
|
|
50
50
|
})
|
|
51
51
|
|
|
52
|
+
/**
|
|
53
|
+
* Strict wire codec carrying BOTH published and checkout faces: the `schema`
|
|
54
|
+
* field feeds the npm-published 0.1.5-rc.2 line, `create` feeds the checkout
|
|
55
|
+
* 0.1.6-alpha.1+ line (the loader materializes the schema lazily on first
|
|
56
|
+
* use). Built through a variable, so neither typecheck ruler flags the other
|
|
57
|
+
* face's field as excess.
|
|
58
|
+
*/
|
|
59
|
+
function strictWire<T>(typeSymbol: string, schema: T) {
|
|
60
|
+
return Object.freeze({ ...{ mode: 'strict' as const, typeSymbol, schema }, create: () => schema })
|
|
61
|
+
}
|
|
62
|
+
|
|
52
63
|
/** The `observe/status` invocation descriptor: the read-only status snapshot. */
|
|
53
64
|
export const OBSERVE_STATUS_DESCRIPTOR = Object.freeze({
|
|
54
65
|
id: 'dsh-observe#observe/status',
|
|
@@ -57,11 +68,7 @@ export const OBSERVE_STATUS_DESCRIPTOR = Object.freeze({
|
|
|
57
68
|
method: 'status',
|
|
58
69
|
invocation: Object.freeze({ kind: 'direct' }),
|
|
59
70
|
parameters: Object.freeze([]),
|
|
60
|
-
result:
|
|
61
|
-
mode: 'strict',
|
|
62
|
-
typeSymbol: 'dsh-observe/types#ObserveStatus',
|
|
63
|
-
schema: OBSERVE_STATUS_SCHEMA,
|
|
64
|
-
}),
|
|
71
|
+
result: strictWire('dsh-observe/types#ObserveStatus', OBSERVE_STATUS_SCHEMA),
|
|
65
72
|
sourceLocation: Object.freeze({ file: 'src/wire.ts', line: 1, column: 1 }),
|
|
66
73
|
} as const) satisfies InvocationDescriptor
|
|
67
74
|
|
|
@@ -76,17 +83,9 @@ export const OBSERVE_SET_ENABLED_DESCRIPTOR = Object.freeze({
|
|
|
76
83
|
name: 'request',
|
|
77
84
|
wire: 'request',
|
|
78
85
|
source: 'json',
|
|
79
|
-
codec:
|
|
80
|
-
mode: 'strict',
|
|
81
|
-
typeSymbol: 'dsh-observe/types#ObserveSetEnabledArgs',
|
|
82
|
-
schema: SET_ENABLED_ARGS_CODEC,
|
|
83
|
-
}),
|
|
86
|
+
codec: strictWire('dsh-observe/types#ObserveSetEnabledArgs', SET_ENABLED_ARGS_CODEC),
|
|
84
87
|
} satisfies InvocationDescriptor['parameters'][number])]),
|
|
85
|
-
result:
|
|
86
|
-
mode: 'strict',
|
|
87
|
-
typeSymbol: 'dsh-observe/types#ObserveSetEnabledResult',
|
|
88
|
-
schema: OBSERVE_SET_ENABLED_RESULT_SCHEMA,
|
|
89
|
-
}),
|
|
88
|
+
result: strictWire('dsh-observe/types#ObserveSetEnabledResult', OBSERVE_SET_ENABLED_RESULT_SCHEMA),
|
|
90
89
|
sourceLocation: Object.freeze({ file: 'src/wire.ts', line: 1, column: 1 }),
|
|
91
90
|
} as const) satisfies InvocationDescriptor
|
|
92
91
|
|