dsh-observe 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -0
- package/LICENSE +201 -0
- package/README.es.md +198 -0
- package/README.hi.md +198 -0
- package/README.md +198 -0
- package/README.pt.md +198 -0
- package/README.zh.md +198 -0
- package/THIRD_PARTY_NOTICES.md +21 -0
- package/cordis.patch.yml +86 -0
- package/lib/index.js +1947 -0
- package/lib/typert.host.js +109 -0
- package/lib/types/backoff.d.ts +17 -0
- package/lib/types/backoff.d.ts.map +1 -0
- package/lib/types/backoff.js +17 -0
- package/lib/types/backoff.js.map +1 -0
- package/lib/types/collector.d.ts +83 -0
- package/lib/types/collector.d.ts.map +1 -0
- package/lib/types/collector.js +506 -0
- package/lib/types/collector.js.map +1 -0
- package/lib/types/config.d.ts +286 -0
- package/lib/types/config.d.ts.map +1 -0
- package/lib/types/config.js +312 -0
- package/lib/types/config.js.map +1 -0
- package/lib/types/ids.d.ts +43 -0
- package/lib/types/ids.d.ts.map +1 -0
- package/lib/types/ids.js +63 -0
- package/lib/types/ids.js.map +1 -0
- package/lib/types/index.d.ts +35 -0
- package/lib/types/index.d.ts.map +1 -0
- package/lib/types/index.js +162 -0
- package/lib/types/index.js.map +1 -0
- package/lib/types/model.d.ts +89 -0
- package/lib/types/model.d.ts.map +1 -0
- package/lib/types/model.js +25 -0
- package/lib/types/model.js.map +1 -0
- package/lib/types/pipeline.d.ts +71 -0
- package/lib/types/pipeline.d.ts.map +1 -0
- package/lib/types/pipeline.js +193 -0
- package/lib/types/pipeline.js.map +1 -0
- package/lib/types/pricing.d.ts +34 -0
- package/lib/types/pricing.d.ts.map +1 -0
- package/lib/types/pricing.js +45 -0
- package/lib/types/pricing.js.map +1 -0
- package/lib/types/project.d.ts +26 -0
- package/lib/types/project.d.ts.map +1 -0
- package/lib/types/project.js +52 -0
- package/lib/types/project.js.map +1 -0
- package/lib/types/remote.d.ts +28 -0
- package/lib/types/remote.d.ts.map +1 -0
- package/lib/types/remote.js +45 -0
- package/lib/types/remote.js.map +1 -0
- package/lib/types/sanitize.d.ts +78 -0
- package/lib/types/sanitize.d.ts.map +1 -0
- package/lib/types/sanitize.js +132 -0
- package/lib/types/sanitize.js.map +1 -0
- package/lib/types/sinks.d.ts +84 -0
- package/lib/types/sinks.d.ts.map +1 -0
- package/lib/types/sinks.js +434 -0
- package/lib/types/sinks.js.map +1 -0
- package/lib/types/spool.d.ts +113 -0
- package/lib/types/spool.d.ts.map +1 -0
- package/lib/types/spool.js +167 -0
- package/lib/types/spool.js.map +1 -0
- package/lib/types/typert.host.d.ts +93 -0
- package/lib/types/typert.host.d.ts.map +1 -0
- package/lib/types/typert.host.js +23 -0
- package/lib/types/typert.host.js.map +1 -0
- package/lib/types/version.d.ts +10 -0
- package/lib/types/version.d.ts.map +1 -0
- package/lib/types/version.js +10 -0
- package/lib/types/version.js.map +1 -0
- package/lib/types/wire.d.ts +206 -0
- package/lib/types/wire.d.ts.map +1 -0
- package/lib/types/wire.js +71 -0
- package/lib/types/wire.js.map +1 -0
- package/package.json +147 -0
- package/src/backoff.ts +20 -0
- package/src/collector.ts +561 -0
- package/src/config.ts +604 -0
- package/src/ids.ts +68 -0
- package/src/index.ts +186 -0
- package/src/model.ts +100 -0
- package/src/pipeline.ts +205 -0
- package/src/pricing.ts +55 -0
- package/src/project.ts +55 -0
- package/src/remote.ts +56 -0
- package/src/sanitize.ts +132 -0
- package/src/sinks.ts +471 -0
- package/src/spool.ts +196 -0
- package/src/typert.host.ts +24 -0
- package/src/version.ts +10 -0
- package/src/wire.ts +97 -0
package/src/spool.ts
ADDED
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The bounded offline buffer: a storage-domain table holding sanitized
|
|
3
|
+
* export batches that could not be delivered. Batches are written when the
|
|
4
|
+
* in-memory queue overflows or after retries are exhausted, and drained on
|
|
5
|
+
* a timer; the bound is a total record count, and pushing past it drops the
|
|
6
|
+
* OLDEST records (reported through the drop callback so the OTLP backend
|
|
7
|
+
* can count them). The buffer survives restarts: the domain reloads from
|
|
8
|
+
* the host's storage backend.
|
|
9
|
+
* @module dsh-observe/spool
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import type { Domain, KvTable } from '@deepseek-ai/dsh-storage-domain'
|
|
13
|
+
import { defineDomain } from '@deepseek-ai/dsh-storage-domain'
|
|
14
|
+
import { z } from 'zod'
|
|
15
|
+
import type { BackendName, ExportRecord } from './model.ts'
|
|
16
|
+
import { isExportRecord } from './model.ts'
|
|
17
|
+
|
|
18
|
+
/** One stored batch: creation time plus sanitized export records. */
|
|
19
|
+
const spoolBatchSchema = z.object({
|
|
20
|
+
createdAt: z.number().int().nonnegative(),
|
|
21
|
+
records: z.array(z.unknown()),
|
|
22
|
+
})
|
|
23
|
+
|
|
24
|
+
/** The durable record type validated by the domain on load. */
|
|
25
|
+
export type SpoolBatchValue = z.infer<typeof spoolBatchSchema>
|
|
26
|
+
|
|
27
|
+
/** The dsh-observe storage domain declaration. */
|
|
28
|
+
export const observeDomainSpec = defineDomain({
|
|
29
|
+
name: 'observe',
|
|
30
|
+
version: 1,
|
|
31
|
+
tables: {
|
|
32
|
+
spool: { valueSchema: spoolBatchSchema },
|
|
33
|
+
},
|
|
34
|
+
})
|
|
35
|
+
|
|
36
|
+
/** Domain spec type for the spool table. */
|
|
37
|
+
export type ObserveDomainSpec = typeof observeDomainSpec
|
|
38
|
+
|
|
39
|
+
/** One batch read back from the spool, validated record-wise. */
|
|
40
|
+
export interface SpoolBatch {
|
|
41
|
+
backend: BackendName
|
|
42
|
+
key: string
|
|
43
|
+
createdAt: number
|
|
44
|
+
records: ExportRecord[]
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** A monotonic, zero-padded key for lexicographically ordered batches. */
|
|
48
|
+
function batchKey(backend: BackendName, seq: number): string {
|
|
49
|
+
return `${backend}:${String(seq).padStart(16, '0')}`
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** Per-backend occupancy snapshot for status surfaces. */
|
|
53
|
+
export interface SpoolCounts {
|
|
54
|
+
batches: number
|
|
55
|
+
records: number
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* The offline buffer over one storage-domain table. The domain itself is
|
|
60
|
+
* opened and closed by the runtime; the spool only owns the table handle.
|
|
61
|
+
*/
|
|
62
|
+
export class Spool {
|
|
63
|
+
private nextSeq = 0
|
|
64
|
+
private bufferedRecords = 0
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* @param table - the domain `spool` table handle.
|
|
68
|
+
* @param maxBufferRecords - total record bound across all backends.
|
|
69
|
+
* @param onDropped - receives the count of records dropped by bound eviction.
|
|
70
|
+
* @param onInvalid - receives the count of stored records rejected by validation.
|
|
71
|
+
*/
|
|
72
|
+
constructor(
|
|
73
|
+
private readonly table: KvTable<string, SpoolBatchValue>,
|
|
74
|
+
private readonly maxBufferRecords: number,
|
|
75
|
+
private readonly onDropped: (count: number) => void,
|
|
76
|
+
private readonly onInvalid: (count: number) => void,
|
|
77
|
+
) {
|
|
78
|
+
for (const [key, value] of this.table.entries()) {
|
|
79
|
+
const match = /^[a-z]+:(\d+)$/u.exec(key)
|
|
80
|
+
if (match !== null && match[1] !== undefined) {
|
|
81
|
+
this.nextSeq = Math.max(this.nextSeq, Number(match[1]) + 1)
|
|
82
|
+
}
|
|
83
|
+
this.bufferedRecords += value.records.length
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** All batches in chronological write order (createdAt, then key). */
|
|
88
|
+
private ordered(): [string, SpoolBatchValue][] {
|
|
89
|
+
return [...this.table.entries()].sort((left, right) =>
|
|
90
|
+
left[1].createdAt - right[1].createdAt || left[0].localeCompare(right[0]))
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Spill one batch into the buffer, enforcing the record bound: when the
|
|
95
|
+
* batch alone exceeds the bound only its newest records are kept, and
|
|
96
|
+
* otherwise the oldest stored batches are evicted until it fits. Drops
|
|
97
|
+
* are reported through {@link onDropped}.
|
|
98
|
+
* @param backend - the owning backend.
|
|
99
|
+
* @param records - sanitized records to persist.
|
|
100
|
+
* @returns resolution after the write and any evictions.
|
|
101
|
+
*/
|
|
102
|
+
async push(backend: BackendName, records: ExportRecord[]): Promise<void> {
|
|
103
|
+
if (records.length === 0) return
|
|
104
|
+
let kept = records
|
|
105
|
+
let dropped = 0
|
|
106
|
+
if (kept.length > this.maxBufferRecords) {
|
|
107
|
+
dropped += kept.length - this.maxBufferRecords
|
|
108
|
+
kept = kept.slice(kept.length - this.maxBufferRecords)
|
|
109
|
+
}
|
|
110
|
+
while (this.bufferedRecords + kept.length > this.maxBufferRecords) {
|
|
111
|
+
const oldest = this.ordered()[0]
|
|
112
|
+
if (oldest === undefined) break
|
|
113
|
+
await this.table.delete(oldest[0])
|
|
114
|
+
this.bufferedRecords -= oldest[1].records.length
|
|
115
|
+
dropped += oldest[1].records.length
|
|
116
|
+
}
|
|
117
|
+
const key = batchKey(backend, this.nextSeq)
|
|
118
|
+
this.nextSeq += 1
|
|
119
|
+
await this.table.put(key, { createdAt: Date.now(), records: kept })
|
|
120
|
+
this.bufferedRecords += kept.length
|
|
121
|
+
if (dropped > 0) this.onDropped(dropped)
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* Read the oldest batches of one backend, up to a record budget. Stored
|
|
126
|
+
* records that fail {@link isExportRecord} are dropped from the returned
|
|
127
|
+
* set (and reported through {@link onInvalid}) but stay on disk until a
|
|
128
|
+
* later `remove` deletes their batch.
|
|
129
|
+
* @param backend - the backend to read.
|
|
130
|
+
* @param maxRecords - record budget for one drain cycle.
|
|
131
|
+
* @returns the batches (fully consumed, delete as a unit) and their records.
|
|
132
|
+
*/
|
|
133
|
+
peek(backend: BackendName, maxRecords: number): { batches: SpoolBatch[]; records: ExportRecord[] } {
|
|
134
|
+
const prefix = `${backend}:`
|
|
135
|
+
const batches: SpoolBatch[] = []
|
|
136
|
+
const records: ExportRecord[] = []
|
|
137
|
+
for (const [key, value] of this.ordered()) {
|
|
138
|
+
if (!key.startsWith(prefix)) continue
|
|
139
|
+
const valid: ExportRecord[] = []
|
|
140
|
+
for (const record of value.records) {
|
|
141
|
+
if (isExportRecord(record)) valid.push(record)
|
|
142
|
+
}
|
|
143
|
+
if (valid.length !== value.records.length) this.onInvalid(value.records.length - valid.length)
|
|
144
|
+
batches.push({ backend, key, createdAt: value.createdAt, records: valid })
|
|
145
|
+
records.push(...valid)
|
|
146
|
+
if (records.length >= maxRecords) break
|
|
147
|
+
}
|
|
148
|
+
return { batches, records }
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* Delete delivered batches.
|
|
153
|
+
* @param batches - the batches that were exported successfully.
|
|
154
|
+
* @returns resolution after every delete.
|
|
155
|
+
*/
|
|
156
|
+
async remove(batches: readonly SpoolBatch[]): Promise<void> {
|
|
157
|
+
await Promise.all(batches.map(batch => this.table.delete(batch.key)))
|
|
158
|
+
for (const batch of batches) {
|
|
159
|
+
this.bufferedRecords -= batch.records.length
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/**
|
|
164
|
+
* Per-backend occupancy.
|
|
165
|
+
* @returns batch and record counts per backend.
|
|
166
|
+
*/
|
|
167
|
+
counts(): Record<BackendName, SpoolCounts> {
|
|
168
|
+
const counts: Record<BackendName, SpoolCounts> = {
|
|
169
|
+
otlp: { batches: 0, records: 0 },
|
|
170
|
+
langfuse: { batches: 0, records: 0 },
|
|
171
|
+
}
|
|
172
|
+
for (const [key, value] of this.table.entries()) {
|
|
173
|
+
const backend: BackendName = key.startsWith('otlp:') ? 'otlp' : 'langfuse'
|
|
174
|
+
counts[backend].batches += 1
|
|
175
|
+
counts[backend].records += value.records.length
|
|
176
|
+
}
|
|
177
|
+
return counts
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/**
|
|
182
|
+
* Open the offline buffer over an already-opened domain.
|
|
183
|
+
* @param domain - the opened `observe` domain.
|
|
184
|
+
* @param maxBufferRecords - total record bound.
|
|
185
|
+
* @param onDropped - drop callback (bound eviction).
|
|
186
|
+
* @param onInvalid - invalid-record callback (durable-boundary validation).
|
|
187
|
+
* @returns the spool handle.
|
|
188
|
+
*/
|
|
189
|
+
export function openSpool(
|
|
190
|
+
domain: Domain<ObserveDomainSpec>,
|
|
191
|
+
maxBufferRecords: number,
|
|
192
|
+
onDropped: (count: number) => void,
|
|
193
|
+
onInvalid: (count: number) => void,
|
|
194
|
+
): Spool {
|
|
195
|
+
return new Spool(domain.table('spool'), maxBufferRecords, onDropped, onInvalid)
|
|
196
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The hand-written Typert HOST manifest for `dsh-observe`, exported as
|
|
3
|
+
* `./typert` so the harness's typert-loader registers the `observe` status /
|
|
4
|
+
* kill-switch invocations automatically when this plugin mounts. Same shape
|
|
5
|
+
* as a generator output (validated by the loader): package face, empty model
|
|
6
|
+
* and schemas, and the canonical invocation list from `src/wire.ts`.
|
|
7
|
+
*
|
|
8
|
+
* @module dsh-observe/typert
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import { OBSERVE_INVOCATIONS } from './wire.ts'
|
|
12
|
+
|
|
13
|
+
/** Host Typert manifest (validated by `@deepseek-ai/dsh-typert-loader`). */
|
|
14
|
+
export const TYPERT = Object.freeze({
|
|
15
|
+
package: 'dsh-observe',
|
|
16
|
+
face: 'host',
|
|
17
|
+
schemas: Object.freeze([]),
|
|
18
|
+
invocations: OBSERVE_INVOCATIONS,
|
|
19
|
+
model: Object.freeze({
|
|
20
|
+
services: Object.freeze([]),
|
|
21
|
+
events: Object.freeze([]),
|
|
22
|
+
objects: Object.freeze([]),
|
|
23
|
+
}),
|
|
24
|
+
})
|
package/src/version.ts
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Package version stamp shared by build artifacts (OTLP instrumentation
|
|
3
|
+
* scope) and the release script. `scripts/release.mjs` bumps this string
|
|
4
|
+
* together with package.json; tests/version.spec.ts trips when the two
|
|
5
|
+
* drift apart.
|
|
6
|
+
* @module dsh-observe/version
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
/** The dsh-observe package version, kept in sync with package.json. */
|
|
10
|
+
export const VERSION = '0.1.0'
|
package/src/wire.ts
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The `observe` Remote wire vocabulary: the status snapshot and the kill
|
|
3
|
+
* switch result, their zod v4 strict schemas, and the invocation descriptors
|
|
4
|
+
* shared by the host `./typert` manifest. One canonical source keeps the two
|
|
5
|
+
* Typert faces from drifting apart.
|
|
6
|
+
* @module dsh-observe/wire
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { z } from 'zod'
|
|
10
|
+
import type { InvocationDescriptor } from '@deepseek-ai/dsh-typert-protocol'
|
|
11
|
+
import type { SpoolCounts } from './spool.ts'
|
|
12
|
+
|
|
13
|
+
/** The status payload returned by `observe/status`. */
|
|
14
|
+
export interface ObserveStatus {
|
|
15
|
+
/** The runtime kill switch state. */
|
|
16
|
+
enabled: boolean
|
|
17
|
+
/** Which backends are configured (not whether they are reachable). */
|
|
18
|
+
backends: { otlp: boolean; langfuse: boolean }
|
|
19
|
+
/** Queued + retry-pending records per backend. */
|
|
20
|
+
queueDepth: { otlp: number; langfuse: number }
|
|
21
|
+
/** Durable offline-buffer occupancy per backend. */
|
|
22
|
+
spool: Record<'otlp' | 'langfuse', SpoolCounts>
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/** Strict wire schema for {@link ObserveStatus} (zod v4). */
|
|
26
|
+
export const OBSERVE_STATUS_SCHEMA = z.object({
|
|
27
|
+
enabled: z.boolean(),
|
|
28
|
+
backends: z.object({ otlp: z.boolean(), langfuse: z.boolean() }),
|
|
29
|
+
queueDepth: z.object({ otlp: z.number().int(), langfuse: z.number().int() }),
|
|
30
|
+
spool: z.object({
|
|
31
|
+
otlp: z.object({ batches: z.number().int(), records: z.number().int() }),
|
|
32
|
+
langfuse: z.object({ batches: z.number().int(), records: z.number().int() }),
|
|
33
|
+
}),
|
|
34
|
+
})
|
|
35
|
+
|
|
36
|
+
/** The result of `observe/setEnabled`: the kill switch state after the write. */
|
|
37
|
+
export interface ObserveSetEnabledResult {
|
|
38
|
+
/** The runtime kill switch state after the change. */
|
|
39
|
+
enabled: boolean
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** Strict wire schema for {@link ObserveSetEnabledResult} (zod v4). */
|
|
43
|
+
export const OBSERVE_SET_ENABLED_RESULT_SCHEMA = z.object({
|
|
44
|
+
enabled: z.boolean(),
|
|
45
|
+
})
|
|
46
|
+
|
|
47
|
+
/** JSON codec for the setEnabled argument. */
|
|
48
|
+
const SET_ENABLED_ARGS_CODEC = z.object({
|
|
49
|
+
enabled: z.boolean(),
|
|
50
|
+
})
|
|
51
|
+
|
|
52
|
+
/** The `observe/status` invocation descriptor: the read-only status snapshot. */
|
|
53
|
+
export const OBSERVE_STATUS_DESCRIPTOR = Object.freeze({
|
|
54
|
+
id: 'dsh-observe#observe/status',
|
|
55
|
+
service: 'dsh-observe',
|
|
56
|
+
namespace: 'observe',
|
|
57
|
+
method: 'status',
|
|
58
|
+
invocation: Object.freeze({ kind: 'direct' }),
|
|
59
|
+
parameters: Object.freeze([]),
|
|
60
|
+
result: Object.freeze({
|
|
61
|
+
mode: 'strict',
|
|
62
|
+
typeSymbol: 'dsh-observe/types#ObserveStatus',
|
|
63
|
+
schema: OBSERVE_STATUS_SCHEMA,
|
|
64
|
+
}),
|
|
65
|
+
sourceLocation: Object.freeze({ file: 'src/wire.ts', line: 1, column: 1 }),
|
|
66
|
+
} as const) satisfies InvocationDescriptor
|
|
67
|
+
|
|
68
|
+
/** The `observe/setEnabled` invocation descriptor: the runtime kill switch. */
|
|
69
|
+
export const OBSERVE_SET_ENABLED_DESCRIPTOR = Object.freeze({
|
|
70
|
+
id: 'dsh-observe#observe/setEnabled',
|
|
71
|
+
service: 'dsh-observe',
|
|
72
|
+
namespace: 'observe',
|
|
73
|
+
method: 'setEnabled',
|
|
74
|
+
invocation: Object.freeze({ kind: 'direct' }),
|
|
75
|
+
parameters: Object.freeze([Object.freeze({
|
|
76
|
+
name: 'request',
|
|
77
|
+
wire: 'request',
|
|
78
|
+
source: 'json',
|
|
79
|
+
codec: Object.freeze({
|
|
80
|
+
mode: 'strict',
|
|
81
|
+
typeSymbol: 'dsh-observe/types#ObserveSetEnabledArgs',
|
|
82
|
+
schema: SET_ENABLED_ARGS_CODEC,
|
|
83
|
+
}),
|
|
84
|
+
} satisfies InvocationDescriptor['parameters'][number])]),
|
|
85
|
+
result: Object.freeze({
|
|
86
|
+
mode: 'strict',
|
|
87
|
+
typeSymbol: 'dsh-observe/types#ObserveSetEnabledResult',
|
|
88
|
+
schema: OBSERVE_SET_ENABLED_RESULT_SCHEMA,
|
|
89
|
+
}),
|
|
90
|
+
sourceLocation: Object.freeze({ file: 'src/wire.ts', line: 1, column: 1 }),
|
|
91
|
+
} as const) satisfies InvocationDescriptor
|
|
92
|
+
|
|
93
|
+
/** The canonical invocation list both Typert faces register. */
|
|
94
|
+
export const OBSERVE_INVOCATIONS = Object.freeze([
|
|
95
|
+
OBSERVE_STATUS_DESCRIPTOR,
|
|
96
|
+
OBSERVE_SET_ENABLED_DESCRIPTOR,
|
|
97
|
+
])
|