dsh-observe 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. package/CHANGELOG.md +25 -0
  2. package/LICENSE +201 -0
  3. package/README.es.md +198 -0
  4. package/README.hi.md +198 -0
  5. package/README.md +198 -0
  6. package/README.pt.md +198 -0
  7. package/README.zh.md +198 -0
  8. package/THIRD_PARTY_NOTICES.md +21 -0
  9. package/cordis.patch.yml +86 -0
  10. package/lib/index.js +1947 -0
  11. package/lib/typert.host.js +109 -0
  12. package/lib/types/backoff.d.ts +17 -0
  13. package/lib/types/backoff.d.ts.map +1 -0
  14. package/lib/types/backoff.js +17 -0
  15. package/lib/types/backoff.js.map +1 -0
  16. package/lib/types/collector.d.ts +83 -0
  17. package/lib/types/collector.d.ts.map +1 -0
  18. package/lib/types/collector.js +506 -0
  19. package/lib/types/collector.js.map +1 -0
  20. package/lib/types/config.d.ts +286 -0
  21. package/lib/types/config.d.ts.map +1 -0
  22. package/lib/types/config.js +312 -0
  23. package/lib/types/config.js.map +1 -0
  24. package/lib/types/ids.d.ts +43 -0
  25. package/lib/types/ids.d.ts.map +1 -0
  26. package/lib/types/ids.js +63 -0
  27. package/lib/types/ids.js.map +1 -0
  28. package/lib/types/index.d.ts +35 -0
  29. package/lib/types/index.d.ts.map +1 -0
  30. package/lib/types/index.js +162 -0
  31. package/lib/types/index.js.map +1 -0
  32. package/lib/types/model.d.ts +89 -0
  33. package/lib/types/model.d.ts.map +1 -0
  34. package/lib/types/model.js +25 -0
  35. package/lib/types/model.js.map +1 -0
  36. package/lib/types/pipeline.d.ts +71 -0
  37. package/lib/types/pipeline.d.ts.map +1 -0
  38. package/lib/types/pipeline.js +193 -0
  39. package/lib/types/pipeline.js.map +1 -0
  40. package/lib/types/pricing.d.ts +34 -0
  41. package/lib/types/pricing.d.ts.map +1 -0
  42. package/lib/types/pricing.js +45 -0
  43. package/lib/types/pricing.js.map +1 -0
  44. package/lib/types/project.d.ts +26 -0
  45. package/lib/types/project.d.ts.map +1 -0
  46. package/lib/types/project.js +52 -0
  47. package/lib/types/project.js.map +1 -0
  48. package/lib/types/remote.d.ts +28 -0
  49. package/lib/types/remote.d.ts.map +1 -0
  50. package/lib/types/remote.js +45 -0
  51. package/lib/types/remote.js.map +1 -0
  52. package/lib/types/sanitize.d.ts +78 -0
  53. package/lib/types/sanitize.d.ts.map +1 -0
  54. package/lib/types/sanitize.js +132 -0
  55. package/lib/types/sanitize.js.map +1 -0
  56. package/lib/types/sinks.d.ts +84 -0
  57. package/lib/types/sinks.d.ts.map +1 -0
  58. package/lib/types/sinks.js +434 -0
  59. package/lib/types/sinks.js.map +1 -0
  60. package/lib/types/spool.d.ts +113 -0
  61. package/lib/types/spool.d.ts.map +1 -0
  62. package/lib/types/spool.js +167 -0
  63. package/lib/types/spool.js.map +1 -0
  64. package/lib/types/typert.host.d.ts +93 -0
  65. package/lib/types/typert.host.d.ts.map +1 -0
  66. package/lib/types/typert.host.js +23 -0
  67. package/lib/types/typert.host.js.map +1 -0
  68. package/lib/types/version.d.ts +10 -0
  69. package/lib/types/version.d.ts.map +1 -0
  70. package/lib/types/version.js +10 -0
  71. package/lib/types/version.js.map +1 -0
  72. package/lib/types/wire.d.ts +206 -0
  73. package/lib/types/wire.d.ts.map +1 -0
  74. package/lib/types/wire.js +71 -0
  75. package/lib/types/wire.js.map +1 -0
  76. package/package.json +147 -0
  77. package/src/backoff.ts +20 -0
  78. package/src/collector.ts +561 -0
  79. package/src/config.ts +604 -0
  80. package/src/ids.ts +68 -0
  81. package/src/index.ts +186 -0
  82. package/src/model.ts +100 -0
  83. package/src/pipeline.ts +205 -0
  84. package/src/pricing.ts +55 -0
  85. package/src/project.ts +55 -0
  86. package/src/remote.ts +56 -0
  87. package/src/sanitize.ts +132 -0
  88. package/src/sinks.ts +471 -0
  89. package/src/spool.ts +196 -0
  90. package/src/typert.host.ts +24 -0
  91. package/src/version.ts +10 -0
  92. package/src/wire.ts +97 -0
package/src/spool.ts ADDED
@@ -0,0 +1,196 @@
1
+ /**
2
+ * The bounded offline buffer: a storage-domain table holding sanitized
3
+ * export batches that could not be delivered. Batches are written when the
4
+ * in-memory queue overflows or after retries are exhausted, and drained on
5
+ * a timer; the bound is a total record count, and pushing past it drops the
6
+ * OLDEST records (reported through the drop callback so the OTLP backend
7
+ * can count them). The buffer survives restarts: the domain reloads from
8
+ * the host's storage backend.
9
+ * @module dsh-observe/spool
10
+ */
11
+
12
+ import type { Domain, KvTable } from '@deepseek-ai/dsh-storage-domain'
13
+ import { defineDomain } from '@deepseek-ai/dsh-storage-domain'
14
+ import { z } from 'zod'
15
+ import type { BackendName, ExportRecord } from './model.ts'
16
+ import { isExportRecord } from './model.ts'
17
+
18
+ /** One stored batch: creation time plus sanitized export records. */
19
+ const spoolBatchSchema = z.object({
20
+ createdAt: z.number().int().nonnegative(),
21
+ records: z.array(z.unknown()),
22
+ })
23
+
24
+ /** The durable record type validated by the domain on load. */
25
+ export type SpoolBatchValue = z.infer<typeof spoolBatchSchema>
26
+
27
+ /** The dsh-observe storage domain declaration. */
28
+ export const observeDomainSpec = defineDomain({
29
+ name: 'observe',
30
+ version: 1,
31
+ tables: {
32
+ spool: { valueSchema: spoolBatchSchema },
33
+ },
34
+ })
35
+
36
+ /** Domain spec type for the spool table. */
37
+ export type ObserveDomainSpec = typeof observeDomainSpec
38
+
39
+ /** One batch read back from the spool, validated record-wise. */
40
+ export interface SpoolBatch {
41
+ backend: BackendName
42
+ key: string
43
+ createdAt: number
44
+ records: ExportRecord[]
45
+ }
46
+
47
+ /** A monotonic, zero-padded key for lexicographically ordered batches. */
48
+ function batchKey(backend: BackendName, seq: number): string {
49
+ return `${backend}:${String(seq).padStart(16, '0')}`
50
+ }
51
+
52
+ /** Per-backend occupancy snapshot for status surfaces. */
53
+ export interface SpoolCounts {
54
+ batches: number
55
+ records: number
56
+ }
57
+
58
+ /**
59
+ * The offline buffer over one storage-domain table. The domain itself is
60
+ * opened and closed by the runtime; the spool only owns the table handle.
61
+ */
62
+ export class Spool {
63
+ private nextSeq = 0
64
+ private bufferedRecords = 0
65
+
66
+ /**
67
+ * @param table - the domain `spool` table handle.
68
+ * @param maxBufferRecords - total record bound across all backends.
69
+ * @param onDropped - receives the count of records dropped by bound eviction.
70
+ * @param onInvalid - receives the count of stored records rejected by validation.
71
+ */
72
+ constructor(
73
+ private readonly table: KvTable<string, SpoolBatchValue>,
74
+ private readonly maxBufferRecords: number,
75
+ private readonly onDropped: (count: number) => void,
76
+ private readonly onInvalid: (count: number) => void,
77
+ ) {
78
+ for (const [key, value] of this.table.entries()) {
79
+ const match = /^[a-z]+:(\d+)$/u.exec(key)
80
+ if (match !== null && match[1] !== undefined) {
81
+ this.nextSeq = Math.max(this.nextSeq, Number(match[1]) + 1)
82
+ }
83
+ this.bufferedRecords += value.records.length
84
+ }
85
+ }
86
+
87
+ /** All batches in chronological write order (createdAt, then key). */
88
+ private ordered(): [string, SpoolBatchValue][] {
89
+ return [...this.table.entries()].sort((left, right) =>
90
+ left[1].createdAt - right[1].createdAt || left[0].localeCompare(right[0]))
91
+ }
92
+
93
+ /**
94
+ * Spill one batch into the buffer, enforcing the record bound: when the
95
+ * batch alone exceeds the bound only its newest records are kept, and
96
+ * otherwise the oldest stored batches are evicted until it fits. Drops
97
+ * are reported through {@link onDropped}.
98
+ * @param backend - the owning backend.
99
+ * @param records - sanitized records to persist.
100
+ * @returns resolution after the write and any evictions.
101
+ */
102
+ async push(backend: BackendName, records: ExportRecord[]): Promise<void> {
103
+ if (records.length === 0) return
104
+ let kept = records
105
+ let dropped = 0
106
+ if (kept.length > this.maxBufferRecords) {
107
+ dropped += kept.length - this.maxBufferRecords
108
+ kept = kept.slice(kept.length - this.maxBufferRecords)
109
+ }
110
+ while (this.bufferedRecords + kept.length > this.maxBufferRecords) {
111
+ const oldest = this.ordered()[0]
112
+ if (oldest === undefined) break
113
+ await this.table.delete(oldest[0])
114
+ this.bufferedRecords -= oldest[1].records.length
115
+ dropped += oldest[1].records.length
116
+ }
117
+ const key = batchKey(backend, this.nextSeq)
118
+ this.nextSeq += 1
119
+ await this.table.put(key, { createdAt: Date.now(), records: kept })
120
+ this.bufferedRecords += kept.length
121
+ if (dropped > 0) this.onDropped(dropped)
122
+ }
123
+
124
+ /**
125
+ * Read the oldest batches of one backend, up to a record budget. Stored
126
+ * records that fail {@link isExportRecord} are dropped from the returned
127
+ * set (and reported through {@link onInvalid}) but stay on disk until a
128
+ * later `remove` deletes their batch.
129
+ * @param backend - the backend to read.
130
+ * @param maxRecords - record budget for one drain cycle.
131
+ * @returns the batches (fully consumed, delete as a unit) and their records.
132
+ */
133
+ peek(backend: BackendName, maxRecords: number): { batches: SpoolBatch[]; records: ExportRecord[] } {
134
+ const prefix = `${backend}:`
135
+ const batches: SpoolBatch[] = []
136
+ const records: ExportRecord[] = []
137
+ for (const [key, value] of this.ordered()) {
138
+ if (!key.startsWith(prefix)) continue
139
+ const valid: ExportRecord[] = []
140
+ for (const record of value.records) {
141
+ if (isExportRecord(record)) valid.push(record)
142
+ }
143
+ if (valid.length !== value.records.length) this.onInvalid(value.records.length - valid.length)
144
+ batches.push({ backend, key, createdAt: value.createdAt, records: valid })
145
+ records.push(...valid)
146
+ if (records.length >= maxRecords) break
147
+ }
148
+ return { batches, records }
149
+ }
150
+
151
+ /**
152
+ * Delete delivered batches.
153
+ * @param batches - the batches that were exported successfully.
154
+ * @returns resolution after every delete.
155
+ */
156
+ async remove(batches: readonly SpoolBatch[]): Promise<void> {
157
+ await Promise.all(batches.map(batch => this.table.delete(batch.key)))
158
+ for (const batch of batches) {
159
+ this.bufferedRecords -= batch.records.length
160
+ }
161
+ }
162
+
163
+ /**
164
+ * Per-backend occupancy.
165
+ * @returns batch and record counts per backend.
166
+ */
167
+ counts(): Record<BackendName, SpoolCounts> {
168
+ const counts: Record<BackendName, SpoolCounts> = {
169
+ otlp: { batches: 0, records: 0 },
170
+ langfuse: { batches: 0, records: 0 },
171
+ }
172
+ for (const [key, value] of this.table.entries()) {
173
+ const backend: BackendName = key.startsWith('otlp:') ? 'otlp' : 'langfuse'
174
+ counts[backend].batches += 1
175
+ counts[backend].records += value.records.length
176
+ }
177
+ return counts
178
+ }
179
+ }
180
+
181
+ /**
182
+ * Open the offline buffer over an already-opened domain.
183
+ * @param domain - the opened `observe` domain.
184
+ * @param maxBufferRecords - total record bound.
185
+ * @param onDropped - drop callback (bound eviction).
186
+ * @param onInvalid - invalid-record callback (durable-boundary validation).
187
+ * @returns the spool handle.
188
+ */
189
+ export function openSpool(
190
+ domain: Domain<ObserveDomainSpec>,
191
+ maxBufferRecords: number,
192
+ onDropped: (count: number) => void,
193
+ onInvalid: (count: number) => void,
194
+ ): Spool {
195
+ return new Spool(domain.table('spool'), maxBufferRecords, onDropped, onInvalid)
196
+ }
@@ -0,0 +1,24 @@
1
+ /**
2
+ * The hand-written Typert HOST manifest for `dsh-observe`, exported as
3
+ * `./typert` so the harness's typert-loader registers the `observe` status /
4
+ * kill-switch invocations automatically when this plugin mounts. Same shape
5
+ * as a generator output (validated by the loader): package face, empty model
6
+ * and schemas, and the canonical invocation list from `src/wire.ts`.
7
+ *
8
+ * @module dsh-observe/typert
9
+ */
10
+
11
+ import { OBSERVE_INVOCATIONS } from './wire.ts'
12
+
13
+ /** Host Typert manifest (validated by `@deepseek-ai/dsh-typert-loader`). */
14
+ export const TYPERT = Object.freeze({
15
+ package: 'dsh-observe',
16
+ face: 'host',
17
+ schemas: Object.freeze([]),
18
+ invocations: OBSERVE_INVOCATIONS,
19
+ model: Object.freeze({
20
+ services: Object.freeze([]),
21
+ events: Object.freeze([]),
22
+ objects: Object.freeze([]),
23
+ }),
24
+ })
package/src/version.ts ADDED
@@ -0,0 +1,10 @@
1
+ /**
2
+ * Package version stamp shared by build artifacts (OTLP instrumentation
3
+ * scope) and the release script. `scripts/release.mjs` bumps this string
4
+ * together with package.json; tests/version.spec.ts trips when the two
5
+ * drift apart.
6
+ * @module dsh-observe/version
7
+ */
8
+
9
+ /** The dsh-observe package version, kept in sync with package.json. */
10
+ export const VERSION = '0.1.0'
package/src/wire.ts ADDED
@@ -0,0 +1,97 @@
1
+ /**
2
+ * The `observe` Remote wire vocabulary: the status snapshot and the kill
3
+ * switch result, their zod v4 strict schemas, and the invocation descriptors
4
+ * shared by the host `./typert` manifest. One canonical source keeps the two
5
+ * Typert faces from drifting apart.
6
+ * @module dsh-observe/wire
7
+ */
8
+
9
+ import { z } from 'zod'
10
+ import type { InvocationDescriptor } from '@deepseek-ai/dsh-typert-protocol'
11
+ import type { SpoolCounts } from './spool.ts'
12
+
13
+ /** The status payload returned by `observe/status`. */
14
+ export interface ObserveStatus {
15
+ /** The runtime kill switch state. */
16
+ enabled: boolean
17
+ /** Which backends are configured (not whether they are reachable). */
18
+ backends: { otlp: boolean; langfuse: boolean }
19
+ /** Queued + retry-pending records per backend. */
20
+ queueDepth: { otlp: number; langfuse: number }
21
+ /** Durable offline-buffer occupancy per backend. */
22
+ spool: Record<'otlp' | 'langfuse', SpoolCounts>
23
+ }
24
+
25
+ /** Strict wire schema for {@link ObserveStatus} (zod v4). */
26
+ export const OBSERVE_STATUS_SCHEMA = z.object({
27
+ enabled: z.boolean(),
28
+ backends: z.object({ otlp: z.boolean(), langfuse: z.boolean() }),
29
+ queueDepth: z.object({ otlp: z.number().int(), langfuse: z.number().int() }),
30
+ spool: z.object({
31
+ otlp: z.object({ batches: z.number().int(), records: z.number().int() }),
32
+ langfuse: z.object({ batches: z.number().int(), records: z.number().int() }),
33
+ }),
34
+ })
35
+
36
+ /** The result of `observe/setEnabled`: the kill switch state after the write. */
37
+ export interface ObserveSetEnabledResult {
38
+ /** The runtime kill switch state after the change. */
39
+ enabled: boolean
40
+ }
41
+
42
+ /** Strict wire schema for {@link ObserveSetEnabledResult} (zod v4). */
43
+ export const OBSERVE_SET_ENABLED_RESULT_SCHEMA = z.object({
44
+ enabled: z.boolean(),
45
+ })
46
+
47
+ /** JSON codec for the setEnabled argument. */
48
+ const SET_ENABLED_ARGS_CODEC = z.object({
49
+ enabled: z.boolean(),
50
+ })
51
+
52
+ /** The `observe/status` invocation descriptor: the read-only status snapshot. */
53
+ export const OBSERVE_STATUS_DESCRIPTOR = Object.freeze({
54
+ id: 'dsh-observe#observe/status',
55
+ service: 'dsh-observe',
56
+ namespace: 'observe',
57
+ method: 'status',
58
+ invocation: Object.freeze({ kind: 'direct' }),
59
+ parameters: Object.freeze([]),
60
+ result: Object.freeze({
61
+ mode: 'strict',
62
+ typeSymbol: 'dsh-observe/types#ObserveStatus',
63
+ schema: OBSERVE_STATUS_SCHEMA,
64
+ }),
65
+ sourceLocation: Object.freeze({ file: 'src/wire.ts', line: 1, column: 1 }),
66
+ } as const) satisfies InvocationDescriptor
67
+
68
+ /** The `observe/setEnabled` invocation descriptor: the runtime kill switch. */
69
+ export const OBSERVE_SET_ENABLED_DESCRIPTOR = Object.freeze({
70
+ id: 'dsh-observe#observe/setEnabled',
71
+ service: 'dsh-observe',
72
+ namespace: 'observe',
73
+ method: 'setEnabled',
74
+ invocation: Object.freeze({ kind: 'direct' }),
75
+ parameters: Object.freeze([Object.freeze({
76
+ name: 'request',
77
+ wire: 'request',
78
+ source: 'json',
79
+ codec: Object.freeze({
80
+ mode: 'strict',
81
+ typeSymbol: 'dsh-observe/types#ObserveSetEnabledArgs',
82
+ schema: SET_ENABLED_ARGS_CODEC,
83
+ }),
84
+ } satisfies InvocationDescriptor['parameters'][number])]),
85
+ result: Object.freeze({
86
+ mode: 'strict',
87
+ typeSymbol: 'dsh-observe/types#ObserveSetEnabledResult',
88
+ schema: OBSERVE_SET_ENABLED_RESULT_SCHEMA,
89
+ }),
90
+ sourceLocation: Object.freeze({ file: 'src/wire.ts', line: 1, column: 1 }),
91
+ } as const) satisfies InvocationDescriptor
92
+
93
+ /** The canonical invocation list both Typert faces register. */
94
+ export const OBSERVE_INVOCATIONS = Object.freeze([
95
+ OBSERVE_STATUS_DESCRIPTOR,
96
+ OBSERVE_SET_ENABLED_DESCRIPTOR,
97
+ ])