@obsunified/collector 2.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (254) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +60 -0
  3. package/dist/auth.d.ts +2 -0
  4. package/dist/auth.js +9 -0
  5. package/dist/auth.js.map +1 -0
  6. package/dist/chunk-UGWDOGGH.js +319 -0
  7. package/dist/chunk-UGWDOGGH.js.map +1 -0
  8. package/dist/chunk-XKQTVIRE.js +671 -0
  9. package/dist/chunk-XKQTVIRE.js.map +1 -0
  10. package/dist/index.d.ts +1197 -0
  11. package/dist/index.js +17707 -0
  12. package/dist/index.js.map +1 -0
  13. package/dist/ingest-auth-HFVO1zIp.d.ts +165 -0
  14. package/dist/openai-FJYOQ7WO.js +384 -0
  15. package/dist/openai-FJYOQ7WO.js.map +1 -0
  16. package/package.json +73 -0
  17. package/src/ai-parity/ai-parity.acceptance.test.ts +449 -0
  18. package/src/analyses/derive.ts +642 -0
  19. package/src/analyses/index.ts +50 -0
  20. package/src/analyses/investigations.ts +277 -0
  21. package/src/analyses/tier0.ts +568 -0
  22. package/src/auth/dashboard-auth.ts +269 -0
  23. package/src/auth/ingest-auth.ts +147 -0
  24. package/src/durable-objects/tail-hub.test.ts +32 -0
  25. package/src/durable-objects/tail-hub.ts +158 -0
  26. package/src/framework/collector.ts +467 -0
  27. package/src/framework/env.ts +90 -0
  28. package/src/framework/logger.ts +81 -0
  29. package/src/index.ts +211 -0
  30. package/src/lib/action-aggregates.ts +1031 -0
  31. package/src/lib/action-identity.ts +106 -0
  32. package/src/lib/actions-store.ts +397 -0
  33. package/src/lib/ai-pricing.ts +80 -0
  34. package/src/lib/ai-store/calls.ts +148 -0
  35. package/src/lib/ai-store/evaluations.test.ts +47 -0
  36. package/src/lib/ai-store/evaluations.ts +125 -0
  37. package/src/lib/ai-store/helpers.ts +37 -0
  38. package/src/lib/ai-store/retention.ts +17 -0
  39. package/src/lib/ai-store/sessions.ts +277 -0
  40. package/src/lib/ai-store/spans.ts +109 -0
  41. package/src/lib/ai-store/types.ts +112 -0
  42. package/src/lib/ai-store.ts +76 -0
  43. package/src/lib/alerts-store.ts +458 -0
  44. package/src/lib/analyses-runner.ts +439 -0
  45. package/src/lib/analyses-store.test.ts +243 -0
  46. package/src/lib/analyses-store.ts +454 -0
  47. package/src/lib/ask.test.ts +302 -0
  48. package/src/lib/ask.ts +390 -0
  49. package/src/lib/blob-store-s3.ts +213 -0
  50. package/src/lib/blob-store.ts +366 -0
  51. package/src/lib/eval-cases-store.test.ts +171 -0
  52. package/src/lib/eval-cases-store.ts +1033 -0
  53. package/src/lib/evidence-references.test.ts +10 -0
  54. package/src/lib/evidence-references.ts +684 -0
  55. package/src/lib/hash.ts +26 -0
  56. package/src/lib/identity-index/action-lookups.ts +805 -0
  57. package/src/lib/identity-index/constants.ts +1 -0
  58. package/src/lib/identity-index/key-lookups.ts +199 -0
  59. package/src/lib/identity-index/mappers.ts +349 -0
  60. package/src/lib/identity-index/types.ts +196 -0
  61. package/src/lib/identity-index/user-lookup.ts +137 -0
  62. package/src/lib/identity-index.test.ts +624 -0
  63. package/src/lib/identity-index.ts +85 -0
  64. package/src/lib/json.ts +39 -0
  65. package/src/lib/llm.ts +263 -0
  66. package/src/lib/logs-store.ts +188 -0
  67. package/src/lib/metrics-store.test.ts +189 -0
  68. package/src/lib/metrics-store.ts +312 -0
  69. package/src/lib/narrate-gate.test.ts +260 -0
  70. package/src/lib/narrate-gate.ts +253 -0
  71. package/src/lib/openai.test.ts +300 -0
  72. package/src/lib/openai.ts +512 -0
  73. package/src/lib/otlp.ts +212 -0
  74. package/src/lib/parse-pprof.test.ts +64 -0
  75. package/src/lib/parse-pprof.ts +184 -0
  76. package/src/lib/payload-capture.ts +22 -0
  77. package/src/lib/projects-store.ts +207 -0
  78. package/src/lib/propagation-metric.test.ts +196 -0
  79. package/src/lib/propagation-metric.ts +175 -0
  80. package/src/lib/sql-db-dialect.test.ts +22 -0
  81. package/src/lib/sql-db-postgres.ts +229 -0
  82. package/src/lib/sql-db.ts +166 -0
  83. package/src/lib/stage6.test.ts +216 -0
  84. package/src/lib/store/export.ts +65 -0
  85. package/src/lib/store/helpers.ts +367 -0
  86. package/src/lib/store/ingest.ts +68 -0
  87. package/src/lib/store/issues.ts +158 -0
  88. package/src/lib/store/overview.ts +127 -0
  89. package/src/lib/store/retention.ts +10 -0
  90. package/src/lib/store/service-map.ts +384 -0
  91. package/src/lib/store/trace-detail.bench.ts +76 -0
  92. package/src/lib/store/trace-detail.test.ts +410 -0
  93. package/src/lib/store/trace-detail.ts +427 -0
  94. package/src/lib/store/trace-queries.test.ts +29 -0
  95. package/src/lib/store/trace-queries.ts +89 -0
  96. package/src/lib/store-overview.test.ts +77 -0
  97. package/src/lib/store.ts +96 -0
  98. package/src/lib/tail-publisher.ts +64 -0
  99. package/src/lib/test-utils/mem-sql-db.test.ts +93 -0
  100. package/src/lib/test-utils/mem-sql-db.ts +86 -0
  101. package/src/lib/ua-parser.ts +61 -0
  102. package/src/lib/usage-store.ts +567 -0
  103. package/src/migrations/001_telemetry_spans.sql +28 -0
  104. package/src/migrations/002_usage_events.sql +24 -0
  105. package/src/migrations/003_usage_analytics_columns.sql +13 -0
  106. package/src/migrations/004_logs.sql +19 -0
  107. package/src/migrations/005_ai_calls.sql +25 -0
  108. package/src/migrations/006_user_profiles.sql +11 -0
  109. package/src/migrations/007_session_replay_metadata.sql +10 -0
  110. package/src/migrations/008_session_replay_storage_bytes.sql +1 -0
  111. package/src/migrations/009_projects_and_keys.sql +22 -0
  112. package/src/migrations/010_signal_project_id.sql +15 -0
  113. package/src/migrations/011_default_project_seed.sql +3 -0
  114. package/src/migrations/012_alert_rules.sql +17 -0
  115. package/src/migrations/013_alert_evaluations.sql +13 -0
  116. package/src/migrations/014_alert_state.sql +9 -0
  117. package/src/migrations/015_metrics.sql +58 -0
  118. package/src/migrations/016_metrics_exp_histogram_summary.sql +5 -0
  119. package/src/migrations/017_span_dropped_counts.sql +11 -0
  120. package/src/migrations/018_log_flags.sql +8 -0
  121. package/src/migrations/019_ai_span_payloads.sql +30 -0
  122. package/src/migrations/020_ai_span_evaluations.sql +33 -0
  123. package/src/migrations/021_ai_span_session.sql +13 -0
  124. package/src/migrations/022_span_log_session_id.sql +13 -0
  125. package/src/migrations/023_service_map_indexes.sql +32 -0
  126. package/src/migrations/024_analysis_results.sql +49 -0
  127. package/src/migrations/025_stage6_autopinning_alerts.sql +38 -0
  128. package/src/migrations/026_obs_dashboard_project_seed.sql +18 -0
  129. package/src/migrations/027_identity_propagation.sql +50 -0
  130. package/src/migrations/028_profile_blobs.sql +57 -0
  131. package/src/migrations/029_telemetry_sdk_name.sql +20 -0
  132. package/src/migrations/030_analysis_claim.sql +18 -0
  133. package/src/migrations/031_agent_action_graph.sql +154 -0
  134. package/src/migrations/032_payload_capture_flags.sql +8 -0
  135. package/src/migrations/033_eval_cases.sql +52 -0
  136. package/src/migrations/034_eval_case_results.sql +23 -0
  137. package/src/migrations/035_metric_exemplars.sql +33 -0
  138. package/src/migrations/036_trace_instrumentation_gaps.sql +19 -0
  139. package/src/migrations/037_eval_runs.sql +37 -0
  140. package/src/migrations/038_tool_call_evidence.sql +11 -0
  141. package/src/migrations/039_drop_trace_instrumentation_gaps.sql +11 -0
  142. package/src/migrations/040_evidence_retrieval_refs.sql +50 -0
  143. package/src/migrations-postgres/001_telemetry_spans.sql +33 -0
  144. package/src/migrations-postgres/002_usage_events.sql +24 -0
  145. package/src/migrations-postgres/003_usage_analytics_columns.sql +13 -0
  146. package/src/migrations-postgres/004_logs.sql +19 -0
  147. package/src/migrations-postgres/005_ai_calls.sql +25 -0
  148. package/src/migrations-postgres/006_user_profiles.sql +11 -0
  149. package/src/migrations-postgres/007_session_replay_metadata.sql +10 -0
  150. package/src/migrations-postgres/008_session_replay_storage_bytes.sql +2 -0
  151. package/src/migrations-postgres/009_projects_and_keys.sql +25 -0
  152. package/src/migrations-postgres/010_signal_project_id.sql +18 -0
  153. package/src/migrations-postgres/011_default_project_seed.sql +5 -0
  154. package/src/migrations-postgres/012_alert_rules.sql +17 -0
  155. package/src/migrations-postgres/013_alert_evaluations.sql +13 -0
  156. package/src/migrations-postgres/014_alert_state.sql +9 -0
  157. package/src/migrations-postgres/015_metrics.sql +47 -0
  158. package/src/migrations-postgres/016_metrics_exp_histogram_summary.sql +1 -0
  159. package/src/migrations-postgres/017_span_dropped_counts.sql +4 -0
  160. package/src/migrations-postgres/018_log_flags.sql +2 -0
  161. package/src/migrations-postgres/019_ai_span_payloads.sql +18 -0
  162. package/src/migrations-postgres/020_ai_span_evaluations.sql +21 -0
  163. package/src/migrations-postgres/021_ai_span_session.sql +7 -0
  164. package/src/migrations-postgres/022_span_log_session_id.sql +7 -0
  165. package/src/migrations-postgres/023_service_map_indexes.sql +9 -0
  166. package/src/migrations-postgres/024_analysis_results.sql +41 -0
  167. package/src/migrations-postgres/025_stage6_autopinning_alerts.sql +17 -0
  168. package/src/migrations-postgres/026_obs_dashboard_project_seed.sql +14 -0
  169. package/src/migrations-postgres/027_identity_propagation.sql +27 -0
  170. package/src/migrations-postgres/028_profile_blobs.sql +48 -0
  171. package/src/migrations-postgres/029_telemetry_sdk_name.sql +5 -0
  172. package/src/migrations-postgres/030_analysis_claim.sql +5 -0
  173. package/src/migrations-postgres/031_agent_action_graph.sql +190 -0
  174. package/src/migrations-postgres/032_payload_capture_flags.sql +8 -0
  175. package/src/migrations-postgres/033_eval_cases.sql +55 -0
  176. package/src/migrations-postgres/034_eval_case_results.sql +26 -0
  177. package/src/migrations-postgres/035_metric_exemplars.sql +31 -0
  178. package/src/migrations-postgres/036_trace_instrumentation_gaps.sql +19 -0
  179. package/src/migrations-postgres/037_eval_runs.sql +40 -0
  180. package/src/migrations-postgres/038_tool_call_evidence.sql +11 -0
  181. package/src/migrations-postgres/039_drop_trace_instrumentation_gaps.sql +11 -0
  182. package/src/migrations-postgres/040_evidence_retrieval_refs.sql +47 -0
  183. package/src/migrations-postgres/README.md +40 -0
  184. package/src/otlp/acceptance.test.ts +373 -0
  185. package/src/otlp/decode/body.ts +127 -0
  186. package/src/otlp/decode/logs.ts +96 -0
  187. package/src/otlp/decode/metrics.ts +402 -0
  188. package/src/otlp/decode/traces.ts +92 -0
  189. package/src/otlp/decode/values.ts +150 -0
  190. package/src/otlp/decode.test.ts +800 -0
  191. package/src/otlp/decode.ts +11 -0
  192. package/src/otlp/gen/opentelemetry/proto/collector/logs/v1/logs_service_pb.ts +132 -0
  193. package/src/otlp/gen/opentelemetry/proto/collector/metrics/v1/metrics_service_pb.ts +158 -0
  194. package/src/otlp/gen/opentelemetry/proto/collector/trace/v1/trace_service_pb.ts +158 -0
  195. package/src/otlp/gen/opentelemetry/proto/common/v1/common_pb.ts +289 -0
  196. package/src/otlp/gen/opentelemetry/proto/logs/v1/logs_pb.ts +397 -0
  197. package/src/otlp/gen/opentelemetry/proto/metrics/v1/metrics_pb.ts +1329 -0
  198. package/src/otlp/gen/opentelemetry/proto/resource/v1/resource_pb.ts +86 -0
  199. package/src/otlp/gen/opentelemetry/proto/trace/v1/trace_pb.ts +710 -0
  200. package/src/otlp/response.test.ts +158 -0
  201. package/src/otlp/response.ts +129 -0
  202. package/src/plugins/_context.ts +15 -0
  203. package/src/plugins/action-graph-processor/enrichers.ts +45 -0
  204. package/src/plugins/action-graph-processor/redaction.ts +84 -0
  205. package/src/plugins/action-graph-processor.test.ts +1023 -0
  206. package/src/plugins/action-graph-processor.ts +1172 -0
  207. package/src/plugins/action-routes.test.ts +716 -0
  208. package/src/plugins/action-routes.ts +343 -0
  209. package/src/plugins/ai-receiver.ts +254 -0
  210. package/src/plugins/ai-span-payloads-processor.test.ts +90 -0
  211. package/src/plugins/ai-span-payloads-processor.ts +180 -0
  212. package/src/plugins/alerts-evaluator.test.ts +91 -0
  213. package/src/plugins/alerts-evaluator.ts +328 -0
  214. package/src/plugins/alerts-routes.ts +101 -0
  215. package/src/plugins/analyses-routes.ts +138 -0
  216. package/src/plugins/ask-routes.ts +182 -0
  217. package/src/plugins/bot-filter.ts +19 -0
  218. package/src/plugins/connected-routes/manifest.ts +531 -0
  219. package/src/plugins/connected-routes.test.ts +1791 -0
  220. package/src/plugins/connected-routes.ts +1103 -0
  221. package/src/plugins/dashboard-routes.ts +41 -0
  222. package/src/plugins/default-span-enrichment.ts +92 -0
  223. package/src/plugins/eval-cases-routes.test.ts +391 -0
  224. package/src/plugins/eval-cases-routes.ts +529 -0
  225. package/src/plugins/evidence-retrieval-routes.benchmark.test.ts +245 -0
  226. package/src/plugins/evidence-retrieval-routes.test.ts +932 -0
  227. package/src/plugins/evidence-retrieval-routes.ts +2179 -0
  228. package/src/plugins/gen-ai-normalizer.test.ts +680 -0
  229. package/src/plugins/gen-ai-normalizer.ts +302 -0
  230. package/src/plugins/identity-receiver.ts +96 -0
  231. package/src/plugins/issue-enrichment.ts +132 -0
  232. package/src/plugins/issue-insights.ts +81 -0
  233. package/src/plugins/logs-receiver.ts +160 -0
  234. package/src/plugins/metrics-receiver.ts +91 -0
  235. package/src/plugins/onboarding-routes.ts +54 -0
  236. package/src/plugins/otlp-receiver.ts +124 -0
  237. package/src/plugins/platform-routes.ts +165 -0
  238. package/src/plugins/profile-routes.ts +468 -0
  239. package/src/plugins/projects-routes.ts +92 -0
  240. package/src/plugins/query-routes.ts +220 -0
  241. package/src/plugins/redaction-processor.ts +132 -0
  242. package/src/plugins/replay-query-routes.test.ts +38 -0
  243. package/src/plugins/replay-query-routes.ts +199 -0
  244. package/src/plugins/replay-receiver.ts +81 -0
  245. package/src/plugins/tail-routes.test.ts +47 -0
  246. package/src/plugins/tail-routes.ts +42 -0
  247. package/src/plugins/timeline-routes.ts +244 -0
  248. package/src/plugins/ua-enrichment.ts +27 -0
  249. package/src/plugins/usage-normalization.ts +46 -0
  250. package/src/plugins/usage-privacy.ts +55 -0
  251. package/src/plugins/usage-query-routes.ts +102 -0
  252. package/src/plugins/usage-receiver.ts +129 -0
  253. package/src/plugins/users-query-routes.ts +78 -0
  254. package/src/plugins/utm-enrichment.ts +36 -0
@@ -0,0 +1,467 @@
1
+ import type { StoredSpan, UsageEventRecord } from "@obsunified/types";
2
+ import { getConfiguredRetentionHours } from "@obsunified/types/constants";
3
+ import type { MiddlewareHandler } from "hono";
4
+ import { Hono } from "hono";
5
+ import { AIStore } from "../lib/ai-store";
6
+ import { runAllDueAnalyses } from "../lib/analyses-runner";
7
+ import { AnalysesStore } from "../lib/analyses-store";
8
+ import { LogsStore } from "../lib/logs-store";
9
+ import { MetricsStore } from "../lib/metrics-store";
10
+ import { aggregatePropagation } from "../lib/propagation-metric";
11
+ import { D1Adapter, dialectFor, type SqlDb, sqlDbFor } from "../lib/sql-db";
12
+ import { TelemetryStore } from "../lib/store";
13
+ import { UsageStore } from "../lib/usage-store";
14
+ import type { CollectorEnv, CollectorRouteContext } from "./env";
15
+ import {
16
+ type ChildSpanRunner,
17
+ consoleLogger,
18
+ type Logger,
19
+ passthroughChildSpan,
20
+ } from "./logger";
21
+
22
+ /**
23
+ * Factory that materializes an `SqlDb` for the request's `env`. Per-request
24
+ * because Cloudflare's `D1Database` binding only exists once `env` is in
25
+ * scope; on Workers this is essentially free since `D1Adapter` is a thin
26
+ * wrapper. Hosts override this to inject a fake DB in tests, a different
27
+ * engine in alternate runtimes, or a tracing wrapper at the storage layer.
28
+ */
29
+ export type SqlDbFactory = (env: CollectorEnv) => SqlDb;
30
+
31
+ const defaultSqlDbFactory: SqlDbFactory = (env) => new D1Adapter(env.DB);
32
+
33
+ export interface SpanProcessorPlugin {
34
+ name: string;
35
+ process(
36
+ spans: StoredSpan[],
37
+ context: CollectorRouteContext,
38
+ ): Promise<StoredSpan[]> | StoredSpan[];
39
+ }
40
+
41
+ export interface UsageEventProcessorPlugin {
42
+ name: string;
43
+ process(
44
+ events: UsageEventRecord[],
45
+ context: CollectorRouteContext,
46
+ ): Promise<UsageEventRecord[]> | UsageEventRecord[];
47
+ }
48
+
49
+ export interface CollectorPluginContext {
50
+ addSpanProcessor(processor: SpanProcessorPlugin): void;
51
+ addUsageEventProcessor(processor: UsageEventProcessorPlugin): void;
52
+ getRegisteredPluginNames(): string[];
53
+ }
54
+
55
+ export interface CollectorPlugin {
56
+ name: string;
57
+ register(
58
+ app: Hono<{ Bindings: CollectorEnv }>,
59
+ context: CollectorRuntime,
60
+ ): void;
61
+ }
62
+
63
+ export interface CollectorAuthConfig {
64
+ /** Single middleware applied to both /v1/* (ingest) and /internal/* (query) routes */
65
+ // biome-ignore lint/suspicious/noExplicitAny: Hono generic middleware variables require any for contravariant compatibility
66
+ middleware?: MiddlewareHandler<{ Bindings: CollectorEnv; Variables: any }>;
67
+ /** @deprecated Use middleware instead. Ingest-only auth for /v1/* routes. */
68
+ // biome-ignore lint/suspicious/noExplicitAny: Hono generic middleware variables require any for contravariant compatibility
69
+ ingest?: MiddlewareHandler<{ Bindings: CollectorEnv; Variables: any }>;
70
+ /** @deprecated Use middleware instead. Query-only auth for /internal/* routes. */
71
+ // biome-ignore lint/suspicious/noExplicitAny: Hono generic middleware variables require any for contravariant compatibility
72
+ query?: MiddlewareHandler<{ Bindings: CollectorEnv; Variables: any }>;
73
+ }
74
+
75
+ export interface CollectorConfig {
76
+ plugins: CollectorPlugin[];
77
+ auth?: CollectorAuthConfig;
78
+ /** Comma-separated allowed origins for CORS on ingest endpoints */
79
+ allowedOrigins?: string;
80
+ /** Register dashboard auth routes and middleware */
81
+ dashboardAuth?: {
82
+ // biome-ignore lint/suspicious/noExplicitAny: Hono generic middleware variables require any for contravariant compatibility
83
+ middleware: MiddlewareHandler<{ Bindings: CollectorEnv; Variables: any }>;
84
+ registerRoutes: (app: Hono<{ Bindings: CollectorEnv }>) => void;
85
+ };
86
+ /**
87
+ * Pluggable structured logger. Defaults to console output. Pass a logger
88
+ * from `@obsunified/telemetry-sdk` to ship the collector's own log output as
89
+ * OTLP — see apps/collector/SELF_INSTRUMENTATION.md.
90
+ */
91
+ logger?: Logger;
92
+ /**
93
+ * Wraps async work in a child span on the active request span. Wire this
94
+ * to `@obsunified/telemetry-sdk`'s `withChildSpan` from the worker entrypoint to
95
+ * surface LLM hops + DB-heavy paths as nested spans. Defaults to
96
+ * pass-through.
97
+ */
98
+ withChildSpan?: ChildSpanRunner;
99
+ /**
100
+ * RFC 0008 — override the storage adapter. Per-request factory because
101
+ * `env.DB` only exists once a request lands. Defaults to wrapping
102
+ * `env.DB` in `D1Adapter`. Override in tests (use `MemSqlDb`) or to add
103
+ * a tracing layer at the storage seam.
104
+ */
105
+ sqlDb?: SqlDbFactory;
106
+ }
107
+
108
+ export class CollectorRuntime implements CollectorPluginContext {
109
+ private readonly spanProcessors: SpanProcessorPlugin[] = [];
110
+ private readonly usageEventProcessors: UsageEventProcessorPlugin[] = [];
111
+ private readonly plugins: string[] = [];
112
+ private readonly sqlDbFactory: SqlDbFactory;
113
+ readonly logger: Logger;
114
+ readonly withChildSpan: ChildSpanRunner;
115
+
116
+ constructor(
117
+ logger: Logger = consoleLogger,
118
+ withChildSpan: ChildSpanRunner = passthroughChildSpan,
119
+ sqlDbFactory: SqlDbFactory = defaultSqlDbFactory,
120
+ ) {
121
+ this.logger = logger;
122
+ this.withChildSpan = withChildSpan;
123
+ this.sqlDbFactory = sqlDbFactory;
124
+ }
125
+
126
+ /**
127
+ * Materialize an `SqlDb` for this request. Stores will accept this
128
+ * directly once Phase 1.5 of RFC 0008 lands; until then plugins that
129
+ * want to bypass `c.env.DB` can call this to opt in early.
130
+ */
131
+ getSqlDb(env: CollectorEnv): SqlDb {
132
+ return this.sqlDbFactory(env);
133
+ }
134
+
135
+ registerPlugin(plugin: CollectorPlugin): void {
136
+ this.plugins.push(plugin.name);
137
+ }
138
+
139
+ addSpanProcessor(processor: SpanProcessorPlugin): void {
140
+ this.spanProcessors.push(processor);
141
+ }
142
+
143
+ addUsageEventProcessor(processor: UsageEventProcessorPlugin): void {
144
+ this.usageEventProcessors.push(processor);
145
+ }
146
+
147
+ getRegisteredPluginNames(): string[] {
148
+ return [...this.plugins];
149
+ }
150
+
151
+ createRouteContext(
152
+ env: CollectorEnv,
153
+ hono: CollectorRouteContext["hono"],
154
+ ): CollectorRouteContext {
155
+ return {
156
+ hono,
157
+ env,
158
+ now: new Date(),
159
+ logger: this.logger,
160
+ };
161
+ }
162
+
163
+ createStore(env: CollectorEnv): TelemetryStore {
164
+ return new TelemetryStore(this.getSqlDb(env));
165
+ }
166
+
167
+ createUsageStore(env: CollectorEnv): UsageStore {
168
+ return new UsageStore(this.getSqlDb(env));
169
+ }
170
+
171
+ async runSpanProcessors(
172
+ spans: StoredSpan[],
173
+ context: CollectorRouteContext,
174
+ ): Promise<StoredSpan[]> {
175
+ let nextSpans = spans;
176
+
177
+ for (const processor of this.spanProcessors) {
178
+ const inputCount = nextSpans.length;
179
+ nextSpans = await this.withChildSpan(
180
+ `process.${processor.name}`,
181
+ async (span) => {
182
+ const out = await processor.process(nextSpans, context);
183
+ span.setAttribute("processor.kind", "span");
184
+ span.setAttribute("processor.name", processor.name);
185
+ span.setAttribute("processor.input_count", inputCount);
186
+ span.setAttribute("processor.output_count", out.length);
187
+ if (out.length !== inputCount)
188
+ span.setAttribute("processor.delta_count", out.length - inputCount);
189
+ return out;
190
+ },
191
+ );
192
+ }
193
+
194
+ return nextSpans;
195
+ }
196
+
197
+ async runUsageEventProcessors(
198
+ events: UsageEventRecord[],
199
+ context: CollectorRouteContext,
200
+ ): Promise<UsageEventRecord[]> {
201
+ let nextEvents = events;
202
+
203
+ for (const processor of this.usageEventProcessors) {
204
+ const inputCount = nextEvents.length;
205
+ nextEvents = await this.withChildSpan(
206
+ `process.${processor.name}`,
207
+ async (span) => {
208
+ const out = await processor.process(nextEvents, context);
209
+ span.setAttribute("processor.kind", "usage");
210
+ span.setAttribute("processor.name", processor.name);
211
+ span.setAttribute("processor.input_count", inputCount);
212
+ span.setAttribute("processor.output_count", out.length);
213
+ if (out.length !== inputCount)
214
+ span.setAttribute("processor.delta_count", out.length - inputCount);
215
+ return out;
216
+ },
217
+ );
218
+ }
219
+
220
+ return nextEvents;
221
+ }
222
+ }
223
+
224
+ export const createTelemetryCollectorApp = (
225
+ pluginsOrConfig: CollectorPlugin[] | CollectorConfig,
226
+ ): Hono<{ Bindings: CollectorEnv }> => {
227
+ const config: CollectorConfig = Array.isArray(pluginsOrConfig)
228
+ ? { plugins: pluginsOrConfig }
229
+ : pluginsOrConfig;
230
+
231
+ const app = new Hono<{ Bindings: CollectorEnv }>();
232
+ const runtime = new CollectorRuntime(
233
+ config.logger,
234
+ config.withChildSpan,
235
+ config.sqlDb,
236
+ );
237
+
238
+ const allowedCorsOrigin = (origin: string | undefined, env: CollectorEnv) => {
239
+ const allowedOrigins = config.allowedOrigins || env.ALLOWED_ORIGINS || "";
240
+ const allowList = allowedOrigins
241
+ .split(",")
242
+ .map((s) => s.trim())
243
+ .filter(Boolean);
244
+ return origin && allowList.includes(origin) ? origin : undefined;
245
+ };
246
+
247
+ // Health endpoint — no auth required. Includes millisecond server time so
248
+ // browser SDKs can estimate clock offset without relying on coarse Date.
249
+ app.get("/health", (c) => {
250
+ const allowedOrigin = allowedCorsOrigin(c.req.header("Origin"), c.env);
251
+ const res = c.json({ status: "ok", serverTimeMs: Date.now() });
252
+ if (allowedOrigin)
253
+ res.headers.set("Access-Control-Allow-Origin", allowedOrigin);
254
+ return res;
255
+ });
256
+
257
+ // CORS for ingest endpoints (browser SDK sends directly)
258
+ app.use("/v1/*", async (c, next) => {
259
+ const origin = c.req.header("Origin");
260
+ const allowedOrigin = allowedCorsOrigin(origin, c.env);
261
+
262
+ if (c.req.method === "OPTIONS") {
263
+ const headers: Record<string, string> = {
264
+ "Access-Control-Allow-Methods": "POST, OPTIONS",
265
+ "Access-Control-Allow-Headers":
266
+ "Content-Type, Authorization, X-API-Key, X-Project-Id, Cache-Control, X-Telemetry-Self, X-Obs-Interaction, X-Obs-Session-Id",
267
+ "Access-Control-Max-Age": "86400",
268
+ };
269
+ if (allowedOrigin) {
270
+ headers["Access-Control-Allow-Origin"] = allowedOrigin;
271
+ }
272
+ return new Response(null, { status: 204, headers });
273
+ }
274
+
275
+ await next();
276
+
277
+ // Mutate the response headers directly rather than using c.header(),
278
+ // which queues the header for a response Hono is about to construct —
279
+ // no-op when downstream middleware already finalized the response via
280
+ // `c.json(...)` (e.g. auth returning 401). Without this, error
281
+ // responses ship CORS-naked and browsers misreport them as CORS
282
+ // blocks instead of the underlying status.
283
+ if (allowedOrigin) {
284
+ c.res.headers.set("Access-Control-Allow-Origin", allowedOrigin);
285
+ }
286
+ });
287
+
288
+ // Ingest auth (API key) — always on /v1/* when configured.
289
+ if (config.auth?.middleware) {
290
+ app.use("/v1/*", config.auth.middleware);
291
+ } else if (config.auth?.ingest) {
292
+ // Legacy: explicit ingest-only middleware
293
+ app.use("/v1/*", config.auth.ingest);
294
+ }
295
+
296
+ // Dashboard auth (session cookie) — owns /internal/* when configured.
297
+ // Routes for login/check/logout are always registered so the dashboard
298
+ // can authenticate regardless of subsequent middleware placement.
299
+ if (config.dashboardAuth) {
300
+ config.dashboardAuth.registerRoutes(app);
301
+ app.use("/dashboard/*", config.dashboardAuth.middleware);
302
+ app.use("/internal/*", config.dashboardAuth.middleware);
303
+ } else if (config.auth?.middleware) {
304
+ // No dashboard auth configured — fall back to applying the unified
305
+ // ingest auth to /internal/* too, preserving the earlier single-auth
306
+ // contract for deployments that haven't adopted dashboard auth yet.
307
+ app.use("/internal/*", config.auth.middleware);
308
+ } else if (config.auth?.query) {
309
+ app.use("/internal/*", config.auth.query);
310
+ }
311
+
312
+ for (const plugin of config.plugins) {
313
+ runtime.registerPlugin(plugin);
314
+ plugin.register(app, runtime);
315
+ }
316
+
317
+ app.get("/internal/collector/plugins", (c) => {
318
+ return c.json({
319
+ plugins: runtime.getRegisteredPluginNames(),
320
+ timestamp: new Date().toISOString(),
321
+ });
322
+ });
323
+
324
+ return app;
325
+ };
326
+
327
+ /**
328
+ * Scheduled handler that purges expired telemetry and usage rows.
329
+ * Wire into wrangler.toml: `[triggers] crons = ["0 * * * *"]`
330
+ * Source: AgentOwl + DecisionOps
331
+ */
332
+ export const createRetentionCleanupHandler = (options?: {
333
+ logger?: Logger;
334
+ }) => {
335
+ const logger = options?.logger ?? consoleLogger;
336
+ return {
337
+ async scheduled(
338
+ _event: ScheduledEvent,
339
+ env: CollectorEnv,
340
+ _ctx: ExecutionContext,
341
+ ): Promise<void> {
342
+ const db = sqlDbFor(env);
343
+ const telemetryStore = new TelemetryStore(db);
344
+ const usageStore = new UsageStore(db);
345
+ const logsStore = new LogsStore(db);
346
+ const aiStore = new AIStore(db);
347
+ const metricsStore = new MetricsStore(db);
348
+ const analysesStore = new AnalysesStore(db);
349
+
350
+ const [
351
+ telemetryPurged,
352
+ usagePurged,
353
+ logsPurged,
354
+ aiPurged,
355
+ metricsPurged,
356
+ analysesPurged,
357
+ ] = await Promise.all([
358
+ telemetryStore.purgeExpired(),
359
+ usageStore.purgeExpired(),
360
+ logsStore.purgeExpired(),
361
+ aiStore.purgeExpired(),
362
+ metricsStore.purgeExpired(),
363
+ analysesStore.purgeExpired(),
364
+ ]);
365
+
366
+ // RFC 0007 Phase 4.10 — profile_blobs retention. Cascade
367
+ // foreign key on profile_trace_index handles the join rows;
368
+ // we still need to delete the R2 blobs, which we do
369
+ // best-effort by reading the URLs first.
370
+ let profilesPurged = 0;
371
+ try {
372
+ const dialect = dialectFor(db);
373
+ const expiredProfiles = await db
374
+ .prepare(
375
+ `SELECT id, blob_url FROM profile_blobs WHERE expires_at < ${dialect.now()} LIMIT 500`,
376
+ )
377
+ .all<{ id: string; blob_url: string }>();
378
+ if (expiredProfiles.results.length > 0 && env.PROFILES_BUCKET) {
379
+ await Promise.allSettled(
380
+ expiredProfiles.results.map((p) =>
381
+ env.PROFILES_BUCKET?.delete(p.blob_url),
382
+ ),
383
+ );
384
+ }
385
+ const deleted = await db
386
+ .prepare(
387
+ `DELETE FROM profile_blobs WHERE expires_at < ${dialect.now()}`,
388
+ )
389
+ .run();
390
+ profilesPurged = deleted.meta.changes;
391
+ } catch (err) {
392
+ logger.error("[retention-cleanup] profile purge failed", {
393
+ error: err instanceof Error ? err.message : String(err),
394
+ });
395
+ }
396
+
397
+ logger.info("[retention-cleanup] purged expired rows", {
398
+ spans: telemetryPurged,
399
+ usage: usagePurged,
400
+ logs: logsPurged,
401
+ ai_calls: aiPurged,
402
+ metric_points: metricsPurged,
403
+ analysis_results: analysesPurged,
404
+ profile_blobs: profilesPurged,
405
+ });
406
+
407
+ // RFC 0004 Phase 1.8 — emit interaction_id propagation counters.
408
+ // Hourly aggregation rides the retention cron because cadence
409
+ // matches and both touch the same projects' tables.
410
+ try {
411
+ const propagation = await aggregatePropagation(db, new Date(), logger);
412
+ logger.info("[propagation-metric] hourly aggregation complete", {
413
+ projects: propagation.projects,
414
+ points_written: propagation.pointsWritten,
415
+ });
416
+ } catch (err) {
417
+ logger.error("[propagation-metric] aggregation failed", {
418
+ error: err instanceof Error ? err.message : String(err),
419
+ });
420
+ }
421
+ },
422
+ };
423
+ };
424
+
425
+ /**
426
+ * RFC 0002 Stage 1 — run any due Analyses. Cheap to call frequently because
427
+ * `runAllDueAnalyses` filters by `last_run + refreshSeconds < now` before
428
+ * touching D1.
429
+ *
430
+ * Wire to a per-minute trigger in wrangler.toml: `crons = ["* * * * *"]`.
431
+ * Kept distinct from retention so the per-minute tick doesn't sweep every
432
+ * retention table every cycle.
433
+ */
434
+ export const createAnalysesRunHandler = (options?: {
435
+ logger?: Logger;
436
+ tracer?: ChildSpanRunner;
437
+ }) => {
438
+ const logger = options?.logger ?? consoleLogger;
439
+ const tracer = options?.tracer;
440
+ return {
441
+ async scheduled(
442
+ _event: ScheduledEvent,
443
+ env: CollectorEnv,
444
+ _ctx: ExecutionContext,
445
+ ): Promise<void> {
446
+ try {
447
+ const retentionHours = getConfiguredRetentionHours(env.RETENTION_HOURS);
448
+ const summary = await runAllDueAnalyses({
449
+ env,
450
+ retentionHours,
451
+ logger,
452
+ tracer,
453
+ });
454
+ logger.info("[analyses] scheduled run summary", {
455
+ refreshed: summary.refreshed,
456
+ ran: summary.ran,
457
+ failed: summary.failed,
458
+ narrated: summary.narrated,
459
+ });
460
+ } catch (error) {
461
+ logger.error("[analyses] scheduled run failed", {
462
+ error: error instanceof Error ? error.message : String(error),
463
+ });
464
+ }
465
+ },
466
+ };
467
+ };
@@ -0,0 +1,90 @@
1
+ import type { Context, Hono } from "hono";
2
+ import type { Logger } from "./logger";
3
+
4
+ /**
5
+ * Collector worker bindings + env vars.
6
+ *
7
+ * Lives here, not in `@obsunified/types`, because it references Cloudflare
8
+ * Workers ambient globals (`D1Database`, `R2Bucket`,
9
+ * `DurableObjectNamespace`) that other packages — notably the web
10
+ * dashboard — should not transitively depend on.
11
+ */
12
+ export interface CollectorEnv {
13
+ DB: D1Database;
14
+ REPLAYS_BUCKET?: R2Bucket;
15
+ /**
16
+ * RFC 0007 — pprof profile blob storage. Keyed by `${projectId}/${ts}/${profileId}.pprof.gz`.
17
+ * Same dispatch pattern as REPLAYS_BUCKET — when unbound, the receiver
18
+ * falls back to filesystem under `PROFILE_BLOB_DIR` (Node) or returns
19
+ * 503 (Workers).
20
+ */
21
+ PROFILES_BUCKET?: R2Bucket;
22
+ /** Durable Object namespace for live-tail SSE pub/sub. Optional — when
23
+ * unbound the /internal/telemetry/tail endpoint returns 503. */
24
+ TAIL_HUB?: DurableObjectNamespace;
25
+ /** Write-only API key for SDK ingest endpoints (/v1/*) */
26
+ INGEST_KEY?: string;
27
+ /** Password for dashboard login */
28
+ DASHBOARD_PASSWORD?: string;
29
+ /** Comma-separated allowed origins for CORS (e.g. "https://my-app.com,https://staging.my-app.com") */
30
+ ALLOWED_ORIGINS?: string;
31
+ /** Set to "true" to allow unauthenticated ingest (local dev only) */
32
+ ALLOW_UNAUTHENTICATED?: string;
33
+ TELEMETRY_REDACT_FIELDS?: string;
34
+ /**
35
+ * Local/test escape hatch for payload capture. Production should prefer the
36
+ * per-project `payload_capture_enabled` flag, which defaults to false.
37
+ */
38
+ OBS_PAYLOAD_CAPTURE_DEFAULT?: string;
39
+ RETENTION_HOURS?: string;
40
+ /**
41
+ * Stage 3 narrative pipeline (RFC 0002).
42
+ * Either ANTHROPIC_API_KEY or OPENAI_API_KEY enables the narrate pass +
43
+ * Ask box. When neither is set, panels remain data-only.
44
+ *
45
+ * If both are set, OpenAI wins — configuring both is presumed deliberate.
46
+ *
47
+ * - ANTHROPIC_API_KEY: enables Anthropic /messages.
48
+ * - OPENAI_API_KEY: enables OpenAI /chat/completions. Use OPENAI_BASE_URL
49
+ * to point at compatible endpoints (openrouter, vLLM, azure, ollama).
50
+ * - NARRATIVE_MODEL: model id. Defaults are provider-specific
51
+ * (claude-haiku-4-5 for anthropic, gpt-4o-mini for openai).
52
+ * - NARRATIVE_BUDGET_PER_HOUR: max narrative writes per project per hour.
53
+ * Acts as a safety rail against a flapping panel running up cost.
54
+ */
55
+ ANTHROPIC_API_KEY?: string;
56
+ OPENAI_API_KEY?: string;
57
+ OPENAI_BASE_URL?: string;
58
+ NARRATIVE_MODEL?: string;
59
+ NARRATIVE_BUDGET_PER_HOUR?: string;
60
+ /**
61
+ * Self-instrumentation — collector dogfoods itself into the `obs-dashboard`
62
+ * project via @obsunified/telemetry-sdk. When unset, self-instrumentation is a
63
+ * no-op (collector still runs normally). See apps/collector/SELF_INSTRUMENTATION.md.
64
+ */
65
+ OBS_DASHBOARD_INGEST_KEY?: string;
66
+ /** Self-URL for cron-time telemetry exports (no inbound request to derive from). */
67
+ OBS_COLLECTOR_SELF_URL?: string;
68
+ /**
69
+ * When "true", the worker's self-instrumentation middleware awaits its
70
+ * span/log export instead of using `ctx.waitUntil`. Miniflare drops
71
+ * waitUntil after the response, losing post-async telemetry. Set in
72
+ * `.dev.vars` only; production CF Workers should leave this unset.
73
+ */
74
+ OBS_SELF_AWAIT_EXPORTS?: string;
75
+ /** @deprecated Use INGEST_KEY instead */
76
+ TELEMETRY_INGEST_TOKEN?: string;
77
+ /** @deprecated Use INGEST_KEY instead */
78
+ USAGE_INGEST_TOKEN?: string;
79
+ /** @deprecated Use DASHBOARD_PASSWORD instead */
80
+ TELEMETRY_QUERY_TOKEN?: string;
81
+ }
82
+
83
+ export interface CollectorRouteContext {
84
+ hono: Context<{ Bindings: CollectorEnv }>;
85
+ env: CollectorEnv;
86
+ now: Date;
87
+ logger: Logger;
88
+ }
89
+
90
+ export type CollectorApp = Hono<{ Bindings: CollectorEnv }>;
@@ -0,0 +1,81 @@
1
+ /**
2
+ * Pluggable logger interface for the collector framework.
3
+ *
4
+ * Structurally compatible with `@obsunified/telemetry-sdk`'s `Logger`, so the worker
5
+ * entrypoint can do:
6
+ *
7
+ * import { createLogger } from "@obsunified/telemetry-sdk";
8
+ * createDefaultCollectorApp({ logger: createLogger("obs-collector") });
9
+ *
10
+ * The framework package intentionally does not import telemetry-sdk to keep
11
+ * the dependency graph one-way (SDK depends on collector contracts, not the
12
+ * other way around). Anything that conforms to this interface works.
13
+ *
14
+ * When no logger is supplied, the framework falls back to a console-backed
15
+ * logger so out-of-the-box deployments keep their existing log output.
16
+ */
17
+ export interface Logger {
18
+ debug(message: string, attributes?: Record<string, unknown>): void;
19
+ info(message: string, attributes?: Record<string, unknown>): void;
20
+ warn(message: string, attributes?: Record<string, unknown>): void;
21
+ error(message: string, attributes?: Record<string, unknown>): void;
22
+ }
23
+
24
+ const formatAttrs = (attributes?: Record<string, unknown>): string => {
25
+ if (!attributes || Object.keys(attributes).length === 0) return "";
26
+ try {
27
+ return ` ${JSON.stringify(attributes)}`;
28
+ } catch {
29
+ return "";
30
+ }
31
+ };
32
+
33
+ export const consoleLogger: Logger = {
34
+ debug: (message, attributes) =>
35
+ console.debug(`${message}${formatAttrs(attributes)}`),
36
+ info: (message, attributes) =>
37
+ console.log(`${message}${formatAttrs(attributes)}`),
38
+ warn: (message, attributes) =>
39
+ console.warn(`${message}${formatAttrs(attributes)}`),
40
+ error: (message, attributes) =>
41
+ console.error(`${message}${formatAttrs(attributes)}`),
42
+ };
43
+
44
+ /**
45
+ * Minimal handle exposed to a `ChildSpanRunner` callback so the wrapped fn
46
+ * can stamp attributes / events / status onto the span as it learns about
47
+ * them (e.g., LLM token counts that are only known after the response).
48
+ *
49
+ * Structurally compatible with `ChildSpan` from `@obsunified/telemetry-sdk` so the
50
+ * worker entrypoint can pass the SDK's child span through directly without
51
+ * a bespoke adapter.
52
+ */
53
+ export interface ChildSpanHandle {
54
+ setAttribute(key: string, value: unknown): void;
55
+ addEvent?(name: string, attributes?: Record<string, unknown>): void;
56
+ setStatus?(code: number, message?: string): void;
57
+ }
58
+
59
+ /**
60
+ * Wraps an async fn in a child span attached to the active parent span.
61
+ * Structurally compatible with `withChildSpan` from `@obsunified/telemetry-sdk`.
62
+ * Default is pass-through — wire from the worker entrypoint to enable.
63
+ *
64
+ * The wrapped fn receives a `ChildSpanHandle` so it can stamp attributes
65
+ * after the fact (e.g., `gen_ai.usage.input_tokens` once the LLM response
66
+ * arrives). When unwired, the handle's setters are no-ops.
67
+ */
68
+ export type ChildSpanRunner = <T>(
69
+ name: string,
70
+ fn: (span: ChildSpanHandle) => Promise<T>,
71
+ attributes?: Record<string, unknown>,
72
+ ) => Promise<T>;
73
+
74
+ const NOOP_HANDLE: ChildSpanHandle = {
75
+ setAttribute() {},
76
+ addEvent() {},
77
+ setStatus() {},
78
+ };
79
+
80
+ export const passthroughChildSpan: ChildSpanRunner = async (_name, fn) =>
81
+ fn(NOOP_HANDLE);