@fml-inc/fml 0.7.0 → 0.8.0-rc.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/.claude-plugin/plugin.json +2 -4
  2. package/README.md +323 -79
  3. package/agent-assets/fml/SKILL.md +180 -0
  4. package/bin/fml +4 -18
  5. package/bin/hook-handler +7 -0
  6. package/bin/mcp-server +4 -18
  7. package/bin/observer +6 -0
  8. package/dist/chunk-3KISRT57.js +140 -0
  9. package/dist/chunk-56TPYMNK.js +4657 -0
  10. package/dist/chunk-7V7G7NVF.js +142 -0
  11. package/dist/chunk-A6HAEVJX.js +1606 -0
  12. package/dist/chunk-DZKYW327.js +1037 -0
  13. package/dist/chunk-ENW2GJNO.js +109 -0
  14. package/dist/{chunk-PZ5AY32C.js → chunk-MLKGABMK.js} +0 -1
  15. package/dist/{chunk-PTUZMKSN.js → chunk-R45CWWZO.js} +34 -23
  16. package/dist/chunk-WZLLO563.js +953 -0
  17. package/dist/chunk-YSU5CS2V.js +3331 -0
  18. package/dist/cli.js +7742 -1072
  19. package/dist/{esm-2EUDFZWP.js → esm-4YKLKFRX.js} +546 -741
  20. package/dist/hook-handler.js +440 -0
  21. package/dist/mcp/server.js +112 -98
  22. package/dist/observer-server.js +1566 -0
  23. package/dist/sync-cycle.js +31 -0
  24. package/dist/{token-store-3IQSLOTQ.js → token-store-BT5THIAJ.js} +10 -3
  25. package/hooks/hooks.json +290 -0
  26. package/package.json +24 -27
  27. package/dist/chunk-PTUZMKSN.js.map +0 -1
  28. package/dist/chunk-PZ5AY32C.js.map +0 -1
  29. package/dist/chunk-S4TJNLEN.js +0 -61
  30. package/dist/chunk-S4TJNLEN.js.map +0 -1
  31. package/dist/chunk-VV4JHH4H.js +0 -437
  32. package/dist/chunk-VV4JHH4H.js.map +0 -1
  33. package/dist/chunk-YH5ROR7G.js +0 -69
  34. package/dist/chunk-YH5ROR7G.js.map +0 -1
  35. package/dist/cli.js.map +0 -1
  36. package/dist/daemon-MSLDHLMH.js +0 -21
  37. package/dist/daemon-MSLDHLMH.js.map +0 -1
  38. package/dist/daemon-utils-OODBIYB7.js +0 -10
  39. package/dist/daemon-utils-OODBIYB7.js.map +0 -1
  40. package/dist/esm-2EUDFZWP.js.map +0 -1
  41. package/dist/mcp/server.js.map +0 -1
  42. package/dist/postinstall.js +0 -16
  43. package/dist/postinstall.js.map +0 -1
  44. package/dist/token-store-3IQSLOTQ.js.map +0 -1
  45. package/scripts/postinstall.cjs +0 -25
@@ -0,0 +1,1566 @@
1
+ #!/usr/bin/env node
2
+ import {
3
+ ObserverStateMutationBusyError,
4
+ claimObserverState,
5
+ createInitialObserverState,
6
+ observerLeaseIsOwned,
7
+ removeObserverStateIfOwned,
8
+ updateObserverState
9
+ } from "./chunk-WZLLO563.js";
10
+ import {
11
+ createHookSyncLoop,
12
+ getActiveHookDestinationBinding
13
+ } from "./chunk-DZKYW327.js";
14
+ import {
15
+ OBSERVER_PROTOCOL_VERSION,
16
+ OBSERVER_SERVICE_NAME,
17
+ ObserverHookSpoolEntryTooLargeError,
18
+ acknowledgeHookInboxEntry,
19
+ createObserverHttpServer,
20
+ drainHookInbox,
21
+ getOrCreateObserverAuthToken,
22
+ normalizeObserverHookCapture,
23
+ openObserverDatabase,
24
+ parseHookCaptureRequest,
25
+ parseHookInboxMetadata,
26
+ readHookInboxStatusAsync,
27
+ retireHookInboxEntry
28
+ } from "./chunk-56TPYMNK.js";
29
+ import "./chunk-7V7G7NVF.js";
30
+ import {
31
+ ObserverSyncCycleAbortedError,
32
+ runObserverSyncCycleSubprocess
33
+ } from "./chunk-3KISRT57.js";
34
+ import {
35
+ createCliRawSync
36
+ } from "./chunk-YSU5CS2V.js";
37
+ import {
38
+ FML_OBSERVER_AUTH_TOKEN_FILE,
39
+ FML_OBSERVER_ENDPOINT_FILE,
40
+ FML_OBSERVER_HOST,
41
+ FML_OBSERVER_LOCK_FILE,
42
+ FML_OBSERVER_PORT,
43
+ FML_OBSERVER_STATE_FILE,
44
+ FML_PANOPTICON_DB_FILE,
45
+ FML_PANOPTICON_DB_OBSERVER_LOCK_FILE,
46
+ FML_PANOPTICON_DB_OBSERVER_STATE_FILE,
47
+ ensureDirs,
48
+ getActiveEnv,
49
+ getSiteUrlForConvexUrl,
50
+ getSyncDestinationAuthBinding,
51
+ getValidToken,
52
+ observerOnlyOverrideWarning
53
+ } from "./chunk-A6HAEVJX.js";
54
+ import "./chunk-MLKGABMK.js";
55
+
56
+ // src/observer-server.ts
57
+ import fs from "fs";
58
+
59
+ // src/otel/sync-loop.ts
60
+ import { createHash } from "crypto";
61
+ var IDLE_INTERVAL_MS = 3e4;
62
+ var AUTH_INTERVAL_MS = 6e4;
63
+ var MIN_RETRY_INTERVAL_MS = 2e3;
64
+ var MAX_RETRY_INTERVAL_MS = 5 * 6e4;
65
+ var MAX_RETRY_AFTER_MS = 24 * 60 * 6e4;
66
+ var OTLP_UPLOAD_TIMEOUT_MS = 45e3;
67
+ var MAX_ERROR_TEXT_BYTES = 1e3;
68
+ var MAX_REJECTION_REASON_BYTES = 3 * 1024;
69
+ var MAX_OTLP_RESPONSE_BYTES = 64 * 1024;
70
+ var MAX_OTLP_UPLOAD_ATTEMPTS = 10;
71
+ var MAX_OTLP_RETRY_AGE_MS = 24 * 60 * 6e4;
72
+ var OUTBOX_BATCH_SIZE = 1;
73
+ var DRAIN_INTERVAL_MS = 25;
74
+ function combineWarnings(...warnings) {
75
+ const present = warnings.filter(
76
+ (warning) => warning !== void 0
77
+ );
78
+ return present.length === 0 ? void 0 : boundedUtf8(present.join("; "), MAX_ERROR_TEXT_BYTES);
79
+ }
80
+ function utf8Prefix(value, maxBytes) {
81
+ let low = 0;
82
+ let high = value.length;
83
+ while (low < high) {
84
+ const middle = Math.ceil((low + high) / 2);
85
+ if (Buffer.byteLength(value.slice(0, middle), "utf8") <= maxBytes) {
86
+ low = middle;
87
+ } else {
88
+ high = middle - 1;
89
+ }
90
+ }
91
+ let end = low;
92
+ if (end > 0 && end < value.length && value.charCodeAt(end - 1) >= 55296 && value.charCodeAt(end - 1) <= 56319 && value.charCodeAt(end) >= 56320 && value.charCodeAt(end) <= 57343) {
93
+ end -= 1;
94
+ }
95
+ return value.slice(0, end);
96
+ }
97
+ function boundedUtf8(value, maxBytes) {
98
+ if (Buffer.byteLength(value, "utf8") <= maxBytes) return value;
99
+ const marker = "...";
100
+ return `${utf8Prefix(value, maxBytes - marker.length)}${marker}`;
101
+ }
102
+ function dropReason(prefix, detail) {
103
+ return boundedUtf8(`${prefix}: ${detail}`, MAX_REJECTION_REASON_BYTES);
104
+ }
105
+ function retryAtMs(response, nowMs) {
106
+ const value = response.headers.get("Retry-After");
107
+ if (!value) return void 0;
108
+ const seconds = Number(value);
109
+ const candidate = Number.isFinite(seconds) ? nowMs + Math.max(0, seconds) * 1e3 : Date.parse(value);
110
+ if (!Number.isFinite(candidate)) return void 0;
111
+ return Math.min(nowMs + MAX_RETRY_AFTER_MS, Math.max(nowMs, candidate));
112
+ }
113
+ function bodySha256(body) {
114
+ return createHash("sha256").update(body).digest("hex");
115
+ }
116
+ function parseVarint(body, offset) {
117
+ let value = 0n;
118
+ for (let index = 0; index < 10 && offset + index < body.length; index += 1) {
119
+ const byte = body[offset + index];
120
+ if (index === 9 && byte > 1) return null;
121
+ value |= BigInt(byte & 127) << BigInt(index * 7);
122
+ if ((byte & 128) === 0) {
123
+ return { value, nextOffset: offset + index + 1 };
124
+ }
125
+ }
126
+ return null;
127
+ }
128
+ function parseLengthDelimited(body, offset) {
129
+ const length = parseVarint(body, offset);
130
+ if (!length || length.value > BigInt(Number.MAX_SAFE_INTEGER)) return null;
131
+ const byteLength = Number(length.value);
132
+ const end = length.nextOffset + byteLength;
133
+ if (end > body.length) return null;
134
+ return { value: body.subarray(length.nextOffset, end), nextOffset: end };
135
+ }
136
+ function validateProtobufPartialSuccess(body) {
137
+ let offset = 0;
138
+ let rejectedRecords = 0n;
139
+ let warning;
140
+ while (offset < body.length) {
141
+ const key = parseVarint(body, offset);
142
+ if (!key) {
143
+ return { error: "OTLP partial-success response has a malformed field" };
144
+ }
145
+ offset = key.nextOffset;
146
+ const fieldNumber = key.value >> 3n;
147
+ const wireType = Number(key.value & 0x07n);
148
+ if (fieldNumber === 1n && wireType === 0) {
149
+ const rejected = parseVarint(body, offset);
150
+ if (!rejected) {
151
+ return {
152
+ error: "OTLP partial-success response has a malformed rejected count"
153
+ };
154
+ }
155
+ offset = rejected.nextOffset;
156
+ if (rejected.value > rejectedRecords) rejectedRecords = rejected.value;
157
+ continue;
158
+ }
159
+ if (fieldNumber === 2n && wireType === 2) {
160
+ const errorMessage = parseLengthDelimited(body, offset);
161
+ if (!errorMessage) {
162
+ return {
163
+ error: "OTLP partial-success response has a malformed error message"
164
+ };
165
+ }
166
+ offset = errorMessage.nextOffset;
167
+ let decoded;
168
+ try {
169
+ decoded = new TextDecoder("utf-8", { fatal: true }).decode(
170
+ errorMessage.value
171
+ );
172
+ } catch {
173
+ return {
174
+ error: "OTLP partial-success response has a non-UTF-8 error message"
175
+ };
176
+ }
177
+ if (decoded.length > 0) {
178
+ warning = boundedUtf8(
179
+ `OTLP destination reported partial success: ${decoded}`,
180
+ MAX_ERROR_TEXT_BYTES
181
+ );
182
+ }
183
+ continue;
184
+ }
185
+ return {
186
+ error: "OTLP partial-success response contains an unexpected field"
187
+ };
188
+ }
189
+ if (rejectedRecords > 0n) {
190
+ return {
191
+ error: `OTLP destination partially rejected ${rejectedRecords.toString()} record(s)`,
192
+ rejectedRecords
193
+ };
194
+ }
195
+ return warning ? { warning } : {};
196
+ }
197
+ function validateProtobufSuccess(body) {
198
+ let offset = 0;
199
+ let warning;
200
+ while (offset < body.length) {
201
+ const key = parseVarint(body, offset);
202
+ if (!key) return { error: "OTLP success response has a malformed field" };
203
+ offset = key.nextOffset;
204
+ const fieldNumber = key.value >> 3n;
205
+ const wireType = Number(key.value & 0x07n);
206
+ if (fieldNumber !== 1n || wireType !== 2) {
207
+ return { error: "OTLP success response contains an unexpected field" };
208
+ }
209
+ const partialSuccess = parseLengthDelimited(body, offset);
210
+ if (!partialSuccess) {
211
+ return {
212
+ error: "OTLP success response has a malformed partial-success field"
213
+ };
214
+ }
215
+ offset = partialSuccess.nextOffset;
216
+ const partial = validateProtobufPartialSuccess(partialSuccess.value);
217
+ if (partial.error) return partial;
218
+ if (partial.warning) warning = partial.warning;
219
+ }
220
+ return warning ? { warning } : {};
221
+ }
222
+ function rejectedJsonCount(value) {
223
+ if (typeof value === "number" && Number.isSafeInteger(value) && value >= 0) {
224
+ return BigInt(value);
225
+ }
226
+ if (typeof value === "string" && value.length <= 20 && /^(?:0|[1-9]\d*)$/.test(value)) {
227
+ return BigInt(value);
228
+ }
229
+ return null;
230
+ }
231
+ function validateJsonSuccess(body, signal) {
232
+ let decoded;
233
+ try {
234
+ decoded = new TextDecoder("utf-8", { fatal: true }).decode(body);
235
+ } catch {
236
+ return { error: "OTLP JSON success response is not UTF-8" };
237
+ }
238
+ let parsed;
239
+ try {
240
+ parsed = JSON.parse(decoded);
241
+ } catch {
242
+ return { error: "OTLP JSON success response is malformed" };
243
+ }
244
+ if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
245
+ return { error: "OTLP JSON success response is not an object" };
246
+ }
247
+ const response = parsed;
248
+ if (Object.keys(response).some((key) => key !== "partialSuccess")) {
249
+ return {
250
+ error: "OTLP JSON success response contains an unexpected field"
251
+ };
252
+ }
253
+ const partialSuccess = response.partialSuccess;
254
+ if (partialSuccess === void 0) return {};
255
+ if (!partialSuccess || typeof partialSuccess !== "object" || Array.isArray(partialSuccess)) {
256
+ return { error: "OTLP JSON partial-success response is not an object" };
257
+ }
258
+ const partial = partialSuccess;
259
+ const rejectedField = signal === "traces" ? "rejectedSpans" : signal === "metrics" ? "rejectedDataPoints" : "rejectedLogRecords";
260
+ if (Object.keys(partial).some(
261
+ (key) => key !== rejectedField && key !== "errorMessage"
262
+ )) {
263
+ return {
264
+ error: "OTLP JSON partial-success response contains an unexpected field"
265
+ };
266
+ }
267
+ if (partial[rejectedField] !== void 0) {
268
+ const rejected = rejectedJsonCount(partial[rejectedField]);
269
+ if (rejected === null) {
270
+ return {
271
+ error: "OTLP JSON partial-success response has an invalid rejected count"
272
+ };
273
+ }
274
+ if (rejected > 0n) {
275
+ return {
276
+ error: `OTLP destination partially rejected ${rejected.toString()} record(s)`,
277
+ rejectedRecords: rejected
278
+ };
279
+ }
280
+ }
281
+ if (partial.errorMessage !== void 0) {
282
+ if (typeof partial.errorMessage !== "string") {
283
+ return {
284
+ error: "OTLP JSON partial-success response has an invalid error message"
285
+ };
286
+ }
287
+ if (partial.errorMessage.length > 0) {
288
+ return {
289
+ warning: boundedUtf8(
290
+ `OTLP destination reported partial success: ${partial.errorMessage}`,
291
+ MAX_ERROR_TEXT_BYTES
292
+ )
293
+ };
294
+ }
295
+ }
296
+ return {};
297
+ }
298
+ async function readBoundedResponseBytes(response, maxBytes) {
299
+ const declaredLength = response.headers.get("Content-Length");
300
+ if (declaredLength !== null && /^\d+$/.test(declaredLength) && Number(declaredLength) > maxBytes) {
301
+ await response.body?.cancel().catch(() => void 0);
302
+ return null;
303
+ }
304
+ if (!response.body) return new Uint8Array();
305
+ const reader = response.body.getReader();
306
+ const chunks = [];
307
+ let totalBytes = 0;
308
+ try {
309
+ while (true) {
310
+ const { done, value } = await reader.read();
311
+ if (done) break;
312
+ totalBytes += value.byteLength;
313
+ if (totalBytes > maxBytes) {
314
+ await reader.cancel().catch(() => void 0);
315
+ return null;
316
+ }
317
+ chunks.push(value);
318
+ }
319
+ } finally {
320
+ reader.releaseLock();
321
+ }
322
+ const body = new Uint8Array(totalBytes);
323
+ let offset = 0;
324
+ for (const chunk of chunks) {
325
+ body.set(chunk, offset);
326
+ offset += chunk.byteLength;
327
+ }
328
+ return body;
329
+ }
330
+ function validateOtlpPayloadSuccess(body, upstreamContentType, envelope) {
331
+ const responseContentType = upstreamContentType?.split(";", 1)[0]?.trim().toLowerCase();
332
+ const validationContentType = responseContentType === "application/json" || responseContentType === "application/x-protobuf" ? responseContentType : envelope.contentType;
333
+ const contentTypeWarning = responseContentType === envelope.contentType ? void 0 : responseContentType === void 0 ? "OTLP success response omitted Content-Type; validated using the request encoding" : responseContentType === validationContentType ? `OTLP success response changed Content-Type to ${validationContentType}` : `OTLP success response used unsupported Content-Type ${responseContentType}; validated using the request encoding`;
334
+ const validation = body.byteLength === 0 ? {} : validationContentType === "application/x-protobuf" ? validateProtobufSuccess(body) : validateJsonSuccess(body, envelope.signal);
335
+ const warning = combineWarnings(contentTypeWarning, validation.warning);
336
+ return {
337
+ ...validation.error ? { error: validation.error } : {},
338
+ ...warning ? { warning } : {},
339
+ ...validation.rejectedRecords !== void 0 ? { rejectedRecords: validation.rejectedRecords } : {}
340
+ };
341
+ }
342
+ async function inspectOtlpSuccess(response, envelope) {
343
+ let body;
344
+ try {
345
+ body = await readBoundedResponseBytes(response, MAX_OTLP_RESPONSE_BYTES);
346
+ } catch {
347
+ return {
348
+ warning: "OTLP success response could not be read; treating the 2xx status as delivered"
349
+ };
350
+ }
351
+ if (body === null) {
352
+ return {
353
+ warning: `OTLP success response exceeds ${MAX_OTLP_RESPONSE_BYTES} bytes; treating the 2xx status as delivered`
354
+ };
355
+ }
356
+ if (body.byteLength === 0) return {};
357
+ const validation = validateOtlpPayloadSuccess(
358
+ body,
359
+ response.headers.get("Content-Type"),
360
+ envelope
361
+ );
362
+ const parseWarning = validation.error ? `${validation.error}; treating the 2xx status as delivered` : void 0;
363
+ const partialRejectionWarning = validation.rejectedRecords !== void 0 && validation.rejectedRecords > 0n ? `OTLP destination partially rejected ${validation.rejectedRecords.toString()} record(s)` : void 0;
364
+ const warning = combineWarnings(
365
+ validation.warning,
366
+ parseWarning,
367
+ partialRejectionWarning
368
+ );
369
+ return {
370
+ ...warning ? { warning } : {}
371
+ };
372
+ }
373
+ async function responseError(response) {
374
+ let text = "";
375
+ try {
376
+ const bytes = await readBoundedResponseBytes(
377
+ response,
378
+ MAX_OTLP_RESPONSE_BYTES
379
+ );
380
+ if (bytes === null) {
381
+ return `OTLP error response exceeds ${MAX_OTLP_RESPONSE_BYTES} bytes`;
382
+ }
383
+ text = new TextDecoder().decode(bytes);
384
+ } catch {
385
+ }
386
+ if (text) {
387
+ try {
388
+ const body = JSON.parse(text);
389
+ if (body && typeof body === "object" && !Array.isArray(body)) {
390
+ const record = body;
391
+ const detail = typeof record.error === "string" ? record.error : typeof record.message === "string" ? record.message : null;
392
+ if (detail) return boundedUtf8(detail, MAX_ERROR_TEXT_BYTES);
393
+ }
394
+ } catch {
395
+ return boundedUtf8(text, MAX_ERROR_TEXT_BYTES);
396
+ }
397
+ }
398
+ return `OTLP request failed with HTTP ${response.status}`;
399
+ }
400
+ async function postOtlpEnvelope(destination, envelope, opts) {
401
+ const body = envelope.body;
402
+ const headers = {
403
+ Authorization: destination.authorization,
404
+ "Content-Type": envelope.contentType,
405
+ "Content-Encoding": envelope.contentEncoding,
406
+ "User-Agent": `fml/${opts.version}`,
407
+ "X-Panopticon-Version": envelope.observerVersion,
408
+ "X-FML-OTLP-Envelope-Id": envelope.envelopeId,
409
+ "X-FML-OTLP-Body-SHA256": envelope.sha256,
410
+ "X-FML-OTLP-Captured-At-Ms": String(envelope.capturedAtMs),
411
+ "X-FML-Observer-Instance-Id": envelope.observerInstanceId,
412
+ ...destination.organizationId ? { "X-FML-Organization-Id": destination.organizationId } : {},
413
+ ...destination.organizationSlug ? { "X-FML-Organization": destination.organizationSlug } : {}
414
+ };
415
+ const request = opts.fetch ?? fetch;
416
+ const timeoutSignal = AbortSignal.timeout(OTLP_UPLOAD_TIMEOUT_MS);
417
+ const signal = opts.signal ? AbortSignal.any([opts.signal, timeoutSignal]) : timeoutSignal;
418
+ const response = await request(
419
+ `${destination.targetUrl.replace(/\/+$/, "")}/v1/${envelope.signal}`,
420
+ {
421
+ method: "POST",
422
+ headers,
423
+ body: body.buffer instanceof ArrayBuffer ? new Uint8Array(body.buffer, body.byteOffset, body.byteLength) : Uint8Array.from(body),
424
+ signal
425
+ }
426
+ );
427
+ if (response.ok) {
428
+ const inspection = await inspectOtlpSuccess(response, envelope);
429
+ return { ok: true, status: response.status, ...inspection };
430
+ }
431
+ const error = await responseError(response);
432
+ const errorCode = response.headers.get("X-FML-OTLP-Error-Code");
433
+ return {
434
+ ok: false,
435
+ status: response.status,
436
+ error,
437
+ ...errorCode ? { errorCode } : {},
438
+ ...response.status === 429 || response.status >= 500 ? { retryAtMs: retryAtMs(response, opts.nowMs()) } : {}
439
+ };
440
+ }
441
+ function activeOtlpDestinationSnapshot() {
442
+ const environment = getActiveEnv();
443
+ const authBinding = getSyncDestinationAuthBinding(environment.name);
444
+ if (!authBinding) {
445
+ return { kind: "unauthenticated", environment: environment.name };
446
+ }
447
+ const baseBinding = {
448
+ environment: environment.name,
449
+ targetUrl: getSiteUrlForConvexUrl(environment.convexUrl),
450
+ destinationIdentity: authBinding.destinationIdentity
451
+ };
452
+ if (authBinding.organizationSelectionRequired) {
453
+ return { kind: "organization_required", binding: baseBinding };
454
+ }
455
+ return {
456
+ kind: "ready",
457
+ binding: {
458
+ ...baseBinding,
459
+ ...authBinding.organizationId && authBinding.organizationSlug ? {
460
+ organizationId: authBinding.organizationId,
461
+ organizationSlug: authBinding.organizationSlug
462
+ } : {}
463
+ }
464
+ };
465
+ }
466
+ function getActiveOtlpDestinationBinding() {
467
+ const snapshot = activeOtlpDestinationSnapshot();
468
+ return snapshot.kind === "unauthenticated" ? null : snapshot.binding;
469
+ }
470
+ function sameDestinationBinding(left, right) {
471
+ return left.environment === right.environment && left.targetUrl === right.targetUrl && left.destinationIdentity === right.destinationIdentity && left.organizationId === right.organizationId && left.organizationSlug === right.organizationSlug;
472
+ }
473
+ async function resolveDestination(forceRefresh = false) {
474
+ const initial2 = activeOtlpDestinationSnapshot();
475
+ if (initial2.kind === "unauthenticated") return initial2;
476
+ if (initial2.kind === "organization_required") {
477
+ return {
478
+ kind: "organization_required",
479
+ environment: initial2.binding.environment
480
+ };
481
+ }
482
+ const destination = initial2.binding;
483
+ const token = await getValidToken({
484
+ env: destination.environment,
485
+ ...forceRefresh ? { forceRefresh: true } : {}
486
+ });
487
+ if (!token) {
488
+ return {
489
+ kind: "unauthenticated",
490
+ environment: destination.environment
491
+ };
492
+ }
493
+ const verified = activeOtlpDestinationSnapshot();
494
+ if (verified.kind !== "ready" || !sameDestinationBinding(destination, verified.binding)) {
495
+ if (verified.kind === "ready") {
496
+ return {
497
+ kind: "unauthenticated",
498
+ environment: verified.binding.environment
499
+ };
500
+ }
501
+ return verified.kind === "organization_required" ? {
502
+ kind: "organization_required",
503
+ environment: verified.binding.environment
504
+ } : verified;
505
+ }
506
+ return {
507
+ kind: "ready",
508
+ destination: {
509
+ ...destination,
510
+ authorization: token.startsWith("Bearer ") ? token : `Bearer ${token}`
511
+ }
512
+ };
513
+ }
514
+ async function runOtlpSyncCycle(opts) {
515
+ const nowMs = opts.nowMs ?? Date.now;
516
+ const cycleStartedAtMs = nowMs();
517
+ const totalPending = opts.database.countPendingOtlpEnvelopes();
518
+ if (totalPending === 0) {
519
+ return { kind: "drained", pending: 0, uploaded: 0 };
520
+ }
521
+ const destinationResolution = await resolveDestination();
522
+ if (destinationResolution.kind !== "ready") {
523
+ return {
524
+ kind: destinationResolution.kind,
525
+ environment: destinationResolution.environment,
526
+ pending: opts.database.countPendingOtlpEnvelopes(),
527
+ uploaded: 0
528
+ };
529
+ }
530
+ const destination = destinationResolution.destination;
531
+ const envelopes = opts.database.claimPendingOtlpEnvelopes(
532
+ destination,
533
+ OUTBOX_BATCH_SIZE,
534
+ cycleStartedAtMs
535
+ );
536
+ if (envelopes.length === 0) {
537
+ const nextAttemptAtMs = opts.database.nextPendingOtlpAttemptAtMs(destination);
538
+ if (nextAttemptAtMs !== void 0) {
539
+ return {
540
+ kind: "completed",
541
+ environment: destination.environment,
542
+ pending: opts.database.countPendingOtlpEnvelopes(),
543
+ uploaded: 0,
544
+ retryAtMs: nextAttemptAtMs
545
+ };
546
+ }
547
+ return {
548
+ kind: "destination_drained",
549
+ environment: destination.environment,
550
+ pending: opts.database.countPendingOtlpEnvelopes(),
551
+ uploaded: 0
552
+ };
553
+ }
554
+ let uploaded = 0;
555
+ let dropped = 0;
556
+ let lastError;
557
+ let lastWarning;
558
+ let nextRetryAtMs;
559
+ const upload = async (uploadDestination, envelope) => {
560
+ try {
561
+ return await postOtlpEnvelope(uploadDestination, envelope, {
562
+ version: opts.version,
563
+ ...opts.signal ? { signal: opts.signal } : {},
564
+ ...opts.fetch ? { fetch: opts.fetch } : {},
565
+ nowMs
566
+ });
567
+ } catch (error) {
568
+ if (opts.signal?.aborted) throw opts.signal.reason;
569
+ const detail = error instanceof Error ? error.message : String(error);
570
+ return {
571
+ ok: false,
572
+ status: 0,
573
+ error: boundedUtf8(
574
+ `OTLP request failed: ${detail}`,
575
+ MAX_ERROR_TEXT_BYTES
576
+ )
577
+ };
578
+ }
579
+ };
580
+ for (const envelope of envelopes) {
581
+ if (opts.signal?.aborted) throw opts.signal.reason;
582
+ if (cycleStartedAtMs - envelope.capturedAtMs >= MAX_OTLP_RETRY_AGE_MS) {
583
+ dropped += opts.database.dropOtlpEnvelopes(
584
+ destination,
585
+ [
586
+ {
587
+ envelopeId: envelope.envelopeId,
588
+ reason: dropReason(
589
+ "otlp_max_age_exceeded",
590
+ `captured ${cycleStartedAtMs - envelope.capturedAtMs}ms ago`
591
+ )
592
+ }
593
+ ],
594
+ nowMs()
595
+ );
596
+ continue;
597
+ }
598
+ const actualSha256 = bodySha256(envelope.body);
599
+ if (actualSha256 !== envelope.sha256) {
600
+ dropped += opts.database.dropOtlpEnvelopes(
601
+ destination,
602
+ [
603
+ {
604
+ envelopeId: envelope.envelopeId,
605
+ reason: dropReason(
606
+ "local_body_hash_mismatch",
607
+ `expected ${envelope.sha256}, received ${actualSha256}`
608
+ )
609
+ }
610
+ ],
611
+ nowMs()
612
+ );
613
+ continue;
614
+ }
615
+ let response = await upload(destination, envelope);
616
+ if (response.status === 401 && response.errorCode !== "UPSTREAM_REJECTED") {
617
+ const refreshed = await resolveDestination(true);
618
+ if (refreshed.kind !== "ready" || !sameDestinationBinding(destination, refreshed.destination) || refreshed.destination.authorization === destination.authorization) {
619
+ return {
620
+ kind: "unauthenticated",
621
+ environment: destination.environment,
622
+ pending: opts.database.countPendingOtlpEnvelopes(),
623
+ uploaded
624
+ };
625
+ }
626
+ response = await upload(refreshed.destination, envelope);
627
+ }
628
+ if (response.ok) {
629
+ uploaded += opts.database.acknowledgeOtlpEnvelopes(destination, [
630
+ envelope.envelopeId
631
+ ]);
632
+ lastWarning = response.warning;
633
+ continue;
634
+ }
635
+ if (response.status === 401 && response.errorCode !== "UPSTREAM_REJECTED") {
636
+ return {
637
+ kind: "unauthenticated",
638
+ environment: destination.environment,
639
+ pending: opts.database.countPendingOtlpEnvelopes(),
640
+ uploaded
641
+ };
642
+ }
643
+ if (response.errorCode === "ORGANIZATION_REQUIRED") {
644
+ return {
645
+ kind: "organization_required",
646
+ environment: destination.environment,
647
+ pending: opts.database.countPendingOtlpEnvelopes(),
648
+ uploaded: 0
649
+ };
650
+ }
651
+ const transient = response.status === 0 || response.status === 429 || response.status >= 500;
652
+ if (!transient) {
653
+ dropped += opts.database.dropOtlpEnvelopes(
654
+ destination,
655
+ [
656
+ {
657
+ envelopeId: envelope.envelopeId,
658
+ reason: dropReason(
659
+ `otlp_terminal_http_${response.status}`,
660
+ response.error ?? "request rejected"
661
+ )
662
+ }
663
+ ],
664
+ nowMs()
665
+ );
666
+ continue;
667
+ }
668
+ const attempt = envelope.retryCount + 1;
669
+ if (attempt >= MAX_OTLP_UPLOAD_ATTEMPTS) {
670
+ dropped += opts.database.dropOtlpEnvelopes(
671
+ destination,
672
+ [
673
+ {
674
+ envelopeId: envelope.envelopeId,
675
+ reason: dropReason(
676
+ "otlp_attempt_limit_exceeded",
677
+ response.error ?? `request failed with status ${response.status}`
678
+ )
679
+ }
680
+ ],
681
+ nowMs()
682
+ );
683
+ continue;
684
+ }
685
+ const failedAtMs = nowMs();
686
+ const retryDelayMs = Math.min(
687
+ MAX_RETRY_INTERVAL_MS,
688
+ MIN_RETRY_INTERVAL_MS * 2 ** Math.min(envelope.retryCount, 8)
689
+ );
690
+ const envelopeRetryAtMs = Math.max(
691
+ failedAtMs + retryDelayMs,
692
+ response.retryAtMs ?? failedAtMs
693
+ );
694
+ opts.database.deferOtlpEnvelopes(
695
+ destination,
696
+ [envelope.envelopeId],
697
+ envelopeRetryAtMs
698
+ );
699
+ lastError = response.error ?? "OTLP upload failed";
700
+ nextRetryAtMs = opts.database.nextPendingOtlpAttemptAtMs(destination) ?? envelopeRetryAtMs;
701
+ break;
702
+ }
703
+ return {
704
+ kind: "completed",
705
+ environment: destination.environment,
706
+ pending: opts.database.countPendingOtlpEnvelopes(),
707
+ uploaded,
708
+ ...dropped > 0 ? { dropped } : {},
709
+ ...lastError ? { error: lastError } : {},
710
+ ...lastWarning ? { warning: lastWarning } : {},
711
+ ...nextRetryAtMs ? { retryAtMs: nextRetryAtMs } : {}
712
+ };
713
+ }
714
+ function createOtlpSyncLoop(opts) {
715
+ const nowMs = opts.nowMs ?? Date.now;
716
+ const runOnce = opts.runOnce ?? runOtlpSyncCycle;
717
+ let stopping = false;
718
+ let wakePending = false;
719
+ let wakeWaiter = null;
720
+ let activeAbort = null;
721
+ let failures = 0;
722
+ let status = {
723
+ phase: "idle",
724
+ pending: opts.database.countPendingOtlpEnvelopes()
725
+ };
726
+ const wake = () => {
727
+ wakePending = true;
728
+ wakeWaiter?.();
729
+ };
730
+ const wait = async (delayMs) => {
731
+ if (stopping || wakePending) {
732
+ wakePending = false;
733
+ return;
734
+ }
735
+ await new Promise((resolve) => {
736
+ const timer = setTimeout(() => {
737
+ wakeWaiter = null;
738
+ resolve();
739
+ }, delayMs);
740
+ wakeWaiter = () => {
741
+ clearTimeout(timer);
742
+ wakeWaiter = null;
743
+ wakePending = false;
744
+ resolve();
745
+ };
746
+ });
747
+ };
748
+ const done = (async () => {
749
+ while (!stopping) {
750
+ const attemptedAtMs = nowMs();
751
+ status = {
752
+ ...status,
753
+ phase: "syncing",
754
+ pending: opts.database.countPendingOtlpEnvelopes(),
755
+ lastAttemptAtMs: attemptedAtMs,
756
+ lastError: void 0,
757
+ nextAttemptAtMs: void 0
758
+ };
759
+ activeAbort = new AbortController();
760
+ let result;
761
+ try {
762
+ result = await runOnce({
763
+ database: opts.database,
764
+ version: opts.version,
765
+ signal: activeAbort.signal,
766
+ nowMs
767
+ });
768
+ } catch (error) {
769
+ activeAbort = null;
770
+ if (stopping) break;
771
+ failures += 1;
772
+ const delayMs2 = Math.min(
773
+ MAX_RETRY_INTERVAL_MS,
774
+ MIN_RETRY_INTERVAL_MS * 2 ** Math.min(failures - 1, 8)
775
+ );
776
+ status = {
777
+ phase: "backing_off",
778
+ pending: opts.database.countPendingOtlpEnvelopes(),
779
+ lastAttemptAtMs: attemptedAtMs,
780
+ lastError: error instanceof Error ? error.message : String(error),
781
+ ...status.lastWarning ? { lastWarning: status.lastWarning } : {},
782
+ nextAttemptAtMs: nowMs() + delayMs2
783
+ };
784
+ await wait(delayMs2);
785
+ continue;
786
+ }
787
+ activeAbort = null;
788
+ if (stopping) break;
789
+ const completedAtMs = nowMs();
790
+ const succeeded = result.kind === "drained" || result.kind === "destination_drained" || result.uploaded > 0 || result.kind === "completed" && (result.dropped ?? 0) > 0;
791
+ failures = result.kind === "completed" && result.error ? failures + 1 : 0;
792
+ const failureDelay = Math.min(
793
+ MAX_RETRY_INTERVAL_MS,
794
+ MIN_RETRY_INTERVAL_MS * 2 ** Math.min(Math.max(0, failures - 1), 8)
795
+ );
796
+ const delayMs = result.kind === "unauthenticated" || result.kind === "organization_required" ? AUTH_INTERVAL_MS : result.kind === "completed" && result.pending > 0 ? result.retryAtMs ? Math.max(
797
+ MIN_RETRY_INTERVAL_MS,
798
+ result.retryAtMs - completedAtMs
799
+ ) : result.error ? failureDelay : DRAIN_INTERVAL_MS : IDLE_INTERVAL_MS;
800
+ status = {
801
+ phase: result.kind === "unauthenticated" ? "waiting_for_auth" : result.kind === "organization_required" ? "waiting_for_org" : result.kind === "completed" && result.error ? "backing_off" : "idle",
802
+ pending: result.pending,
803
+ lastAttemptAtMs: attemptedAtMs,
804
+ ...succeeded ? { lastSuccessAtMs: completedAtMs } : {},
805
+ ...result.kind === "completed" && result.error ? { lastError: result.error } : result.kind === "unauthenticated" ? { lastError: "FML login required or token rejected" } : result.kind === "organization_required" ? {
806
+ lastError: "FML organization selection required; run `fml org`"
807
+ } : {},
808
+ ...result.kind === "completed" && result.warning ? { lastWarning: result.warning } : status.lastWarning ? { lastWarning: status.lastWarning } : {},
809
+ nextAttemptAtMs: completedAtMs + delayMs
810
+ };
811
+ await wait(delayMs);
812
+ }
813
+ })();
814
+ return {
815
+ done,
816
+ wake,
817
+ stop: () => {
818
+ if (stopping) return;
819
+ stopping = true;
820
+ activeAbort?.abort();
821
+ wake();
822
+ },
823
+ status: () => status
824
+ };
825
+ }
826
+
827
+ // src/hooks/spool-config.ts
828
+ var BYTE_SUFFIXES = {
829
+ b: 1,
830
+ kib: 1024,
831
+ mib: 1024 * 1024,
832
+ gib: 1024 * 1024 * 1024
833
+ };
834
+ var MAX_BYTES_PATTERN = /^([1-9]\d*)(b|kib|mib|gib)?$/i;
835
+ var POSITIVE_INTEGER_PATTERN = /^[1-9]\d*$/;
836
+ var MILLISECONDS_PER_DAY = 24 * 60 * 60 * 1e3;
837
+ var DEFAULT_HOOK_QUARANTINE_MAX_AGE_DAYS = 30;
838
+ function parseHookSpoolMaxBytes(configured) {
839
+ if (configured === void 0) return void 0;
840
+ const value = configured.trim();
841
+ if (value.toLowerCase() === "unbounded") return void 0;
842
+ const match = MAX_BYTES_PATTERN.exec(value);
843
+ if (!match) {
844
+ throw new Error(
845
+ "FML_HOOK_SPOOL_MAX_BYTES must be `unbounded` or a positive integer optionally followed by B, KiB, MiB, or GiB"
846
+ );
847
+ }
848
+ const quantity = Number(match[1]);
849
+ const suffix = match[2]?.toLowerCase() ?? "b";
850
+ const bytes = quantity * BYTE_SUFFIXES[suffix];
851
+ if (!Number.isSafeInteger(bytes)) {
852
+ throw new Error("FML_HOOK_SPOOL_MAX_BYTES exceeds the safe integer range");
853
+ }
854
+ return bytes;
855
+ }
856
+ function parseHookQuarantineMaxAgeMs(configured) {
857
+ if (configured === void 0) {
858
+ return DEFAULT_HOOK_QUARANTINE_MAX_AGE_DAYS * MILLISECONDS_PER_DAY;
859
+ }
860
+ const value = configured.trim();
861
+ if (value.toLowerCase() === "unbounded") return void 0;
862
+ if (!POSITIVE_INTEGER_PATTERN.test(value)) {
863
+ throw new Error(
864
+ "FML_HOOK_QUARANTINE_MAX_AGE_DAYS must be `unbounded` or a positive integer number of days"
865
+ );
866
+ }
867
+ const milliseconds = Number(value) * MILLISECONDS_PER_DAY;
868
+ if (!Number.isSafeInteger(milliseconds)) {
869
+ throw new Error(
870
+ "FML_HOOK_QUARANTINE_MAX_AGE_DAYS exceeds the safe integer range"
871
+ );
872
+ }
873
+ return milliseconds;
874
+ }
875
+ function parseOtlpSpoolMaxBytes(configured) {
876
+ if (configured === void 0) return void 0;
877
+ const value = configured.trim();
878
+ const match = MAX_BYTES_PATTERN.exec(value);
879
+ if (!match) {
880
+ throw new Error(
881
+ "FML_OTLP_SPOOL_MAX_BYTES must be a positive integer optionally followed by B, KiB, MiB, or GiB"
882
+ );
883
+ }
884
+ const quantity = Number(match[1]);
885
+ const suffix = match[2]?.toLowerCase() ?? "b";
886
+ const bytes = quantity * BYTE_SUFFIXES[suffix];
887
+ if (!Number.isSafeInteger(bytes)) {
888
+ throw new Error("FML_OTLP_SPOOL_MAX_BYTES exceeds the safe integer range");
889
+ }
890
+ return bytes;
891
+ }
892
+
893
+ // src/observer/sync-loop.ts
894
+ var OBSERVER_IDLE_INTERVAL_MS = 3e4;
895
+ var OBSERVER_BUSY_INTERVAL_MS = 2e3;
896
+ var OBSERVER_AUTH_INTERVAL_MS = 6e4;
897
+ var OBSERVER_MAX_TIMER_DELAY_MS = 2147483647;
898
+ function truncateError(error) {
899
+ const message = error instanceof Error ? error.message : String(error);
900
+ return message.length > 1e3 ? `${message.slice(0, 997)}...` : message;
901
+ }
902
+ function delayForResult(result, nowMs) {
903
+ if (result.kind === "unauthenticated") return OBSERVER_AUTH_INTERVAL_MS;
904
+ if (result.kind !== "completed") return OBSERVER_IDLE_INTERVAL_MS;
905
+ if (result.outcome === "busy" || result.outcome === "pending") {
906
+ return OBSERVER_BUSY_INTERVAL_MS;
907
+ }
908
+ if (result.outcome === "throttled" && result.retryAtMs !== void 0) {
909
+ return Math.max(1e3, result.retryAtMs - nowMs);
910
+ }
911
+ if (result.outcome === "backing_off" && result.nextRetryAtMs !== void 0) {
912
+ return Math.min(
913
+ OBSERVER_IDLE_INTERVAL_MS,
914
+ Math.max(1e3, result.nextRetryAtMs - nowMs)
915
+ );
916
+ }
917
+ return OBSERVER_IDLE_INTERVAL_MS;
918
+ }
919
+ function phaseForResult(result) {
920
+ if (result.kind === "unauthenticated") return "waiting_for_auth";
921
+ if (result.kind !== "completed") return "idle";
922
+ return result.outcome === "failed" || result.outcome === "exhausted" || result.outcome === "backing_off" || result.outcome === "throttled" || result.outcome === "busy" ? "backing_off" : "idle";
923
+ }
924
+ function outcomeForResult(result) {
925
+ return result.kind === "completed" ? result.outcome : result.kind;
926
+ }
927
+ function createObserverSyncLoop(opts) {
928
+ const nowMs = opts.nowMs ?? Date.now;
929
+ const runOnce = opts.runOnce ?? runObserverSyncCycleSubprocess;
930
+ let stopping = false;
931
+ let wakePending = false;
932
+ let wakeWaiter = null;
933
+ let activeAbort = null;
934
+ const wake = () => {
935
+ wakePending = true;
936
+ activeAbort?.abort();
937
+ wakeWaiter?.();
938
+ };
939
+ const wait = async (delayMs) => {
940
+ if (stopping || wakePending) {
941
+ wakePending = false;
942
+ return "woken";
943
+ }
944
+ return await new Promise((resolve) => {
945
+ const timer = setTimeout(
946
+ () => {
947
+ wakeWaiter = null;
948
+ resolve("elapsed");
949
+ },
950
+ Math.min(delayMs, OBSERVER_MAX_TIMER_DELAY_MS)
951
+ );
952
+ wakeWaiter = () => {
953
+ clearTimeout(timer);
954
+ wakeWaiter = null;
955
+ wakePending = false;
956
+ resolve("woken");
957
+ };
958
+ });
959
+ };
960
+ const waitUntil = async (deadlineMs) => {
961
+ while (!stopping) {
962
+ const remainingMs = deadlineMs - nowMs();
963
+ if (remainingMs <= 0) return;
964
+ if (await wait(remainingMs) === "woken") return;
965
+ }
966
+ };
967
+ const done = (async () => {
968
+ while (!stopping) {
969
+ const attemptedAtMs = nowMs();
970
+ opts.onUpdate({
971
+ phase: "syncing",
972
+ heartbeatAtMs: attemptedAtMs,
973
+ lastAttemptAtMs: attemptedAtMs,
974
+ nextAttemptAtMs: void 0,
975
+ lastError: void 0
976
+ });
977
+ activeAbort = new AbortController();
978
+ let result;
979
+ try {
980
+ result = await runOnce({ signal: activeAbort.signal });
981
+ } catch (error) {
982
+ activeAbort = null;
983
+ if (stopping) break;
984
+ if (wakePending) {
985
+ wakePending = false;
986
+ if (error instanceof ObserverSyncCycleAbortedError) {
987
+ continue;
988
+ }
989
+ opts.onUpdate({
990
+ phase: "backing_off",
991
+ heartbeatAtMs: nowMs(),
992
+ lastOutcome: "error",
993
+ lastError: truncateError(error),
994
+ nextAttemptAtMs: nowMs()
995
+ });
996
+ continue;
997
+ }
998
+ const nextAttemptAtMs = nowMs() + OBSERVER_IDLE_INTERVAL_MS;
999
+ opts.onUpdate({
1000
+ phase: "backing_off",
1001
+ heartbeatAtMs: nowMs(),
1002
+ lastOutcome: "error",
1003
+ lastError: truncateError(error),
1004
+ nextAttemptAtMs
1005
+ });
1006
+ await waitUntil(nextAttemptAtMs);
1007
+ continue;
1008
+ }
1009
+ activeAbort = null;
1010
+ if (stopping) break;
1011
+ const completedAtMs = nowMs();
1012
+ opts.onCycle?.(result, {
1013
+ startedAtMs: attemptedAtMs,
1014
+ completedAtMs
1015
+ });
1016
+ const delayMs = delayForResult(result, completedAtMs);
1017
+ const successful = result.kind === "completed" && (result.outcome === "drained" || result.outcome === "pending");
1018
+ opts.onUpdate({
1019
+ phase: phaseForResult(result),
1020
+ ...result.kind !== "disabled" ? { environment: result.environment } : {},
1021
+ heartbeatAtMs: completedAtMs,
1022
+ ...successful ? { lastSuccessAtMs: completedAtMs } : {},
1023
+ lastOutcome: outcomeForResult(result),
1024
+ lastError: result.kind === "unauthenticated" ? "FML login required" : result.kind === "completed" && (result.outcome === "failed" || result.outcome === "exhausted") ? `${result.failed} failed, ${result.exhausted} exhausted` : void 0,
1025
+ nextAttemptAtMs: completedAtMs + delayMs
1026
+ });
1027
+ if (result.kind === "completed" && (result.uploadedBytes > 0 || result.outcome !== "drained")) {
1028
+ console.log(
1029
+ `${new Date(completedAtMs).toISOString()} sync ${result.outcome}: ${result.uploadedBytes} bytes, ${result.failed} failed`
1030
+ );
1031
+ }
1032
+ await waitUntil(completedAtMs + delayMs);
1033
+ }
1034
+ })();
1035
+ return {
1036
+ done,
1037
+ wake,
1038
+ stop: () => {
1039
+ if (stopping) return;
1040
+ stopping = true;
1041
+ activeAbort?.abort();
1042
+ wake();
1043
+ }
1044
+ };
1045
+ }
1046
+
1047
+ // src/observer-server.ts
1048
+ var HEARTBEAT_INTERVAL_MS = 5e3;
1049
+ var STATE_UPDATE_RETRY_MS = 50;
1050
+ var STARTUP_STATE_UPDATE_TIMEOUT_MS = 5e3;
1051
+ var PORT_FALLBACK_COUNT = 10;
1052
+ var HOOK_REJECTION_WARNING_INTERVAL_MS = 6e4;
1053
+ var HOOK_INBOX_DRAIN_INTERVAL_MS = 1e3;
1054
+ var HOOK_QUARANTINE_PRUNE_INTERVAL_MS = 5 * 60 * 1e3;
1055
+ var OBSERVER_DIRECTORY_LEASE = {
1056
+ stateFile: FML_OBSERVER_STATE_FILE,
1057
+ lockFile: FML_OBSERVER_LOCK_FILE
1058
+ };
1059
+ var DATABASE_LEASE = {
1060
+ stateFile: FML_PANOPTICON_DB_OBSERVER_STATE_FILE,
1061
+ lockFile: FML_PANOPTICON_DB_OBSERVER_LOCK_FILE
1062
+ };
1063
+ function removeInitialLeaseBestEffort(instanceId, lease) {
1064
+ try {
1065
+ removeObserverStateIfOwned(instanceId, lease);
1066
+ } catch {
1067
+ }
1068
+ }
1069
+ function removeInitialLeasesBestEffort(instanceId) {
1070
+ removeInitialLeaseBestEffort(instanceId, DATABASE_LEASE);
1071
+ removeInitialLeaseBestEffort(instanceId, OBSERVER_DIRECTORY_LEASE);
1072
+ }
1073
+ function persistedPort() {
1074
+ try {
1075
+ const parsed = JSON.parse(
1076
+ fs.readFileSync(FML_OBSERVER_ENDPOINT_FILE, "utf8")
1077
+ );
1078
+ if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
1079
+ return null;
1080
+ }
1081
+ const port = parsed.port;
1082
+ return typeof port === "number" && Number.isInteger(port) && port > 0 && port <= 65535 ? port : null;
1083
+ } catch {
1084
+ return null;
1085
+ }
1086
+ }
1087
+ function writeEndpoint(port) {
1088
+ const tempFile = `${FML_OBSERVER_ENDPOINT_FILE}.${process.pid}.tmp`;
1089
+ fs.writeFileSync(
1090
+ tempFile,
1091
+ `${JSON.stringify({
1092
+ service: OBSERVER_SERVICE_NAME,
1093
+ protocolVersion: OBSERVER_PROTOCOL_VERSION,
1094
+ host: FML_OBSERVER_HOST,
1095
+ port
1096
+ })}
1097
+ `,
1098
+ { mode: 384 }
1099
+ );
1100
+ fs.renameSync(tempFile, FML_OBSERVER_ENDPOINT_FILE);
1101
+ }
1102
+ async function tryListen(server2, port) {
1103
+ return await new Promise((resolve, reject) => {
1104
+ const onError = (error) => {
1105
+ server2.off("listening", onListening);
1106
+ if (error.code === "EADDRINUSE") resolve(false);
1107
+ else reject(error);
1108
+ };
1109
+ const onListening = () => {
1110
+ server2.off("error", onError);
1111
+ resolve(true);
1112
+ };
1113
+ server2.once("error", onError);
1114
+ server2.once("listening", onListening);
1115
+ server2.listen(port, FML_OBSERVER_HOST);
1116
+ });
1117
+ }
1118
+ async function listenObserver(server2) {
1119
+ const explicitPort = process.env.FML_OBSERVER_PORT !== void 0 || process.env.PANOPTICON_PORT !== void 0;
1120
+ const preferredPort = explicitPort ? FML_OBSERVER_PORT : persistedPort() ?? FML_OBSERVER_PORT;
1121
+ const attempts = explicitPort ? 1 : PORT_FALLBACK_COUNT;
1122
+ for (let offset = 0; offset < attempts; offset += 1) {
1123
+ const port = preferredPort + offset;
1124
+ if (port > 65535) break;
1125
+ if (await tryListen(server2, port)) return port;
1126
+ }
1127
+ throw new Error(
1128
+ explicitPort ? `FML observer port ${preferredPort} is already in use` : `No available FML observer port in ${preferredPort}-${Math.min(65535, preferredPort + attempts - 1)}`
1129
+ );
1130
+ }
1131
+ function cycleOutcome(result) {
1132
+ if (result.kind === "completed") {
1133
+ return {
1134
+ environment: result.environment,
1135
+ outcome: result.outcome,
1136
+ uploadedBytes: result.uploadedBytes,
1137
+ failed: result.failed,
1138
+ ...result.failed > 0 ? { error: `${result.failed} upload attempt(s) failed` } : {}
1139
+ };
1140
+ }
1141
+ return {
1142
+ ...result.kind === "unauthenticated" ? { environment: result.environment } : {},
1143
+ outcome: result.kind,
1144
+ uploadedBytes: 0,
1145
+ failed: 0,
1146
+ ...result.kind === "unauthenticated" ? { error: "FML login required" } : {}
1147
+ };
1148
+ }
1149
+ var hookSpoolMaxBytes = parseHookSpoolMaxBytes(
1150
+ process.env.FML_HOOK_SPOOL_MAX_BYTES
1151
+ );
1152
+ var hookQuarantineMaxAgeMs = parseHookQuarantineMaxAgeMs(
1153
+ process.env.FML_HOOK_QUARANTINE_MAX_AGE_DAYS
1154
+ );
1155
+ var otlpSpoolMaxBytes = parseOtlpSpoolMaxBytes(
1156
+ process.env.FML_OTLP_SPOOL_MAX_BYTES
1157
+ );
1158
+ ensureDirs();
1159
+ var isolationWarning = observerOnlyOverrideWarning();
1160
+ if (isolationWarning) console.warn(isolationWarning);
1161
+ var initial = createInitialObserverState();
1162
+ if (!claimObserverState(initial, OBSERVER_DIRECTORY_LEASE)) {
1163
+ console.error("FML observer is already running");
1164
+ process.exit(1);
1165
+ }
1166
+ var databaseLeaseClaimed = false;
1167
+ try {
1168
+ databaseLeaseClaimed = claimObserverState(initial, {
1169
+ ...DATABASE_LEASE,
1170
+ liveOwnerPolicy: "process_identity"
1171
+ });
1172
+ } catch (error) {
1173
+ removeInitialLeasesBestEffort(initial.instanceId);
1174
+ console.error(
1175
+ `FML observer could not claim database lease for ${FML_PANOPTICON_DB_FILE}: ${error instanceof Error ? error.message : String(error)}`
1176
+ );
1177
+ process.exit(1);
1178
+ }
1179
+ if (!databaseLeaseClaimed) {
1180
+ removeInitialLeaseBestEffort(initial.instanceId, OBSERVER_DIRECTORY_LEASE);
1181
+ console.error(
1182
+ `FML database ${FML_PANOPTICON_DB_FILE} is already in use by another observer. Stop that observer using the same FML_STATE_DIR, or select a different absolute FML_STATE_DIR.`
1183
+ );
1184
+ process.exit(1);
1185
+ }
1186
+ var state = initial;
1187
+ var shuttingDown = false;
1188
+ var fatalShutdownScheduled = false;
1189
+ var heartbeatTimer = null;
1190
+ var hookInboxDrainTimer = null;
1191
+ var hookInboxDrainPromise = null;
1192
+ var hookQuarantinePruneTimer = null;
1193
+ var stateUpdateRetryTimer = null;
1194
+ var syncLoop = null;
1195
+ var hookSyncLoop = null;
1196
+ var otlpSyncLoop = null;
1197
+ var pendingStateUpdate = null;
1198
+ var authToken;
1199
+ var database;
1200
+ try {
1201
+ authToken = getOrCreateObserverAuthToken(FML_OBSERVER_AUTH_TOKEN_FILE);
1202
+ database = openObserverDatabase(FML_PANOPTICON_DB_FILE, {
1203
+ // FML_STATE_DIR can be caller-managed. Keep SQLite payload files private
1204
+ // without changing permissions on the shared parent directory.
1205
+ restrictDatabaseDirectory: false,
1206
+ ...hookSpoolMaxBytes === void 0 ? {} : { maxHookSpoolBytes: hookSpoolMaxBytes },
1207
+ ...hookQuarantineMaxAgeMs === void 0 ? {} : { maxHookQuarantineAgeMs: hookQuarantineMaxAgeMs },
1208
+ ...otlpSpoolMaxBytes === void 0 ? {} : { maxOtlpSpoolBytes: otlpSpoolMaxBytes }
1209
+ });
1210
+ } catch (error) {
1211
+ removeInitialLeasesBestEffort(initial.instanceId);
1212
+ console.error(
1213
+ `FML observer could not open database ${FML_PANOPTICON_DB_FILE}: ${error instanceof Error ? error.message : String(error)}`
1214
+ );
1215
+ process.exit(1);
1216
+ }
1217
+ var rejectedHookCaptures = 0;
1218
+ var lastRejectedAtMs;
1219
+ var lastRejectedReason;
1220
+ var lastRejectedTarget;
1221
+ var hookRejectionWarnings = /* @__PURE__ */ new Map();
1222
+ var unsupportedInboxSchemaReported = false;
1223
+ var recordHookRejection = (reason, target) => {
1224
+ const now = Date.now();
1225
+ rejectedHookCaptures += 1;
1226
+ lastRejectedAtMs = now;
1227
+ lastRejectedReason = reason;
1228
+ lastRejectedTarget = target;
1229
+ const lastWarningAtMs = hookRejectionWarnings.get(reason) ?? 0;
1230
+ if (now - lastWarningAtMs < HOOK_REJECTION_WARNING_INTERVAL_MS) return;
1231
+ hookRejectionWarnings.set(reason, now);
1232
+ console.warn(
1233
+ `[fml] Rejected hook capture: reason=${reason}${target ? ` target=${target}` : ""}`
1234
+ );
1235
+ };
1236
+ var hookRejectionStatus = () => ({
1237
+ rejected: rejectedHookCaptures,
1238
+ ...lastRejectedAtMs !== void 0 ? { lastRejectedAtMs } : {},
1239
+ ...lastRejectedReason !== void 0 ? { lastRejectedReason } : {},
1240
+ ...lastRejectedTarget !== void 0 ? { lastRejectedTarget } : {}
1241
+ });
1242
+ var packageVersion = true ? "0.8.0-rc.2+bed6eaed" : "dev";
1243
+ var health = () => ({
1244
+ status: "ok",
1245
+ service: OBSERVER_SERVICE_NAME,
1246
+ protocolVersion: OBSERVER_PROTOCOL_VERSION,
1247
+ instanceId: state.instanceId,
1248
+ pid: state.pid,
1249
+ port: state.port,
1250
+ version: packageVersion,
1251
+ capabilities: ["hook-capture-v1", "otlp-http-capture-v1", "panopticon-db-v1"]
1252
+ });
1253
+ var publicStatus = async () => ({
1254
+ ...health(),
1255
+ observer: state,
1256
+ sync: createCliRawSync({ continuous: true }).status(),
1257
+ hooks: {
1258
+ ...hookSyncLoop?.status() ?? {
1259
+ phase: "idle",
1260
+ pending: database.countPendingHookEvents()
1261
+ },
1262
+ quarantined: database.countQuarantinedHookEvents(),
1263
+ spool: database.getHookSpoolStatus(),
1264
+ inbox: await readHookInboxStatusAsync(),
1265
+ ...hookRejectionStatus()
1266
+ },
1267
+ otlp: {
1268
+ ...otlpSyncLoop?.status() ?? {
1269
+ phase: "idle",
1270
+ pending: database.countPendingOtlpEnvelopes()
1271
+ },
1272
+ spool: database.getOtlpSpoolStatus()
1273
+ }
1274
+ });
1275
+ function bothObserverLeasesAreOwned(instanceId) {
1276
+ return observerLeaseIsOwned(instanceId, OBSERVER_DIRECTORY_LEASE.lockFile) && observerLeaseIsOwned(instanceId, DATABASE_LEASE.lockFile);
1277
+ }
1278
+ function updateBothObserverStates(current, update) {
1279
+ updateObserverState(
1280
+ current,
1281
+ update,
1282
+ OBSERVER_DIRECTORY_LEASE.stateFile,
1283
+ OBSERVER_DIRECTORY_LEASE.lockFile
1284
+ );
1285
+ const databaseState = updateObserverState(
1286
+ current,
1287
+ update,
1288
+ DATABASE_LEASE.stateFile,
1289
+ DATABASE_LEASE.lockFile
1290
+ );
1291
+ return databaseState;
1292
+ }
1293
+ function removeBothObserverStates(instanceId) {
1294
+ try {
1295
+ removeObserverStateIfOwned(instanceId, DATABASE_LEASE);
1296
+ } finally {
1297
+ removeObserverStateIfOwned(instanceId, OBSERVER_DIRECTORY_LEASE);
1298
+ }
1299
+ }
1300
+ var server;
1301
+ var shutdown = async () => {
1302
+ if (shuttingDown) return;
1303
+ shuttingDown = true;
1304
+ if (heartbeatTimer) clearInterval(heartbeatTimer);
1305
+ if (hookInboxDrainTimer) clearInterval(hookInboxDrainTimer);
1306
+ if (hookQuarantinePruneTimer) clearInterval(hookQuarantinePruneTimer);
1307
+ if (stateUpdateRetryTimer) clearTimeout(stateUpdateRetryTimer);
1308
+ if (bothObserverLeasesAreOwned(state.instanceId)) {
1309
+ try {
1310
+ state = updateBothObserverStates(state, {
1311
+ phase: "stopping",
1312
+ heartbeatAtMs: Date.now(),
1313
+ nextAttemptAtMs: void 0
1314
+ });
1315
+ } catch {
1316
+ }
1317
+ }
1318
+ syncLoop?.stop();
1319
+ hookSyncLoop?.stop();
1320
+ otlpSyncLoop?.stop();
1321
+ await new Promise((resolve) => server.close(() => resolve()));
1322
+ await syncLoop?.done.catch(() => {
1323
+ });
1324
+ await hookSyncLoop?.done.catch(() => {
1325
+ });
1326
+ await otlpSyncLoop?.done.catch(() => {
1327
+ });
1328
+ await hookInboxDrainPromise?.catch(() => {
1329
+ });
1330
+ try {
1331
+ database.close();
1332
+ } finally {
1333
+ removeBothObserverStates(state.instanceId);
1334
+ }
1335
+ };
1336
+ var scheduleFatalShutdown = (error) => {
1337
+ console.error(error);
1338
+ if (fatalShutdownScheduled) return;
1339
+ fatalShutdownScheduled = true;
1340
+ setImmediate(() => {
1341
+ void shutdown().finally(() => process.exit(1));
1342
+ });
1343
+ };
1344
+ var flushStateUpdate = () => {
1345
+ stateUpdateRetryTimer = null;
1346
+ if (!pendingStateUpdate || shuttingDown || fatalShutdownScheduled) {
1347
+ return false;
1348
+ }
1349
+ const update = pendingStateUpdate;
1350
+ try {
1351
+ state = updateBothObserverStates(state, update);
1352
+ pendingStateUpdate = null;
1353
+ database.setRuntimeState("observer", state);
1354
+ return true;
1355
+ } catch (error) {
1356
+ if (error instanceof ObserverStateMutationBusyError) {
1357
+ stateUpdateRetryTimer = setTimeout(
1358
+ flushStateUpdate,
1359
+ STATE_UPDATE_RETRY_MS
1360
+ );
1361
+ stateUpdateRetryTimer.unref();
1362
+ return false;
1363
+ }
1364
+ scheduleFatalShutdown(error);
1365
+ return false;
1366
+ }
1367
+ };
1368
+ var updateState = (update) => {
1369
+ pendingStateUpdate = { ...pendingStateUpdate, ...update };
1370
+ if (stateUpdateRetryTimer) return false;
1371
+ return flushStateUpdate();
1372
+ };
1373
+ var updateStartupState = async (update) => {
1374
+ const deadlineMs = Date.now() + STARTUP_STATE_UPDATE_TIMEOUT_MS;
1375
+ while (true) {
1376
+ try {
1377
+ state = updateBothObserverStates(state, update);
1378
+ return;
1379
+ } catch (error) {
1380
+ if (!(error instanceof ObserverStateMutationBusyError) || Date.now() >= deadlineMs) {
1381
+ throw error;
1382
+ }
1383
+ await new Promise(
1384
+ (resolve) => setTimeout(resolve, STATE_UPDATE_RETRY_MS)
1385
+ );
1386
+ }
1387
+ }
1388
+ };
1389
+ var terminalHookCaptureError = (error) => {
1390
+ if (error instanceof ObserverHookSpoolEntryTooLargeError) return true;
1391
+ if (!(error instanceof Error)) return false;
1392
+ return /^(?:Hook event|Hook destination) .+ (?:exceeds|must (?:not )?be)/u.test(
1393
+ error.message
1394
+ );
1395
+ };
1396
+ var captureHookRequest = (request, metadata) => {
1397
+ const event = normalizeObserverHookCapture(
1398
+ request,
1399
+ metadata ? {
1400
+ nowMs: () => metadata.capturedAtMs,
1401
+ syncId: () => metadata.syncId,
1402
+ ...metadata.repository ? { repository: metadata.repository } : {}
1403
+ } : {}
1404
+ );
1405
+ if (!event) {
1406
+ recordHookRejection("unnormalizable_event", request.target);
1407
+ return "terminal_rejected";
1408
+ }
1409
+ let destination = null;
1410
+ if (metadata) {
1411
+ destination = metadata.destination ?? null;
1412
+ } else {
1413
+ try {
1414
+ destination = getActiveHookDestinationBinding();
1415
+ } catch {
1416
+ }
1417
+ }
1418
+ try {
1419
+ database.captureHookEvent({
1420
+ ...event,
1421
+ ...destination ? { destination } : {}
1422
+ });
1423
+ } catch (error) {
1424
+ const terminal = terminalHookCaptureError(error);
1425
+ recordHookRejection(
1426
+ error instanceof ObserverHookSpoolEntryTooLargeError ? "spool_entry_too_large" : "capture_failed",
1427
+ request.target
1428
+ );
1429
+ return terminal ? "terminal_rejected" : "retry";
1430
+ }
1431
+ hookSyncLoop?.wake();
1432
+ return "committed";
1433
+ };
1434
+ var captureHookInboxEntry = (entry) => captureHookRequest(entry.request, entry);
1435
+ var drainPendingHookInbox = () => {
1436
+ if (hookInboxDrainPromise) return hookInboxDrainPromise;
1437
+ const drainPromise = (async () => {
1438
+ const result = await drainHookInbox(captureHookInboxEntry);
1439
+ if (result.malformed > 0) recordHookRejection("inbox_malformed");
1440
+ if (result.failed > 0) recordHookRejection("inbox_drain_failed");
1441
+ if (result.unsupported > 0 && !unsupportedInboxSchemaReported) {
1442
+ recordHookRejection("inbox_unsupported_schema");
1443
+ unsupportedInboxSchemaReported = true;
1444
+ } else if (result.unsupported === 0) {
1445
+ unsupportedInboxSchemaReported = false;
1446
+ }
1447
+ })();
1448
+ hookInboxDrainPromise = drainPromise;
1449
+ void drainPromise.then(
1450
+ () => {
1451
+ if (hookInboxDrainPromise === drainPromise) hookInboxDrainPromise = null;
1452
+ },
1453
+ () => {
1454
+ if (hookInboxDrainPromise === drainPromise) hookInboxDrainPromise = null;
1455
+ }
1456
+ );
1457
+ return drainPromise;
1458
+ };
1459
+ server = createObserverHttpServer({
1460
+ authToken,
1461
+ getHealth: health,
1462
+ getStatus: publicStatus,
1463
+ onHook: (payload) => {
1464
+ const request = parseHookCaptureRequest(payload);
1465
+ if (!request) {
1466
+ recordHookRejection("invalid_envelope");
1467
+ return;
1468
+ }
1469
+ const metadata = parseHookInboxMetadata(payload) ?? void 0;
1470
+ const outcome = captureHookRequest(request, metadata);
1471
+ if (metadata && outcome === "committed") {
1472
+ acknowledgeHookInboxEntry(metadata);
1473
+ } else if (metadata && outcome === "terminal_rejected") {
1474
+ retireHookInboxEntry(metadata, "terminal_rejected");
1475
+ }
1476
+ },
1477
+ onOtlp: (envelope) => {
1478
+ let destination = null;
1479
+ try {
1480
+ destination = getActiveOtlpDestinationBinding();
1481
+ } catch {
1482
+ }
1483
+ database.captureOtlpEnvelope({
1484
+ ...envelope,
1485
+ ...destination ? { destination } : {}
1486
+ });
1487
+ otlpSyncLoop?.wake();
1488
+ },
1489
+ onWake: () => {
1490
+ syncLoop?.wake();
1491
+ hookSyncLoop?.wake(true);
1492
+ otlpSyncLoop?.wake();
1493
+ },
1494
+ onShutdown: () => {
1495
+ void shutdown().finally(() => process.exit(0));
1496
+ }
1497
+ });
1498
+ try {
1499
+ const port = await listenObserver(server);
1500
+ await updateStartupState({
1501
+ port,
1502
+ phase: "idle",
1503
+ heartbeatAtMs: Date.now()
1504
+ });
1505
+ writeEndpoint(port);
1506
+ database.setRuntimeState("observer", state);
1507
+ console.log(
1508
+ `${(/* @__PURE__ */ new Date()).toISOString()} FML observer listening on ${FML_OBSERVER_HOST}:${port}`
1509
+ );
1510
+ heartbeatTimer = setInterval(() => {
1511
+ updateState({ heartbeatAtMs: Date.now() });
1512
+ }, HEARTBEAT_INTERVAL_MS);
1513
+ heartbeatTimer.unref();
1514
+ await drainPendingHookInbox();
1515
+ hookInboxDrainTimer = setInterval(() => {
1516
+ void drainPendingHookInbox();
1517
+ }, HOOK_INBOX_DRAIN_INTERVAL_MS);
1518
+ hookInboxDrainTimer.unref();
1519
+ hookQuarantinePruneTimer = setInterval(() => {
1520
+ try {
1521
+ database.countQuarantinedHookEvents();
1522
+ } catch (error) {
1523
+ scheduleFatalShutdown(error);
1524
+ }
1525
+ }, HOOK_QUARANTINE_PRUNE_INTERVAL_MS);
1526
+ hookQuarantinePruneTimer.unref();
1527
+ hookSyncLoop = createHookSyncLoop({
1528
+ database,
1529
+ version: packageVersion
1530
+ });
1531
+ void hookSyncLoop.done.catch((error) => {
1532
+ if (!shuttingDown) scheduleFatalShutdown(error);
1533
+ });
1534
+ otlpSyncLoop = createOtlpSyncLoop({
1535
+ database,
1536
+ version: packageVersion
1537
+ });
1538
+ void otlpSyncLoop.done.catch((error) => {
1539
+ if (!shuttingDown) scheduleFatalShutdown(error);
1540
+ });
1541
+ syncLoop = createObserverSyncLoop({
1542
+ onUpdate: (update) => {
1543
+ updateState(update);
1544
+ },
1545
+ onCycle: (result, timing) => {
1546
+ try {
1547
+ database.recordSyncRun({ ...timing, ...cycleOutcome(result) });
1548
+ } catch (error) {
1549
+ scheduleFatalShutdown(error);
1550
+ }
1551
+ }
1552
+ });
1553
+ void syncLoop.done.catch((error) => {
1554
+ if (!shuttingDown) scheduleFatalShutdown(error);
1555
+ });
1556
+ } catch (error) {
1557
+ console.error(error);
1558
+ await shutdown().catch(() => {
1559
+ });
1560
+ process.exit(1);
1561
+ }
1562
+ for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"]) {
1563
+ process.once(signal, () => {
1564
+ void shutdown().finally(() => process.exit(0));
1565
+ });
1566
+ }