@x12i/memorix-service 3.9.2 → 3.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/dist/analytics/definition.d.ts +158 -0
  2. package/dist/analytics/definition.d.ts.map +1 -0
  3. package/dist/analytics/definition.js +439 -0
  4. package/dist/analytics/definition.js.map +1 -0
  5. package/dist/analytics/engine.d.ts +63 -0
  6. package/dist/analytics/engine.d.ts.map +1 -0
  7. package/dist/analytics/engine.js +181 -0
  8. package/dist/analytics/engine.js.map +1 -0
  9. package/dist/analytics/host.d.ts +15 -0
  10. package/dist/analytics/host.d.ts.map +1 -0
  11. package/dist/analytics/host.js +129 -0
  12. package/dist/analytics/host.js.map +1 -0
  13. package/dist/analytics/materializer.d.ts +88 -0
  14. package/dist/analytics/materializer.d.ts.map +1 -0
  15. package/dist/analytics/materializer.js +358 -0
  16. package/dist/analytics/materializer.js.map +1 -0
  17. package/dist/analytics/pipeline-port.d.ts +10 -0
  18. package/dist/analytics/pipeline-port.d.ts.map +1 -0
  19. package/dist/analytics/pipeline-port.js +42 -0
  20. package/dist/analytics/pipeline-port.js.map +1 -0
  21. package/dist/analytics/projector.d.ts +60 -0
  22. package/dist/analytics/projector.d.ts.map +1 -0
  23. package/dist/analytics/projector.js +262 -0
  24. package/dist/analytics/projector.js.map +1 -0
  25. package/dist/analytics/state.d.ts +83 -0
  26. package/dist/analytics/state.d.ts.map +1 -0
  27. package/dist/analytics/state.js +273 -0
  28. package/dist/analytics/state.js.map +1 -0
  29. package/dist/analytics/tracker.d.ts +182 -0
  30. package/dist/analytics/tracker.d.ts.map +1 -0
  31. package/dist/analytics/tracker.js +1302 -0
  32. package/dist/analytics/tracker.js.map +1 -0
  33. package/dist/app.d.ts.map +1 -1
  34. package/dist/app.js +4 -0
  35. package/dist/app.js.map +1 -1
  36. package/dist/content-census-countex.d.ts +15 -0
  37. package/dist/content-census-countex.d.ts.map +1 -0
  38. package/dist/content-census-countex.js +18 -0
  39. package/dist/content-census-countex.js.map +1 -0
  40. package/dist/datasource-backfill.d.ts +2 -0
  41. package/dist/datasource-backfill.d.ts.map +1 -1
  42. package/dist/datasource-backfill.js +5 -1
  43. package/dist/datasource-backfill.js.map +1 -1
  44. package/dist/index.d.ts +4 -0
  45. package/dist/index.d.ts.map +1 -1
  46. package/dist/index.js +3 -0
  47. package/dist/index.js.map +1 -1
  48. package/dist/openapi/openapi.json +955 -210
  49. package/dist/platform.d.ts +6 -0
  50. package/dist/platform.d.ts.map +1 -1
  51. package/dist/platform.js +54 -2
  52. package/dist/platform.js.map +1 -1
  53. package/dist/routes/analytics-counters.d.ts +8 -0
  54. package/dist/routes/analytics-counters.d.ts.map +1 -0
  55. package/dist/routes/analytics-counters.js +100 -0
  56. package/dist/routes/analytics-counters.js.map +1 -0
  57. package/dist/routes/datasource-backfill.d.ts.map +1 -1
  58. package/dist/routes/datasource-backfill.js +8 -1
  59. package/dist/routes/datasource-backfill.js.map +1 -1
  60. package/openapi/openapi.json +955 -210
  61. package/package.json +5 -5
@@ -0,0 +1,1302 @@
1
+ import { ServiceError } from "../errors.js";
2
+ import { abstractTypesOf, definitionHash, usesDatasource, validateCounterDefinition, } from "./definition.js";
3
+ import { countexScope } from "./engine.js";
4
+ import { buildEnvelope, hasScopePlaceholder, resolveRecordIds, scopeValuesOf, summarizePair, topLeaderIds, } from "./materializer.js";
5
+ import { diffFacts, metricKey, projectSubject, SEP, transitionFacts, } from "./projector.js";
6
+ /** Datasource service stamped on materialized records; writes carrying it are not counted. */
7
+ export const ANALYTICS_DATASOURCE_SERVICE = "memorix-analytics";
8
+ /** Built-in organization-wide census of current heads by data source. */
9
+ export const DATASOURCES_COUNTER_ID = "@datasources";
10
+ const UNATTRIBUTED = { system: "unattributed", name: "Unattributed" };
11
+ function errorCode(err) {
12
+ return err && typeof err === "object" && "code" in err ? String(err.code) : "";
13
+ }
14
+ function isGone(err) {
15
+ const code = errorCode(err);
16
+ return code === "READ_NOT_FOUND" || code === "READ_ABSTRACT_MEMBER_MISSING";
17
+ }
18
+ /** Attribution order: `sources[0]`, then the declared `origin`, then the writing `service`. */
19
+ export function datasourceOfRecord(record) {
20
+ const ds = record?._system?._datasource;
21
+ if (!ds || typeof ds !== "object")
22
+ return null;
23
+ const text = (v) => (typeof v === "string" ? v.trim() : "");
24
+ const src = Array.isArray(ds.sources) ? ds.sources[0] : undefined;
25
+ const connector = text(src?.system);
26
+ if (connector)
27
+ return { system: connector, name: text(src?.name) || connector };
28
+ const system = text(ds.origin) || text(ds.service);
29
+ return system ? { system, name: system } : null;
30
+ }
31
+ function sameJson(a, b) {
32
+ return JSON.stringify(a ?? null) === JSON.stringify(b ?? null);
33
+ }
34
+ function sortedAgents(agentIds) {
35
+ return [...new Set(agentIds)].sort((a, b) => a.localeCompare(b));
36
+ }
37
+ export class AnalyticsTracker {
38
+ o;
39
+ orgs = new Map();
40
+ orgInit = new Map();
41
+ queue = new Map();
42
+ drainTimer = null;
43
+ chain = Promise.resolve();
44
+ dirty = new Map();
45
+ backfills = new Map();
46
+ inflight = new Set();
47
+ closed = false;
48
+ constructor(options) {
49
+ this.o = {
50
+ debounceMs: 1_000,
51
+ batchMs: 25,
52
+ pendingStaleMs: 120_000,
53
+ backfillPageSize: 200,
54
+ refreshMs: 15_000,
55
+ ...options,
56
+ };
57
+ }
58
+ now() {
59
+ return this.o.now?.() ?? new Date();
60
+ }
61
+ log(message, extra) {
62
+ this.o.log?.(message, extra);
63
+ }
64
+ track(p) {
65
+ this.inflight.add(p);
66
+ void p.finally(() => this.inflight.delete(p)).catch(() => undefined);
67
+ return p;
68
+ }
69
+ serial(fn) {
70
+ const run = this.chain.then(fn);
71
+ this.chain = run.catch(() => undefined);
72
+ return run;
73
+ }
74
+ // ---------------------------------------------------------------- org registry cache
75
+ async counters(orgId, fresh = false) {
76
+ await this.ensureOrg(orgId);
77
+ const cached = this.orgs.get(orgId);
78
+ if (!fresh && cached && this.now().getTime() - cached.loadedAt < this.o.refreshMs)
79
+ return cached.counters;
80
+ const counters = await this.o.stateFor(orgId).listCounters();
81
+ this.orgs.set(orgId, { counters, loadedAt: this.now().getTime() });
82
+ return counters;
83
+ }
84
+ async reload(orgId) {
85
+ const counters = await this.o.stateFor(orgId).listCounters();
86
+ this.orgs.set(orgId, { counters, loadedAt: this.now().getTime() });
87
+ }
88
+ ensureOrg(orgId) {
89
+ let p = this.orgInit.get(orgId);
90
+ if (!p) {
91
+ p = this.initOrg(orgId).catch((err) => {
92
+ this.orgInit.delete(orgId);
93
+ throw err;
94
+ });
95
+ this.orgInit.set(orgId, p);
96
+ }
97
+ return p;
98
+ }
99
+ async initOrg(orgId) {
100
+ const state = this.o.stateFor(orgId);
101
+ await this.recoverPending(orgId);
102
+ const timer = setTimeout(() => void this.track(this.recoverPending(orgId)), this.o.pendingStaleMs + 1_000);
103
+ timer.unref?.();
104
+ if (this.o.datasources !== false) {
105
+ const census = await state.getCounter(DATASOURCES_COUNTER_ID);
106
+ if (!census) {
107
+ const at = this.now().toISOString();
108
+ await state.saveCounter({
109
+ id: DATASOURCES_COUNTER_ID,
110
+ orgId,
111
+ origin: "builtin",
112
+ agentIds: [],
113
+ definition: {
114
+ id: DATASOURCES_COUNTER_ID,
115
+ name: "Data sources",
116
+ target: { objectType: "", contentType: "", recordId: "" },
117
+ },
118
+ hash: "builtin-1",
119
+ generation: 1,
120
+ status: "backfilling",
121
+ backfill: {},
122
+ registeredAt: at,
123
+ updatedAt: at,
124
+ });
125
+ }
126
+ }
127
+ const counters = await state.listCounters();
128
+ this.orgs.set(orgId, { counters, loadedAt: this.now().getTime() });
129
+ for (const c of counters) {
130
+ if (c.status === "backfilling")
131
+ this.startBackfill(orgId, c.id);
132
+ }
133
+ }
134
+ async recoverPending(orgId) {
135
+ if (this.closed)
136
+ return;
137
+ const state = this.o.stateFor(orgId);
138
+ const stale = await state.stalePending(new Date(this.now().getTime() - this.o.pendingStaleMs));
139
+ for (const id of stale) {
140
+ this.log("analytics: rebuilding counter left pending by an interrupted flush", { orgId, counterId: id });
141
+ await this.rebuildInternal(orgId, id).catch((err) => this.log("analytics: recovery rebuild failed", { orgId, counterId: id, error: String(err) }));
142
+ }
143
+ }
144
+ // ---------------------------------------------------------------- write hook
145
+ /** Called by the writer decorator after every successful write or remove. */
146
+ onWrite(scope, event) {
147
+ if (this.closed || event.datasourceService === ANALYTICS_DATASOURCE_SERVICE)
148
+ return;
149
+ void this.track(this.fanOut(scope, event).catch((err) => this.log("analytics: write fan-out failed", { orgId: scope.orgId, error: String(err) })));
150
+ }
151
+ async fanOut(scope, event) {
152
+ const counters = await this.counters(scope.orgId);
153
+ for (const c of counters) {
154
+ if (c.status === "error")
155
+ continue;
156
+ if (c.origin === "builtin") {
157
+ if (c.id === DATASOURCES_COUNTER_ID) {
158
+ this.enqueue({
159
+ kind: "head",
160
+ orgId: scope.orgId,
161
+ agentIds: scope.agentIds,
162
+ objectType: event.objectType,
163
+ contentType: event.contentType,
164
+ recordId: event.recordId,
165
+ attempts: 0,
166
+ });
167
+ }
168
+ continue;
169
+ }
170
+ const def = c.definition;
171
+ if (def.target.objectType === event.objectType && def.target.contentType === event.contentType)
172
+ continue;
173
+ const view = this.o.metadataFor(c.agentIds);
174
+ for (const abstractType of abstractTypesOf(def)) {
175
+ if (!view.members(abstractType).some((m) => m.objectType === event.objectType))
176
+ continue;
177
+ this.enqueue({
178
+ kind: "abstract",
179
+ orgId: scope.orgId,
180
+ counterId: c.id,
181
+ abstractType,
182
+ recordId: event.recordId,
183
+ attempts: 0,
184
+ });
185
+ }
186
+ }
187
+ }
188
+ taskKey(t) {
189
+ return t.kind === "abstract"
190
+ ? `a${SEP}${t.orgId}${SEP}${t.counterId}${SEP}${t.abstractType}${SEP}${t.recordId}`
191
+ : `h${SEP}${t.orgId}${SEP}${t.objectType}${SEP}${t.contentType}${SEP}${t.recordId}`;
192
+ }
193
+ enqueue(task, delayMs) {
194
+ if (this.closed)
195
+ return;
196
+ const key = this.taskKey(task);
197
+ if (!this.queue.has(key))
198
+ this.queue.set(key, task);
199
+ this.scheduleDrain(delayMs ?? this.o.batchMs);
200
+ }
201
+ scheduleDrain(delayMs) {
202
+ if (this.drainTimer)
203
+ return;
204
+ this.drainTimer = setTimeout(() => {
205
+ this.drainTimer = null;
206
+ void this.track(this.serial(() => this.drain()));
207
+ }, delayMs);
208
+ this.drainTimer.unref?.();
209
+ }
210
+ async drain() {
211
+ while (this.queue.size > 0 && !this.closed) {
212
+ const batch = [...this.queue.entries()].slice(0, 500);
213
+ for (const [k] of batch)
214
+ this.queue.delete(k);
215
+ const groups = new Map();
216
+ for (const [, t] of batch) {
217
+ const g = t.kind === "abstract" ? `${t.orgId}${SEP}${t.counterId}` : `${t.orgId}${SEP}${DATASOURCES_COUNTER_ID}`;
218
+ const list = groups.get(g);
219
+ if (list)
220
+ list.push(t);
221
+ else
222
+ groups.set(g, [t]);
223
+ }
224
+ for (const tasks of groups.values()) {
225
+ try {
226
+ if (tasks[0].kind === "abstract")
227
+ await this.processAbstract(tasks);
228
+ else
229
+ await this.processHeads(tasks);
230
+ }
231
+ catch (err) {
232
+ this.log("analytics: reconcile failed; retrying", { error: String(err) });
233
+ for (const t of tasks)
234
+ this.retry(t);
235
+ }
236
+ }
237
+ }
238
+ }
239
+ retry(task) {
240
+ const attempts = task.attempts + 1;
241
+ if (attempts > 5) {
242
+ this.log("analytics: dropping task after retries", { task: this.taskKey(task) });
243
+ return;
244
+ }
245
+ const delay = Math.min(30_000, 250 * 2 ** attempts);
246
+ const timer = setTimeout(() => this.enqueue({ ...task, attempts }), delay);
247
+ timer.unref?.();
248
+ }
249
+ // ---------------------------------------------------------------- reconcile primitive
250
+ async applyEntries(orgId, counter, entries) {
251
+ if (entries.length === 0)
252
+ return;
253
+ const state = this.o.stateFor(orgId);
254
+ const gen = counter.generation;
255
+ const nowIso = this.now().toISOString();
256
+ const ledger = await state.readLedger(counter.id, gen, [...new Set(entries.map((e) => e.key))]);
257
+ const intents = [];
258
+ const incsByKey = new Map();
259
+ const retries = new Map();
260
+ const pairs = new Map();
261
+ const touched = new Set();
262
+ let rebuild = false;
263
+ for (const e of entries) {
264
+ const row = ledger.get(e.key);
265
+ if (incsByKey.has(e.key))
266
+ continue;
267
+ if (row?.pending) {
268
+ const age = Date.parse(row.pendingAt ?? nowIso);
269
+ if (this.now().getTime() - age >= this.o.pendingStaleMs)
270
+ rebuild = true;
271
+ else {
272
+ const timer = setTimeout(() => e.retry(), 1_000);
273
+ timer.unref?.();
274
+ }
275
+ continue;
276
+ }
277
+ const prev = row
278
+ ? { scope: row.scope, at: row.at, facts: row.facts, track: row.track }
279
+ : null;
280
+ const { state: next, once, pair } = e.next(prev);
281
+ if (!prev && !next)
282
+ continue;
283
+ const incs = [];
284
+ const push = (s, facts, delta, at = s.at) => {
285
+ for (const f of facts) {
286
+ incs.push({
287
+ scope: s.scope,
288
+ at,
289
+ metric: f.metric,
290
+ subject: f.subject,
291
+ ...(f.split !== undefined ? { split: f.split } : {}),
292
+ ...(f.value !== undefined ? { value: f.value } : {}),
293
+ delta,
294
+ });
295
+ }
296
+ };
297
+ if (prev && next && prev.scope === next.scope && prev.at === next.at) {
298
+ const { removed, added } = diffFacts(prev.facts, next.facts);
299
+ push(prev, removed, -1);
300
+ push(next, added, 1);
301
+ }
302
+ else {
303
+ if (prev)
304
+ push(prev, prev.facts, -1);
305
+ if (next)
306
+ push(next, next.facts, 1);
307
+ }
308
+ if (next && once?.length)
309
+ push(next, once, 1, nowIso);
310
+ const moved = prev?.scope !== next?.scope || prev?.at !== next?.at;
311
+ if (incs.length === 0 && !moved && sameJson(prev?.track, next?.track))
312
+ continue;
313
+ intents.push({ key: e.key, expectedSeq: row ? row.seq : null, next });
314
+ incsByKey.set(e.key, incs);
315
+ retries.set(e.key, e.retry);
316
+ if (prev)
317
+ touched.add(prev.scope);
318
+ if (next) {
319
+ touched.add(next.scope);
320
+ const known = pairs.get(next.scope);
321
+ if (!known || next.at < known.firstSeenAt) {
322
+ pairs.set(next.scope, {
323
+ scope: next.scope,
324
+ counterId: counter.id,
325
+ generation: gen,
326
+ pair: pair ?? null,
327
+ firstSeenAt: next.at,
328
+ });
329
+ }
330
+ }
331
+ }
332
+ if (rebuild) {
333
+ await this.rebuildInternal(orgId, counter.id);
334
+ return;
335
+ }
336
+ if (intents.length === 0)
337
+ return;
338
+ const conflicts = new Set(await state.writeIntents(counter.id, gen, intents, nowIso));
339
+ const ok = intents.filter((i) => !conflicts.has(i.key)).map((i) => i.key);
340
+ for (const key of conflicts)
341
+ retries.get(key)?.();
342
+ const increments = ok.flatMap((k) => incsByKey.get(k) ?? []);
343
+ await this.o.engine.ingest(increments);
344
+ await this.o.engine.flush();
345
+ await state.commitIntents(counter.id, gen, ok);
346
+ for (const p of pairs.values())
347
+ await state.touchPair(p);
348
+ if (counter.origin !== "builtin")
349
+ this.markDirty(orgId, counter.id, touched);
350
+ }
351
+ subjectEntry(orgId, counter, abstractType, recordId, subject, retry) {
352
+ const def = counter.definition;
353
+ return {
354
+ key: `${abstractType}${SEP}${recordId}`,
355
+ retry,
356
+ next: (prev) => {
357
+ if (!subject)
358
+ return { state: null };
359
+ const proj = projectSubject(def, abstractType, subject);
360
+ if (!proj.inScope)
361
+ return { state: null };
362
+ const scope = countexScope({ orgId, counterId: counter.id, generation: counter.generation, pair: proj.pair });
363
+ const at = proj.clock ?? (prev && prev.scope === scope ? prev.at : this.now().toISOString());
364
+ return {
365
+ state: { scope, at, facts: proj.facts, track: proj.track },
366
+ once: prev ? transitionFacts(def, prev.track, proj.track) : [],
367
+ pair: proj.pair,
368
+ };
369
+ },
370
+ };
371
+ }
372
+ async loadSubject(reader, scope, def, abstractType, recordId) {
373
+ let item;
374
+ try {
375
+ item = (await reader.getAbstract(scope, { abstractType, recordId }));
376
+ }
377
+ catch (err) {
378
+ if (isGone(err))
379
+ return null;
380
+ throw err;
381
+ }
382
+ return this.toSubject(reader, scope, def, item);
383
+ }
384
+ async toSubject(reader, scope, def, item) {
385
+ let datasource = null;
386
+ if (usesDatasource(def)) {
387
+ try {
388
+ const record = await reader.getContent(scope, {
389
+ objectType: item.objectType,
390
+ contentType: item.contentType,
391
+ recordId: item.recordId,
392
+ });
393
+ datasource = datasourceOfRecord(record) ?? UNATTRIBUTED;
394
+ }
395
+ catch (err) {
396
+ if (!isGone(err))
397
+ throw err;
398
+ }
399
+ }
400
+ return { recordId: item.recordId, properties: item.properties ?? {}, datasource };
401
+ }
402
+ async processAbstract(tasks) {
403
+ const { orgId, counterId } = tasks[0];
404
+ const counter = await this.o.stateFor(orgId).getCounter(counterId);
405
+ if (!counter || counter.status === "error")
406
+ return;
407
+ const scope = { orgId, agentIds: counter.agentIds };
408
+ const { reader } = this.o.portsFor(scope);
409
+ const entries = await Promise.all(tasks.map(async (t) => {
410
+ const subject = await this.loadSubject(reader, scope, counter.definition, t.abstractType, t.recordId);
411
+ return this.subjectEntry(orgId, counter, t.abstractType, t.recordId, subject, () => this.retry(t));
412
+ }));
413
+ await this.applyEntries(orgId, counter, entries);
414
+ }
415
+ // ---------------------------------------------------------------- @datasources census
416
+ headEntry(orgId, counter, recordId, observations, retry) {
417
+ return {
418
+ key: `rec${SEP}${recordId}`,
419
+ retry,
420
+ next: (prev) => {
421
+ const track = prev?.track
422
+ ? structuredClone(prev.track)
423
+ : { objectType: observations[0].objectType, heads: {} };
424
+ for (const o of observations) {
425
+ track.objectType = o.objectType;
426
+ if (o.datasource === "gone" || o.datasource?.system === ANALYTICS_DATASOURCE_SERVICE) {
427
+ delete track.heads[o.contentType];
428
+ }
429
+ else {
430
+ const ds = o.datasource ?? UNATTRIBUTED;
431
+ track.heads[o.contentType] = { system: ds.system, name: ds.name ?? ds.system };
432
+ }
433
+ }
434
+ const contentTypes = Object.keys(track.heads).sort();
435
+ if (contentTypes.length === 0)
436
+ return { state: null };
437
+ const facts = [];
438
+ const systems = new Set();
439
+ for (const ct of contentTypes) {
440
+ const h = track.heads[ct];
441
+ facts.push({ metric: `dh${SEP}${h.system}`, subject: track.objectType, split: ct });
442
+ facts.push({ metric: `dl${SEP}${h.system}`, subject: h.name });
443
+ systems.add(h.system);
444
+ }
445
+ for (const s of [...systems].sort())
446
+ facts.push({ metric: `dsub${SEP}${s}`, subject: track.objectType });
447
+ const scope = countexScope({ orgId, counterId: counter.id, generation: counter.generation, pair: null });
448
+ return {
449
+ state: { scope, at: prev?.at ?? this.now().toISOString(), facts, track: track },
450
+ };
451
+ },
452
+ };
453
+ }
454
+ async processHeads(tasks) {
455
+ const orgId = tasks[0].orgId;
456
+ const counter = await this.o.stateFor(orgId).getCounter(DATASOURCES_COUNTER_ID);
457
+ if (!counter || counter.status === "error")
458
+ return;
459
+ const byRecord = new Map();
460
+ for (const t of tasks) {
461
+ const scope = { orgId, agentIds: t.agentIds };
462
+ let datasource;
463
+ try {
464
+ const record = await this.o.portsFor(scope).reader.getContent(scope, {
465
+ objectType: t.objectType,
466
+ contentType: t.contentType,
467
+ recordId: t.recordId,
468
+ });
469
+ datasource = datasourceOfRecord(record);
470
+ }
471
+ catch (err) {
472
+ if (!isGone(err))
473
+ throw err;
474
+ datasource = "gone";
475
+ }
476
+ const e = byRecord.get(t.recordId) ?? { tasks: [], obs: [] };
477
+ e.tasks.push(t);
478
+ e.obs.push({ objectType: t.objectType, contentType: t.contentType, datasource });
479
+ byRecord.set(t.recordId, e);
480
+ }
481
+ const entries = [...byRecord.entries()].map(([recordId, e]) => this.headEntry(orgId, counter, recordId, e.obs, () => e.tasks.forEach((t) => this.retry(t))));
482
+ await this.applyEntries(orgId, counter, entries);
483
+ }
484
+ async sourceCensus(scope) {
485
+ const timestamp = this.now().toISOString();
486
+ if (this.o.datasources === false) {
487
+ return { ok: true, censusId: null, timestamp, status: "disabled", subjects: [], heads: [] };
488
+ }
489
+ await this.ensureOrg(scope.orgId);
490
+ await this.chain;
491
+ const counter = await this.o.stateFor(scope.orgId).getCounter(DATASOURCES_COUNTER_ID);
492
+ if (!counter || counter.status !== "live") {
493
+ return {
494
+ ok: true,
495
+ censusId: null,
496
+ timestamp,
497
+ status: counter?.status ?? "backfilling",
498
+ subjects: [],
499
+ heads: [],
500
+ };
501
+ }
502
+ const cells = await this.o.engine.readCells(countexScope({ orgId: scope.orgId, counterId: counter.id, generation: counter.generation, pair: null }));
503
+ const view = this.o.metadataFor(scope.agentIds);
504
+ const names = new Map();
505
+ const grain = (prefix) => {
506
+ const bySystem = new Map();
507
+ for (const c of cells) {
508
+ if (c.count <= 0 || !c.metric.startsWith(prefix))
509
+ continue;
510
+ const system = c.metric.slice(prefix.length);
511
+ const kinds = bySystem.get(system) ?? new Map();
512
+ kinds.set(c.subject, (kinds.get(c.subject) ?? 0) + c.count);
513
+ bySystem.set(system, kinds);
514
+ }
515
+ return bySystem;
516
+ };
517
+ for (const c of cells) {
518
+ if (c.count <= 0 || !c.metric.startsWith(`dl${SEP}`))
519
+ continue;
520
+ const system = c.metric.slice(3);
521
+ const best = names.get(system);
522
+ if (!best || c.count > best.n)
523
+ names.set(system, { name: c.subject, n: c.count });
524
+ }
525
+ const shape = (bySystem) => [...bySystem.entries()]
526
+ .map(([system, kinds]) => {
527
+ const list = [...kinds.entries()]
528
+ .map(([kind, count]) => ({ kind, label: view.objectTypeName(kind) ?? kind, count }))
529
+ .sort((a, b) => b.count - a.count || a.kind.localeCompare(b.kind));
530
+ return {
531
+ system,
532
+ name: names.get(system)?.name ?? system,
533
+ total: list.reduce((n, k) => n + k.count, 0),
534
+ kinds: list,
535
+ };
536
+ })
537
+ .sort((a, b) => b.total - a.total || a.system.localeCompare(b.system));
538
+ return {
539
+ ok: true,
540
+ censusId: `${DATASOURCES_COUNTER_ID}#${counter.generation}`,
541
+ timestamp: counter.liveAt ?? timestamp,
542
+ status: "live",
543
+ subjects: shape(grain(`dsub${SEP}`)),
544
+ heads: shape(grain(`dh${SEP}`)),
545
+ };
546
+ }
547
+ // ---------------------------------------------------------------- registry
548
+ /** Registers (or updates) the analytics[] items of the scope's installed packs. */
549
+ async syncPack(scope) {
550
+ if (this.closed)
551
+ return;
552
+ const agentIds = sortedAgents(scope.agentIds);
553
+ const view = this.o.metadataFor(agentIds);
554
+ const counters = await this.counters(scope.orgId, true);
555
+ const seen = new Set();
556
+ for (const raw of view.analytics()) {
557
+ const id = raw && typeof raw === "object" ? String(raw.id ?? "") : "";
558
+ if (!id)
559
+ continue;
560
+ seen.add(id);
561
+ const existing = counters.find((c) => c.id === id);
562
+ if (existing && existing.origin !== "pack")
563
+ continue;
564
+ const hash = definitionHash(raw);
565
+ if (existing && existing.hash === hash && existing.status !== "error")
566
+ continue;
567
+ try {
568
+ await this.register(scope, raw, { origin: "pack", replace: true });
569
+ }
570
+ catch (err) {
571
+ this.log("analytics: pack counter rejected", { orgId: scope.orgId, id, error: String(err) });
572
+ const at = this.now().toISOString();
573
+ const issues = err instanceof ServiceError ? err.details : undefined;
574
+ if (!existing) {
575
+ await this.o.stateFor(scope.orgId).saveCounter({
576
+ id,
577
+ orgId: scope.orgId,
578
+ origin: "pack",
579
+ agentIds,
580
+ definition: raw,
581
+ hash,
582
+ generation: 1,
583
+ status: "error",
584
+ backfill: {},
585
+ registeredAt: at,
586
+ updatedAt: at,
587
+ error: `${err instanceof Error ? err.message : String(err)}${issues ? ` ${JSON.stringify(issues)}` : ""}`,
588
+ });
589
+ }
590
+ }
591
+ }
592
+ for (const c of counters) {
593
+ if (c.origin === "pack" && !seen.has(c.id) && sameJson(sortedAgents(c.agentIds), agentIds)) {
594
+ await this.unregister(scope.orgId, c.id, { allowPack: true });
595
+ }
596
+ }
597
+ await this.reload(scope.orgId);
598
+ }
599
+ validate(scope, raw) {
600
+ return validateCounterDefinition(raw, this.o.metadataFor(sortedAgents(scope.agentIds)).catalog);
601
+ }
602
+ async register(scope, raw, opts) {
603
+ const agentIds = sortedAgents(scope.agentIds);
604
+ const issues = this.validate(scope, raw);
605
+ if (issues.length > 0) {
606
+ throw new ServiceError("ANALYTICS_DEFINITION_INVALID", "analytics definition is invalid", 400, { issues });
607
+ }
608
+ const def = raw;
609
+ if (def.id.startsWith("@")) {
610
+ throw new ServiceError("ANALYTICS_DEFINITION_INVALID", "counter ids starting with @ are reserved", 400);
611
+ }
612
+ const state = this.o.stateFor(scope.orgId);
613
+ const counters = await this.counters(scope.orgId, true);
614
+ const existing = counters.find((c) => c.id === def.id) ?? null;
615
+ const hash = definitionHash(def);
616
+ if (existing) {
617
+ if (existing.origin !== opts.origin) {
618
+ throw new ServiceError("ANALYTICS_COUNTER_CONFLICT", `counter "${def.id}" is registered by ${existing.origin}; pick another id`, 409);
619
+ }
620
+ if (existing.hash === hash && sameJson(existing.agentIds, agentIds) && existing.status !== "error") {
621
+ return existing;
622
+ }
623
+ if (!opts.replace) {
624
+ throw new ServiceError("ANALYTICS_COUNTER_EXISTS", `counter "${def.id}" exists with a different definition; pass replace: true to rebuild it`, 409);
625
+ }
626
+ }
627
+ const clash = counters.find((c) => c.id !== def.id &&
628
+ c.origin !== "builtin" &&
629
+ c.definition.target.objectType === def.target.objectType &&
630
+ c.definition.target.contentType === def.target.contentType &&
631
+ c.definition.target.recordId === def.target.recordId);
632
+ if (clash) {
633
+ throw new ServiceError("ANALYTICS_TARGET_CONFLICT", `counter "${clash.id}" already materializes into this target`, 409);
634
+ }
635
+ const at = this.now().toISOString();
636
+ const record = {
637
+ id: def.id,
638
+ orgId: scope.orgId,
639
+ origin: opts.origin,
640
+ agentIds,
641
+ definition: def,
642
+ hash,
643
+ generation: (existing?.generation ?? 0) + 1,
644
+ status: "backfilling",
645
+ backfill: {},
646
+ registeredAt: existing?.registeredAt ?? at,
647
+ updatedAt: at,
648
+ registeredBy: opts.registeredBy ?? existing?.registeredBy ?? null,
649
+ liveAt: null,
650
+ lastMaterializedAt: existing?.lastMaterializedAt ?? null,
651
+ error: null,
652
+ };
653
+ await state.saveCounter(record);
654
+ if (existing)
655
+ await this.dropGeneration(scope.orgId, existing);
656
+ await this.reload(scope.orgId);
657
+ this.writePlaceholder(record);
658
+ this.startBackfill(scope.orgId, record.id);
659
+ return record;
660
+ }
661
+ async unregister(orgId, id, opts = {}) {
662
+ const state = this.o.stateFor(orgId);
663
+ const existing = await state.getCounter(id);
664
+ if (!existing)
665
+ return false;
666
+ if (existing.origin === "builtin" || (existing.origin === "pack" && !opts.allowPack)) {
667
+ throw new ServiceError("ANALYTICS_COUNTER_NOT_REMOVABLE", existing.origin === "pack"
668
+ ? `counter "${id}" comes from an installed pack; remove it from the pack's analytics[]`
669
+ : `counter "${id}" is built in`, 409);
670
+ }
671
+ await state.deleteCounter(id);
672
+ await state.dropGenerations(id);
673
+ await this.o.dropCells?.(orgId, `${encodeURIComponent(orgId)}|${encodeURIComponent(id)}#`);
674
+ const d = this.dirty.get(`${orgId}${SEP}${id}`);
675
+ if (d) {
676
+ clearTimeout(d.timer);
677
+ this.dirty.delete(`${orgId}${SEP}${id}`);
678
+ }
679
+ await this.reload(orgId);
680
+ return true;
681
+ }
682
+ /**
683
+ * Data-source stamps changed outside the writer (datasource backfill): rebuild the counters
684
+ * that count by `@datasource`, including the built-in census. Returns their ids.
685
+ */
686
+ async datasourcesRestamped(orgId) {
687
+ const rebuilt = [];
688
+ for (const c of await this.counters(orgId, true)) {
689
+ const depends = c.origin === "builtin" ? c.id === DATASOURCES_COUNTER_ID : usesDatasource(c.definition);
690
+ if (!depends)
691
+ continue;
692
+ await this.rebuildInternal(orgId, c.id);
693
+ rebuilt.push(c.id);
694
+ }
695
+ return rebuilt;
696
+ }
697
+ async rebuild(orgId, id) {
698
+ await this.ensureOrg(orgId);
699
+ const rec = await this.rebuildInternal(orgId, id);
700
+ if (!rec)
701
+ throw new ServiceError("ANALYTICS_COUNTER_NOT_FOUND", `counter "${id}" not found`, 404);
702
+ return rec;
703
+ }
704
+ async rebuildInternal(orgId, id) {
705
+ const state = this.o.stateFor(orgId);
706
+ const existing = await state.getCounter(id);
707
+ if (!existing)
708
+ return null;
709
+ const next = {
710
+ ...existing,
711
+ generation: existing.generation + 1,
712
+ status: "backfilling",
713
+ backfill: {},
714
+ liveAt: null,
715
+ error: null,
716
+ updatedAt: this.now().toISOString(),
717
+ };
718
+ await state.saveCounter(next);
719
+ await this.dropGeneration(orgId, existing);
720
+ await this.reload(orgId);
721
+ this.writePlaceholder(next);
722
+ this.startBackfill(orgId, id);
723
+ return next;
724
+ }
725
+ async dropGeneration(orgId, old) {
726
+ const state = this.o.stateFor(orgId);
727
+ await state.dropGenerations(old.id, old.generation + 1);
728
+ await this.o
729
+ .dropCells?.(orgId, `${encodeURIComponent(orgId)}|${encodeURIComponent(old.id)}#${old.generation}|`)
730
+ .catch((err) => this.log("analytics: dropping old cells failed", { orgId, error: String(err) }));
731
+ }
732
+ async list(scope) {
733
+ return [...(await this.counters(scope.orgId, true))].sort((a, b) => a.id.localeCompare(b.id));
734
+ }
735
+ async get(scope, id) {
736
+ await this.ensureOrg(scope.orgId);
737
+ return this.o.stateFor(scope.orgId).getCounter(id);
738
+ }
739
+ // ---------------------------------------------------------------- backfill
740
+ startBackfill(orgId, id) {
741
+ const key = `${orgId}${SEP}${id}`;
742
+ if (this.backfills.has(key) || this.closed)
743
+ return;
744
+ const state = this.o.stateFor(orgId);
745
+ let ranGeneration = null;
746
+ const run = this.runBackfill(orgId, id, (g) => (ranGeneration = g))
747
+ .catch(async (err) => {
748
+ this.log("analytics: backfill failed", { orgId, counterId: id, error: String(err) });
749
+ const c = await state.getCounter(id);
750
+ if (c) {
751
+ await state.patchCounter(id, { status: "error", error: err instanceof Error ? err.message : String(err), updatedAt: this.now().toISOString() }, c.generation);
752
+ await this.reload(orgId);
753
+ }
754
+ })
755
+ .then(async () => {
756
+ this.backfills.delete(key);
757
+ const c = await state.getCounter(id);
758
+ if (c?.status === "backfilling" && c.generation !== ranGeneration)
759
+ this.startBackfill(orgId, id);
760
+ })
761
+ .catch(() => {
762
+ this.backfills.delete(key);
763
+ });
764
+ this.backfills.set(key, run);
765
+ void this.track(run);
766
+ }
767
+ async runBackfill(orgId, id, started) {
768
+ const state = this.o.stateFor(orgId);
769
+ let counter = await state.getCounter(id);
770
+ if (!counter || counter.status !== "backfilling")
771
+ return;
772
+ const gen = counter.generation;
773
+ started(gen);
774
+ const stillCurrent = async () => {
775
+ const c = await state.getCounter(id);
776
+ return c && c.generation === gen && c.status === "backfilling" && !this.closed ? c : null;
777
+ };
778
+ const saveProgress = async (key, progress) => {
779
+ counter.backfill[key] = progress;
780
+ await state.patchCounter(id, { backfill: counter.backfill, updatedAt: this.now().toISOString() }, gen);
781
+ };
782
+ if (counter.origin === "builtin") {
783
+ if (!this.o.heads) {
784
+ this.log("analytics: no head source; @datasources counts new writes only", { orgId });
785
+ }
786
+ else {
787
+ for (const coll of await this.o.heads.collections(orgId)) {
788
+ let progress = counter.backfill[coll.key] ?? { cursor: null, done: false, seen: 0 };
789
+ while (!progress.done) {
790
+ const current = await stillCurrent();
791
+ if (!current)
792
+ return;
793
+ counter = current;
794
+ const rows = await this.o.heads.page(orgId, coll.key, progress.cursor, this.o.backfillPageSize);
795
+ const byRecord = new Map();
796
+ for (const r of rows) {
797
+ const list = byRecord.get(r.recordId) ?? [];
798
+ list.push({ objectType: coll.objectType, contentType: coll.contentType, datasource: r.datasource });
799
+ byRecord.set(r.recordId, list);
800
+ }
801
+ const c = counter;
802
+ await this.serial(() => this.applyEntries(orgId, c, [...byRecord.entries()].map(([rid, obs]) => this.headEntry(orgId, c, rid, obs, () => undefined))));
803
+ progress = {
804
+ cursor: rows.length ? rows[rows.length - 1].id : progress.cursor,
805
+ done: rows.length < this.o.backfillPageSize,
806
+ seen: progress.seen + rows.length,
807
+ };
808
+ await saveProgress(coll.key, progress);
809
+ }
810
+ }
811
+ }
812
+ }
813
+ else {
814
+ const def = counter.definition;
815
+ const scope = { orgId, agentIds: counter.agentIds };
816
+ const { reader } = this.o.portsFor(scope);
817
+ for (const abstractType of abstractTypesOf(def)) {
818
+ let progress = counter.backfill[abstractType] ?? { cursor: null, done: false, seen: 0 };
819
+ while (!progress.done) {
820
+ const current = await stillCurrent();
821
+ if (!current)
822
+ return;
823
+ counter = current;
824
+ const page = await reader.listAbstract(scope, {
825
+ abstractType,
826
+ limit: this.o.backfillPageSize,
827
+ ...(progress.cursor ? { cursor: progress.cursor } : {}),
828
+ });
829
+ const c = counter;
830
+ const entries = [];
831
+ for (const item of page.items) {
832
+ const subject = await this.toSubject(reader, scope, def, item);
833
+ entries.push(this.subjectEntry(orgId, c, abstractType, item.recordId, subject, () => this.enqueue({ kind: "abstract", orgId, counterId: id, abstractType, recordId: item.recordId, attempts: 0 })));
834
+ }
835
+ await this.serial(() => this.applyEntries(orgId, c, entries));
836
+ progress = {
837
+ cursor: page.nextCursor ?? null,
838
+ done: !page.nextCursor,
839
+ seen: progress.seen + page.items.length,
840
+ };
841
+ await saveProgress(abstractType, progress);
842
+ }
843
+ }
844
+ }
845
+ const at = this.now().toISOString();
846
+ const promoted = await state.patchCounter(id, { status: "live", liveAt: at, updatedAt: at }, gen);
847
+ await this.reload(orgId);
848
+ if (promoted && counter.origin !== "builtin") {
849
+ await this.serial(() => this.materialize(orgId, id, "all", false));
850
+ }
851
+ }
852
+ // ---------------------------------------------------------------- materialization
853
+ markDirty(orgId, counterId, scopes) {
854
+ const key = `${orgId}${SEP}${counterId}`;
855
+ const d = this.dirty.get(key);
856
+ if (d) {
857
+ for (const s of scopes)
858
+ d.scopes.add(s);
859
+ return;
860
+ }
861
+ const entry = {
862
+ orgId,
863
+ counterId,
864
+ scopes: new Set(scopes),
865
+ timer: setTimeout(() => {
866
+ this.dirty.delete(key);
867
+ void this.track(this.serial(() => this.materialize(orgId, counterId, entry.scopes, false)).catch((err) => this.log("analytics: materialize failed", { orgId, counterId, error: String(err) })));
868
+ }, this.o.debounceMs),
869
+ };
870
+ entry.timer.unref?.();
871
+ this.dirty.set(key, entry);
872
+ }
873
+ /**
874
+ * Materializes the counter now (pipeline step). With covering-scope values, that pair's
875
+ * history slot is rewritten as well as the current record.
876
+ */
877
+ async materializeNow(scope, id, scopeValues) {
878
+ await this.ensureOrg(scope.orgId);
879
+ await this.chain;
880
+ const counter = await this.o.stateFor(scope.orgId).getCounter(id);
881
+ const dirty = new Set();
882
+ const pairDef = counter?.origin !== "builtin" ? counter?.definition.scope : undefined;
883
+ if (counter && pairDef && scopeValues) {
884
+ const a = scopeValues[pairDef[0].key];
885
+ const b = scopeValues[pairDef[1].key];
886
+ if (typeof a === "string" && a && typeof b === "string" && b) {
887
+ dirty.add(countexScope({ orgId: scope.orgId, counterId: id, generation: counter.generation, pair: [a, b] }));
888
+ }
889
+ }
890
+ return this.serial(() => this.materialize(scope.orgId, id, dirty, false));
891
+ }
892
+ /** Publishes an all-`produced: false` body while the counter backfills. Never awaited: callers may hold the chain. */
893
+ writePlaceholder(counter) {
894
+ if (counter.origin === "builtin")
895
+ return;
896
+ const def = counter.definition;
897
+ if (hasScopePlaceholder(def, def.target.recordId))
898
+ return;
899
+ void this.track(this.serial(() => this.materialize(counter.orgId, counter.id, new Set(), true)).catch((err) => this.log("analytics: placeholder write failed", { counterId: counter.id, error: String(err) })));
900
+ }
901
+ async materialize(orgId, id, dirtyScopes, allowNotLive) {
902
+ const state = this.o.stateFor(orgId);
903
+ const counter = await state.getCounter(id);
904
+ if (!counter)
905
+ return { materialized: false, recordId: null, historyRecordId: null, status: "missing" };
906
+ if (counter.origin === "builtin") {
907
+ return { materialized: false, recordId: null, historyRecordId: null, status: counter.status };
908
+ }
909
+ const live = counter.status === "live";
910
+ if (!live && !allowNotLive) {
911
+ return { materialized: false, recordId: null, historyRecordId: null, status: counter.status };
912
+ }
913
+ const def = counter.definition;
914
+ const scope = { orgId, agentIds: counter.agentIds };
915
+ const generatedAt = this.now().toISOString();
916
+ const historyLimit = def.history?.limit ?? 12;
917
+ const needsSeries = (def.trends ?? []).length > 0;
918
+ const orgWide = !def.scope;
919
+ let pairs;
920
+ if (orgWide) {
921
+ pairs = [
922
+ {
923
+ scope: countexScope({ orgId, counterId: id, generation: counter.generation, pair: null }),
924
+ counterId: id,
925
+ generation: counter.generation,
926
+ pair: null,
927
+ firstSeenAt: counter.liveAt ?? generatedAt,
928
+ },
929
+ ];
930
+ }
931
+ else {
932
+ pairs = live ? await state.listPairs(id, counter.generation, Math.max(2, needsSeries ? historyLimit : 2)) : [];
933
+ if (pairs.length === 0) {
934
+ if (hasScopePlaceholder(def, def.target.recordId)) {
935
+ return { materialized: false, recordId: null, historyRecordId: null, status: counter.status };
936
+ }
937
+ pairs = [
938
+ {
939
+ scope: countexScope({ orgId, counterId: id, generation: counter.generation, pair: null }),
940
+ counterId: id,
941
+ generation: counter.generation,
942
+ pair: null,
943
+ firstSeenAt: generatedAt,
944
+ },
945
+ ];
946
+ }
947
+ }
948
+ const cellCache = new Map();
949
+ const cellsOf = (s) => {
950
+ let p = cellCache.get(s);
951
+ if (!p) {
952
+ p = live ? this.o.engine.readCells(s) : Promise.resolve([]);
953
+ cellCache.set(s, p);
954
+ }
955
+ return p;
956
+ };
957
+ const summaryCache = new Map();
958
+ const summaryOf = (p) => {
959
+ let s = summaryCache.get(p.scope);
960
+ if (!s) {
961
+ s = cellsOf(p.scope).then((cells) => summarizePair(def, cells, { live, organizationWide: orgWide }));
962
+ summaryCache.set(p.scope, s);
963
+ }
964
+ return s;
965
+ };
966
+ const envelopeFor = async (index) => {
967
+ const pairRow = pairs[index];
968
+ const current = await summaryOf(pairRow);
969
+ const previous = pairs[index + 1] ? await summaryOf(pairs[index + 1]) : null;
970
+ const windows = {};
971
+ if (live) {
972
+ const wanted = new Set((def.measures ?? []).flatMap((m) => m.windows ?? []));
973
+ for (const w of wanted)
974
+ windows[w] = await this.o.engine.readWindow(pairRow.scope, w, this.now());
975
+ }
976
+ const trends = await this.trends(def, pairs.slice(index), summaryOf, live, orgWide);
977
+ const leaders = live ? await this.leaders(scope, def, await cellsOf(pairRow.scope), pairRow.pair) : this.emptyLeaders(def);
978
+ return buildEnvelope({
979
+ definition: def,
980
+ counter: {
981
+ id,
982
+ generation: counter.generation,
983
+ status: counter.status,
984
+ hash: counter.hash,
985
+ origin: counter.origin,
986
+ },
987
+ generatedAt,
988
+ pair: pairRow.pair,
989
+ current,
990
+ previous,
991
+ windows,
992
+ trends,
993
+ leaders,
994
+ });
995
+ };
996
+ const currentEnvelope = await envelopeFor(0);
997
+ const ids = resolveRecordIds(def, pairs[0].pair);
998
+ await this.writeRecord(scope, def, ids.current, def.name, currentEnvelope);
999
+ let historyRecordId = null;
1000
+ if (def.history && ids.history) {
1001
+ historyRecordId = ids.history;
1002
+ const targets = new Map([[ids.history, 0]]);
1003
+ if (hasScopePlaceholder(def, def.history.recordId)) {
1004
+ pairs.forEach((p, i) => {
1005
+ if (i > 0 && (dirtyScopes === "all" || dirtyScopes.has(p.scope))) {
1006
+ const hid = resolveRecordIds(def, p.pair).history;
1007
+ if (!targets.has(hid))
1008
+ targets.set(hid, i);
1009
+ }
1010
+ });
1011
+ }
1012
+ for (const [recordId, index] of targets) {
1013
+ const body = index === 0 ? currentEnvelope : await envelopeFor(index);
1014
+ await this.writeRecord(scope, def, recordId, `${def.name} history`, {
1015
+ ...body,
1016
+ historySlot: true,
1017
+ coveringScope: scopeValuesOf(def, pairs[index].pair),
1018
+ });
1019
+ }
1020
+ }
1021
+ await state.patchCounter(id, { lastMaterializedAt: generatedAt }, counter.generation);
1022
+ return { materialized: true, recordId: ids.current, historyRecordId, status: counter.status };
1023
+ }
1024
+ emptyLeaders(def) {
1025
+ return Object.fromEntries((def.leaders ?? []).map((l) => [l.key, []]));
1026
+ }
1027
+ async leaders(scope, def, cells, pair) {
1028
+ const out = {};
1029
+ const { reader } = this.o.portsFor(scope);
1030
+ const prop = (item, p) => p === "recordId" ? item.recordId : item.properties?.[p];
1031
+ for (const l of def.leaders ?? []) {
1032
+ const limit = l.limit ?? 10;
1033
+ try {
1034
+ if (l.countOf) {
1035
+ const top = topLeaderIds(cells, l.key, limit);
1036
+ const items = new Map();
1037
+ if (top.length > 0) {
1038
+ if (l.idProperty === "recordId") {
1039
+ const found = await Promise.all(top.map((t) => reader
1040
+ .getAbstract(scope, { abstractType: l.abstractType, recordId: t.id })
1041
+ .then((it) => it)
1042
+ .catch(() => null)));
1043
+ for (const it of found)
1044
+ if (it)
1045
+ items.set(it.recordId, it);
1046
+ }
1047
+ else {
1048
+ const page = await reader
1049
+ .listAbstract(scope, {
1050
+ abstractType: l.abstractType,
1051
+ filters: [{ field: l.idProperty, op: "in", value: top.map((t) => t.id) }],
1052
+ limit: top.length,
1053
+ })
1054
+ .catch(() => ({ items: [] }));
1055
+ for (const it of page.items) {
1056
+ items.set(String(prop(it, l.idProperty)), it);
1057
+ }
1058
+ }
1059
+ }
1060
+ out[l.key] = top.map((t) => {
1061
+ const it = items.get(t.id);
1062
+ const row = { id: t.id, label: (it && prop(it, l.labelProperty)) ?? t.id, count: t.count };
1063
+ for (const c of l.columns ?? []) {
1064
+ if (c.kind === "count")
1065
+ row[c.key] = t.count;
1066
+ else if (c.property && it)
1067
+ row[c.key] = prop(it, c.property);
1068
+ }
1069
+ return row;
1070
+ });
1071
+ }
1072
+ else {
1073
+ const filters = (def.scope ?? [])
1074
+ .map((s, i) => ({ s, value: pair?.[i] }))
1075
+ .filter(({ s, value }) => s.abstractType === l.abstractType && value !== undefined)
1076
+ .map(({ s, value }) => ({ field: s.property, op: "eq", value }));
1077
+ const page = await reader.listAbstract(scope, {
1078
+ abstractType: l.abstractType,
1079
+ ...(filters.length ? { filters } : {}),
1080
+ sort: (l.sort ?? []).map((s) => ({ field: s.property, direction: s.direction ?? "desc" })),
1081
+ limit,
1082
+ });
1083
+ out[l.key] = page.items.map((it) => {
1084
+ const row = { id: prop(it, l.idProperty), label: prop(it, l.labelProperty) };
1085
+ for (const c of l.columns ?? []) {
1086
+ if (c.kind === "count")
1087
+ row[c.key] = 1;
1088
+ else if (c.property)
1089
+ row[c.key] = prop(it, c.property);
1090
+ }
1091
+ return row;
1092
+ });
1093
+ }
1094
+ }
1095
+ catch (err) {
1096
+ this.log("analytics: leader read failed", { leader: l.key, error: String(err) });
1097
+ out[l.key] = [];
1098
+ }
1099
+ }
1100
+ return out;
1101
+ }
1102
+ async trends(def, pairs, summaryOf, live, orgWide) {
1103
+ const out = [];
1104
+ const historyLimit = def.history?.limit ?? 12;
1105
+ const now = this.now();
1106
+ for (const t of def.trends ?? []) {
1107
+ const m = (def.measures ?? []).find((x) => x.key === t.measure);
1108
+ for (const window of t.windows) {
1109
+ if (!live || !m) {
1110
+ out.push({ key: t.key, label: t.label, window, points: [] });
1111
+ continue;
1112
+ }
1113
+ if (window === "snapshots") {
1114
+ const points = [];
1115
+ for (const p of pairs.slice(0, historyLimit)) {
1116
+ const v = (await summaryOf(p)).measures[t.measure];
1117
+ if (v?.produced && typeof v.value === "number") {
1118
+ points.push({ at: p.firstSeenAt, scope: scopeValuesOf(def, p.pair), value: v.value });
1119
+ }
1120
+ }
1121
+ out.push({ key: t.key, label: t.label, window, points });
1122
+ continue;
1123
+ }
1124
+ const days = window === "7d" ? 7 : 30;
1125
+ if (def.clock?.abstractType === m.abstractType && m.kind === "count") {
1126
+ const series = await this.o.engine.readDaily(pairs[0].scope, metricKey.measure(m.key), days, now);
1127
+ const byDay = new Map(series.map((p) => [p.at, p.value]));
1128
+ const points = [];
1129
+ for (let i = days - 1; i >= 0; i--) {
1130
+ const day = new Date(now.getTime() - i * 86_400_000).toISOString().slice(0, 10);
1131
+ points.push({ at: day, value: byDay.get(day) ?? 0 });
1132
+ }
1133
+ out.push({ key: t.key, label: t.label, window, points });
1134
+ continue;
1135
+ }
1136
+ if (orgWide) {
1137
+ out.push({ key: t.key, label: t.label, window, points: [] });
1138
+ continue;
1139
+ }
1140
+ const byDay = new Map();
1141
+ for (const p of [...pairs].reverse()) {
1142
+ const age = (now.getTime() - Date.parse(p.firstSeenAt)) / 86_400_000;
1143
+ if (!(age >= 0 && age <= days))
1144
+ continue;
1145
+ const v = (await summaryOf(p)).measures[t.measure];
1146
+ if (v?.produced && typeof v.value === "number")
1147
+ byDay.set(p.firstSeenAt.slice(0, 10), v.value);
1148
+ }
1149
+ out.push({
1150
+ key: t.key,
1151
+ label: t.label,
1152
+ window,
1153
+ points: [...byDay.entries()].sort(([a], [b]) => a.localeCompare(b)).map(([at, value]) => ({ at, value })),
1154
+ });
1155
+ }
1156
+ }
1157
+ return out;
1158
+ }
1159
+ async writeRecord(scope, def, recordId, title, data) {
1160
+ const { reader, writer } = this.o.portsFor(scope);
1161
+ const address = { objectType: def.target.objectType, contentType: def.target.contentType, recordId };
1162
+ const draft = {
1163
+ format: "memorix-record/2",
1164
+ recordId,
1165
+ objectType: address.objectType,
1166
+ contentType: address.contentType,
1167
+ dataCategory: this.o.metadataFor(scope.agentIds).dataCategoryOf(address.objectType),
1168
+ concept: {
1169
+ title,
1170
+ identifiers: { primary: { kind: "recordId", value: recordId, issuer: ANALYTICS_DATASOURCE_SERVICE } },
1171
+ },
1172
+ data,
1173
+ };
1174
+ const datasource = { service: ANALYTICS_DATASOURCE_SERVICE, declared: false };
1175
+ for (let attempt = 0;; attempt++) {
1176
+ let revision = null;
1177
+ try {
1178
+ revision = (await reader.getContent(scope, address)).revision;
1179
+ }
1180
+ catch (err) {
1181
+ if (!isGone(err))
1182
+ throw err;
1183
+ }
1184
+ try {
1185
+ if (revision === null) {
1186
+ await writer.add({ scope, precondition: { kind: "absent" }, record: draft, datasource });
1187
+ }
1188
+ else {
1189
+ await writer.upsert({
1190
+ scope,
1191
+ precondition: { kind: "revision", revision },
1192
+ record: { ...draft, revision },
1193
+ datasource,
1194
+ });
1195
+ }
1196
+ return;
1197
+ }
1198
+ catch (err) {
1199
+ const code = errorCode(err);
1200
+ if (attempt < 3 && (code === "WRITE_NOT_ABSENT" || code === "WRITE_STALE_REVISION"))
1201
+ continue;
1202
+ throw err;
1203
+ }
1204
+ }
1205
+ }
1206
+ // ---------------------------------------------------------------- lifecycle
1207
+ /** Resolves once queued reconciles, backfills, and debounced materializations have run. */
1208
+ async idle(timeoutMs = 60_000) {
1209
+ const deadline = Date.now() + timeoutMs;
1210
+ for (;;) {
1211
+ await this.chain;
1212
+ await Promise.allSettled([...this.inflight]);
1213
+ await this.chain;
1214
+ const busy = this.queue.size > 0 || this.drainTimer !== null || this.dirty.size > 0 || this.backfills.size > 0 || this.inflight.size > 0;
1215
+ if (!busy)
1216
+ return;
1217
+ if (Date.now() > deadline)
1218
+ throw new Error("analytics tracker did not settle");
1219
+ await new Promise((r) => setTimeout(r, Math.max(5, Math.min(this.o.batchMs, this.o.debounceMs))));
1220
+ }
1221
+ }
1222
+ async close() {
1223
+ this.closed = true;
1224
+ if (this.drainTimer)
1225
+ clearTimeout(this.drainTimer);
1226
+ this.drainTimer = null;
1227
+ for (const d of this.dirty.values())
1228
+ clearTimeout(d.timer);
1229
+ this.dirty.clear();
1230
+ this.queue.clear();
1231
+ await this.chain;
1232
+ await Promise.allSettled([...this.backfills.values()]);
1233
+ await this.o.engine.flush().catch(() => undefined);
1234
+ }
1235
+ }
1236
+ /** Wraps the shared content writer so every successful write reaches the tracker. */
1237
+ export function withAnalyticsTracking(writer, onWrite) {
1238
+ const emit = (scope, objectType, contentType, recordId, datasource) => {
1239
+ try {
1240
+ onWrite(scope, { objectType, contentType, recordId, datasourceService: datasource?.service });
1241
+ }
1242
+ catch {
1243
+ /* tracking never fails a write */
1244
+ }
1245
+ };
1246
+ return {
1247
+ async add(req) {
1248
+ const res = await writer.add(req);
1249
+ emit(req.scope, res.record.objectType, res.record.contentType, res.record.recordId, req.datasource);
1250
+ return res;
1251
+ },
1252
+ async upsert(req) {
1253
+ const res = await writer.upsert(req);
1254
+ emit(req.scope, res.record.objectType, res.record.contentType, res.record.recordId, req.datasource);
1255
+ return res;
1256
+ },
1257
+ async replace(req) {
1258
+ const res = await writer.replace(req);
1259
+ emit(req.scope, res.record.objectType, res.record.contentType, res.record.recordId, req.datasource);
1260
+ return res;
1261
+ },
1262
+ async patch(req) {
1263
+ const res = await writer.patch(req);
1264
+ emit(req.scope, req.objectType, req.contentType, req.recordId, req.datasource);
1265
+ return res;
1266
+ },
1267
+ async moveFields(req) {
1268
+ const res = await writer.moveFields(req);
1269
+ emit(req.scope, req.objectType, req.contentType, req.recordId, req.datasource);
1270
+ return res;
1271
+ },
1272
+ async remove(req) {
1273
+ const res = await writer.remove(req);
1274
+ emit(req.scope, req.objectType, req.contentType, req.recordId);
1275
+ return res;
1276
+ },
1277
+ reconcileDirectory(scope, recordId) {
1278
+ return writer.reconcileDirectory(scope, recordId);
1279
+ },
1280
+ };
1281
+ }
1282
+ /** Abstract reverse-writes go through their own concrete writer; report them too. */
1283
+ export function withAbstractTracking(abstractWriter, onWrite) {
1284
+ return {
1285
+ async patchAbstract(req) {
1286
+ const res = await abstractWriter.patchAbstract(req);
1287
+ try {
1288
+ onWrite(req.scope, {
1289
+ objectType: res.objectType,
1290
+ contentType: res.contentType,
1291
+ recordId: res.record.recordId,
1292
+ datasourceService: req.datasource?.service,
1293
+ });
1294
+ }
1295
+ catch {
1296
+ /* tracking never fails a write */
1297
+ }
1298
+ return res;
1299
+ },
1300
+ };
1301
+ }
1302
+ //# sourceMappingURL=tracker.js.map