@x12i/memorix-service 3.10.0 → 3.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/analytics/content-allow.d.ts +17 -0
- package/dist/analytics/content-allow.d.ts.map +1 -0
- package/dist/analytics/content-allow.js +39 -0
- package/dist/analytics/content-allow.js.map +1 -0
- package/dist/analytics/definition.d.ts +27 -6
- package/dist/analytics/definition.d.ts.map +1 -1
- package/dist/analytics/definition.js +110 -25
- package/dist/analytics/definition.js.map +1 -1
- package/dist/analytics/host.d.ts.map +1 -1
- package/dist/analytics/host.js +14 -0
- package/dist/analytics/host.js.map +1 -1
- package/dist/analytics/index-store.d.ts +98 -0
- package/dist/analytics/index-store.d.ts.map +1 -0
- package/dist/analytics/index-store.js +295 -0
- package/dist/analytics/index-store.js.map +1 -0
- package/dist/analytics/materializer.d.ts +1 -1
- package/dist/analytics/materializer.d.ts.map +1 -1
- package/dist/analytics/materializer.js +18 -12
- package/dist/analytics/materializer.js.map +1 -1
- package/dist/analytics/projector.d.ts +1 -1
- package/dist/analytics/projector.d.ts.map +1 -1
- package/dist/analytics/projector.js +19 -11
- package/dist/analytics/projector.js.map +1 -1
- package/dist/analytics/property-index.d.ts +87 -0
- package/dist/analytics/property-index.d.ts.map +1 -0
- package/dist/analytics/property-index.js +537 -0
- package/dist/analytics/property-index.js.map +1 -0
- package/dist/analytics/tracker.d.ts +34 -1
- package/dist/analytics/tracker.d.ts.map +1 -1
- package/dist/analytics/tracker.js +222 -9
- package/dist/analytics/tracker.js.map +1 -1
- package/dist/app.d.ts.map +1 -1
- package/dist/app.js +2 -0
- package/dist/app.js.map +1 -1
- package/dist/openapi/openapi.json +474 -2
- package/dist/platform.d.ts.map +1 -1
- package/dist/platform.js +15 -2
- package/dist/platform.js.map +1 -1
- package/dist/routes/analytics-indexes.d.ts +8 -0
- package/dist/routes/analytics-indexes.d.ts.map +1 -0
- package/dist/routes/analytics-indexes.js +80 -0
- package/dist/routes/analytics-indexes.js.map +1 -0
- package/dist/routes/data.d.ts.map +1 -1
- package/dist/routes/data.js +22 -0
- package/dist/routes/data.js.map +1 -1
- package/openapi/openapi.json +474 -2
- package/package.json +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tracker.d.ts","sourceRoot":"","sources":["../../src/analytics/tracker.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,KAAK,EACV,qBAAqB,EACrB,oBAAoB,EACpB,oBAAoB,
|
|
1
|
+
{"version":3,"file":"tracker.d.ts","sourceRoot":"","sources":["../../src/analytics/tracker.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,KAAK,EACV,gBAAgB,EAChB,qBAAqB,EACrB,oBAAoB,EACpB,oBAAoB,EAEpB,WAAW,EACZ,MAAM,oBAAoB,CAAC;AAE5B,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAEhD,OAAO,EAOL,KAAK,cAAc,EAEpB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAoC,KAAK,gBAAgB,EAA2B,MAAM,aAAa,CAAC;AAY/G,OAAO,EAOL,KAAK,iBAAiB,EAEvB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAkC,KAAK,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAE3F,OAAO,KAAK,EACV,mBAAmB,EAEnB,aAAa,EACb,aAAa,EAId,MAAM,YAAY,CAAC;AAEpB,8FAA8F;AAC9F,eAAO,MAAM,4BAA4B,sBAAsB,CAAC;AAChE,yEAAyE;AACzE,eAAO,MAAM,sBAAsB,iBAAiB,CAAC;AAErD,MAAM,MAAM,qBAAqB,GAAG;IAClC,oFAAoF;IACpF,SAAS,IAAI,OAAO,EAAE,CAAC;IACvB,OAAO,EAAE,cAAc,CAAC;IACxB,uEAAuE;IACvE,OAAO,CAAC,YAAY,EAAE,MAAM,GAAG,KAAK,CAAC;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAClF,cAAc,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;IAClD,cAAc,CAAC,UAAU,EAAE,MAAM,GAAG,QAAQ,GAAG,OAAO,GAAG,WAAW,CAAC;CACtE,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,MAAM,CAAA;CAAE,CAAC;AACtF,MAAM,MAAM,OAAO,GAAG;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,iBAAiB,GAAG,IAAI,CAAA;CAAE,CAAC;AAE7F,wEAAwE;AACxE,MAAM,MAAM,UAAU,GAAG;IACvB,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC;IACtD,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;CAC7F,CAAC;AAEF,MAAM,MAAM,UAAU,GAAG;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,CAAC;AAEF,MAAM,MAAM,uBAAuB,GAAG;IACpC,MAAM,EAAE,gBAAgB,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,mBAAmB,CAAC;IAC7C,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,qBAAqB,CAAC;IACvD,QAAQ,CAAC,KAAK,EAAE,YAAY,GAAG;QAAE,MAAM,EAAE,oBAAoB,CAAC;QAAC,MAAM,EAAE,oBAAoB,CAAA;KAAE,CAAC;IAC9F,8FAA8F;IAC9F,UAAU,CAAC,CAAC,KAAK,EAAE,MAAM,GAAG,kBAAkB,CAAC;IAC/C,uFAAuF;IACvF,SAAS,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9D,KAAK,CAAC,EAAE,UAAU,GAAG,IAAI,CAAC;IAC1B,kDAAkD;IAClD,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;IACjB,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CAClE,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC;AAC9E,MAAM,MAAM,kBAAkB,GAAG;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,gBAAgB,EAAE,CAAA;CAAE,CAAC;AAC5G,MAAM,MAAM,mBAAmB,GAAG;IAChC,EAAE,EAAE,IAAI,CAAC;IACT,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,GAAG,aAAa,GAAG,OAAO,GAAG,UAAU,CAAC;IACtD,QAAQ,EAAE,kBAAkB,EAAE,CAAC;IAC/B,KAAK,EAAE,kBAAkB,EAAE,CAAC;CAC7B,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,YAAY,EAAE,OAAO,CAAC;IACtB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,MAAM,EAAE,aAAa,CAAC,QAAQ,CAAC,GAAG,SAAS,CAAC;CAC7C,CAAC;AA2CF,+FAA+F;AAC/F,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,OAAO,GAAG,iBAAiB,GAAG,IAAI,CAW5E;AAUD,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,QAAQ,CAAC,CAAC,CAGQ;IAC1B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAsE;IAC3F,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAoC;IAC5D,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA2B;IACjD,OAAO,CAAC,UAAU,CAA8C;IAChE,OAAO,CAAC,KAAK,CAAuC;IACpD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAGlB;IACJ,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAoC;IAC9D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA+B;IACxD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAyC;IACvE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAuB;IAC/C,OAAO,CAAC,MAAM,CAAS;gBAEX,OAAO,EAAE,uBAAuB;IAqB5C,OAAO,CAAC,UAAU;IAUlB,OAAO,CAAC,GAAG;IAIX,OAAO,CAAC,GAAG;IAIX,OAAO,CAAC,KAAK;IAMb,OAAO,CAAC,MAAM;YAQA,QAAQ;YASR,MAAM;IAMpB,4FAA4F;IACtF,OAAO,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAI3C,OAAO,CAAC,UAAU;IASlB,OAAO,CAAC,SAAS;YAYH,OAAO;YAoCP,cAAc;IAc5B,6EAA6E;IAC7E,OAAO,CAAC,KAAK,EAAE,YAAY,EAAE,KAAK,EAAE,UAAU,GAAG,IAAI;YASvC,MAAM;IAoDpB,OAAO,CAAC,OAAO;IAUf,OAAO,CAAC,OAAO;IAOf,OAAO,CAAC,aAAa;YASP,KAAK;IA2BnB,OAAO,CAAC,KAAK;YAaC,YAAY;IA4F1B,OAAO,CAAC,YAAY;YA2BN,WAAW;YAiBX,SAAS;YAsBT,eAAe;IAe7B,OAAO,CAAC,iBAAiB;YAWX,kBAAkB;YAkBlB,cAAc;IAyB5B,OAAO,CAAC,SAAS;YA0CH,YAAY;IA8BpB,YAAY,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,mBAAmB,CAAC;IAkErE,mFAAmF;IAC7E,QAAQ,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IA8ClD,QAAQ,CAAC,KAAK,EAAE,YAAY,EAAE,GAAG,EAAE,OAAO;IAIpC,QAAQ,CACZ,KAAK,EAAE,YAAY,EACnB,GAAG,EAAE,OAAO,EACZ,IAAI,EAAE;QAAE,MAAM,EAAE,OAAO,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;QAAC,OAAO,CAAC,EAAE,OAAO,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,GACnG,OAAO,CAAC,aAAa,CAAC;IA0EnB,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,GAAE;QAAE,SAAS,CAAC,EAAE,OAAO,CAAA;KAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IAyBjG;;;OAGG;IACG,oBAAoB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAWtD,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC;YAOlD,eAAe;IAqB7B,8FAA8F;YAChF,gBAAgB;YAmBhB,cAAc;IAQtB,IAAI,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IAInD,GAAG,CAAC,KAAK,EAAE,YAAY,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IAKzE,aAAa,CAAC,KAAK,EAAE,YAAY,EAAE,GAAG,EAAE,OAAO;IAI/C,aAAa,CACX,KAAK,EAAE,YAAY,EACnB,GAAG,EAAE,OAAO,EACZ,IAAI,GAAE;QAAE,OAAO,CAAC,EAAE,OAAO,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;KAAO;IAKhE,WAAW,CAAC,KAAK,EAAE,YAAY;IAI/B,QAAQ,CAAC,KAAK,EAAE,YAAY,EAAE,EAAE,EAAE,MAAM;IAIxC,WAAW,CAAC,KAAK,EAAE,YAAY,EAAE,EAAE,EAAE,MAAM;IAI3C,YAAY,CAAC,KAAK,EAAE,YAAY,EAAE,KAAK,EAAE;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,WAAW,EAAE,CAAA;KAAE;IAI7G,cAAc,CAAC,KAAK,EAAE,YAAY,EAAE,KAAK,EAAE;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,WAAW,EAAE,CAAA;KAAE;IAI/G,aAAa,CAAC,KAAK,EAAE,YAAY,EAAE,KAAK,EAAE,gBAAgB,EAAE,MAAM,EAAE,oBAAoB;IAMxF,OAAO,CAAC,aAAa;YA8BP,WAAW;IAkLzB,OAAO,CAAC,SAAS;IAwBjB;;;OAGG;IACG,cAAc,CAClB,KAAK,EAAE,YAAY,EACnB,EAAE,EAAE,MAAM,EACV,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GACpC,OAAO,CAAC,iBAAiB,CAAC;IAgB7B,sHAAsH;IACtH,OAAO,CAAC,gBAAgB;YAWV,WAAW;IAkIzB,OAAO,CAAC,YAAY;YAIN,OAAO;YA6EP,MAAM;YA8DN,WAAW;IAmDzB,2FAA2F;IACrF,IAAI,CAAC,SAAS,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IAmBvC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAW7B;AAED,qFAAqF;AACrF,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,oBAAoB,EAC5B,OAAO,EAAE,CAAC,KAAK,EAAE,YAAY,EAAE,KAAK,EAAE,UAAU,KAAK,IAAI,GACxD,oBAAoB,CA2CtB;AAED,qFAAqF;AACrF,wBAAgB,oBAAoB,CAClC,cAAc,EAAE,qBAAqB,EACrC,OAAO,EAAE,CAAC,KAAK,EAAE,YAAY,EAAE,KAAK,EAAE,UAAU,KAAK,IAAI,GACxD,qBAAqB,CAiBvB"}
|
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
import { ServiceError } from "../errors.js";
|
|
2
|
-
import {
|
|
2
|
+
import { replaceAnalyticsContentGrants } from "./content-allow.js";
|
|
3
|
+
import { abstractTypesOf, contentSelections, definitionHash, populationKey, usesDatasource, validateCounterDefinition, } from "./definition.js";
|
|
3
4
|
import { countexScope } from "./engine.js";
|
|
4
5
|
import { buildEnvelope, hasScopePlaceholder, resolveRecordIds, scopeValuesOf, summarizePair, topLeaderIds, } from "./materializer.js";
|
|
5
6
|
import { diffFacts, metricKey, projectSubject, SEP, transitionFacts, } from "./projector.js";
|
|
7
|
+
import { createMemoryPropertyIndexStore } from "./index-store.js";
|
|
8
|
+
import { PropertyIndexService, contentPaths, recordFromIndexValues } from "./property-index.js";
|
|
6
9
|
/** Datasource service stamped on materialized records; writes carrying it are not counted. */
|
|
7
10
|
export const ANALYTICS_DATASOURCE_SERVICE = "memorix-analytics";
|
|
8
11
|
/** Built-in organization-wide census of current heads by data source. */
|
|
@@ -44,6 +47,8 @@ export class AnalyticsTracker {
|
|
|
44
47
|
dirty = new Map();
|
|
45
48
|
backfills = new Map();
|
|
46
49
|
inflight = new Set();
|
|
50
|
+
memoryIndexes = new Map();
|
|
51
|
+
indexes;
|
|
47
52
|
closed = false;
|
|
48
53
|
constructor(options) {
|
|
49
54
|
this.o = {
|
|
@@ -54,6 +59,26 @@ export class AnalyticsTracker {
|
|
|
54
59
|
refreshMs: 15_000,
|
|
55
60
|
...options,
|
|
56
61
|
};
|
|
62
|
+
this.indexes = new PropertyIndexService({
|
|
63
|
+
storeFor: (orgId) => this.indexStore(orgId),
|
|
64
|
+
catalogFor: (agentIds) => this.o.metadataFor(agentIds).catalog,
|
|
65
|
+
readerFor: (scope) => this.o.portsFor(scope).reader,
|
|
66
|
+
backfillPageSize: this.o.backfillPageSize,
|
|
67
|
+
now: () => this.now(),
|
|
68
|
+
log: (message, extra) => this.log(message, extra),
|
|
69
|
+
isClosed: () => this.closed,
|
|
70
|
+
onLive: (orgId, index) => this.rebuildFromIndex(orgId, index),
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
indexStore(orgId) {
|
|
74
|
+
if (this.o.indexesFor)
|
|
75
|
+
return this.o.indexesFor(orgId);
|
|
76
|
+
let store = this.memoryIndexes.get(orgId);
|
|
77
|
+
if (!store) {
|
|
78
|
+
store = createMemoryPropertyIndexStore();
|
|
79
|
+
this.memoryIndexes.set(orgId, store);
|
|
80
|
+
}
|
|
81
|
+
return store;
|
|
57
82
|
}
|
|
58
83
|
now() {
|
|
59
84
|
return this.o.now?.() ?? new Date();
|
|
@@ -84,6 +109,16 @@ export class AnalyticsTracker {
|
|
|
84
109
|
async reload(orgId) {
|
|
85
110
|
const counters = await this.o.stateFor(orgId).listCounters();
|
|
86
111
|
this.orgs.set(orgId, { counters, loadedAt: this.now().getTime() });
|
|
112
|
+
this.syncGrants(orgId, counters);
|
|
113
|
+
}
|
|
114
|
+
/** Loads the org registry so target content types are readable before a content request. */
|
|
115
|
+
async prepare(orgId) {
|
|
116
|
+
await this.ensureOrg(orgId);
|
|
117
|
+
}
|
|
118
|
+
syncGrants(orgId, counters) {
|
|
119
|
+
replaceAnalyticsContentGrants(orgId, counters
|
|
120
|
+
.filter((c) => c.origin !== "builtin" && c.status !== "error")
|
|
121
|
+
.map((c) => c.definition.target));
|
|
87
122
|
}
|
|
88
123
|
ensureOrg(orgId) {
|
|
89
124
|
let p = this.orgInit.get(orgId);
|
|
@@ -126,6 +161,7 @@ export class AnalyticsTracker {
|
|
|
126
161
|
}
|
|
127
162
|
const counters = await state.listCounters();
|
|
128
163
|
this.orgs.set(orgId, { counters, loadedAt: this.now().getTime() });
|
|
164
|
+
this.syncGrants(orgId, counters);
|
|
129
165
|
for (const c of counters) {
|
|
130
166
|
if (c.status === "backfilling")
|
|
131
167
|
this.startBackfill(orgId, c.id);
|
|
@@ -149,6 +185,12 @@ export class AnalyticsTracker {
|
|
|
149
185
|
void this.track(this.fanOut(scope, event).catch((err) => this.log("analytics: write fan-out failed", { orgId: scope.orgId, error: String(err) })));
|
|
150
186
|
}
|
|
151
187
|
async fanOut(scope, event) {
|
|
188
|
+
try {
|
|
189
|
+
await this.indexes.applyWrite(scope, event);
|
|
190
|
+
}
|
|
191
|
+
catch (err) {
|
|
192
|
+
this.log("analytics: index update failed", { orgId: scope.orgId, error: String(err) });
|
|
193
|
+
}
|
|
152
194
|
const counters = await this.counters(scope.orgId);
|
|
153
195
|
for (const c of counters) {
|
|
154
196
|
if (c.status === "error")
|
|
@@ -183,12 +225,29 @@ export class AnalyticsTracker {
|
|
|
183
225
|
attempts: 0,
|
|
184
226
|
});
|
|
185
227
|
}
|
|
228
|
+
for (const sel of contentSelections(def)) {
|
|
229
|
+
if (sel.objectType !== event.objectType || sel.contentType !== event.contentType)
|
|
230
|
+
continue;
|
|
231
|
+
this.enqueue({
|
|
232
|
+
kind: "content",
|
|
233
|
+
orgId: scope.orgId,
|
|
234
|
+
counterId: c.id,
|
|
235
|
+
objectType: sel.objectType,
|
|
236
|
+
contentType: sel.contentType,
|
|
237
|
+
recordId: event.recordId,
|
|
238
|
+
attempts: 0,
|
|
239
|
+
});
|
|
240
|
+
}
|
|
186
241
|
}
|
|
187
242
|
}
|
|
188
243
|
taskKey(t) {
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
244
|
+
if (t.kind === "abstract") {
|
|
245
|
+
return `a${SEP}${t.orgId}${SEP}${t.counterId}${SEP}${t.abstractType}${SEP}${t.recordId}`;
|
|
246
|
+
}
|
|
247
|
+
if (t.kind === "content") {
|
|
248
|
+
return `c${SEP}${t.orgId}${SEP}${t.counterId}${SEP}${t.objectType}${SEP}${t.contentType}${SEP}${t.recordId}`;
|
|
249
|
+
}
|
|
250
|
+
return `h${SEP}${t.orgId}${SEP}${t.objectType}${SEP}${t.contentType}${SEP}${t.recordId}`;
|
|
192
251
|
}
|
|
193
252
|
enqueue(task, delayMs) {
|
|
194
253
|
if (this.closed)
|
|
@@ -214,7 +273,9 @@ export class AnalyticsTracker {
|
|
|
214
273
|
this.queue.delete(k);
|
|
215
274
|
const groups = new Map();
|
|
216
275
|
for (const [, t] of batch) {
|
|
217
|
-
const g = t.kind === "
|
|
276
|
+
const g = t.kind === "head"
|
|
277
|
+
? `${t.orgId}${SEP}${DATASOURCES_COUNTER_ID}`
|
|
278
|
+
: `${t.kind}${SEP}${t.orgId}${SEP}${t.counterId}`;
|
|
218
279
|
const list = groups.get(g);
|
|
219
280
|
if (list)
|
|
220
281
|
list.push(t);
|
|
@@ -225,6 +286,8 @@ export class AnalyticsTracker {
|
|
|
225
286
|
try {
|
|
226
287
|
if (tasks[0].kind === "abstract")
|
|
227
288
|
await this.processAbstract(tasks);
|
|
289
|
+
else if (tasks[0].kind === "content")
|
|
290
|
+
await this.processContent(tasks);
|
|
228
291
|
else
|
|
229
292
|
await this.processHeads(tasks);
|
|
230
293
|
}
|
|
@@ -412,6 +475,44 @@ export class AnalyticsTracker {
|
|
|
412
475
|
}));
|
|
413
476
|
await this.applyEntries(orgId, counter, entries);
|
|
414
477
|
}
|
|
478
|
+
subjectFromStored(def, record) {
|
|
479
|
+
if (datasourceOfRecord(record)?.system === ANALYTICS_DATASOURCE_SERVICE)
|
|
480
|
+
return null;
|
|
481
|
+
let datasource = datasourceOfRecord(record);
|
|
482
|
+
if (!datasource && usesDatasource(def))
|
|
483
|
+
datasource = UNATTRIBUTED;
|
|
484
|
+
return {
|
|
485
|
+
recordId: record.recordId,
|
|
486
|
+
properties: record,
|
|
487
|
+
datasource,
|
|
488
|
+
};
|
|
489
|
+
}
|
|
490
|
+
async loadContentSubject(reader, scope, def, objectType, contentType, recordId) {
|
|
491
|
+
let record;
|
|
492
|
+
try {
|
|
493
|
+
record = await reader.getContent(scope, { objectType, contentType, recordId });
|
|
494
|
+
}
|
|
495
|
+
catch (err) {
|
|
496
|
+
if (isGone(err))
|
|
497
|
+
return null;
|
|
498
|
+
throw err;
|
|
499
|
+
}
|
|
500
|
+
return this.subjectFromStored(def, record);
|
|
501
|
+
}
|
|
502
|
+
async processContent(tasks) {
|
|
503
|
+
const { orgId, counterId } = tasks[0];
|
|
504
|
+
const counter = await this.o.stateFor(orgId).getCounter(counterId);
|
|
505
|
+
if (!counter || counter.status === "error")
|
|
506
|
+
return;
|
|
507
|
+
const scope = { orgId, agentIds: counter.agentIds };
|
|
508
|
+
const { reader } = this.o.portsFor(scope);
|
|
509
|
+
const entries = await Promise.all(tasks.map(async (t) => {
|
|
510
|
+
const population = populationKey({ objectType: t.objectType, contentType: t.contentType });
|
|
511
|
+
const subject = await this.loadContentSubject(reader, scope, counter.definition, t.objectType, t.contentType, t.recordId);
|
|
512
|
+
return this.subjectEntry(orgId, counter, population, t.recordId, subject, () => this.retry(t));
|
|
513
|
+
}));
|
|
514
|
+
await this.applyEntries(orgId, counter, entries);
|
|
515
|
+
}
|
|
415
516
|
// ---------------------------------------------------------------- @datasources census
|
|
416
517
|
headEntry(orgId, counter, recordId, observations, retry) {
|
|
417
518
|
return {
|
|
@@ -722,6 +823,22 @@ export class AnalyticsTracker {
|
|
|
722
823
|
this.startBackfill(orgId, id);
|
|
723
824
|
return next;
|
|
724
825
|
}
|
|
826
|
+
/** A live index replaces the scan for counters of the same agent that read only its paths. */
|
|
827
|
+
async rebuildFromIndex(orgId, index) {
|
|
828
|
+
const agents = sortedAgents(index.agentIds);
|
|
829
|
+
for (const counter of await this.counters(orgId, true)) {
|
|
830
|
+
if (counter.origin === "builtin" || counter.status === "error")
|
|
831
|
+
continue;
|
|
832
|
+
if (!sameJson(sortedAgents(counter.agentIds), agents))
|
|
833
|
+
continue;
|
|
834
|
+
const populations = contentSelections(counter.definition).filter((sel) => sel.objectType === index.definition.objectType && sel.contentType === index.definition.contentType);
|
|
835
|
+
if (populations.length === 0)
|
|
836
|
+
continue;
|
|
837
|
+
const covered = populations.some((sel) => contentPaths(counter.definition, sel.objectType, sel.contentType).every((path) => index.definition.properties.includes(path)));
|
|
838
|
+
if (covered)
|
|
839
|
+
await this.rebuildInternal(orgId, counter.id);
|
|
840
|
+
}
|
|
841
|
+
}
|
|
725
842
|
async dropGeneration(orgId, old) {
|
|
726
843
|
const state = this.o.stateFor(orgId);
|
|
727
844
|
await state.dropGenerations(old.id, old.generation + 1);
|
|
@@ -736,6 +853,30 @@ export class AnalyticsTracker {
|
|
|
736
853
|
await this.ensureOrg(scope.orgId);
|
|
737
854
|
return this.o.stateFor(scope.orgId).getCounter(id);
|
|
738
855
|
}
|
|
856
|
+
validateIndex(scope, raw) {
|
|
857
|
+
return this.indexes.validate(sortedAgents(scope.agentIds), raw);
|
|
858
|
+
}
|
|
859
|
+
registerIndex(scope, raw, opts = {}) {
|
|
860
|
+
return this.indexes.register(scope, raw, opts);
|
|
861
|
+
}
|
|
862
|
+
listIndexes(scope) {
|
|
863
|
+
return this.indexes.list(scope);
|
|
864
|
+
}
|
|
865
|
+
getIndex(scope, id) {
|
|
866
|
+
return this.indexes.get(scope, id);
|
|
867
|
+
}
|
|
868
|
+
deleteIndex(scope, id) {
|
|
869
|
+
return this.indexes.remove(scope, id);
|
|
870
|
+
}
|
|
871
|
+
countIndexed(scope, query) {
|
|
872
|
+
return this.indexes.countIndexed(scope, query);
|
|
873
|
+
}
|
|
874
|
+
explainIndexed(scope, query) {
|
|
875
|
+
return this.indexes.explainIndexed(scope, query);
|
|
876
|
+
}
|
|
877
|
+
listFromIndex(scope, query, reader) {
|
|
878
|
+
return this.indexes.listFromIndex(scope, query, reader);
|
|
879
|
+
}
|
|
739
880
|
// ---------------------------------------------------------------- backfill
|
|
740
881
|
startBackfill(orgId, id) {
|
|
741
882
|
const key = `${orgId}${SEP}${id}`;
|
|
@@ -841,6 +982,73 @@ export class AnalyticsTracker {
|
|
|
841
982
|
await saveProgress(abstractType, progress);
|
|
842
983
|
}
|
|
843
984
|
}
|
|
985
|
+
for (const sel of contentSelections(def)) {
|
|
986
|
+
const population = populationKey(sel);
|
|
987
|
+
let progress = counter.backfill[population] ?? { cursor: null, done: false, seen: 0 };
|
|
988
|
+
while (!progress.done) {
|
|
989
|
+
const current = await stillCurrent();
|
|
990
|
+
if (!current)
|
|
991
|
+
return;
|
|
992
|
+
counter = current;
|
|
993
|
+
const source = await this.indexes.populationSource(orgId, counter.agentIds, sel.objectType, sel.contentType, def);
|
|
994
|
+
if (source.mode === "pending") {
|
|
995
|
+
if (this.closed)
|
|
996
|
+
return;
|
|
997
|
+
await this.indexes.waitUntilCovered(orgId, counter.agentIds, sel.objectType, sel.contentType, def);
|
|
998
|
+
continue;
|
|
999
|
+
}
|
|
1000
|
+
if (source.mode === "indexed" && progress.cursor?.startsWith("mrxq1.")) {
|
|
1001
|
+
await this.rebuildInternal(orgId, id);
|
|
1002
|
+
return;
|
|
1003
|
+
}
|
|
1004
|
+
const c = counter;
|
|
1005
|
+
const retry = (recordId) => this.enqueue({
|
|
1006
|
+
kind: "content",
|
|
1007
|
+
orgId,
|
|
1008
|
+
counterId: id,
|
|
1009
|
+
objectType: sel.objectType,
|
|
1010
|
+
contentType: sel.contentType,
|
|
1011
|
+
recordId,
|
|
1012
|
+
attempts: 0,
|
|
1013
|
+
});
|
|
1014
|
+
const entries = [];
|
|
1015
|
+
let seen = 0;
|
|
1016
|
+
let nextCursor = null;
|
|
1017
|
+
let done = false;
|
|
1018
|
+
if (source.mode === "indexed") {
|
|
1019
|
+
const page = await this.indexes.pageSubjects(orgId, source.index.id, source.index.generation, progress.cursor, this.o.backfillPageSize);
|
|
1020
|
+
for (const subjectRow of page.subjects) {
|
|
1021
|
+
const subject = this.subjectFromStored(def, recordFromIndexValues(subjectRow.recordId, sel.objectType, sel.contentType, subjectRow.values));
|
|
1022
|
+
if (!subject)
|
|
1023
|
+
continue;
|
|
1024
|
+
entries.push(this.subjectEntry(orgId, c, population, subjectRow.recordId, subject, () => retry(subjectRow.recordId)));
|
|
1025
|
+
}
|
|
1026
|
+
seen = page.subjects.length;
|
|
1027
|
+
nextCursor = page.nextCursor;
|
|
1028
|
+
done = !page.nextCursor;
|
|
1029
|
+
}
|
|
1030
|
+
else {
|
|
1031
|
+
const page = await reader.listContent(scope, {
|
|
1032
|
+
objectType: sel.objectType,
|
|
1033
|
+
contentType: sel.contentType,
|
|
1034
|
+
limit: this.o.backfillPageSize,
|
|
1035
|
+
...(progress.cursor ? { cursor: progress.cursor } : {}),
|
|
1036
|
+
});
|
|
1037
|
+
for (const record of page.items) {
|
|
1038
|
+
const subject = this.subjectFromStored(def, record);
|
|
1039
|
+
if (!subject)
|
|
1040
|
+
continue;
|
|
1041
|
+
entries.push(this.subjectEntry(orgId, c, population, record.recordId, subject, () => retry(record.recordId)));
|
|
1042
|
+
}
|
|
1043
|
+
seen = page.items.length;
|
|
1044
|
+
nextCursor = page.nextCursor ?? null;
|
|
1045
|
+
done = !page.nextCursor;
|
|
1046
|
+
}
|
|
1047
|
+
await this.serial(() => this.applyEntries(orgId, c, entries));
|
|
1048
|
+
progress = { cursor: nextCursor, done, seen: progress.seen + seen };
|
|
1049
|
+
await saveProgress(population, progress);
|
|
1050
|
+
}
|
|
1051
|
+
}
|
|
844
1052
|
}
|
|
845
1053
|
const at = this.now().toISOString();
|
|
846
1054
|
const promoted = await state.patchCounter(id, { status: "live", liveAt: at, updatedAt: at }, gen);
|
|
@@ -1122,7 +1330,7 @@ export class AnalyticsTracker {
|
|
|
1122
1330
|
continue;
|
|
1123
1331
|
}
|
|
1124
1332
|
const days = window === "7d" ? 7 : 30;
|
|
1125
|
-
if (def.clock
|
|
1333
|
+
if (def.clock && populationKey(def.clock) === populationKey(m) && m.kind === "count") {
|
|
1126
1334
|
const series = await this.o.engine.readDaily(pairs[0].scope, metricKey.measure(m.key), days, now);
|
|
1127
1335
|
const byDay = new Map(series.map((p) => [p.at, p.value]));
|
|
1128
1336
|
const points = [];
|
|
@@ -1209,9 +1417,14 @@ export class AnalyticsTracker {
|
|
|
1209
1417
|
const deadline = Date.now() + timeoutMs;
|
|
1210
1418
|
for (;;) {
|
|
1211
1419
|
await this.chain;
|
|
1212
|
-
await Promise.allSettled([...this.inflight]);
|
|
1420
|
+
await Promise.allSettled([...this.inflight, ...this.indexes.pending()]);
|
|
1213
1421
|
await this.chain;
|
|
1214
|
-
const busy = this.queue.size > 0 ||
|
|
1422
|
+
const busy = this.queue.size > 0 ||
|
|
1423
|
+
this.drainTimer !== null ||
|
|
1424
|
+
this.dirty.size > 0 ||
|
|
1425
|
+
this.backfills.size > 0 ||
|
|
1426
|
+
this.inflight.size > 0 ||
|
|
1427
|
+
this.indexes.busy();
|
|
1215
1428
|
if (!busy)
|
|
1216
1429
|
return;
|
|
1217
1430
|
if (Date.now() > deadline)
|
|
@@ -1229,7 +1442,7 @@ export class AnalyticsTracker {
|
|
|
1229
1442
|
this.dirty.clear();
|
|
1230
1443
|
this.queue.clear();
|
|
1231
1444
|
await this.chain;
|
|
1232
|
-
await Promise.allSettled([...this.backfills.values()]);
|
|
1445
|
+
await Promise.allSettled([...this.backfills.values(), ...this.indexes.pending()]);
|
|
1233
1446
|
await this.o.engine.flush().catch(() => undefined);
|
|
1234
1447
|
}
|
|
1235
1448
|
}
|