@x12i/countex 1.8.5 → 1.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. package/LICENSE +21 -21
  2. package/README.md +867 -860
  3. package/dist/cli.js +6 -6
  4. package/dist/core/countex.d.ts +61 -1
  5. package/dist/core/countex.d.ts.map +1 -1
  6. package/dist/core/countex.js +241 -0
  7. package/dist/core/countex.js.map +1 -1
  8. package/dist/core/managed.d.ts +200 -0
  9. package/dist/core/managed.d.ts.map +1 -0
  10. package/dist/core/managed.js +431 -0
  11. package/dist/core/managed.js.map +1 -0
  12. package/dist/core/timeframe.d.ts +5 -0
  13. package/dist/core/timeframe.d.ts.map +1 -1
  14. package/dist/core/timeframe.js +36 -0
  15. package/dist/core/timeframe.js.map +1 -1
  16. package/dist/index.d.ts +6 -2
  17. package/dist/index.d.ts.map +1 -1
  18. package/dist/index.js +9 -1
  19. package/dist/index.js.map +1 -1
  20. package/dist/insights/advisors/resource-advisor.d.ts.map +1 -1
  21. package/dist/insights/advisors/resource-advisor.js +5 -1
  22. package/dist/insights/advisors/resource-advisor.js.map +1 -1
  23. package/dist/insights/types.d.ts +4 -0
  24. package/dist/insights/types.d.ts.map +1 -1
  25. package/dist/storage/managed-memory.d.ts +33 -0
  26. package/dist/storage/managed-memory.d.ts.map +1 -0
  27. package/dist/storage/managed-memory.js +211 -0
  28. package/dist/storage/managed-memory.js.map +1 -0
  29. package/dist/storage/memory.d.ts +7 -4
  30. package/dist/storage/memory.d.ts.map +1 -1
  31. package/dist/storage/memory.js.map +1 -1
  32. package/dist/storage/postgres.d.ts +123 -0
  33. package/dist/storage/postgres.d.ts.map +1 -0
  34. package/dist/storage/postgres.js +1325 -0
  35. package/dist/storage/postgres.js.map +1 -0
  36. package/dist/storage/redis.d.ts +6 -0
  37. package/dist/storage/redis.d.ts.map +1 -1
  38. package/dist/storage/redis.js +21 -0
  39. package/dist/storage/redis.js.map +1 -1
  40. package/dist/types/index.d.ts +63 -5
  41. package/dist/types/index.d.ts.map +1 -1
  42. package/dist/utils/errors.d.ts +12 -0
  43. package/dist/utils/errors.d.ts.map +1 -1
  44. package/dist/utils/errors.js +22 -1
  45. package/dist/utils/errors.js.map +1 -1
  46. package/docs/DEPLOYMENT.md +399 -375
  47. package/docs/END_TO_END_EXAMPLE.md +316 -316
  48. package/docs/INSIGHTS.md +59 -59
  49. package/docs/INSIGHTS_ANOMALY_DETECTION.md +33 -33
  50. package/docs/INSIGHTS_LLM_COST_AND_TOKENS.md +26 -26
  51. package/docs/INSIGHTS_RESOURCE_ADVISOR.md +33 -31
  52. package/docs/countex_gap_analysis_complete_checklist.md +1340 -1340
  53. package/docs/guides/README.md +37 -34
  54. package/docs/guides/adapter-memory.md +60 -60
  55. package/docs/guides/adapter-mongo.md +64 -62
  56. package/docs/guides/adapter-postgres.md +92 -0
  57. package/docs/guides/adapter-redis.md +89 -89
  58. package/docs/guides/backup-export.md +135 -135
  59. package/docs/guides/cardinality.md +50 -50
  60. package/docs/guides/cli.md +53 -53
  61. package/docs/guides/client-events.md +65 -65
  62. package/docs/guides/counter-maps.md +160 -160
  63. package/docs/guides/idempotency.md +39 -39
  64. package/docs/guides/ingestion-and-flush.md +67 -67
  65. package/docs/guides/insights-advisors-and-optimization.md +39 -39
  66. package/docs/guides/insights-anomalies-and-rules.md +48 -48
  67. package/docs/guides/insights-comparison-and-trends.md +47 -47
  68. package/docs/guides/insights-forecasting-and-budget.md +61 -61
  69. package/docs/guides/insights-overview.md +82 -82
  70. package/docs/guides/insights-ranking.md +81 -81
  71. package/docs/guides/managed-commit.md +119 -0
  72. package/docs/guides/measures.md +72 -71
  73. package/docs/guides/migrating-mongo-to-postgres.md +127 -0
  74. package/docs/guides/preview.md +92 -92
  75. package/docs/guides/querying.md +178 -178
  76. package/docs/guides/simulation-profiles.md +33 -33
  77. package/docs/guides/simulation-resource-estimation.md +31 -30
  78. package/docs/guides/simulation.md +36 -36
  79. package/docs/guides/timeframes-and-bucketing.md +55 -55
  80. package/docs/internals/README.md +21 -21
  81. package/docs/internals/internal-architecture-overview.md +65 -65
  82. package/docs/internals/internal-backup-stream.md +23 -23
  83. package/docs/internals/internal-buffer-merge-and-flush.md +51 -51
  84. package/docs/internals/internal-cardinality.md +30 -30
  85. package/docs/internals/internal-dimension-hash.md +34 -34
  86. package/docs/internals/internal-idempotency.md +36 -36
  87. package/docs/internals/internal-ingest-pipeline.md +52 -52
  88. package/docs/internals/internal-insights-layer.md +36 -36
  89. package/docs/internals/internal-mapping-validation-and-extraction.md +59 -59
  90. package/docs/internals/internal-measures-and-scales.md +35 -35
  91. package/docs/internals/internal-query-path.md +46 -46
  92. package/docs/internals/internal-storage-memory.md +13 -13
  93. package/docs/internals/internal-storage-mongo.md +44 -44
  94. package/docs/internals/internal-storage-postgres.md +32 -0
  95. package/docs/internals/internal-storage-redis.md +63 -63
  96. package/docs/internals/internal-timeframe-bucketing.md +33 -33
  97. package/docs/use-cases/README.md +56 -56
  98. package/docs/use-cases/audit-security-log-counts.md +550 -550
  99. package/docs/use-cases/llm-cost-and-token-metering.md +255 -255
  100. package/docs/use-cases/multi-tenant-isolation.md +203 -203
  101. package/docs/use-cases/ops-dashboard-daily-rollups.md +255 -255
  102. package/docs/use-cases/processing-workflow-activity.md +165 -165
  103. package/docs/use-cases/product-feature-usage.md +240 -240
  104. package/docs/use-cases/webhook-idempotent-ingestion.md +188 -188
  105. package/package.json +190 -170
@@ -1,240 +1,240 @@
1
- # Technical playbook: Product and feature activity
2
-
3
- You run a **product analytics** pipeline: in-app events describe **what happened**, **who** did it, under **which account**, with **feature/plan** context. You want rollups suitable for MAU-style counts, feature adoption, and breakdowns — stored in **your** infra via Countex.
4
-
5
- ---
6
-
7
- ## 1. Problem statement
8
-
9
- ### 1.1 Outcomes
10
-
11
- - Per **account** (`scope`), count activity by **user** (`subject`) and **named event** (`metric`).
12
- - Slice by **`feature`** and **`plan`** dimensions.
13
- - Buckets: **`hour`** for near-real-time, **`day`** / **`month`** for exec summaries.
14
-
15
- ### 1.2 Constraints
16
-
17
- - Event names (`metrics`) must remain **bounded cardinality** — treat them like an enum registry.
18
- - User ids may be large but stable; dictionary handles string→int per scope.
19
-
20
- ---
21
-
22
- ## 2. Primitives mapping
23
-
24
- | Primitive | JSON path | Notes |
25
- |-----------|-----------|--------|
26
- | scope | `$.account.id` | Tenant = billing account. |
27
- | eventTime | `$.event.createdAt` | UTC bucketing. |
28
- | subject `user` | `$.user.id` | Required — anonymous traffic must be filtered **before** ingest or mapped to a stable anon key you accept. |
29
- | metric | `$.event.name` | Single-valued path (no wildcard) → one metric per event unless you add fan-out. |
30
- | dim `feature` | `$.product.feature` | Literal dimension. |
31
- | dim `plan` | `$.product.plan` | Literal dimension. |
32
- | delta | `1` | One occurrence per event. |
33
-
34
- ---
35
-
36
- ## 3. Canonical payload
37
-
38
- ```json
39
- {
40
- "account": { "id": "acct_123" },
41
- "user": { "id": "user_42" },
42
- "event": {
43
- "name": "risk_dashboard_viewed",
44
- "createdAt": "2026-05-08T16:00:00.000Z"
45
- },
46
- "product": {
47
- "feature": "risk_dashboard",
48
- "plan": "enterprise"
49
- }
50
- }
51
- ```
52
-
53
- ### 3.1 Required-field matrix
54
-
55
- | Path | Required | Effect if absent |
56
- |------|----------|------------------|
57
- | `$.account.id` | Yes | Skip event for map. |
58
- | `$.event.createdAt` | Yes | Skip. |
59
- | `$.user.id` | Yes | Skip (`required: true` subject). |
60
- | `$.event.name` | Yes | Skip (`metrics.required: true`). |
61
- | `$.product.feature` | No | Optional dimension omitted by default (`skipMissingOptionalDimensions: true`). |
62
- | `$.product.plan` | No | Same. |
63
-
64
- ---
65
-
66
- ## 4. Counter map (exact)
67
-
68
- `src/countex/maps/productActivityMap.ts`:
69
-
70
- ```ts
71
- import type { CounterMap } from "@x12i/countex";
72
-
73
- export const productActivityMap: CounterMap = {
74
- counterId: "product_activity",
75
-
76
- scope: { path: "$.account.id" },
77
- eventTime: { path: "$.event.createdAt" },
78
-
79
- subjects: [{ type: "user", path: "$.user.id", required: true }],
80
-
81
- metrics: [{ path: "$.event.name", required: true }],
82
-
83
- dimensions: [
84
- { name: "feature", path: "$.product.feature" },
85
- { name: "plan", path: "$.product.plan" },
86
- ],
87
-
88
- delta: { default: 1 },
89
- timeframes: ["hour", "day", "month"],
90
- };
91
- ```
92
-
93
- ### 4.1 Optional: weighted events
94
-
95
- If product sends an integer weight:
96
-
97
- ```ts
98
- delta: { path: "$.event.weight", default: 1 },
99
- ```
100
-
101
- **Requirement:** path must resolve to a finite number Countex can truncate to integer per extraction logic.
102
-
103
- ---
104
-
105
- ## 5. Fan-out
106
-
107
- Single subject × single metric × **T** timeframes:
108
-
109
- ```
110
- R = 1 × 1 × T
111
- ```
112
-
113
- For `["hour","day","month"]`, **R = 3** increments per accepted event.
114
-
115
- ---
116
-
117
- ## 6. Implementation steps
118
-
119
- ### Step 6.1 — Local verification (memory adapters)
120
-
121
- Use this **exact** program first — no Mongo/Redis:
122
-
123
- ```ts
124
- import { createCountex } from "@x12i/countex";
125
- import { MemoryDurable, MemoryCache } from "@x12i/countex/adapters/memory";
126
- import { productActivityMap } from "./maps/productActivityMap.js";
127
-
128
- const countex = createCountex({
129
- maps: [productActivityMap],
130
- durable: new MemoryDurable(),
131
- cache: new MemoryCache(),
132
- });
133
-
134
- const event = {
135
- account: { id: "acct_123" },
136
- user: { id: "user_42" },
137
- event: { name: "risk_dashboard_viewed", createdAt: "2026-05-08T16:00:00.000Z" },
138
- product: { feature: "risk_dashboard", plan: "enterprise" },
139
- };
140
-
141
- await countex.ingest(event);
142
- await countex.flush();
143
-
144
- const q = await countex.query({
145
- scope: "acct_123",
146
- counterId: "product_activity",
147
- timeframe: { type: "day", from: "2026-05-08", to: "2026-05-08" },
148
- groupBy: ["subjectType", "subject", "metric", "dim:feature"],
149
- format: "table",
150
- });
151
-
152
- console.log(q.rows);
153
- await countex.shutdown();
154
- ```
155
-
156
- **Expected:** `rows` non-empty; `keys.subjectType === "user"`; `keys.metric` is numeric; `dim:feature` readable string.
157
-
158
- ### Step 6.2 — Production wiring
159
-
160
- Follow the same structural pattern as the audit playbook:
161
-
162
- 1. `MongoDurable` + optional `RedisCache`.
163
- 2. `await countex.init()` at boot.
164
- 3. HTTP/batch worker calls `ingest` / `ingestBatch`.
165
- 4. `flush.everyMs` tuned to interaction with your SLA (see README buffering section).
166
-
167
- ### Step 6.3 — Instrumentation (exact events)
168
-
169
- At minimum:
170
-
171
- ```ts
172
- countex.on("flush", ({ cells, durationMs }) => { /* metrics histogram */ });
173
- countex.on("flush_error", ({ error }) => { /* page */ });
174
- ```
175
-
176
- ---
177
-
178
- ## 7. Queries
179
-
180
- ### 7.1 Feature adoption for one account (day)
181
-
182
- ```ts
183
- const adoption = await countex.query({
184
- scope: "acct_123",
185
- counterId: "product_activity",
186
- timeframe: { type: "day", from: "2026-05-08", to: "2026-05-08" },
187
- groupBy: ["dim:feature", "metric"],
188
- format: "table",
189
- orderBy: "-count",
190
- });
191
- ```
192
-
193
- ### 7.2 Filter to one event name
194
-
195
- ```ts
196
- metrics: ["risk_dashboard_viewed"],
197
- ```
198
-
199
- Strings match **dictionary source keys** before resolution.
200
-
201
- ### 7.3 Hourly timeseries (sparkline)
202
-
203
- ```ts
204
- const ts = await countex.query({
205
- scope: "acct_123",
206
- counterId: "product_activity",
207
- timeframe: { type: "hour", from: "2026-05-08T00", to: "2026-05-08T23" },
208
- groupBy: [],
209
- format: "timeseries",
210
- });
211
- ```
212
-
213
- **Bucket strings** for hour keys follow `YYYY-MM-DDTHH` UTC (see `formatHour` in `src/core/timeframe.ts`).
214
-
215
- ---
216
-
217
- ## 8. Verification
218
-
219
- | Check | Procedure |
220
- |-------|-----------|
221
- | Enum discipline | Count distinct metric source keys in staging → must stay within product-defined catalog. |
222
- | Scope isolation | Query with wrong `scope` → empty rows, not neighbor data. |
223
- | UTC correctness | Compare bucket keys to clickhouse/raw logs for same event timestamps. |
224
-
225
- ---
226
-
227
- ## 9. Failure modes
228
-
229
- | Symptom | Cause | Fix |
230
- |---------|-------|-----|
231
- | Metric explosion | Typos in `event.name` | Validate against catalog pre-ingest; reject unknown names. |
232
- | Missing optional dims | Expected | Use `dimensions: { feature: ["risk_dashboard"] }` filter only when dim guaranteed present. |
233
- | High write amplification | Too many timeframes | Drop `hour` in low-value environments. |
234
-
235
- ---
236
-
237
- ## 10. References
238
-
239
- - [`docs/guides/timeframes-and-bucketing.md`](../guides/timeframes-and-bucketing.md)
240
- - [`docs/guides/querying.md`](../guides/querying.md)
1
+ # Technical playbook: Product and feature activity
2
+
3
+ You run a **product analytics** pipeline: in-app events describe **what happened**, **who** did it, under **which account**, with **feature/plan** context. You want rollups suitable for MAU-style counts, feature adoption, and breakdowns — stored in **your** infra via Countex.
4
+
5
+ ---
6
+
7
+ ## 1. Problem statement
8
+
9
+ ### 1.1 Outcomes
10
+
11
+ - Per **account** (`scope`), count activity by **user** (`subject`) and **named event** (`metric`).
12
+ - Slice by **`feature`** and **`plan`** dimensions.
13
+ - Buckets: **`hour`** for near-real-time, **`day`** / **`month`** for exec summaries.
14
+
15
+ ### 1.2 Constraints
16
+
17
+ - Event names (`metrics`) must remain **bounded cardinality** — treat them like an enum registry.
18
+ - User ids may be large but stable; dictionary handles string→int per scope.
19
+
20
+ ---
21
+
22
+ ## 2. Primitives mapping
23
+
24
+ | Primitive | JSON path | Notes |
25
+ |-----------|-----------|--------|
26
+ | scope | `$.account.id` | Tenant = billing account. |
27
+ | eventTime | `$.event.createdAt` | UTC bucketing. |
28
+ | subject `user` | `$.user.id` | Required — anonymous traffic must be filtered **before** ingest or mapped to a stable anon key you accept. |
29
+ | metric | `$.event.name` | Single-valued path (no wildcard) → one metric per event unless you add fan-out. |
30
+ | dim `feature` | `$.product.feature` | Literal dimension. |
31
+ | dim `plan` | `$.product.plan` | Literal dimension. |
32
+ | delta | `1` | One occurrence per event. |
33
+
34
+ ---
35
+
36
+ ## 3. Canonical payload
37
+
38
+ ```json
39
+ {
40
+ "account": { "id": "acct_123" },
41
+ "user": { "id": "user_42" },
42
+ "event": {
43
+ "name": "risk_dashboard_viewed",
44
+ "createdAt": "2026-05-08T16:00:00.000Z"
45
+ },
46
+ "product": {
47
+ "feature": "risk_dashboard",
48
+ "plan": "enterprise"
49
+ }
50
+ }
51
+ ```
52
+
53
+ ### 3.1 Required-field matrix
54
+
55
+ | Path | Required | Effect if absent |
56
+ |------|----------|------------------|
57
+ | `$.account.id` | Yes | Skip event for map. |
58
+ | `$.event.createdAt` | Yes | Skip. |
59
+ | `$.user.id` | Yes | Skip (`required: true` subject). |
60
+ | `$.event.name` | Yes | Skip (`metrics.required: true`). |
61
+ | `$.product.feature` | No | Optional dimension omitted by default (`skipMissingOptionalDimensions: true`). |
62
+ | `$.product.plan` | No | Same. |
63
+
64
+ ---
65
+
66
+ ## 4. Counter map (exact)
67
+
68
+ `src/countex/maps/productActivityMap.ts`:
69
+
70
+ ```ts
71
+ import type { CounterMap } from "@x12i/countex";
72
+
73
+ export const productActivityMap: CounterMap = {
74
+ counterId: "product_activity",
75
+
76
+ scope: { path: "$.account.id" },
77
+ eventTime: { path: "$.event.createdAt" },
78
+
79
+ subjects: [{ type: "user", path: "$.user.id", required: true }],
80
+
81
+ metrics: [{ path: "$.event.name", required: true }],
82
+
83
+ dimensions: [
84
+ { name: "feature", path: "$.product.feature" },
85
+ { name: "plan", path: "$.product.plan" },
86
+ ],
87
+
88
+ delta: { default: 1 },
89
+ timeframes: ["hour", "day", "month"],
90
+ };
91
+ ```
92
+
93
+ ### 4.1 Optional: weighted events
94
+
95
+ If product sends an integer weight:
96
+
97
+ ```ts
98
+ delta: { path: "$.event.weight", default: 1 },
99
+ ```
100
+
101
+ **Requirement:** path must resolve to a finite number Countex can truncate to integer per extraction logic.
102
+
103
+ ---
104
+
105
+ ## 5. Fan-out
106
+
107
+ Single subject × single metric × **T** timeframes:
108
+
109
+ ```
110
+ R = 1 × 1 × T
111
+ ```
112
+
113
+ For `["hour","day","month"]`, **R = 3** increments per accepted event.
114
+
115
+ ---
116
+
117
+ ## 6. Implementation steps
118
+
119
+ ### Step 6.1 — Local verification (memory adapters)
120
+
121
+ Use this **exact** program first — no Mongo/Redis:
122
+
123
+ ```ts
124
+ import { createCountex } from "@x12i/countex";
125
+ import { MemoryDurable, MemoryCache } from "@x12i/countex/adapters/memory";
126
+ import { productActivityMap } from "./maps/productActivityMap.js";
127
+
128
+ const countex = createCountex({
129
+ maps: [productActivityMap],
130
+ durable: new MemoryDurable(),
131
+ cache: new MemoryCache(),
132
+ });
133
+
134
+ const event = {
135
+ account: { id: "acct_123" },
136
+ user: { id: "user_42" },
137
+ event: { name: "risk_dashboard_viewed", createdAt: "2026-05-08T16:00:00.000Z" },
138
+ product: { feature: "risk_dashboard", plan: "enterprise" },
139
+ };
140
+
141
+ await countex.ingest(event);
142
+ await countex.flush();
143
+
144
+ const q = await countex.query({
145
+ scope: "acct_123",
146
+ counterId: "product_activity",
147
+ timeframe: { type: "day", from: "2026-05-08", to: "2026-05-08" },
148
+ groupBy: ["subjectType", "subject", "metric", "dim:feature"],
149
+ format: "table",
150
+ });
151
+
152
+ console.log(q.rows);
153
+ await countex.shutdown();
154
+ ```
155
+
156
+ **Expected:** `rows` non-empty; `keys.subjectType === "user"`; `keys.metric` is numeric; `dim:feature` readable string.
157
+
158
+ ### Step 6.2 — Production wiring
159
+
160
+ Follow the same structural pattern as the audit playbook:
161
+
162
+ 1. `MongoDurable` + optional `RedisCache`.
163
+ 2. `await countex.init()` at boot.
164
+ 3. HTTP/batch worker calls `ingest` / `ingestBatch`.
165
+ 4. `flush.everyMs` tuned to interaction with your SLA (see README buffering section).
166
+
167
+ ### Step 6.3 — Instrumentation (exact events)
168
+
169
+ At minimum:
170
+
171
+ ```ts
172
+ countex.on("flush", ({ cells, durationMs }) => { /* metrics histogram */ });
173
+ countex.on("flush_error", ({ error }) => { /* page */ });
174
+ ```
175
+
176
+ ---
177
+
178
+ ## 7. Queries
179
+
180
+ ### 7.1 Feature adoption for one account (day)
181
+
182
+ ```ts
183
+ const adoption = await countex.query({
184
+ scope: "acct_123",
185
+ counterId: "product_activity",
186
+ timeframe: { type: "day", from: "2026-05-08", to: "2026-05-08" },
187
+ groupBy: ["dim:feature", "metric"],
188
+ format: "table",
189
+ orderBy: "-count",
190
+ });
191
+ ```
192
+
193
+ ### 7.2 Filter to one event name
194
+
195
+ ```ts
196
+ metrics: ["risk_dashboard_viewed"],
197
+ ```
198
+
199
+ Strings match **dictionary source keys** before resolution.
200
+
201
+ ### 7.3 Hourly timeseries (sparkline)
202
+
203
+ ```ts
204
+ const ts = await countex.query({
205
+ scope: "acct_123",
206
+ counterId: "product_activity",
207
+ timeframe: { type: "hour", from: "2026-05-08T00", to: "2026-05-08T23" },
208
+ groupBy: [],
209
+ format: "timeseries",
210
+ });
211
+ ```
212
+
213
+ **Bucket strings** for hour keys follow `YYYY-MM-DDTHH` UTC (see `formatHour` in `src/core/timeframe.ts`).
214
+
215
+ ---
216
+
217
+ ## 8. Verification
218
+
219
+ | Check | Procedure |
220
+ |-------|-----------|
221
+ | Enum discipline | Count distinct metric source keys in staging → must stay within product-defined catalog. |
222
+ | Scope isolation | Query with wrong `scope` → empty rows, not neighbor data. |
223
+ | UTC correctness | Compare bucket keys to clickhouse/raw logs for same event timestamps. |
224
+
225
+ ---
226
+
227
+ ## 9. Failure modes
228
+
229
+ | Symptom | Cause | Fix |
230
+ |---------|-------|-----|
231
+ | Metric explosion | Typos in `event.name` | Validate against catalog pre-ingest; reject unknown names. |
232
+ | Missing optional dims | Expected | Use `dimensions: { feature: ["risk_dashboard"] }` filter only when dim guaranteed present. |
233
+ | High write amplification | Too many timeframes | Drop `hour` in low-value environments. |
234
+
235
+ ---
236
+
237
+ ## 10. References
238
+
239
+ - [`docs/guides/timeframes-and-bucketing.md`](../guides/timeframes-and-bucketing.md)
240
+ - [`docs/guides/querying.md`](../guides/querying.md)