@x12i/countex 1.2.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -1
- package/dist/backup/manifest.d.ts +49 -0
- package/dist/backup/manifest.d.ts.map +1 -0
- package/dist/backup/manifest.js +103 -0
- package/dist/backup/manifest.js.map +1 -0
- package/dist/backup/snapshot-encoding.d.ts +8 -0
- package/dist/backup/snapshot-encoding.d.ts.map +1 -0
- package/dist/backup/snapshot-encoding.js +27 -0
- package/dist/backup/snapshot-encoding.js.map +1 -0
- package/dist/cli.js +16 -0
- package/dist/cli.js.map +1 -1
- package/dist/core/countex.d.ts +14 -1
- package/dist/core/countex.d.ts.map +1 -1
- package/dist/core/countex.js +141 -45
- package/dist/core/countex.js.map +1 -1
- package/dist/core/extraction.d.ts +1 -5
- package/dist/core/extraction.d.ts.map +1 -1
- package/dist/core/extraction.js.map +1 -1
- package/dist/core/mapping.d.ts +5 -0
- package/dist/core/mapping.d.ts.map +1 -1
- package/dist/core/mapping.js +27 -0
- package/dist/core/mapping.js.map +1 -1
- package/dist/index.d.ts +4 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +13 -1
- package/dist/index.js.map +1 -1
- package/dist/query/formatter.d.ts +2 -2
- package/dist/query/formatter.d.ts.map +1 -1
- package/dist/query/formatter.js +9 -6
- package/dist/query/formatter.js.map +1 -1
- package/dist/query/planner.d.ts.map +1 -1
- package/dist/query/planner.js +16 -2
- package/dist/query/planner.js.map +1 -1
- package/dist/simulation/context.d.ts +5 -0
- package/dist/simulation/context.d.ts.map +1 -0
- package/dist/simulation/context.js +56 -0
- package/dist/simulation/context.js.map +1 -0
- package/dist/simulation/events.d.ts +3 -0
- package/dist/simulation/events.d.ts.map +1 -0
- package/dist/simulation/events.js +67 -0
- package/dist/simulation/events.js.map +1 -0
- package/dist/simulation/index.d.ts +7 -0
- package/dist/simulation/index.d.ts.map +1 -0
- package/dist/simulation/index.js +16 -0
- package/dist/simulation/index.js.map +1 -0
- package/dist/simulation/prng.d.ts +4 -0
- package/dist/simulation/prng.d.ts.map +1 -0
- package/dist/simulation/prng.js +18 -0
- package/dist/simulation/prng.js.map +1 -0
- package/dist/simulation/resource-estimate.d.ts +6 -0
- package/dist/simulation/resource-estimate.d.ts.map +1 -0
- package/dist/simulation/resource-estimate.js +11 -0
- package/dist/simulation/resource-estimate.js.map +1 -0
- package/dist/simulation/simulator.d.ts +15 -0
- package/dist/simulation/simulator.d.ts.map +1 -0
- package/dist/simulation/simulator.js +67 -0
- package/dist/simulation/simulator.js.map +1 -0
- package/dist/simulation/tracker.d.ts +8 -0
- package/dist/simulation/tracker.d.ts.map +1 -0
- package/dist/simulation/tracker.js +18 -0
- package/dist/simulation/tracker.js.map +1 -0
- package/dist/simulation/types.d.ts +49 -0
- package/dist/simulation/types.d.ts.map +1 -0
- package/dist/simulation/types.js +21 -0
- package/dist/simulation/types.js.map +1 -0
- package/dist/storage/memory.d.ts +3 -0
- package/dist/storage/memory.d.ts.map +1 -1
- package/dist/storage/memory.js +35 -0
- package/dist/storage/memory.js.map +1 -1
- package/dist/storage/mongo.d.ts +5 -0
- package/dist/storage/mongo.d.ts.map +1 -1
- package/dist/storage/mongo.js +68 -1
- package/dist/storage/mongo.js.map +1 -1
- package/dist/storage/redis.d.ts +6 -2
- package/dist/storage/redis.d.ts.map +1 -1
- package/dist/storage/redis.js +18 -8
- package/dist/storage/redis.js.map +1 -1
- package/dist/types/index.d.ts +51 -0
- package/dist/types/index.d.ts.map +1 -1
- package/docs/DEPLOYMENT.md +40 -2
- package/docs/countex_gap_analysis_complete_checklist.md +1340 -0
- package/docs/guides/README.md +5 -0
- package/docs/guides/adapter-mongo.md +12 -0
- package/docs/guides/adapter-redis.md +6 -0
- package/docs/guides/backup-export.md +46 -2
- package/docs/guides/cli.md +8 -0
- package/docs/guides/counter-maps.md +6 -0
- package/docs/guides/insights-overview.md +6 -0
- package/docs/guides/measures.md +4 -0
- package/docs/guides/querying.md +40 -0
- package/docs/guides/simulation-profiles.md +33 -0
- package/docs/guides/simulation-resource-estimation.md +30 -0
- package/docs/guides/simulation.md +36 -0
- package/docs/internals/README.md +1 -0
- package/docs/internals/internal-dimension-hash.md +34 -0
- package/docs/use-cases/README.md +56 -0
- package/docs/use-cases/audit-security-log-counts.md +550 -0
- package/docs/use-cases/llm-cost-and-token-metering.md +255 -0
- package/docs/use-cases/multi-tenant-isolation.md +203 -0
- package/docs/use-cases/ops-dashboard-daily-rollups.md +255 -0
- package/docs/use-cases/processing-workflow-activity.md +165 -0
- package/docs/use-cases/product-feature-usage.md +240 -0
- package/docs/use-cases/webhook-idempotent-ingestion.md +188 -0
- package/package.json +18 -2
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
# Technical playbook: Processing and workflow activity
|
|
2
|
+
|
|
3
|
+
You operate **pipelines**, **queues**, **agents**, or **batch jobs**. Events describe steps, statuses, errors, and durations. You want rollups by job/workflow, task type, outcome, and timeframe — without Countex knowing your orchestrator brand.
|
|
4
|
+
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
## 1. Problem statement
|
|
8
|
+
|
|
9
|
+
### 1.1 Outcomes
|
|
10
|
+
|
|
11
|
+
- Per **tenant** (`scope`), count activity by **job or run** (`subject`) and **step/task kind** (`metric`).
|
|
12
|
+
- Slice by **status**, **worker pool**, or **region** (`dimensions`).
|
|
13
|
+
- Track **duration** and **retry counts** as **measures** where useful.
|
|
14
|
+
|
|
15
|
+
### 1.2 Constraints
|
|
16
|
+
|
|
17
|
+
- Step names (`metrics`) should stay **bounded** — register allowed task types like an enum.
|
|
18
|
+
- Job ids may be high cardinality; dictionary maps string job ids per scope.
|
|
19
|
+
|
|
20
|
+
---
|
|
21
|
+
|
|
22
|
+
## 2. Primitives mapping
|
|
23
|
+
|
|
24
|
+
| Primitive | JSON path | Notes |
|
|
25
|
+
|-----------|-----------|--------|
|
|
26
|
+
| scope | `$.tenant.id` | Isolation boundary. |
|
|
27
|
+
| eventTime | `$.run.recordedAt` | UTC bucketing. |
|
|
28
|
+
| subject `job` | `$.run.jobId` | Required — one counter row per job run key. |
|
|
29
|
+
| metric | `$.task.name` | Task/step identifier (single path or controlled fan-out). |
|
|
30
|
+
| dim `status` | `$.task.status` | Literal dimension (`success`, `failed`, …). |
|
|
31
|
+
| dim `pool` | `$.worker.pool` | Optional. |
|
|
32
|
+
| delta | `1` | One logical step completion per event (adjust if one event batches many steps). |
|
|
33
|
+
| measure `durationMs` | `$.task.durationMs` | Optional `sum` / `count` / `min` / `max`. |
|
|
34
|
+
|
|
35
|
+
---
|
|
36
|
+
|
|
37
|
+
## 3. Canonical payload
|
|
38
|
+
|
|
39
|
+
```json
|
|
40
|
+
{
|
|
41
|
+
"tenant": { "id": "org_42" },
|
|
42
|
+
"run": {
|
|
43
|
+
"jobId": "job_9f3a",
|
|
44
|
+
"recordedAt": "2026-05-08T18:22:01.000Z"
|
|
45
|
+
},
|
|
46
|
+
"task": {
|
|
47
|
+
"name": "extract_documents",
|
|
48
|
+
"status": "success",
|
|
49
|
+
"durationMs": 842
|
|
50
|
+
},
|
|
51
|
+
"worker": { "pool": "default" }
|
|
52
|
+
}
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
### 3.1 Required-field matrix
|
|
56
|
+
|
|
57
|
+
| Path | Required | Effect if absent |
|
|
58
|
+
|------|----------|------------------|
|
|
59
|
+
| `$.tenant.id` | Yes | Skip for map. |
|
|
60
|
+
| `$.run.recordedAt` | Yes | Skip. |
|
|
61
|
+
| `$.run.jobId` | Yes | Skip (`required: true` subject). |
|
|
62
|
+
| `$.task.name` | Yes | Skip (`metrics.required: true`). |
|
|
63
|
+
| `$.task.status` | No | Optional dimension omitted if missing. |
|
|
64
|
+
| `$.worker.pool` | No | Same. |
|
|
65
|
+
| `$.task.durationMs` | No | Measure skipped per `onMissing`. |
|
|
66
|
+
|
|
67
|
+
---
|
|
68
|
+
|
|
69
|
+
## 4. Counter map (exact)
|
|
70
|
+
|
|
71
|
+
```ts
|
|
72
|
+
import type { CounterMap } from "@x12i/countex";
|
|
73
|
+
|
|
74
|
+
export const processingWorkflowMap: CounterMap = {
|
|
75
|
+
counterId: "processing_workflow",
|
|
76
|
+
|
|
77
|
+
scope: { path: "$.tenant.id" },
|
|
78
|
+
eventTime: { path: "$.run.recordedAt" },
|
|
79
|
+
|
|
80
|
+
subjects: [{ type: "job", path: "$.run.jobId", required: true }],
|
|
81
|
+
|
|
82
|
+
metrics: [{ path: "$.task.name", required: true }],
|
|
83
|
+
|
|
84
|
+
dimensions: [
|
|
85
|
+
{ name: "status", path: "$.task.status", required: false },
|
|
86
|
+
{ name: "pool", path: "$.worker.pool", required: false },
|
|
87
|
+
],
|
|
88
|
+
|
|
89
|
+
delta: { default: 1 },
|
|
90
|
+
timeframes: ["hour", "day", "month"],
|
|
91
|
+
|
|
92
|
+
measures: [
|
|
93
|
+
{
|
|
94
|
+
name: "durationMs",
|
|
95
|
+
path: "$.task.durationMs",
|
|
96
|
+
aggregations: ["sum", "count", "min", "max"],
|
|
97
|
+
onMissing: "skip",
|
|
98
|
+
},
|
|
99
|
+
],
|
|
100
|
+
};
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
---
|
|
104
|
+
|
|
105
|
+
## 5. Fan-out arithmetic
|
|
106
|
+
|
|
107
|
+
One subject × one metric × `timeframes.length` increments per event unless paths fan out.
|
|
108
|
+
|
|
109
|
+
Example: **1 × 1 × 3 = 3** cells updated per event for `hour`, `day`, `month`.
|
|
110
|
+
|
|
111
|
+
---
|
|
112
|
+
|
|
113
|
+
## 6. Implementation steps
|
|
114
|
+
|
|
115
|
+
1. Validate payloads at your ingress (schema or optional `validateMap(map, samples)` with fixtures).
|
|
116
|
+
2. `createCountex({ maps: [processingWorkflowMap], durable, cache })`.
|
|
117
|
+
3. `ingest(event)` on each completed step or aggregated batch boundary.
|
|
118
|
+
4. `flush()` on interval or shutdown.
|
|
119
|
+
|
|
120
|
+
---
|
|
121
|
+
|
|
122
|
+
## 7. Query specifications
|
|
123
|
+
|
|
124
|
+
```ts
|
|
125
|
+
await countex.query({
|
|
126
|
+
scope: "org_42",
|
|
127
|
+
counterId: "processing_workflow",
|
|
128
|
+
timeframe: { type: "day", from: "2026-05-08", to: "2026-05-08" },
|
|
129
|
+
groupBy: ["metric"],
|
|
130
|
+
measures: ["durationMs"],
|
|
131
|
+
format: "table",
|
|
132
|
+
});
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
---
|
|
136
|
+
|
|
137
|
+
## 8. Production hardening
|
|
138
|
+
|
|
139
|
+
- **Idempotency** if the same step may be reported twice (dedupe on `event.id`).
|
|
140
|
+
- **Cardinality** on `task.name` — alert when unknown names appear.
|
|
141
|
+
- **Minute/hour** buckets only if you need SLA dashboards; they multiply storage.
|
|
142
|
+
|
|
143
|
+
---
|
|
144
|
+
|
|
145
|
+
## 9. Verification
|
|
146
|
+
|
|
147
|
+
- Dry-run mapping with **`preview`** on sample payloads (see package release notes / API).
|
|
148
|
+
- Compare simulator **`audit-log`** / **`processing-activity`** profiles against staging counters.
|
|
149
|
+
|
|
150
|
+
---
|
|
151
|
+
|
|
152
|
+
## 10. Failure modes
|
|
153
|
+
|
|
154
|
+
| Symptom | Cause | Mitigation |
|
|
155
|
+
|---------|-------|------------|
|
|
156
|
+
| Missing rollups | Required paths absent | Fix producer or relax `required`. |
|
|
157
|
+
| Cardinality blow-up | Unbounded `task.name` | Registry + overflow dimension policy. |
|
|
158
|
+
| Slow queries | Too-wide timeframe + huge groups | Narrow `bucketRange`, pre-aggregate in map if needed. |
|
|
159
|
+
|
|
160
|
+
---
|
|
161
|
+
|
|
162
|
+
## Related
|
|
163
|
+
|
|
164
|
+
- [Product & feature activity](./product-feature-usage.md) — analogous pattern for user-facing events.
|
|
165
|
+
- [Simulation profiles](../guides/simulation-profiles.md) — `processing-activity` profile.
|
|
@@ -0,0 +1,240 @@
|
|
|
1
|
+
# Technical playbook: Product and feature activity
|
|
2
|
+
|
|
3
|
+
You run a **product analytics** pipeline: in-app events describe **what happened**, **who** did it, under **which account**, with **feature/plan** context. You want rollups suitable for MAU-style counts, feature adoption, and breakdowns — stored in **your** infra via Countex.
|
|
4
|
+
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
## 1. Problem statement
|
|
8
|
+
|
|
9
|
+
### 1.1 Outcomes
|
|
10
|
+
|
|
11
|
+
- Per **account** (`scope`), count activity by **user** (`subject`) and **named event** (`metric`).
|
|
12
|
+
- Slice by **`feature`** and **`plan`** dimensions.
|
|
13
|
+
- Buckets: **`hour`** for near-real-time, **`day`** / **`month`** for exec summaries.
|
|
14
|
+
|
|
15
|
+
### 1.2 Constraints
|
|
16
|
+
|
|
17
|
+
- Event names (`metrics`) must remain **bounded cardinality** — treat them like an enum registry.
|
|
18
|
+
- User ids may be large but stable; dictionary handles string→int per scope.
|
|
19
|
+
|
|
20
|
+
---
|
|
21
|
+
|
|
22
|
+
## 2. Primitives mapping
|
|
23
|
+
|
|
24
|
+
| Primitive | JSON path | Notes |
|
|
25
|
+
|-----------|-----------|--------|
|
|
26
|
+
| scope | `$.account.id` | Tenant = billing account. |
|
|
27
|
+
| eventTime | `$.event.createdAt` | UTC bucketing. |
|
|
28
|
+
| subject `user` | `$.user.id` | Required — anonymous traffic must be filtered **before** ingest or mapped to a stable anon key you accept. |
|
|
29
|
+
| metric | `$.event.name` | Single-valued path (no wildcard) → one metric per event unless you add fan-out. |
|
|
30
|
+
| dim `feature` | `$.product.feature` | Literal dimension. |
|
|
31
|
+
| dim `plan` | `$.product.plan` | Literal dimension. |
|
|
32
|
+
| delta | `1` | One occurrence per event. |
|
|
33
|
+
|
|
34
|
+
---
|
|
35
|
+
|
|
36
|
+
## 3. Canonical payload
|
|
37
|
+
|
|
38
|
+
```json
|
|
39
|
+
{
|
|
40
|
+
"account": { "id": "acct_123" },
|
|
41
|
+
"user": { "id": "user_42" },
|
|
42
|
+
"event": {
|
|
43
|
+
"name": "risk_dashboard_viewed",
|
|
44
|
+
"createdAt": "2026-05-08T16:00:00.000Z"
|
|
45
|
+
},
|
|
46
|
+
"product": {
|
|
47
|
+
"feature": "risk_dashboard",
|
|
48
|
+
"plan": "enterprise"
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
### 3.1 Required-field matrix
|
|
54
|
+
|
|
55
|
+
| Path | Required | Effect if absent |
|
|
56
|
+
|------|----------|------------------|
|
|
57
|
+
| `$.account.id` | Yes | Skip event for map. |
|
|
58
|
+
| `$.event.createdAt` | Yes | Skip. |
|
|
59
|
+
| `$.user.id` | Yes | Skip (`required: true` subject). |
|
|
60
|
+
| `$.event.name` | Yes | Skip (`metrics.required: true`). |
|
|
61
|
+
| `$.product.feature` | No | Optional dimension omitted by default (`skipMissingOptionalDimensions: true`). |
|
|
62
|
+
| `$.product.plan` | No | Same. |
|
|
63
|
+
|
|
64
|
+
---
|
|
65
|
+
|
|
66
|
+
## 4. Counter map (exact)
|
|
67
|
+
|
|
68
|
+
`src/countex/maps/productActivityMap.ts`:
|
|
69
|
+
|
|
70
|
+
```ts
|
|
71
|
+
import type { CounterMap } from "@x12i/countex";
|
|
72
|
+
|
|
73
|
+
export const productActivityMap: CounterMap = {
|
|
74
|
+
counterId: "product_activity",
|
|
75
|
+
|
|
76
|
+
scope: { path: "$.account.id" },
|
|
77
|
+
eventTime: { path: "$.event.createdAt" },
|
|
78
|
+
|
|
79
|
+
subjects: [{ type: "user", path: "$.user.id", required: true }],
|
|
80
|
+
|
|
81
|
+
metrics: [{ path: "$.event.name", required: true }],
|
|
82
|
+
|
|
83
|
+
dimensions: [
|
|
84
|
+
{ name: "feature", path: "$.product.feature" },
|
|
85
|
+
{ name: "plan", path: "$.product.plan" },
|
|
86
|
+
],
|
|
87
|
+
|
|
88
|
+
delta: { default: 1 },
|
|
89
|
+
timeframes: ["hour", "day", "month"],
|
|
90
|
+
};
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
### 4.1 Optional: weighted events
|
|
94
|
+
|
|
95
|
+
If product sends an integer weight:
|
|
96
|
+
|
|
97
|
+
```ts
|
|
98
|
+
delta: { path: "$.event.weight", default: 1 },
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
**Requirement:** path must resolve to a finite number Countex can truncate to integer per extraction logic.
|
|
102
|
+
|
|
103
|
+
---
|
|
104
|
+
|
|
105
|
+
## 5. Fan-out
|
|
106
|
+
|
|
107
|
+
Single subject × single metric × **T** timeframes:
|
|
108
|
+
|
|
109
|
+
```
|
|
110
|
+
R = 1 × 1 × T
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
For `["hour","day","month"]`, **R = 3** increments per accepted event.
|
|
114
|
+
|
|
115
|
+
---
|
|
116
|
+
|
|
117
|
+
## 6. Implementation steps
|
|
118
|
+
|
|
119
|
+
### Step 6.1 — Local verification (memory adapters)
|
|
120
|
+
|
|
121
|
+
Use this **exact** program first — no Mongo/Redis:
|
|
122
|
+
|
|
123
|
+
```ts
|
|
124
|
+
import { createCountex } from "@x12i/countex";
|
|
125
|
+
import { MemoryDurable, MemoryCache } from "@x12i/countex/adapters/memory";
|
|
126
|
+
import { productActivityMap } from "./maps/productActivityMap.js";
|
|
127
|
+
|
|
128
|
+
const countex = createCountex({
|
|
129
|
+
maps: [productActivityMap],
|
|
130
|
+
durable: new MemoryDurable(),
|
|
131
|
+
cache: new MemoryCache(),
|
|
132
|
+
});
|
|
133
|
+
|
|
134
|
+
const event = {
|
|
135
|
+
account: { id: "acct_123" },
|
|
136
|
+
user: { id: "user_42" },
|
|
137
|
+
event: { name: "risk_dashboard_viewed", createdAt: "2026-05-08T16:00:00.000Z" },
|
|
138
|
+
product: { feature: "risk_dashboard", plan: "enterprise" },
|
|
139
|
+
};
|
|
140
|
+
|
|
141
|
+
await countex.ingest(event);
|
|
142
|
+
await countex.flush();
|
|
143
|
+
|
|
144
|
+
const q = await countex.query({
|
|
145
|
+
scope: "acct_123",
|
|
146
|
+
counterId: "product_activity",
|
|
147
|
+
timeframe: { type: "day", from: "2026-05-08", to: "2026-05-08" },
|
|
148
|
+
groupBy: ["subjectType", "subject", "metric", "dim:feature"],
|
|
149
|
+
format: "table",
|
|
150
|
+
});
|
|
151
|
+
|
|
152
|
+
console.log(q.rows);
|
|
153
|
+
await countex.shutdown();
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
**Expected:** `rows` non-empty; `keys.subjectType === "user"`; `keys.metric` is numeric; `dim:feature` readable string.
|
|
157
|
+
|
|
158
|
+
### Step 6.2 — Production wiring
|
|
159
|
+
|
|
160
|
+
Follow the same structural pattern as the audit playbook:
|
|
161
|
+
|
|
162
|
+
1. `MongoDurable` + optional `RedisCache`.
|
|
163
|
+
2. `await countex.init()` at boot.
|
|
164
|
+
3. HTTP/batch worker calls `ingest` / `ingestBatch`.
|
|
165
|
+
4. `flush.everyMs` tuned to interaction with your SLA (see README buffering section).
|
|
166
|
+
|
|
167
|
+
### Step 6.3 — Instrumentation (exact events)
|
|
168
|
+
|
|
169
|
+
At minimum:
|
|
170
|
+
|
|
171
|
+
```ts
|
|
172
|
+
countex.on("flush", ({ cells, durationMs }) => { /* metrics histogram */ });
|
|
173
|
+
countex.on("flush_error", ({ error }) => { /* page */ });
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
---
|
|
177
|
+
|
|
178
|
+
## 7. Queries
|
|
179
|
+
|
|
180
|
+
### 7.1 Feature adoption for one account (day)
|
|
181
|
+
|
|
182
|
+
```ts
|
|
183
|
+
const adoption = await countex.query({
|
|
184
|
+
scope: "acct_123",
|
|
185
|
+
counterId: "product_activity",
|
|
186
|
+
timeframe: { type: "day", from: "2026-05-08", to: "2026-05-08" },
|
|
187
|
+
groupBy: ["dim:feature", "metric"],
|
|
188
|
+
format: "table",
|
|
189
|
+
orderBy: "-count",
|
|
190
|
+
});
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
### 7.2 Filter to one event name
|
|
194
|
+
|
|
195
|
+
```ts
|
|
196
|
+
metrics: ["risk_dashboard_viewed"],
|
|
197
|
+
```
|
|
198
|
+
|
|
199
|
+
Strings match **dictionary source keys** before resolution.
|
|
200
|
+
|
|
201
|
+
### 7.3 Hourly timeseries (sparkline)
|
|
202
|
+
|
|
203
|
+
```ts
|
|
204
|
+
const ts = await countex.query({
|
|
205
|
+
scope: "acct_123",
|
|
206
|
+
counterId: "product_activity",
|
|
207
|
+
timeframe: { type: "hour", from: "2026-05-08T00", to: "2026-05-08T23" },
|
|
208
|
+
groupBy: [],
|
|
209
|
+
format: "timeseries",
|
|
210
|
+
});
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
**Bucket strings** for hour keys follow `YYYY-MM-DDTHH` UTC (see `formatHour` in `src/core/timeframe.ts`).
|
|
214
|
+
|
|
215
|
+
---
|
|
216
|
+
|
|
217
|
+
## 8. Verification
|
|
218
|
+
|
|
219
|
+
| Check | Procedure |
|
|
220
|
+
|-------|-----------|
|
|
221
|
+
| Enum discipline | Count distinct metric source keys in staging → must stay within product-defined catalog. |
|
|
222
|
+
| Scope isolation | Query with wrong `scope` → empty rows, not neighbor data. |
|
|
223
|
+
| UTC correctness | Compare bucket keys to clickhouse/raw logs for same event timestamps. |
|
|
224
|
+
|
|
225
|
+
---
|
|
226
|
+
|
|
227
|
+
## 9. Failure modes
|
|
228
|
+
|
|
229
|
+
| Symptom | Cause | Fix |
|
|
230
|
+
|---------|-------|-----|
|
|
231
|
+
| Metric explosion | Typos in `event.name` | Validate against catalog pre-ingest; reject unknown names. |
|
|
232
|
+
| Missing optional dims | Expected | Use `dimensions: { feature: ["risk_dashboard"] }` filter only when dim guaranteed present. |
|
|
233
|
+
| High write amplification | Too many timeframes | Drop `hour` in low-value environments. |
|
|
234
|
+
|
|
235
|
+
---
|
|
236
|
+
|
|
237
|
+
## 10. References
|
|
238
|
+
|
|
239
|
+
- [`docs/guides/timeframes-and-bucketing.md`](../guides/timeframes-and-bucketing.md)
|
|
240
|
+
- [`docs/guides/querying.md`](../guides/querying.md)
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
# Technical playbook: Idempotent webhooks and queues
|
|
2
|
+
|
|
3
|
+
HTTP webhooks and brokers (SQS, Pub/Sub, Rabbit) usually deliver **at least once**. Retries and consumer crashes re-deliver the same logical message. Without dedupe, Countex **double-counts**.
|
|
4
|
+
|
|
5
|
+
This playbook specifies **exact** configuration and handler semantics using Countex **idempotency** + Redis **`markEventSeen`**.
|
|
6
|
+
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
## 1. Preconditions
|
|
10
|
+
|
|
11
|
+
| Requirement | Detail |
|
|
12
|
+
|-------------|--------|
|
|
13
|
+
| Node.js | ≥ 18 |
|
|
14
|
+
| Packages | `@x12i/countex`, `mongodb` (durable), `ioredis` (cache + dedupe) |
|
|
15
|
+
| Stable id | Unique string per logical delivery, available **before** ingest |
|
|
16
|
+
|
|
17
|
+
---
|
|
18
|
+
|
|
19
|
+
## 2. Idempotency mechanics (internal contract)
|
|
20
|
+
|
|
21
|
+
1. During `extractIncrements`, Countex reads `eventIdPath` → string `eventId` (optional warning if missing).
|
|
22
|
+
2. Before buffering, `ingestBatch` calls `cache.markEventSeen(scope, dedupeKey, ttl)` where:
|
|
23
|
+
|
|
24
|
+
```
|
|
25
|
+
dedupeKey = `${counterId}:${eventId}`
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
See `src/core/countex.ts` — dedupe is **per map** so two maps on same raw event do not share keys incorrectly.
|
|
29
|
+
|
|
30
|
+
3. If `markEventSeen` returns **`false`**, increments for that map slice are cleared and event **`duplicate`** fires.
|
|
31
|
+
|
|
32
|
+
4. Redis adapter stores `countex:{scope}:dedupe:{eventId}` — note the adapter receives **already-namespaced** `eventId` from Countex (includes `counterId:` prefix).
|
|
33
|
+
|
|
34
|
+
---
|
|
35
|
+
|
|
36
|
+
## 3. Canonical payload
|
|
37
|
+
|
|
38
|
+
```json
|
|
39
|
+
{
|
|
40
|
+
"tenant": { "id": "org_1" },
|
|
41
|
+
"delivery": { "id": "wh_8f3a2c9e1d004f2a" },
|
|
42
|
+
"at": "2026-05-08T18:00:00.000Z",
|
|
43
|
+
"userId": "u_1",
|
|
44
|
+
"kind": "invoice.paid"
|
|
45
|
+
}
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
**Invariant:** `delivery.id` is unique **for all time** or unique within TTL horizon — if ids recycle after TTL expiry, duplicates may re-count.
|
|
49
|
+
|
|
50
|
+
---
|
|
51
|
+
|
|
52
|
+
## 4. Counter map (exact)
|
|
53
|
+
|
|
54
|
+
`src/countex/maps/billingWebhookMap.ts`:
|
|
55
|
+
|
|
56
|
+
```ts
|
|
57
|
+
import type { CounterMap } from "@x12i/countex";
|
|
58
|
+
|
|
59
|
+
export const billingWebhookMap: CounterMap = {
|
|
60
|
+
counterId: "billing_events",
|
|
61
|
+
|
|
62
|
+
scope: { path: "$.tenant.id" },
|
|
63
|
+
eventTime: { path: "$.at" },
|
|
64
|
+
|
|
65
|
+
subjects: [{ type: "user", path: "$.userId", required: true }],
|
|
66
|
+
metrics: [{ path: "$.kind", required: true }],
|
|
67
|
+
|
|
68
|
+
delta: { default: 1 },
|
|
69
|
+
timeframes: ["day"],
|
|
70
|
+
|
|
71
|
+
idempotency: {
|
|
72
|
+
enabled: true,
|
|
73
|
+
eventIdPath: "$.delivery.id",
|
|
74
|
+
ttlSeconds: 604800, // 7d — MUST exceed max retry horizon
|
|
75
|
+
},
|
|
76
|
+
};
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
### 4.1 Choosing `ttlSeconds`
|
|
80
|
+
|
|
81
|
+
Set:
|
|
82
|
+
|
|
83
|
+
```
|
|
84
|
+
ttlSeconds ≥ (max broker retention) + (max consumer lag) + safety_margin
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
Example: SQS visibility timeout retries for 4 days → use ≥ 5–7 days.
|
|
88
|
+
|
|
89
|
+
---
|
|
90
|
+
|
|
91
|
+
## 5. Factory (exact)
|
|
92
|
+
|
|
93
|
+
```ts
|
|
94
|
+
import { MongoClient } from "mongodb";
|
|
95
|
+
import IORedis from "ioredis";
|
|
96
|
+
import { createCountex } from "@x12i/countex";
|
|
97
|
+
import { MongoDurable } from "@x12i/countex/adapters/mongo";
|
|
98
|
+
import { RedisCache } from "@x12i/countex/adapters/redis";
|
|
99
|
+
import { billingWebhookMap } from "./maps/billingWebhookMap.js";
|
|
100
|
+
|
|
101
|
+
export async function createWebhookCountex() {
|
|
102
|
+
const mongo = new MongoClient(process.env.MONGO_URI!);
|
|
103
|
+
await mongo.connect();
|
|
104
|
+
const redis = new IORedis(process.env.REDIS_URL!);
|
|
105
|
+
|
|
106
|
+
const durable = new MongoDurable({ client: mongo, dbResolver: ({ scope }) => `countex_${scope}` });
|
|
107
|
+
const cache = new RedisCache({ client: redis, keyPrefix: "countex", ttlSeconds: 14 * 24 * 3600 });
|
|
108
|
+
|
|
109
|
+
const countex = createCountex({
|
|
110
|
+
maps: [billingWebhookMap],
|
|
111
|
+
durable,
|
|
112
|
+
cache,
|
|
113
|
+
flush: { everyMs: 5_000, maxBufferedCells: 20_000, maxBufferedEvents: 10_000, flushOnShutdown: true, onOverflow: "block" },
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
await countex.init();
|
|
117
|
+
return { countex, mongo, redis };
|
|
118
|
+
}
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
**Critical:** Without `cache` implementing `markEventSeen`, idempotency **does nothing** — duplicates still buffer.
|
|
122
|
+
|
|
123
|
+
---
|
|
124
|
+
|
|
125
|
+
## 6. Consumer handler (exact steps)
|
|
126
|
+
|
|
127
|
+
### Step 6.1 — Parse
|
|
128
|
+
|
|
129
|
+
```ts
|
|
130
|
+
const payload = JSON.parse(messageBody) as unknown;
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
### Step 6.2 — Validate schema (your code)
|
|
134
|
+
|
|
135
|
+
Reject before ingest if `delivery.id` missing while idempotency enabled — otherwise you get warnings and **non-deduped** increments.
|
|
136
|
+
|
|
137
|
+
### Step 6.3 — Ingest
|
|
138
|
+
|
|
139
|
+
```ts
|
|
140
|
+
await countex.ingest(payload);
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
### Step 6.4 — Observe duplicates
|
|
144
|
+
|
|
145
|
+
```ts
|
|
146
|
+
countex.on("duplicate", ({ scopeId, eventId }) => {
|
|
147
|
+
// eventId here is the NAMESPACED key "billing_events:wh_8f3a2c..."
|
|
148
|
+
});
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
Return **200 OK / ack message** even on duplicate — delivery succeeded idempotently.
|
|
152
|
+
|
|
153
|
+
### Step 6.5 — Durability boundary
|
|
154
|
+
|
|
155
|
+
Either:
|
|
156
|
+
|
|
157
|
+
- Rely on **`flush.everyMs`**, or
|
|
158
|
+
- **`await countex.flush()`** after batch if webhook framework requires strong durability before ack (latency tradeoff).
|
|
159
|
+
|
|
160
|
+
**Ack ordering policy:** industry-standard is **process → flush OR tolerate timer → ack**; never ack before deserialize if you cannot handle poison pills.
|
|
161
|
+
|
|
162
|
+
---
|
|
163
|
+
|
|
164
|
+
## 7. Verification tests
|
|
165
|
+
|
|
166
|
+
| Test | Procedure | Expected |
|
|
167
|
+
|------|-----------|----------|
|
|
168
|
+
| Dedupe works | Same payload twice | Second `duplicate`; Mongo count unchanged. |
|
|
169
|
+
| TTL expiry | Simulate key delete | Third delivery increments again — expected. |
|
|
170
|
+
| Multi-map | Two maps, same raw `delivery.id` | Distinct dedupe keys (`counterId:` prefix) → both run once each. |
|
|
171
|
+
|
|
172
|
+
---
|
|
173
|
+
|
|
174
|
+
## 8. Failure modes
|
|
175
|
+
|
|
176
|
+
| Symptom | Cause | Fix |
|
|
177
|
+
|---------|-------|-----|
|
|
178
|
+
| Duplicates still increment | Memory-only cache adapter | Use `RedisCache` or implement `markEventSeen`. |
|
|
179
|
+
| Redis down | `markEventSeen` throws | Decide fail-open vs fail-closed; today ingest path uses cache — surface errors to DLQ. |
|
|
180
|
+
| Mass replays after TTL | TTL too short | Raise `ttlSeconds`. |
|
|
181
|
+
|
|
182
|
+
---
|
|
183
|
+
|
|
184
|
+
## 9. References
|
|
185
|
+
|
|
186
|
+
- [`docs/guides/idempotency.md`](../guides/idempotency.md)
|
|
187
|
+
- [`docs/internals/internal-idempotency.md`](../internals/internal-idempotency.md)
|
|
188
|
+
- `src/storage/redis.ts` dedupe key format
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@x12i/countex",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.5.0",
|
|
4
4
|
"description": "Generic sparse counter matrices for JSON events. Dynamic counting, aggregation, and retrieval from mapped JSON inputs.",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -73,6 +73,11 @@
|
|
|
73
73
|
"import": "./dist/insights/advisors-subpath.js",
|
|
74
74
|
"require": "./dist/insights/advisors-subpath.js"
|
|
75
75
|
},
|
|
76
|
+
"./simulation": {
|
|
77
|
+
"types": "./dist/simulation/index.d.ts",
|
|
78
|
+
"import": "./dist/simulation/index.js",
|
|
79
|
+
"require": "./dist/simulation/index.js"
|
|
80
|
+
},
|
|
76
81
|
"./docs/deployment": "./docs/DEPLOYMENT.md",
|
|
77
82
|
"./docs/end-to-end-example": "./docs/END_TO_END_EXAMPLE.md",
|
|
78
83
|
"./docs/guides": "./docs/guides/README.md",
|
|
@@ -94,7 +99,18 @@
|
|
|
94
99
|
"./docs/guides/insights-anomalies-and-rules": "./docs/guides/insights-anomalies-and-rules.md",
|
|
95
100
|
"./docs/guides/insights-forecasting-and-budget": "./docs/guides/insights-forecasting-and-budget.md",
|
|
96
101
|
"./docs/guides/insights-advisors-and-optimization": "./docs/guides/insights-advisors-and-optimization.md",
|
|
97
|
-
"./docs/guides/cli": "./docs/guides/cli.md"
|
|
102
|
+
"./docs/guides/cli": "./docs/guides/cli.md",
|
|
103
|
+
"./docs/guides/simulation": "./docs/guides/simulation.md",
|
|
104
|
+
"./docs/guides/simulation-profiles": "./docs/guides/simulation-profiles.md",
|
|
105
|
+
"./docs/guides/simulation-resource-estimation": "./docs/guides/simulation-resource-estimation.md",
|
|
106
|
+
"./docs/use-cases": "./docs/use-cases/README.md",
|
|
107
|
+
"./docs/use-cases/audit-security-log-counts": "./docs/use-cases/audit-security-log-counts.md",
|
|
108
|
+
"./docs/use-cases/product-feature-usage": "./docs/use-cases/product-feature-usage.md",
|
|
109
|
+
"./docs/use-cases/llm-cost-and-token-metering": "./docs/use-cases/llm-cost-and-token-metering.md",
|
|
110
|
+
"./docs/use-cases/multi-tenant-isolation": "./docs/use-cases/multi-tenant-isolation.md",
|
|
111
|
+
"./docs/use-cases/webhook-idempotent-ingestion": "./docs/use-cases/webhook-idempotent-ingestion.md",
|
|
112
|
+
"./docs/use-cases/ops-dashboard-daily-rollups": "./docs/use-cases/ops-dashboard-daily-rollups.md",
|
|
113
|
+
"./docs/use-cases/processing-workflow-activity": "./docs/use-cases/processing-workflow-activity.md"
|
|
98
114
|
},
|
|
99
115
|
"files": [
|
|
100
116
|
"dist",
|