@x12i/countex 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +520 -0
- package/dist/backup/export.d.ts +19 -0
- package/dist/backup/export.d.ts.map +1 -0
- package/dist/backup/export.js +36 -0
- package/dist/backup/export.js.map +1 -0
- package/dist/core/buffer.d.ts +27 -0
- package/dist/core/buffer.d.ts.map +1 -0
- package/dist/core/buffer.js +63 -0
- package/dist/core/buffer.js.map +1 -0
- package/dist/core/cardinality.d.ts +43 -0
- package/dist/core/cardinality.d.ts.map +1 -0
- package/dist/core/cardinality.js +93 -0
- package/dist/core/cardinality.js.map +1 -0
- package/dist/core/countex.d.ts +49 -0
- package/dist/core/countex.d.ts.map +1 -0
- package/dist/core/countex.js +331 -0
- package/dist/core/countex.js.map +1 -0
- package/dist/core/extraction.d.ts +39 -0
- package/dist/core/extraction.d.ts.map +1 -0
- package/dist/core/extraction.js +176 -0
- package/dist/core/extraction.js.map +1 -0
- package/dist/core/mapping.d.ts +8 -0
- package/dist/core/mapping.d.ts.map +1 -0
- package/dist/core/mapping.js +90 -0
- package/dist/core/mapping.js.map +1 -0
- package/dist/core/normalization.d.ts +15 -0
- package/dist/core/normalization.d.ts.map +1 -0
- package/dist/core/normalization.js +43 -0
- package/dist/core/normalization.js.map +1 -0
- package/dist/core/timeframe.d.ts +30 -0
- package/dist/core/timeframe.d.ts.map +1 -0
- package/dist/core/timeframe.js +151 -0
- package/dist/core/timeframe.js.map +1 -0
- package/dist/index.d.ts +17 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +28 -0
- package/dist/index.js.map +1 -0
- package/dist/query/formatter.d.ts +9 -0
- package/dist/query/formatter.d.ts.map +1 -0
- package/dist/query/formatter.js +119 -0
- package/dist/query/formatter.js.map +1 -0
- package/dist/query/planner.d.ts +14 -0
- package/dist/query/planner.d.ts.map +1 -0
- package/dist/query/planner.js +32 -0
- package/dist/query/planner.js.map +1 -0
- package/dist/storage/memory.d.ts +36 -0
- package/dist/storage/memory.d.ts.map +1 -0
- package/dist/storage/memory.js +150 -0
- package/dist/storage/memory.js.map +1 -0
- package/dist/storage/mongo.d.ts +56 -0
- package/dist/storage/mongo.d.ts.map +1 -0
- package/dist/storage/mongo.js +323 -0
- package/dist/storage/mongo.js.map +1 -0
- package/dist/storage/redis.d.ts +44 -0
- package/dist/storage/redis.d.ts.map +1 -0
- package/dist/storage/redis.js +193 -0
- package/dist/storage/redis.js.map +1 -0
- package/dist/types/index.d.ts +317 -0
- package/dist/types/index.d.ts.map +1 -0
- package/dist/types/index.js +9 -0
- package/dist/types/index.js.map +1 -0
- package/dist/utils/errors.d.ts +20 -0
- package/dist/utils/errors.d.ts.map +1 -0
- package/dist/utils/errors.js +44 -0
- package/dist/utils/errors.js.map +1 -0
- package/dist/utils/hash.d.ts +30 -0
- package/dist/utils/hash.d.ts.map +1 -0
- package/dist/utils/hash.js +61 -0
- package/dist/utils/hash.js.map +1 -0
- package/dist/utils/jsonpath.d.ts +40 -0
- package/dist/utils/jsonpath.d.ts.map +1 -0
- package/dist/utils/jsonpath.js +162 -0
- package/dist/utils/jsonpath.js.map +1 -0
- package/dist/utils/logger.d.ts +7 -0
- package/dist/utils/logger.d.ts.map +1 -0
- package/dist/utils/logger.js +24 -0
- package/dist/utils/logger.js.map +1 -0
- package/docs/DEPLOYMENT.md +244 -0
- package/docs/END_TO_END_EXAMPLE.md +316 -0
- package/package.json +91 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 x12i
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,520 @@
|
|
|
1
|
+
# @x12i/countex
|
|
2
|
+
|
|
3
|
+
> Generic sparse counter matrices for JSON events.
|
|
4
|
+
> Dynamic counting, aggregation, and retrieval from mapped JSON inputs.
|
|
5
|
+
|
|
6
|
+
`@x12i/countex` is a source-agnostic counting engine. You give it any JSON event, a declarative mapping config, and storage adapters; it projects events into sparse counter cells, batches deltas locally, and persists them atomically to a durable store (MongoDB) and a hot cache (Redis).
|
|
7
|
+
|
|
8
|
+
It does **not** know about users, organizations, documents, labels, or events. It only knows generic primitives: **scope, subject, metric, dimensions, timeframe, delta**. Whatever your domain calls those things — Countex doesn't care.
|
|
9
|
+
|
|
10
|
+
```
|
|
11
|
+
event JSON ──► map ──► raw increments ──► buffer ──► flush ──► Mongo (durable)
|
|
12
|
+
└───► Redis (hot cache)
|
|
13
|
+
|
|
14
|
+
◄── query ◄── Redis (full coverage) or ◄── Mongo (fallback) ──◄
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
---
|
|
18
|
+
|
|
19
|
+
## Install
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
npm install @x12i/countex
|
|
23
|
+
# Bring your own clients — peer deps, both optional:
|
|
24
|
+
npm install mongodb ioredis
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
Node 18+. Zero runtime dependencies in the core package.
|
|
28
|
+
|
|
29
|
+
### Documentation shipped with the package
|
|
30
|
+
|
|
31
|
+
These guides are included in the published tarball under `docs/` and exposed as stable import subpaths (resolve to `.md` files for editors and tooling):
|
|
32
|
+
|
|
33
|
+
| Topic | Package subpath | File |
|
|
34
|
+
|-------|-----------------|------|
|
|
35
|
+
| Release & production deployment | `@x12i/countex/docs/deployment` | `docs/DEPLOYMENT.md` |
|
|
36
|
+
| Tutorial — install → map → ingest → query | `@x12i/countex/docs/end-to-end-example` | `docs/END_TO_END_EXAMPLE.md` |
|
|
37
|
+
|
|
38
|
+
After install: `node_modules/@x12i/countex/docs/`.
|
|
39
|
+
|
|
40
|
+
---
|
|
41
|
+
|
|
42
|
+
## The mental model
|
|
43
|
+
|
|
44
|
+
Countex turns mapped JSON events into sparse aggregate cells.
|
|
45
|
+
|
|
46
|
+
A counter cell is uniquely identified by:
|
|
47
|
+
|
|
48
|
+
```
|
|
49
|
+
(scope, counterId, timeframeType, timeframeKey, subjectType, subjectId, metricId, dimensionsHash)
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
Each event can produce one or many cell increments. A counter increment is defined by:
|
|
53
|
+
|
|
54
|
+
- **scope** — your tenant / org / account boundary
|
|
55
|
+
- **counterId** — which counter family this is (e.g. `access_by_subject_metric`)
|
|
56
|
+
- **timeframe** — `day` / `week` / `month` / `quarter` / `year` / `all`
|
|
57
|
+
- **subject** — the thing being counted against (e.g. an actor or a target)
|
|
58
|
+
- **metric** — the count category / label
|
|
59
|
+
- **dimensions** — arbitrary additional breakdowns
|
|
60
|
+
- **delta** — `+1`, `-1`, or any integer
|
|
61
|
+
|
|
62
|
+
Countex buffers increments locally, flushes atomic deltas to durable + cache, and serves queries from cache with durable fallback.
|
|
63
|
+
|
|
64
|
+
---
|
|
65
|
+
|
|
66
|
+
## Quick start
|
|
67
|
+
|
|
68
|
+
```ts
|
|
69
|
+
import { createCountex } from "@x12i/countex";
|
|
70
|
+
import { MemoryDurable, MemoryCache } from "@x12i/countex/adapters/memory";
|
|
71
|
+
|
|
72
|
+
const accessMap = {
|
|
73
|
+
counterId: "access_by_subject_metric",
|
|
74
|
+
|
|
75
|
+
scope: { path: "$.tenant.id" },
|
|
76
|
+
eventTime: { path: "$.event.createdAt" },
|
|
77
|
+
|
|
78
|
+
subjects: [
|
|
79
|
+
{ type: "actor", path: "$.actor.id", required: true },
|
|
80
|
+
{ type: "target", path: "$.target.id", required: false },
|
|
81
|
+
],
|
|
82
|
+
|
|
83
|
+
metrics: [
|
|
84
|
+
{ path: "$.labels[*]", required: true },
|
|
85
|
+
],
|
|
86
|
+
|
|
87
|
+
dimensions: [
|
|
88
|
+
{ name: "eventType", path: "$.event.type" },
|
|
89
|
+
{ name: "situation", path: "$.event.situation" },
|
|
90
|
+
{ name: "targetType", path: "$.target.type" },
|
|
91
|
+
],
|
|
92
|
+
|
|
93
|
+
delta: { default: 1 },
|
|
94
|
+
timeframes: ["day", "week", "month", "quarter", "year"],
|
|
95
|
+
} as const;
|
|
96
|
+
|
|
97
|
+
const countex = createCountex({
|
|
98
|
+
maps: [accessMap],
|
|
99
|
+
durable: new MemoryDurable(),
|
|
100
|
+
cache: new MemoryCache(),
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
await countex.ingest({
|
|
104
|
+
tenant: { id: "org_123" },
|
|
105
|
+
event: { type: "share", situation: "external", createdAt: "2026-05-08T14:33:00Z" },
|
|
106
|
+
actor: { id: "user_42" },
|
|
107
|
+
target: { id: "doc_99", type: "document" },
|
|
108
|
+
labels: ["PCI", "PII"],
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
await countex.flush();
|
|
112
|
+
|
|
113
|
+
const result = await countex.query({
|
|
114
|
+
scope: "org_123",
|
|
115
|
+
counterId: "access_by_subject_metric",
|
|
116
|
+
timeframe: { type: "day", from: "2026-05-01", to: "2026-05-08" },
|
|
117
|
+
groupBy: ["subject", "metric", "dim:eventType"],
|
|
118
|
+
format: "table",
|
|
119
|
+
orderBy: "-count",
|
|
120
|
+
limit: 100,
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
console.log(result.rows);
|
|
124
|
+
|
|
125
|
+
await countex.shutdown();
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
That single ingest produces:
|
|
129
|
+
|
|
130
|
+
- 1 actor + 1 target = **2 subjects**
|
|
131
|
+
- 2 labels = **2 metrics**
|
|
132
|
+
- 5 timeframes
|
|
133
|
+
- → **2 × 2 × 5 = 20 cell increments**
|
|
134
|
+
|
|
135
|
+
Each cell uses an atomic `$inc` on durable and `HINCRBY` on the cache, so 20 ingest workers can safely run in parallel.
|
|
136
|
+
|
|
137
|
+
---
|
|
138
|
+
|
|
139
|
+
## Production setup (Mongo + Redis)
|
|
140
|
+
|
|
141
|
+
```ts
|
|
142
|
+
import { MongoClient } from "mongodb";
|
|
143
|
+
import IORedis from "ioredis";
|
|
144
|
+
import { createCountex } from "@x12i/countex";
|
|
145
|
+
import { MongoDurable } from "@x12i/countex/adapters/mongo";
|
|
146
|
+
import { RedisCache } from "@x12i/countex/adapters/redis";
|
|
147
|
+
|
|
148
|
+
const mongo = new MongoClient(process.env.MONGO_URI!);
|
|
149
|
+
await mongo.connect();
|
|
150
|
+
const redis = new IORedis(process.env.REDIS_URL!);
|
|
151
|
+
|
|
152
|
+
const countex = createCountex({
|
|
153
|
+
maps: [accessMap],
|
|
154
|
+
durable: new MongoDurable({
|
|
155
|
+
client: mongo,
|
|
156
|
+
dbResolver: ({ scope }) => `countex_${scope}`,
|
|
157
|
+
}),
|
|
158
|
+
cache: new RedisCache({
|
|
159
|
+
client: redis,
|
|
160
|
+
keyPrefix: "countex",
|
|
161
|
+
ttlSeconds: 92 * 24 * 3600, // ~3 months hot window
|
|
162
|
+
}),
|
|
163
|
+
flush: {
|
|
164
|
+
everyMs: 60_000,
|
|
165
|
+
maxBufferedCells: 100_000,
|
|
166
|
+
maxBufferedEvents: 50_000,
|
|
167
|
+
flushOnShutdown: true,
|
|
168
|
+
},
|
|
169
|
+
cardinality: {
|
|
170
|
+
maxDimensionValuesPerName: 10_000,
|
|
171
|
+
onLimit: "overflow",
|
|
172
|
+
overflowValue: "__OTHER__",
|
|
173
|
+
},
|
|
174
|
+
});
|
|
175
|
+
|
|
176
|
+
await countex.init();
|
|
177
|
+
|
|
178
|
+
process.on("SIGTERM", async () => { await countex.shutdown(); });
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
---
|
|
182
|
+
|
|
183
|
+
## Mapping reference
|
|
184
|
+
|
|
185
|
+
A `CounterMap` is the heart of the package. Everything Countex knows about your data is in here.
|
|
186
|
+
|
|
187
|
+
| Field | Type | Notes |
|
|
188
|
+
|---------------|----------------------------|-------|
|
|
189
|
+
| `counterId` | `string` | Stable identifier for this counter family. |
|
|
190
|
+
| `scope` | `{ path }` | Path into the event for the tenant/scope id. Required. |
|
|
191
|
+
| `eventTime` | `{ path }` | Path to a Date / ISO string / epoch ms. Drives bucket selection. |
|
|
192
|
+
| `subjects` | `SubjectMapping[]` | One or many. Each has `type` and `path`; multi-value paths fan out. |
|
|
193
|
+
| `metrics` | `MetricMapping[]` | Same fanout semantics. `$.labels[*]` is common. |
|
|
194
|
+
| `dimensions` | `DimensionMapping[]` | Named extra breakdowns. Optional by default. |
|
|
195
|
+
| `delta` | `{ path?, default? }` | Numeric value at runtime. Default `1`. Negative deltas are fine. |
|
|
196
|
+
| `timeframes` | `TimeframeType[]` | Subset of `day` / `week` / `month` / `quarter` / `year` / `all`. |
|
|
197
|
+
| `idempotency` | `IdempotencyPolicy` | Optional; needs a cache adapter that implements `markEventSeen`. |
|
|
198
|
+
| `cardinality` | `CardinalityPolicy` | Optional; per-map override of engine-level defaults. |
|
|
199
|
+
| `options` | `MappingOptions` | `skipMissingOptionalDimensions` (default true), `normalizeValues` (default true). |
|
|
200
|
+
|
|
201
|
+
### Path syntax
|
|
202
|
+
|
|
203
|
+
A small JSONPath subset:
|
|
204
|
+
|
|
205
|
+
| Syntax | Meaning |
|
|
206
|
+
|-------------------|--------------------------------------------------|
|
|
207
|
+
| `$` | Root. |
|
|
208
|
+
| `.name` | Property. |
|
|
209
|
+
| `['name']` | Property (bracket form, allows arbitrary chars). |
|
|
210
|
+
| `[N]` | Numeric array index (negatives allowed). |
|
|
211
|
+
| `[*]` | Wildcard — fan out across array elems / object values. |
|
|
212
|
+
|
|
213
|
+
Examples:
|
|
214
|
+
|
|
215
|
+
```
|
|
216
|
+
$.tenant.id
|
|
217
|
+
$.actor.id
|
|
218
|
+
$.labels[*]
|
|
219
|
+
$.items[*].id
|
|
220
|
+
$['weird.key.with.dots']
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
### Fanout
|
|
224
|
+
|
|
225
|
+
A single event with 2 subjects and 3 labels (metrics) and 5 selected timeframes produces **2 × 3 × 5 = 30** atomic increments. This is intentional — read your mapping carefully. Countex emits a `dropped` event when cardinality protection bites, so you'll know.
|
|
226
|
+
|
|
227
|
+
---
|
|
228
|
+
|
|
229
|
+
## Storage adapters
|
|
230
|
+
|
|
231
|
+
Countex defines two contracts and ships three adapters.
|
|
232
|
+
|
|
233
|
+
```ts
|
|
234
|
+
interface DurableStorage {
|
|
235
|
+
applyDeltas(deltas: PendingDelta[]): Promise<void>;
|
|
236
|
+
readCells(spec: QuerySpec): Promise<CounterCell[]>;
|
|
237
|
+
resolveIds(lookups: DictionaryLookup[]): Promise<Map<string, number>>;
|
|
238
|
+
streamCells(filter): AsyncIterable<CounterCell>;
|
|
239
|
+
init?(): Promise<void>;
|
|
240
|
+
close?(): Promise<void>;
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
interface CacheStorage {
|
|
244
|
+
applyDeltas(deltas: PendingDelta[]): Promise<void>;
|
|
245
|
+
readCells(spec: QuerySpec): Promise<{ cells: CounterCell[]; coverage: "full" | "partial" | "miss" }>;
|
|
246
|
+
warm(cells: CounterCell[]): Promise<void>;
|
|
247
|
+
markEventSeen?(scopeId, eventId, ttlSeconds): Promise<boolean>;
|
|
248
|
+
init?(): Promise<void>;
|
|
249
|
+
close?(): Promise<void>;
|
|
250
|
+
}
|
|
251
|
+
```
|
|
252
|
+
|
|
253
|
+
### Built-in adapters
|
|
254
|
+
|
|
255
|
+
| Module | Purpose |
|
|
256
|
+
|-----------------------------------|--------------------------------------|
|
|
257
|
+
| `@x12i/countex/adapters/memory` | In-process; tests / local dev only. |
|
|
258
|
+
| `@x12i/countex/adapters/mongo` | Durable. Requires `mongodb` peer. |
|
|
259
|
+
| `@x12i/countex/adapters/redis` | Hot cache. Requires `ioredis` peer. |
|
|
260
|
+
|
|
261
|
+
Bring your own clients — Countex doesn't open or close them. That keeps lifecycle in the host app's hands.
|
|
262
|
+
|
|
263
|
+
### Mongo schema
|
|
264
|
+
|
|
265
|
+
`countex_cells`:
|
|
266
|
+
|
|
267
|
+
```json
|
|
268
|
+
{
|
|
269
|
+
"scopeId": "org_123",
|
|
270
|
+
"counterId": "access_by_subject_metric",
|
|
271
|
+
"timeframeType": "day",
|
|
272
|
+
"timeframeKey": "2026-05-08",
|
|
273
|
+
"subjectType": "actor",
|
|
274
|
+
"subjectId": 10001,
|
|
275
|
+
"metricId": 41,
|
|
276
|
+
"dimensionsHash": "a8f21c0b1d2e",
|
|
277
|
+
"dimensions": { "eventType": "share", "situation": "external" },
|
|
278
|
+
"count": 42,
|
|
279
|
+
"createdAt": "...",
|
|
280
|
+
"updatedAt": "..."
|
|
281
|
+
}
|
|
282
|
+
```
|
|
283
|
+
|
|
284
|
+
Indexes (created automatically on first write):
|
|
285
|
+
|
|
286
|
+
- `uniq_cell_identity` (unique, on the full identity tuple)
|
|
287
|
+
- `by_timeframe` — for range scans
|
|
288
|
+
- `by_subject` — for subject filters
|
|
289
|
+
- `by_metric` — for metric filters
|
|
290
|
+
|
|
291
|
+
`countex_dict` — the per-scope subject/metric dictionary, with monotonic ids allocated via `countex_seq`.
|
|
292
|
+
|
|
293
|
+
### Redis layout
|
|
294
|
+
|
|
295
|
+
```
|
|
296
|
+
countex:{scope}:{counterId}:{timeframeType}:{timeframeKey} → HASH
|
|
297
|
+
field = "{subjectType}|{subjectId}|{metricId}|{dimensionsHash}"
|
|
298
|
+
value = count
|
|
299
|
+
|
|
300
|
+
countex:{scope}:{counterId}:...:{timeframeKey}:meta:{field} → JSON of dimensions
|
|
301
|
+
|
|
302
|
+
countex:{scope}:dedupe:{eventId} → "1" with TTL
|
|
303
|
+
```
|
|
304
|
+
|
|
305
|
+
One hash per `(scope, counterId, bucket)` keeps Redis key counts small and per-bucket reads to a single `HGETALL`.
|
|
306
|
+
|
|
307
|
+
---
|
|
308
|
+
|
|
309
|
+
## Query API
|
|
310
|
+
|
|
311
|
+
```ts
|
|
312
|
+
const result = await countex.query({
|
|
313
|
+
scope: "org_123",
|
|
314
|
+
counterId: "access_by_subject_metric",
|
|
315
|
+
|
|
316
|
+
timeframe: {
|
|
317
|
+
type: "day",
|
|
318
|
+
from: "2026-05-01",
|
|
319
|
+
to: "2026-05-08",
|
|
320
|
+
},
|
|
321
|
+
|
|
322
|
+
// Filters — all optional
|
|
323
|
+
subject: { type: "actor", ids: [10001, 10002] },
|
|
324
|
+
metrics: [41, 42],
|
|
325
|
+
dimensions: { eventType: ["share", "download"], situation: ["external"] },
|
|
326
|
+
|
|
327
|
+
// Shape
|
|
328
|
+
groupBy: ["subject", "metric", "dim:eventType"],
|
|
329
|
+
format: "table", // "cells" | "table" | "matrix" | "timeseries"
|
|
330
|
+
orderBy: "-count",
|
|
331
|
+
limit: 100,
|
|
332
|
+
|
|
333
|
+
// Diagnostics
|
|
334
|
+
bypassCache: false,
|
|
335
|
+
signal: AbortSignal.timeout(5_000),
|
|
336
|
+
});
|
|
337
|
+
```
|
|
338
|
+
|
|
339
|
+
Result:
|
|
340
|
+
|
|
341
|
+
```ts
|
|
342
|
+
{
|
|
343
|
+
format: "table",
|
|
344
|
+
rows: [
|
|
345
|
+
{ keys: { subject: 10001, metric: 41, "dim:eventType": "share" }, count: 142 },
|
|
346
|
+
...
|
|
347
|
+
],
|
|
348
|
+
total: 1234,
|
|
349
|
+
source: "cache" | "durable" | "merged",
|
|
350
|
+
}
|
|
351
|
+
```
|
|
352
|
+
|
|
353
|
+
`groupBy` accepts:
|
|
354
|
+
- `"subject"`, `"metric"`, `"timeframeKey"`
|
|
355
|
+
- `"dim:NAME"` — group by a dimension value
|
|
356
|
+
|
|
357
|
+
---
|
|
358
|
+
|
|
359
|
+
## Events
|
|
360
|
+
|
|
361
|
+
The Countex client is an `EventEmitter`. Subscribe to observe what's happening:
|
|
362
|
+
|
|
363
|
+
```ts
|
|
364
|
+
countex.on("flush", ({ cells, durationMs }) => {});
|
|
365
|
+
countex.on("flush_error", ({ error }) => {});
|
|
366
|
+
countex.on("cardinality_warning", ({ kind, count, limit })=> {});
|
|
367
|
+
countex.on("overflow", ({ kind, sourceKey }) => {});
|
|
368
|
+
countex.on("dropped", ({ reason }) => {});
|
|
369
|
+
countex.on("duplicate", ({ scopeId, eventId }) => {});
|
|
370
|
+
```
|
|
371
|
+
|
|
372
|
+
All event payloads are typed via `CountexEvents`.
|
|
373
|
+
|
|
374
|
+
---
|
|
375
|
+
|
|
376
|
+
## Operational notes
|
|
377
|
+
|
|
378
|
+
### Buffering and flushing
|
|
379
|
+
|
|
380
|
+
Each Countex instance keeps only a delta buffer in memory:
|
|
381
|
+
|
|
382
|
+
```
|
|
383
|
+
Map<cellKey, delta>
|
|
384
|
+
```
|
|
385
|
+
|
|
386
|
+
Flush triggers (any of):
|
|
387
|
+
|
|
388
|
+
- `flush.everyMs` timer (default 60s)
|
|
389
|
+
- `flush.maxBufferedCells` reached (default 100k)
|
|
390
|
+
- `flush.maxBufferedEvents` reached (default 50k)
|
|
391
|
+
- `shutdown()` if `flushOnShutdown` is true (default)
|
|
392
|
+
- explicit `flush()`
|
|
393
|
+
|
|
394
|
+
Flush is **two-phase**: durable first, cache second. If durable fails, the snapshot is merged back into the buffer — counts are not lost. If cache fails after durable succeeded, it's logged and continues; the next read warms cache from durable.
|
|
395
|
+
|
|
396
|
+
At most one flush runs at a time. Concurrent callers receive the same in-flight promise.
|
|
397
|
+
|
|
398
|
+
### Cardinality protection
|
|
399
|
+
|
|
400
|
+
A bad mapping (e.g. a free-form URL as a dimension) can balloon the dictionary. Countex tracks per-`(scope, kind, qualifier)` how many distinct keys it has minted, and applies a policy when limits hit:
|
|
401
|
+
|
|
402
|
+
- `overflow` (default) — rewrite the source key to `__OTHER__`. Emits `overflow`.
|
|
403
|
+
- `drop` — silently drop the increment. Emits `dropped`.
|
|
404
|
+
- `error` — throw `CardinalityLimitError`.
|
|
405
|
+
|
|
406
|
+
Defaults are conservative; tune them per workload. A warning fires at 80% of the limit (`cardinality_warning`).
|
|
407
|
+
|
|
408
|
+
### Idempotency
|
|
409
|
+
|
|
410
|
+
Optional and per-map. Requires a cache adapter that implements `markEventSeen` (the included `RedisCache` and `MemoryCache` both do).
|
|
411
|
+
|
|
412
|
+
```ts
|
|
413
|
+
idempotency: {
|
|
414
|
+
enabled: true,
|
|
415
|
+
eventIdPath: "$.event.id",
|
|
416
|
+
ttlSeconds: 86_400,
|
|
417
|
+
}
|
|
418
|
+
```
|
|
419
|
+
|
|
420
|
+
Dedupe keys are namespaced by `counterId`, so different maps with their own idempotency don't collide on shared event ids.
|
|
421
|
+
|
|
422
|
+
### Late events
|
|
423
|
+
|
|
424
|
+
Bucket selection always uses the **event time**, never the ingestion time. Late-arriving events go into the correct historical bucket — `$inc` doesn't care about ordering.
|
|
425
|
+
|
|
426
|
+
### Concurrency safety
|
|
427
|
+
|
|
428
|
+
All writes are atomic per cell:
|
|
429
|
+
|
|
430
|
+
- Mongo: `bulkWrite` of `updateOne` with `$inc` and `upsert: true`
|
|
431
|
+
- Redis: `HINCRBY`
|
|
432
|
+
|
|
433
|
+
20+ Countex workers can safely run in parallel against the same scope. Reads are eventually consistent across the two stores; if Redis is behind durable, the planner will detect partial coverage and read durable.
|
|
434
|
+
|
|
435
|
+
### Backup / rebuild
|
|
436
|
+
|
|
437
|
+
```ts
|
|
438
|
+
import { exportCells, rebuildCache } from "@x12i/countex";
|
|
439
|
+
|
|
440
|
+
// Stream all cells in a window — bounded async iteration.
|
|
441
|
+
for await (const cell of exportCells(durable, {
|
|
442
|
+
scope: "org_123",
|
|
443
|
+
counterId: "access_by_subject_metric",
|
|
444
|
+
timeframe: { type: "day", from: "2026-02-01", to: "2026-05-08" },
|
|
445
|
+
})) {
|
|
446
|
+
// ndjson, S3, whatever you want
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
// Re-warm cache from durable.
|
|
450
|
+
await rebuildCache(durable, cache, {
|
|
451
|
+
scope: "org_123",
|
|
452
|
+
counterId: "access_by_subject_metric",
|
|
453
|
+
timeframe: { type: "day", from: "2026-02-01", to: "2026-05-08" },
|
|
454
|
+
});
|
|
455
|
+
```
|
|
456
|
+
|
|
457
|
+
---
|
|
458
|
+
|
|
459
|
+
## Errors
|
|
460
|
+
|
|
461
|
+
All errors thrown by Countex extend `CountexError`:
|
|
462
|
+
|
|
463
|
+
| Class | When |
|
|
464
|
+
|--------------------------|------|
|
|
465
|
+
| `MappingError` | Invalid mapping config (caught at engine startup). |
|
|
466
|
+
| `ExtractionError` | Malformed path or value during extraction. |
|
|
467
|
+
| `CardinalityLimitError` | Cardinality limit hit when policy is `error`. |
|
|
468
|
+
| `StorageError` | Wrapping a storage-adapter failure. |
|
|
469
|
+
|
|
470
|
+
---
|
|
471
|
+
|
|
472
|
+
## Versioning & scope
|
|
473
|
+
|
|
474
|
+
`v0.x` is for stabilizing the public API. Once it's hardened:
|
|
475
|
+
|
|
476
|
+
- exact unique counts (HLL / sets) are an opt-in extension, not core
|
|
477
|
+
- complex expression deltas are not in core
|
|
478
|
+
- distributed locks / exactly-once: not Countex's job; durable storage is the source of truth and `$inc` makes worker-count irrelevant
|
|
479
|
+
|
|
480
|
+
---
|
|
481
|
+
|
|
482
|
+
## Repository layout
|
|
483
|
+
|
|
484
|
+
| Path | Role |
|
|
485
|
+
|------|------|
|
|
486
|
+
| `src/index.ts` | Public exports (`createCountex`, types, errors). |
|
|
487
|
+
| `src/core/` | Engine: ingest pipeline, buffer, extraction, mapping validation, time buckets, cardinality. |
|
|
488
|
+
| `src/query/` | Query planner and result formatting. |
|
|
489
|
+
| `src/storage/` | Adapters: `memory` (in-process), `mongo` (durable), `redis` (cache). |
|
|
490
|
+
| `src/utils/` | JSONPath subset, hashing, logging. |
|
|
491
|
+
| `src/backup/` | Export / cache rebuild helpers. |
|
|
492
|
+
| `examples/` | Runnable demos (`npx tsx examples/basic.ts`). |
|
|
493
|
+
| `docs/` | Deployment guide and end-to-end tutorial (published in the npm package). |
|
|
494
|
+
| `tests/unit/` | Fast Node test runner tests (`npm test`). |
|
|
495
|
+
| `tests/integration/` | Opt-in live tests against real services (`npm run test:integration`). |
|
|
496
|
+
|
|
497
|
+
---
|
|
498
|
+
|
|
499
|
+
## Publishing (npm)
|
|
500
|
+
|
|
501
|
+
This package is scoped as **`@x12i/countex`** and is configured for **public** registry access (`publishConfig.access` in `package.json`).
|
|
502
|
+
|
|
503
|
+
1. Ensure you are logged in and allowed to publish under the **`@x12i`** scope on [npm](https://www.npmjs.com/).
|
|
504
|
+
2. Put auth tokens only in **`~/.npmrc`** — do not commit project `.npmrc` with credentials (it is listed in `.gitignore`).
|
|
505
|
+
3. From the repo root:
|
|
506
|
+
|
|
507
|
+
```bash
|
|
508
|
+
npm test
|
|
509
|
+
npm publish
|
|
510
|
+
```
|
|
511
|
+
|
|
512
|
+
`prepublishOnly` runs a clean build so the tarball always contains a fresh `dist/`.
|
|
513
|
+
|
|
514
|
+
The **`docs/`** directory (`DEPLOYMENT.md`, `END_TO_END_EXAMPLE.md`) is part of the published package (`files` in `package.json`).
|
|
515
|
+
|
|
516
|
+
---
|
|
517
|
+
|
|
518
|
+
## License
|
|
519
|
+
|
|
520
|
+
MIT — see `LICENSE`.
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Backup helpers.
|
|
3
|
+
*
|
|
4
|
+
* exportCells(filter) async iterable of cells from durable
|
|
5
|
+
* rebuildCache(filter) re-warm the cache from durable for a window
|
|
6
|
+
*
|
|
7
|
+
* Both are bounded by the (scope, counterId, timeframe) filter the caller
|
|
8
|
+
* provides; we never iterate the entire collection.
|
|
9
|
+
*/
|
|
10
|
+
import type { CacheStorage, CounterCell, DurableStorage, QuerySpec } from "../types/index.js";
|
|
11
|
+
export type ExportFilter = Pick<QuerySpec, "scope" | "counterId" | "timeframe">;
|
|
12
|
+
export declare function exportCells(durable: DurableStorage, filter: ExportFilter, signal?: AbortSignal): AsyncIterable<CounterCell>;
|
|
13
|
+
export declare function rebuildCache(durable: DurableStorage, cache: CacheStorage, filter: ExportFilter, options?: {
|
|
14
|
+
batchSize?: number;
|
|
15
|
+
signal?: AbortSignal;
|
|
16
|
+
}): Promise<{
|
|
17
|
+
cellsWarmed: number;
|
|
18
|
+
}>;
|
|
19
|
+
//# sourceMappingURL=export.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"export.d.ts","sourceRoot":"","sources":["../../src/backup/export.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EACV,YAAY,EACZ,WAAW,EACX,cAAc,EACd,SAAS,EACV,MAAM,mBAAmB,CAAC;AAE3B,MAAM,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,EAAE,OAAO,GAAG,WAAW,GAAG,WAAW,CAAC,CAAC;AAEhF,wBAAgB,WAAW,CACzB,OAAO,EAAE,cAAc,EACvB,MAAM,EAAE,YAAY,EACpB,MAAM,CAAC,EAAE,WAAW,GACnB,aAAa,CAAC,WAAW,CAAC,CAE5B;AAED,wBAAsB,YAAY,CAChC,OAAO,EAAE,cAAc,EACvB,KAAK,EAAE,YAAY,EACnB,MAAM,EAAE,YAAY,EACpB,OAAO,GAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,WAAW,CAAA;CAAO,GACzD,OAAO,CAAC;IAAE,WAAW,EAAE,MAAM,CAAA;CAAE,CAAC,CAoBlC"}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Backup helpers.
|
|
4
|
+
*
|
|
5
|
+
* exportCells(filter) async iterable of cells from durable
|
|
6
|
+
* rebuildCache(filter) re-warm the cache from durable for a window
|
|
7
|
+
*
|
|
8
|
+
* Both are bounded by the (scope, counterId, timeframe) filter the caller
|
|
9
|
+
* provides; we never iterate the entire collection.
|
|
10
|
+
*/
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.exportCells = exportCells;
|
|
13
|
+
exports.rebuildCache = rebuildCache;
|
|
14
|
+
function exportCells(durable, filter, signal) {
|
|
15
|
+
return durable.streamCells(filter, signal ? { signal } : undefined);
|
|
16
|
+
}
|
|
17
|
+
async function rebuildCache(durable, cache, filter, options = {}) {
|
|
18
|
+
const batchSize = options.batchSize ?? 1000;
|
|
19
|
+
let warmed = 0;
|
|
20
|
+
let batch = [];
|
|
21
|
+
const opts = options.signal ? { signal: options.signal } : undefined;
|
|
22
|
+
for await (const cell of durable.streamCells(filter, opts)) {
|
|
23
|
+
batch.push(cell);
|
|
24
|
+
if (batch.length >= batchSize) {
|
|
25
|
+
await cache.warm(batch, opts);
|
|
26
|
+
warmed += batch.length;
|
|
27
|
+
batch = [];
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
if (batch.length > 0) {
|
|
31
|
+
await cache.warm(batch, opts);
|
|
32
|
+
warmed += batch.length;
|
|
33
|
+
}
|
|
34
|
+
return { cellsWarmed: warmed };
|
|
35
|
+
}
|
|
36
|
+
//# sourceMappingURL=export.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"export.js","sourceRoot":"","sources":["../../src/backup/export.ts"],"names":[],"mappings":";AAAA;;;;;;;;GAQG;;AAWH,kCAMC;AAED,oCAyBC;AAjCD,SAAgB,WAAW,CACzB,OAAuB,EACvB,MAAoB,EACpB,MAAoB;IAEpB,OAAO,OAAO,CAAC,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;AACtE,CAAC;AAEM,KAAK,UAAU,YAAY,CAChC,OAAuB,EACvB,KAAmB,EACnB,MAAoB,EACpB,UAAwD,EAAE;IAE1D,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC;IAC5C,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,KAAK,GAAkB,EAAE,CAAC;IAE9B,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IAErE,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,OAAO,CAAC,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC;QAC3D,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjB,IAAI,KAAK,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;YAC9B,MAAM,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC;YACvB,KAAK,GAAG,EAAE,CAAC;QACb,CAAC;IACH,CAAC;IACD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,MAAM,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC;IACzB,CAAC;IACD,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC;AACjC,CAAC"}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Local delta buffer.
|
|
3
|
+
*
|
|
4
|
+
* Each app instance keeps just a Map<cellKey, delta>, not full state. On flush
|
|
5
|
+
* we drain the buffer into a snapshot, send it to durable storage, and only
|
|
6
|
+
* then forget those deltas. If the durable write fails, we merge the snapshot
|
|
7
|
+
* back into the buffer so no counts are lost.
|
|
8
|
+
*/
|
|
9
|
+
import type { CellIdentity, PendingDelta } from "../types/index.js";
|
|
10
|
+
export declare class DeltaBuffer {
|
|
11
|
+
private readonly map;
|
|
12
|
+
/** Number of source events accumulated since last flush (for flush triggers). */
|
|
13
|
+
private events;
|
|
14
|
+
size(): number;
|
|
15
|
+
eventCount(): number;
|
|
16
|
+
noteEvent(): void;
|
|
17
|
+
/** Add a delta to the cell identified by `identity`. */
|
|
18
|
+
add(identity: CellIdentity, delta: number): void;
|
|
19
|
+
/**
|
|
20
|
+
* Take the current contents as a flush snapshot. The buffer is cleared
|
|
21
|
+
* atomically — if you need to re-merge on failure, call `mergeBack`.
|
|
22
|
+
*/
|
|
23
|
+
drain(): PendingDelta[];
|
|
24
|
+
/** Re-merge a previously drained snapshot back into the buffer (failure recovery). */
|
|
25
|
+
mergeBack(snapshot: PendingDelta[]): void;
|
|
26
|
+
}
|
|
27
|
+
//# sourceMappingURL=buffer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"buffer.d.ts","sourceRoot":"","sources":["../../src/core/buffer.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAQpE,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAyC;IAC7D,iFAAiF;IACjF,OAAO,CAAC,MAAM,CAAK;IAEnB,IAAI,IAAI,MAAM;IAId,UAAU,IAAI,MAAM;IAIpB,SAAS,IAAI,IAAI;IAIjB,wDAAwD;IACxD,GAAG,CAAC,QAAQ,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAahD;;;OAGG;IACH,KAAK,IAAI,YAAY,EAAE;IAUvB,sFAAsF;IACtF,SAAS,CAAC,QAAQ,EAAE,YAAY,EAAE,GAAG,IAAI;CAK1C"}
|