@x12i/countex 1.5.0 → 1.6.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +138 -21
- package/dist/backup/manifest.d.ts +22 -1
- package/dist/backup/manifest.d.ts.map +1 -1
- package/dist/backup/manifest.js +34 -6
- package/dist/backup/manifest.js.map +1 -1
- package/dist/core/countex.d.ts +66 -1
- package/dist/core/countex.d.ts.map +1 -1
- package/dist/core/countex.js +113 -4
- package/dist/core/countex.js.map +1 -1
- package/dist/index.d.ts +4 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -1
- package/dist/index.js.map +1 -1
- package/dist/query/planner.d.ts +15 -1
- package/dist/query/planner.d.ts.map +1 -1
- package/dist/query/planner.js +136 -1
- package/dist/query/planner.js.map +1 -1
- package/dist/storage/coverage.d.ts +13 -0
- package/dist/storage/coverage.d.ts.map +1 -0
- package/dist/storage/coverage.js +37 -0
- package/dist/storage/coverage.js.map +1 -0
- package/dist/storage/memory.d.ts +20 -5
- package/dist/storage/memory.d.ts.map +1 -1
- package/dist/storage/memory.js +127 -3
- package/dist/storage/memory.js.map +1 -1
- package/dist/storage/redis.d.ts +36 -11
- package/dist/storage/redis.d.ts.map +1 -1
- package/dist/storage/redis.js +189 -41
- package/dist/storage/redis.js.map +1 -1
- package/dist/types/index.d.ts +114 -7
- package/dist/types/index.d.ts.map +1 -1
- package/docs/DEPLOYMENT.md +87 -3
- package/docs/guides/adapter-redis.md +40 -4
- package/docs/guides/backup-export.md +84 -32
- package/docs/guides/counter-maps.md +8 -0
- package/docs/guides/preview.md +92 -0
- package/docs/guides/querying.md +30 -0
- package/docs/internals/internal-storage-mongo.md +24 -4
- package/docs/internals/internal-storage-redis.md +41 -9
- package/package.json +14 -2
package/docs/DEPLOYMENT.md
CHANGED
|
@@ -158,7 +158,12 @@ On **`init()`**, the adapter **ensures indexes** on each DB it touches (unless *
|
|
|
158
158
|
|
|
159
159
|
1. Provide an **`ioredis`** client instance.
|
|
160
160
|
2. Set **`keyPrefix`** to avoid collisions with other apps on shared Redis.
|
|
161
|
-
3. Set **`ttlSeconds`** for cache eviction consistent with your retention (example code uses ~3 months).
|
|
161
|
+
3. Set **`ttlSeconds`** for cache eviction consistent with your retention (example code uses ~3 months), or pass `retention.redis` on `createCountex` to forward per-timeframe TTLs to the cache adapter.
|
|
162
|
+
|
|
163
|
+
**Required Redis configuration:**
|
|
164
|
+
|
|
165
|
+
- **`maxmemory-policy noeviction`** — silent eviction breaks per-bucket coverage assumptions: a half-evicted bucket would still report `:cov.status=complete` while the cells hash is gone, producing incorrect "full" coverage reads. Use `noeviction` and provision Redis memory above your hot working set; rely on TTLs (not LRU) for expiry.
|
|
166
|
+
- Plan memory headroom = (cells per hot bucket × bucket count for hottest timeframe × per-cell overhead) + 25%.
|
|
162
167
|
|
|
163
168
|
**Idempotency:** If your map defines `idempotency` with `eventIdPath`, Countex calls **`cache.markEventSeen`** before buffering. Ensure Redis is available for dedupe semantics you rely on.
|
|
164
169
|
|
|
@@ -211,13 +216,66 @@ Subscribe to **`Countex`** events (`countex.on(...)`) for:
|
|
|
211
216
|
- **`flush_error`** — durable failure (buffer re-merged).
|
|
212
217
|
- **`duplicate`** — idempotent skip.
|
|
213
218
|
- **`cardinality_warning`**, **`overflow`**, **`dropped`** — mapping/cardinality/dictionary issues.
|
|
219
|
+
- **`measure_invalid`** — per-measure extraction failures (missing, non-numeric, non-finite, negative-for-unsigned, precision-truncated).
|
|
214
220
|
|
|
215
221
|
Wire these to your logger/metrics stack.
|
|
216
222
|
|
|
223
|
+
**Recommended monitoring signals:**
|
|
224
|
+
|
|
225
|
+
| Signal | What to watch |
|
|
226
|
+
|---|---|
|
|
227
|
+
| `flush.cells` per minute | Workload volume |
|
|
228
|
+
| `flush.durationMs` p50 / p95 | Mongo write latency under load |
|
|
229
|
+
| `flush_error` rate | Durable failover; deltas re-merged into the buffer |
|
|
230
|
+
| `dropped.reason` cardinality vs cardinality:* | Map cardinality budgets |
|
|
231
|
+
| `duplicate` rate | Idempotency dedupe effectiveness; webhook replay rate |
|
|
232
|
+
| `countex.diag().bufferCells` | Backpressure signal — rising = flush can't keep up |
|
|
233
|
+
| Redis `maxmemory_used` | Stay below maxmemory; rising eviction count means policy drift |
|
|
234
|
+
| Mongo `countex_cells` doc count growth per timeframe | Capacity planning, retention triggers |
|
|
235
|
+
| Query `result.diagnostics.cacheCoverage` distribution | Cache effectiveness; lots of `partial`/`miss` means undersized hot window |
|
|
236
|
+
|
|
237
|
+
**Worker crash-loss window:** the only state lost on a hard worker crash is **unflushed deltas in memory**. Bound this window with `flush.everyMs`, `flush.maxBufferedCells`, and `flush.maxBufferedEvents`. With `flush.everyMs: 60_000` and `maxBufferedCells: 100_000`, expected loss is `min(60s of writes, 100k cells)`. Tune lower for high-volume / low-tolerance workloads.
|
|
238
|
+
|
|
217
239
|
### 9. Retention (Mongo + Redis)
|
|
218
240
|
|
|
219
|
-
|
|
220
|
-
|
|
241
|
+
Prefer the unified **`CountexConfig.retention`** policy:
|
|
242
|
+
|
|
243
|
+
```ts
|
|
244
|
+
const countex = createCountex({
|
|
245
|
+
// …
|
|
246
|
+
retention: {
|
|
247
|
+
redis: {
|
|
248
|
+
minute: "2d",
|
|
249
|
+
hour: "30d",
|
|
250
|
+
day: "92d",
|
|
251
|
+
week: "52w",
|
|
252
|
+
month: "24mo",
|
|
253
|
+
quarter: "12q",
|
|
254
|
+
year: "7y",
|
|
255
|
+
all: "forever",
|
|
256
|
+
},
|
|
257
|
+
// Mongo entries are documented today, not enforced.
|
|
258
|
+
mongo: { day: "365d", month: "60mo", all: "forever" },
|
|
259
|
+
},
|
|
260
|
+
});
|
|
261
|
+
```
|
|
262
|
+
|
|
263
|
+
- **Redis** is ephemeral: the engine forwards `retention.redis` to the cache adapter via `setRetentionPolicy(...)`, overriding any constructor-time `ttlSecondsByTimeframe`. After TTL expiry, the bucket disappears, reads fall back to Mongo, and the planner re-warms the cache. Combined with `noeviction`, this is the only authoritative way buckets leave Redis.
|
|
264
|
+
- **MongoDB** is the system of record. Plan archival or TTL **per your compliance window** — Countex does not delete Mongo rows automatically. Operators implement Mongo retention themselves:
|
|
265
|
+
|
|
266
|
+
```js
|
|
267
|
+
// Option 1: TTL index on updatedAt (per-collection, drops cells after N seconds).
|
|
268
|
+
db.countex_cells.createIndex({ updatedAt: 1 }, { expireAfterSeconds: 365 * 86400 });
|
|
269
|
+
|
|
270
|
+
// Option 2: scheduled prune by timeframeKey range (per-counter retention window).
|
|
271
|
+
db.countex_cells.deleteMany({
|
|
272
|
+
counterId: "events",
|
|
273
|
+
timeframeType: "day",
|
|
274
|
+
timeframeKey: { $lt: "2025-05-01" },
|
|
275
|
+
});
|
|
276
|
+
```
|
|
277
|
+
|
|
278
|
+
Large tenants may use **per-scope databases** via `dbResolver` and drop/archive databases independently.
|
|
221
279
|
|
|
222
280
|
Cell documents written by `MongoDurable` include optional **`schemaVersion`** for future migrations (`MONGO_CELL_SCHEMA_VERSION` export).
|
|
223
281
|
|
|
@@ -240,6 +298,32 @@ Stream **`exportCells`**, attach a **`BackupManifest`** (`computeCellsFingerprin
|
|
|
240
298
|
|
|
241
299
|
Cross-link: [`docs/use-cases/multi-tenant-isolation.md`](./use-cases/multi-tenant-isolation.md) — pin **`scope`** on every query, isolate Mongo via **`dbResolver`**, and use separate Redis key prefixes per environment.
|
|
242
300
|
|
|
301
|
+
**Tenant DB naming rules** (when using `dbResolver`):
|
|
302
|
+
|
|
303
|
+
- Mongo DB names are 1–63 characters, must not contain `/ \ . " $ * < > : | ?` or null bytes, and on Windows are also case-insensitive (so two scopes that differ only in case collide). Sanitize **before** passing to `dbResolver`:
|
|
304
|
+
|
|
305
|
+
```ts
|
|
306
|
+
function sanitizeScope(s: string): string {
|
|
307
|
+
return s.toLowerCase().replace(/[^a-z0-9_-]/g, "_").slice(0, 50);
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
const durable = new MongoDurable({
|
|
311
|
+
client: mongo,
|
|
312
|
+
dbResolver: ({ scope }) => `countex_${sanitizeScope(scope)}`,
|
|
313
|
+
});
|
|
314
|
+
```
|
|
315
|
+
|
|
316
|
+
- For very high tenant counts (>1k tenants), prefer a **single shared DB with indexed `scopeId`** over DB-per-tenant — Mongo has per-DB metadata overhead (`countex_cells`, `countex_dict`, `countex_seq` ≈ 6 indexes per tenant DB).
|
|
317
|
+
- For untrusted scope sources (raw user input, URLs, emails) prefer **hashing** rather than sanitizing:
|
|
318
|
+
|
|
319
|
+
```ts
|
|
320
|
+
import crypto from "node:crypto";
|
|
321
|
+
const hashScope = (s: string) => crypto.createHash("sha1").update(s).digest("hex").slice(0, 16);
|
|
322
|
+
dbResolver: ({ scope }) => `countex_${hashScope(scope)}`;
|
|
323
|
+
```
|
|
324
|
+
|
|
325
|
+
- Sanity-check that two distinct sanitized names cannot collide in your dataset (e.g. `org/123` and `org_123` both → `org_123`). Hash if collision is plausible.
|
|
326
|
+
|
|
243
327
|
### 13. Failure modes (short)
|
|
244
328
|
|
|
245
329
|
| Situation | Behavior |
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Guide: Redis cache adapter
|
|
2
2
|
|
|
3
|
-
`RedisCache` stores hot counter cells for fast reads. It pairs with a durable store (typically Mongo).
|
|
3
|
+
`RedisCache` stores hot counter cells for fast reads. It pairs with a durable store (typically Mongo) and tracks per-bucket coverage so the query planner only treats fully-warmed buckets as cache-authoritative.
|
|
4
4
|
|
|
5
5
|
## Step 1: Install peer dependency
|
|
6
6
|
|
|
@@ -26,6 +26,8 @@ const cache = new RedisCache({
|
|
|
26
26
|
hour: 14 * 24 * 3600,
|
|
27
27
|
day: 92 * 24 * 3600,
|
|
28
28
|
},
|
|
29
|
+
// Optional opaque map version stamped onto bucket coverage on warm()
|
|
30
|
+
mapVersion: 1,
|
|
29
31
|
});
|
|
30
32
|
|
|
31
33
|
const countex = createCountex({
|
|
@@ -37,15 +39,49 @@ const countex = createCountex({
|
|
|
37
39
|
await countex.init();
|
|
38
40
|
```
|
|
39
41
|
|
|
40
|
-
## Step 3:
|
|
42
|
+
## Step 3: Unified retention via `CountexConfig.retention`
|
|
41
43
|
|
|
42
|
-
|
|
44
|
+
Prefer the engine-level `retention` config so the same policy is enforced regardless of which cache adapter is used:
|
|
43
45
|
|
|
44
|
-
|
|
46
|
+
```ts
|
|
47
|
+
const countex = createCountex({
|
|
48
|
+
maps: [myMap],
|
|
49
|
+
durable,
|
|
50
|
+
cache,
|
|
51
|
+
retention: {
|
|
52
|
+
redis: {
|
|
53
|
+
minute: "2d",
|
|
54
|
+
hour: "30d",
|
|
55
|
+
day: "92d",
|
|
56
|
+
week: "52w",
|
|
57
|
+
month: "24mo",
|
|
58
|
+
quarter: "12q",
|
|
59
|
+
year: "7y",
|
|
60
|
+
all: "forever",
|
|
61
|
+
},
|
|
62
|
+
// Mongo retention is operator-managed today (TTL index or scheduled prune).
|
|
63
|
+
},
|
|
64
|
+
});
|
|
65
|
+
```
|
|
45
66
|
|
|
67
|
+
The engine forwards the resolved per-timeframe TTLs to `cache.setRetentionPolicy(...)` at construction. Engine policy overrides the adapter's own `ttlSecondsByTimeframe`.
|
|
68
|
+
|
|
69
|
+
Accepted duration strings: `<n><unit>` where unit is `s|m|h|d|w|mo|q|y`, or `"forever"` for no expiry. Plain numbers are seconds.
|
|
70
|
+
|
|
71
|
+
## Step 4: Operations
|
|
72
|
+
|
|
73
|
+
- **Production Redis must use `maxmemory-policy noeviction`.** Silent eviction breaks the per-bucket "complete" coverage assumption — a half-evicted bucket can no longer be served from cache, and silent eviction means the `:cov` key may report `complete` while the cells hash is gone.
|
|
46
74
|
- Cache misses fall back to durable reads; results may warm the cache afterward.
|
|
47
75
|
- Cache flush failures during write path are **non-fatal** (durable is source of truth).
|
|
48
76
|
|
|
77
|
+
## Step 5: Idempotency hook
|
|
78
|
+
|
|
79
|
+
When maps declare **idempotency**, the cache adapter implements **`markEventSeen`** so duplicates are skipped before buffering.
|
|
80
|
+
|
|
81
|
+
## Step 6: Per-bucket coverage
|
|
82
|
+
|
|
83
|
+
`RedisCache` maintains a `:cov` HASH per bucket with `status` (`complete | partial | rebuilding`), `lastWarmAt`, `cellCount`, `schemaVersion`, and optional `mapVersion`. The query planner reads it to know which buckets the cache can answer with confidence; details in [internals/internal-storage-redis.md](../internals/internal-storage-redis.md).
|
|
84
|
+
|
|
49
85
|
## See also
|
|
50
86
|
|
|
51
87
|
- [Idempotency](./idempotency.md)
|
|
@@ -1,8 +1,10 @@
|
|
|
1
|
-
# Guide: Backup
|
|
1
|
+
# Guide: Backup, manifest, verify, restore, and cache rebuild
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
`@x12i/countex` ships export, fingerprint, manifest, verify, restore, and cache-rebuild helpers. Use them as free functions or via the `countex.backup` namespace bound to your engine.
|
|
4
4
|
|
|
5
|
-
## Step 1: Export cells
|
|
5
|
+
## Step 1: Export cells (stream)
|
|
6
|
+
|
|
7
|
+
Free function:
|
|
6
8
|
|
|
7
9
|
```ts
|
|
8
10
|
import { exportCells } from "@x12i/countex";
|
|
@@ -12,36 +14,34 @@ for await (const cell of exportCells(durable, {
|
|
|
12
14
|
counterId: "events",
|
|
13
15
|
timeframe: { type: "day", from: "2026-05-01", to: "2026-05-07" },
|
|
14
16
|
})) {
|
|
15
|
-
// serialize
|
|
17
|
+
// serialize to NDJSON / S3 / cold storage…
|
|
16
18
|
}
|
|
17
19
|
```
|
|
18
20
|
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
## Step 2: Rebuild cache after outage
|
|
21
|
+
Namespace:
|
|
22
22
|
|
|
23
23
|
```ts
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
const { cellsWarmed } = await rebuildCache(durable, cache, {
|
|
24
|
+
for await (const cell of countex.backup.export({
|
|
27
25
|
scope: "org_1",
|
|
28
26
|
counterId: "events",
|
|
29
27
|
timeframe: { type: "day", from: "2026-05-01", to: "2026-05-07" },
|
|
30
|
-
}
|
|
28
|
+
})) {
|
|
29
|
+
// …
|
|
30
|
+
}
|
|
31
31
|
```
|
|
32
32
|
|
|
33
|
-
|
|
33
|
+
Streaming is bounded by the `(scope, counterId, timeframe)` filter you provide.
|
|
34
|
+
|
|
35
|
+
## Step 2: Fingerprint + manifest
|
|
36
|
+
|
|
37
|
+
The fingerprint sorts canonical JSON lines in memory and computes:
|
|
34
38
|
|
|
35
|
-
|
|
39
|
+
- `cellCount` — number of cells in the stream.
|
|
40
|
+
- `contentSha256` — SHA-256 of the sorted canonical lines.
|
|
41
|
+
- `countSum` — sum of `count` across all cells (sanity-check field).
|
|
36
42
|
|
|
37
43
|
```ts
|
|
38
|
-
import {
|
|
39
|
-
computeCellsFingerprint,
|
|
40
|
-
createBackupManifest,
|
|
41
|
-
verifyBackupAgainstManifest,
|
|
42
|
-
exportCells,
|
|
43
|
-
COUNTEX_CELL_SCHEMA_VERSION,
|
|
44
|
-
} from "@x12i/countex";
|
|
44
|
+
import { computeCellsFingerprint, createBackupManifest, COUNTEX_CELL_SCHEMA_VERSION } from "@x12i/countex";
|
|
45
45
|
|
|
46
46
|
const filter = {
|
|
47
47
|
scope: "org_1",
|
|
@@ -51,33 +51,85 @@ const filter = {
|
|
|
51
51
|
|
|
52
52
|
const fp = await computeCellsFingerprint(exportCells(durable, filter));
|
|
53
53
|
const manifest = createBackupManifest(filter, fp, COUNTEX_CELL_SCHEMA_VERSION);
|
|
54
|
-
// persist manifest JSON
|
|
54
|
+
// persist manifest JSON next to the NDJSON dump…
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
Resulting manifest:
|
|
58
|
+
|
|
59
|
+
```json
|
|
60
|
+
{
|
|
61
|
+
"manifestVersion": 1,
|
|
62
|
+
"schemaVersion": 1,
|
|
63
|
+
"createdAtIso": "2026-05-09T08:00:00.000Z",
|
|
64
|
+
"filter": { "scope": "org_1", "counterId": "events", "timeframe": { "type": "day", "from": "2026-05-01", "to": "2026-05-07" } },
|
|
65
|
+
"cellCount": 1234,
|
|
66
|
+
"contentSha256": "…",
|
|
67
|
+
"countSum": 98765
|
|
68
|
+
}
|
|
69
|
+
```
|
|
55
70
|
|
|
56
|
-
|
|
71
|
+
## Step 3: Verify
|
|
72
|
+
|
|
73
|
+
```ts
|
|
74
|
+
import { verifyBackupAgainstManifest } from "@x12i/countex";
|
|
75
|
+
|
|
76
|
+
const check = await verifyBackupAgainstManifest(manifest, exportCells(durable, manifest.filter));
|
|
57
77
|
if (!check.ok) throw new Error(check.detail);
|
|
58
78
|
```
|
|
59
79
|
|
|
60
|
-
|
|
80
|
+
Or via the namespace, which uses the engine's durable adapter:
|
|
61
81
|
|
|
62
|
-
|
|
82
|
+
```ts
|
|
83
|
+
const check = await countex.backup.verify(manifest);
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
`verify` checks `cellCount`, `contentSha256`, and (when present) `countSum`. Any mismatch returns `{ ok: false, detail }`.
|
|
87
|
+
|
|
88
|
+
## Step 4: Restore (modes)
|
|
89
|
+
|
|
90
|
+
```ts
|
|
91
|
+
import { restoreSnapshotCells } from "@x12i/countex";
|
|
92
|
+
|
|
93
|
+
const { restored, mode } = await restoreSnapshotCells(
|
|
94
|
+
targetDurable,
|
|
95
|
+
exportCells(sourceDurable, filter),
|
|
96
|
+
{ mode: "replace-window", batchSize: 500 }
|
|
97
|
+
);
|
|
98
|
+
```
|
|
63
99
|
|
|
64
|
-
|
|
100
|
+
Or via the namespace:
|
|
65
101
|
|
|
66
102
|
```ts
|
|
67
|
-
|
|
103
|
+
await countex.backup.restore(countex.backup.export(filter), { mode: "replace-window" });
|
|
104
|
+
```
|
|
68
105
|
|
|
69
|
-
|
|
70
|
-
|
|
106
|
+
Modes:
|
|
107
|
+
|
|
108
|
+
| Mode | Behavior |
|
|
109
|
+
|---|---|
|
|
110
|
+
| `validate-only` | Count cells without writing. Equivalent to legacy `dryRun: true`. |
|
|
111
|
+
| `replace-window` | Replace each cell identity in the stream via `importSnapshotCells`. Default. |
|
|
112
|
+
| `merge-deltas` | **Reserved.** Throws `NotImplementedError` today. |
|
|
113
|
+
|
|
114
|
+
After a `replace-window` restore, **rebuild Redis** for the affected windows so coverage flips back to `complete`.
|
|
115
|
+
|
|
116
|
+
## Step 5: Cache rebuild
|
|
117
|
+
|
|
118
|
+
```ts
|
|
119
|
+
const { cellsWarmed } = await countex.backup.rebuildCache({
|
|
120
|
+
scope: "org_1",
|
|
121
|
+
counterId: "events",
|
|
122
|
+
timeframe: { type: "day", from: "2026-05-01", to: "2026-05-07" },
|
|
71
123
|
});
|
|
72
|
-
// dryRun: true counts cells without writing
|
|
73
124
|
```
|
|
74
125
|
|
|
75
|
-
|
|
126
|
+
`rebuildCache` streams `streamCells` from durable in batches and calls `cache.warm(batch)`. Each warmed bucket's `:cov` hash flips to `status=complete`.
|
|
76
127
|
|
|
77
|
-
##
|
|
128
|
+
## Cancellation
|
|
78
129
|
|
|
79
|
-
Pass `AbortSignal` where supported (`streamCells` / `warm` /
|
|
130
|
+
Pass `AbortSignal` where supported (`streamCells` / `warm` / `restoreSnapshotCells` options propagate through adapters).
|
|
80
131
|
|
|
81
132
|
## See also
|
|
82
133
|
|
|
83
134
|
- Internal: [Backup streaming](../internals/internal-backup-stream.md)
|
|
135
|
+
- Internal: [Mongo durable](../internals/internal-storage-mongo.md) — `importSnapshotCells` semantics.
|
|
@@ -114,9 +114,17 @@ Total raw increments per event ≈
|
|
|
114
114
|
|
|
115
115
|
Plan capacity accordingly.
|
|
116
116
|
|
|
117
|
+
## Preview before ingesting
|
|
118
|
+
|
|
119
|
+
Run `countex.preview(event, { counterId })` on a sample event to see exactly what cells your map would generate — without any dictionary writes or buffering. See [Preview a map](./preview.md) for the full result shape (`subjectSourceKey`, `metricSourceKey`, `dimensionsHash`, `timeframes`, `delta`, `measures`, plus `extractionWarnings` and `droppedAfterCardinality`).
|
|
120
|
+
|
|
121
|
+
Dimension keys/values feeding into `dimensionsHash` are canonicalized: keys sorted lexicographically, values type-tagged, hashed (SHA-1 truncated to 12 hex). Any two workers always produce the same hash for the same logical dimension tuple. See [internal-dimension-hash.md](../internals/internal-dimension-hash.md) for the exact recipe.
|
|
122
|
+
|
|
117
123
|
## See also
|
|
118
124
|
|
|
119
125
|
- [Timeframes](./timeframes-and-bucketing.md)
|
|
120
126
|
- [Measures](./measures.md)
|
|
121
127
|
- [Cardinality](./cardinality.md)
|
|
128
|
+
- [Preview a map](./preview.md)
|
|
122
129
|
- Internal: [Mapping validation & extraction](../internals/internal-mapping-validation-and-extraction.md)
|
|
130
|
+
- Internal: [Dimension hash](../internals/internal-dimension-hash.md)
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
# Guide: Preview a map (dry-run)
|
|
2
|
+
|
|
3
|
+
`countex.preview(event, options?)` runs **extraction + cardinality** for one event against your maps and returns the increments it _would_ produce — without dictionary I/O, without buffering, without any storage writes. Use it to debug a counter map before ingesting millions of events.
|
|
4
|
+
|
|
5
|
+
## Step 1: Construct a Countex instance
|
|
6
|
+
|
|
7
|
+
```ts
|
|
8
|
+
import { createCountex } from "@x12i/countex";
|
|
9
|
+
import { MemoryDurable } from "@x12i/countex/adapters/memory";
|
|
10
|
+
|
|
11
|
+
const countex = createCountex({
|
|
12
|
+
maps: [accessMap],
|
|
13
|
+
durable: new MemoryDurable(),
|
|
14
|
+
flush: { everyMs: 0 },
|
|
15
|
+
});
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
A real durable adapter is fine too — preview never calls it.
|
|
19
|
+
|
|
20
|
+
## Step 2: Run preview
|
|
21
|
+
|
|
22
|
+
```ts
|
|
23
|
+
const previews = countex.preview(event, {
|
|
24
|
+
counterId: "access_by_subject_metric", // optional; defaults to all maps
|
|
25
|
+
});
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
`previews` is an array of `MapPreviewResult`, one per map.
|
|
29
|
+
|
|
30
|
+
## Result shape
|
|
31
|
+
|
|
32
|
+
```ts
|
|
33
|
+
interface MapPreviewResult {
|
|
34
|
+
counterId: string;
|
|
35
|
+
skipped: boolean;
|
|
36
|
+
extractionWarnings: ExtractionWarning[];
|
|
37
|
+
cells: PreviewIncrementCell[];
|
|
38
|
+
/** Increments dropped by cardinality guard. Preview runs silently — no `dropped` events. */
|
|
39
|
+
droppedAfterCardinality: number;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
interface PreviewIncrementCell {
|
|
43
|
+
scopeId: string;
|
|
44
|
+
counterId: string;
|
|
45
|
+
timeframeType: TimeframeType;
|
|
46
|
+
timeframeKey: string;
|
|
47
|
+
subjectType: string;
|
|
48
|
+
subjectSourceKey: string;
|
|
49
|
+
metricSourceKey: string;
|
|
50
|
+
dimensions: Dimensions;
|
|
51
|
+
dimensionsHash: string;
|
|
52
|
+
delta: number;
|
|
53
|
+
measures: Record<string, { value: number; aggs: MeasureAgg[] }>;
|
|
54
|
+
}
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
`subjectSourceKey` and `metricSourceKey` are the raw strings extracted from the event — preview does not allocate dictionary ids. `dimensionsHash` matches the canonical form computed during real ingestion (see [internal-dimension-hash.md](../internals/internal-dimension-hash.md)).
|
|
58
|
+
|
|
59
|
+
## Counting fan-out
|
|
60
|
+
|
|
61
|
+
```ts
|
|
62
|
+
const total = previews.reduce((s, p) => s + p.cells.length, 0);
|
|
63
|
+
console.log(`This event would generate ${total} cell increments.`);
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
For the canonical example (1 actor × 2 metrics × 5 timeframes), `total === 10`.
|
|
67
|
+
|
|
68
|
+
## Diagnostics: warnings and skips
|
|
69
|
+
|
|
70
|
+
`extractionWarnings` includes things like `"measure_invalid"`, `"measure_missing"`, missing required dimensions, and bad path resolutions. Use them to spot a misconfigured map without having to ingest and watch event listeners.
|
|
71
|
+
|
|
72
|
+
```ts
|
|
73
|
+
for (const p of previews) {
|
|
74
|
+
if (p.skipped) console.warn(`map "${p.counterId}" skipped this event`);
|
|
75
|
+
for (const w of p.extractionWarnings) {
|
|
76
|
+
console.warn(`extraction warning: ${w.reason}`, w.meta);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
## Caveats
|
|
82
|
+
|
|
83
|
+
- **Cardinality guard state is shared with the live engine.** Preview runs the same `CardinalityGuard.check`, so calling preview repeatedly counts toward the per-scope cardinality limits. For isolated dry-runs, construct a separate `Countex` instance just for previewing.
|
|
84
|
+
- **No idempotency check.** Preview ignores `idempotency` — same event can be previewed any number of times.
|
|
85
|
+
- **No measure scaling on output.** `measures[name].value` is the raw extracted number; on the live ingest path it would be multiplied by `measure.scale` before persistence.
|
|
86
|
+
- **No dropped/overflow events emitted.** Use `droppedAfterCardinality` and `extractionWarnings` instead.
|
|
87
|
+
|
|
88
|
+
## See also
|
|
89
|
+
|
|
90
|
+
- [Counter maps guide](./counter-maps.md)
|
|
91
|
+
- [Cardinality guide](./cardinality.md)
|
|
92
|
+
- Internal: [Mapping validation & extraction](../internals/internal-mapping-validation-and-extraction.md)
|
package/docs/guides/querying.md
CHANGED
|
@@ -81,9 +81,39 @@ Result includes `source`: `"cache"`, `"durable"`, or `"merged"`.
|
|
|
81
81
|
| `cacheAttempted` | `true` if the engine consulted the cache adapter (cache configured, not `bypassCache`, and the planner allowed cache for this query shape). |
|
|
82
82
|
| `cacheCoverage` | When `cacheAttempted` is true: `"full"` if the cache answered completely; `"partial"` or `"miss"` if the planner fell through to durable for authoritative cells. |
|
|
83
83
|
| `cellsMaterialized` | Number of counter cells loaded from storage before grouping/formatting. |
|
|
84
|
+
| `bucketsRequested` | Bucket keys the query expanded to (after `bucketRange`). |
|
|
85
|
+
| `cacheBucketsHit` | Number of buckets the cache reported as `complete`. |
|
|
86
|
+
| `durableFallback` | `true` when the planner had to read durable for some/all buckets. |
|
|
87
|
+
| `plannedCellsEstimate` | Heuristic estimate of cells before grouping (matches `cellsMaterialized` after execution). |
|
|
88
|
+
| `groupByCost` | `"trivial"` (no groupBy), `"low"`, or `"high"` (3+ keys or 2+ dimensions). |
|
|
89
|
+
| `expensive` | Heuristic flag — `true` for big windows, high groupByCost, or `format: "matrix"` with many cells. |
|
|
84
90
|
|
|
85
91
|
Use this for tests, dashboards, and verifying Redis vs Mongo paths.
|
|
86
92
|
|
|
93
|
+
### Explain a query without running it
|
|
94
|
+
|
|
95
|
+
`countex.explainQuery(spec)` returns a {@link QueryPlan} without touching durable storage and only probing per-bucket cache coverage:
|
|
96
|
+
|
|
97
|
+
```ts
|
|
98
|
+
const plan = await countex.explainQuery({
|
|
99
|
+
scope: "org_1",
|
|
100
|
+
counterId: "events",
|
|
101
|
+
timeframe: { type: "day", from: "2026-05-01", to: "2026-05-31" },
|
|
102
|
+
groupBy: ["dim:operation", "dim:situation", "subject"],
|
|
103
|
+
});
|
|
104
|
+
// {
|
|
105
|
+
// bucketsRequested: ["2026-05-01", …, "2026-05-31"],
|
|
106
|
+
// cacheEligible: true,
|
|
107
|
+
// cacheBucketsExpectedHit: 18,
|
|
108
|
+
// durableFallbackPredicted: true,
|
|
109
|
+
// plannedCellsEstimate: 1_240,
|
|
110
|
+
// groupByCost: "high",
|
|
111
|
+
// expensive: true,
|
|
112
|
+
// }
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
`cacheBucketsExpectedHit` and `plannedCellsEstimate` are populated when the cache adapter implements `probeBucketCoverage` (the bundled `RedisCache` and `MemoryCache` both do).
|
|
116
|
+
|
|
87
117
|
## Step 6: Totals, event volume, and “distinct entities”
|
|
88
118
|
|
|
89
119
|
### Grand total (sum all matching cells)
|
|
@@ -4,21 +4,41 @@
|
|
|
4
4
|
|
|
5
5
|
- `applyDeltas` — upserts with `$inc` style operations for counts and compatible measure fields.
|
|
6
6
|
- `readCells` — satisfies `QuerySpec` filters using compound indexes.
|
|
7
|
-
- `resolveIds` —
|
|
7
|
+
- `resolveIds` — concurrency-safe allocation of dictionary ids (`countex_dict`, `countex_seq`).
|
|
8
8
|
- `streamCells` — bounded iteration for backup/export.
|
|
9
|
+
- `importSnapshotCells` — replace-window restore target (called by `restoreSnapshotCells`).
|
|
9
10
|
|
|
10
11
|
## Write path interaction
|
|
11
12
|
|
|
12
|
-
Flush snapshots arrive as arrays of `PendingDelta`; Mongo applies them idempotently per cell identity tuple
|
|
13
|
+
Flush snapshots arrive as arrays of `PendingDelta`; Mongo applies them idempotently per cell identity tuple via `bulkWrite` of `updateOne` operations with `$inc` and `upsert: true`.
|
|
13
14
|
|
|
14
15
|
## Indexes
|
|
15
16
|
|
|
16
|
-
Created lazily on first write paths
|
|
17
|
+
Created lazily on first write paths:
|
|
18
|
+
|
|
19
|
+
- `countex_cells` — `uniq_cell_identity` (unique on the full identity tuple), `by_timeframe`, `by_subject`, `by_metric`.
|
|
20
|
+
- `countex_dict` — `uniq_dict_key` (unique on `scopeId, kind, qualifier, sourceKey`).
|
|
21
|
+
|
|
22
|
+
## Dictionary allocation under concurrency
|
|
23
|
+
|
|
24
|
+
`resolveIds` follows a strict five-step recipe so 20+ workers seeing the same new `sourceKey` at once still converge on a single internal id:
|
|
25
|
+
|
|
26
|
+
1. **Read** existing dict rows for the requested `(scopeId, kind, qualifier, sourceKey)` batch in one round-trip.
|
|
27
|
+
2. **Allocate** a contiguous block of ids from the per-scope sequence document via `findOneAndUpdate({ _id }, { $inc: { seq: count } })` — atomic per sequence document.
|
|
28
|
+
3. **Insert** the new dict rows with `insertMany({ ordered: false })`. The `uniq_dict_key` index forbids duplicates.
|
|
29
|
+
4. On **duplicate-key error** (another worker won the race), **re-read** the dict for the same identities — the winning row is now visible.
|
|
30
|
+
5. **Cache** the resolved id in the local LRU and (optionally) Redis.
|
|
31
|
+
|
|
32
|
+
Bad outcome avoided: two workers each minting `user_42 → 10001` and `user_42 → 10002`. With this protocol one insert wins, the loser re-reads the winner's id, and counts never fragment across two ids.
|
|
33
|
+
|
|
34
|
+
The user-facing recipe lives in [`docs/guides/adapter-mongo.md`](../guides/adapter-mongo.md) §"Dictionary allocation (`resolveIds`) and concurrency".
|
|
17
35
|
|
|
18
36
|
## Dictionary vs dimensions
|
|
19
37
|
|
|
20
|
-
Subjects/metrics map to integers
|
|
38
|
+
Subjects/metrics map to integers via `countex_dict`. Dimension VALUES are stored as raw values on cells (not via the dictionary), keeping cells self-describing and avoiding dictionary churn for low-cardinality dimensions.
|
|
21
39
|
|
|
22
40
|
## See also
|
|
23
41
|
|
|
24
42
|
- [Backup streaming](./internal-backup-stream.md)
|
|
43
|
+
- [Dimension hash](./internal-dimension-hash.md) — how the `dimensionsHash` part of the cell identity is computed.
|
|
44
|
+
- [Mongo adapter guide](../guides/adapter-mongo.md)
|
|
@@ -1,18 +1,49 @@
|
|
|
1
1
|
# Internal: Redis cache adapter
|
|
2
2
|
|
|
3
|
-
`RedisCache` (`src/storage/redis.ts`) maps each `(scope, counterId, timeframeType, timeframeKey)` bucket to a **HASH
|
|
3
|
+
`RedisCache` (`src/storage/redis.ts`) maps each `(scope, counterId, timeframeType, timeframeKey)` bucket to a **HASH** of cell entries plus a sibling **`:cov` HASH** of coverage metadata:
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
|
|
5
|
+
```
|
|
6
|
+
countex:{scope}:{counterId}:{tf}:{bucket} → HASH (cells)
|
|
7
|
+
field = "{subjectType}|{subjectId}|{metricId}|{dimensionsHash}"
|
|
8
|
+
value = count (integer)
|
|
9
|
+
field = "{base}:m:{measureName}:sum" → measure sum
|
|
10
|
+
field = "{base}:m:{measureName}:count" → measure event count
|
|
7
11
|
|
|
8
|
-
|
|
12
|
+
countex:{scope}:{counterId}:{tf}:{bucket}:cov → HASH (coverage meta)
|
|
13
|
+
status = "complete" | "partial" | "rebuilding"
|
|
14
|
+
lastWarmAt = epoch ms
|
|
15
|
+
cellCount = number of cells last warmed (set on warm)
|
|
16
|
+
schemaVersion = REDIS_CACHE_SCHEMA_VERSION
|
|
17
|
+
mapVersion = optional opaque integer
|
|
18
|
+
|
|
19
|
+
countex:{scope}:{counterId}:{tf}:{bucket}:meta:{baseField} → JSON of dimensions
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
Per-bucket coverage gives the planner authoritative information about which buckets the cache can answer with confidence, instead of guessing from "did `HGETALL` return anything?".
|
|
23
|
+
|
|
24
|
+
## Coverage state machine
|
|
25
|
+
|
|
26
|
+
| Triggered by | Effect on `:cov.status` |
|
|
27
|
+
|---|---|
|
|
28
|
+
| First `applyDeltas` to a previously-empty bucket | `partial` (via `HSETNX`) — never downgrades a `complete` or `rebuilding` bucket |
|
|
29
|
+
| Subsequent `applyDeltas` | unchanged status; `lastWarmAt` updated |
|
|
30
|
+
| `warm()` | `complete`, with `cellCount`, `schemaVersion`, `mapVersion`, `lastWarmAt` written atomically |
|
|
31
|
+
| `markBucketRebuilding()` | `rebuilding` (called by `rebuildCache` before refilling) |
|
|
32
|
+
| TTL expiry | key disappears → `missing` on next read |
|
|
9
33
|
|
|
10
34
|
## Read coverage
|
|
11
35
|
|
|
12
|
-
`readCells`
|
|
36
|
+
`readCells` pipelines both `HGETALL bucket` and `HGETALL :cov`, then derives per-bucket coverage from the meta:
|
|
37
|
+
|
|
38
|
+
- All buckets `complete` → aggregate `full`.
|
|
39
|
+
- Any `complete | partial | rebuilding` mixed with the rest → aggregate `partial`.
|
|
40
|
+
- All `missing` → aggregate `miss`.
|
|
41
|
+
|
|
42
|
+
When the planner supplies `expectedMapVersion` or `expectedSchemaVersion` and the bucket's stored version differs, `complete` is demoted to `partial` (cache is treated as stale for that bucket). The planner then falls back to durable.
|
|
43
|
+
|
|
44
|
+
## Probe-only path
|
|
13
45
|
|
|
14
|
-
|
|
15
|
-
- `"partial"` / `"miss"` otherwise — planner falls back to durable
|
|
46
|
+
`probeBucketCoverage(scope, counterId, tf, [keys])` returns just the `:cov` hashes without touching cell data. The `Countex.explainQuery(spec)` API uses this to predict the cache hit rate without materializing rows.
|
|
16
47
|
|
|
17
48
|
## Dedupe keys
|
|
18
49
|
|
|
@@ -20,12 +51,13 @@ Metadata hashes may store dimension JSON for reconstruction.
|
|
|
20
51
|
|
|
21
52
|
## Warm path
|
|
22
53
|
|
|
23
|
-
`warm` hydrates hashes from `CounterCell[]` after durable reads.
|
|
54
|
+
`warm` hydrates hashes from `CounterCell[]` after durable reads, then sets `:cov.status=complete`. Used by `rebuildCache` and by the query planner's best-effort warm-after-fallback.
|
|
24
55
|
|
|
25
56
|
## Error stance
|
|
26
57
|
|
|
27
|
-
Apply failures during flush are swallowed at Countex level after durable success — subsequent reads rebuild cache.
|
|
58
|
+
Apply failures during flush are swallowed at Countex level after durable success — subsequent reads rebuild cache. Coverage meta is best-effort: a stale `:cov.status=complete` only causes a temporary divergence (caller sees stale-but-consistent reads until TTL expires).
|
|
28
59
|
|
|
29
60
|
## See also
|
|
30
61
|
|
|
31
62
|
- [Query path](./internal-query-path.md)
|
|
63
|
+
- [Storage memory](./internal-storage-memory.md) — mirrors the same coverage contract for tests.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@x12i/countex",
|
|
3
|
-
"version": "1.5
|
|
3
|
+
"version": "1.6.5",
|
|
4
4
|
"description": "Generic sparse counter matrices for JSON events. Dynamic counting, aggregation, and retrieval from mapped JSON inputs.",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -122,7 +122,18 @@
|
|
|
122
122
|
"build": "tsc",
|
|
123
123
|
"clean": "rm -rf dist",
|
|
124
124
|
"test": "node --test --import=tsx tests/unit/*.test.ts",
|
|
125
|
-
"
|
|
125
|
+
"docker:up": "docker compose -f docker/docker-compose.integration.yml up -d mongo redis minio minio-setup",
|
|
126
|
+
"docker:down": "docker compose -f docker/docker-compose.integration.yml down",
|
|
127
|
+
"docker:reset": "docker compose -f docker/docker-compose.integration.yml down -v && docker compose -f docker/docker-compose.integration.yml up -d mongo redis minio minio-setup",
|
|
128
|
+
"test:integration": "tsx tests/integration/run.ts",
|
|
129
|
+
"test:integration:exec": "tsx tests/integration/exec-integration-tests.ts",
|
|
130
|
+
"integration:up": "tsx scripts/integration-up.ts",
|
|
131
|
+
"integration:down": "tsx scripts/integration-down.ts",
|
|
132
|
+
"integration:reset": "tsx scripts/integration-reset.ts",
|
|
133
|
+
"integration:run": "tsx scripts/integration-run.ts",
|
|
134
|
+
"integration:report": "tsx scripts/integration-report.ts",
|
|
135
|
+
"test:integration:docker": "docker compose -f docker/docker-compose.integration.yml run --rm test-runner",
|
|
136
|
+
"test:e2e": "tsx scripts/integration-e2e.ts",
|
|
126
137
|
"prepublishOnly": "npm run clean && npm run build"
|
|
127
138
|
},
|
|
128
139
|
"peerDependencies": {
|
|
@@ -138,6 +149,7 @@
|
|
|
138
149
|
}
|
|
139
150
|
},
|
|
140
151
|
"devDependencies": {
|
|
152
|
+
"@aws-sdk/client-s3": "^3.787.0",
|
|
141
153
|
"@types/node": "^20.19.40",
|
|
142
154
|
"ioredis": "^5.10.1",
|
|
143
155
|
"mongodb": "^7.2.0",
|