@x12i/countex 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +520 -0
  3. package/dist/backup/export.d.ts +19 -0
  4. package/dist/backup/export.d.ts.map +1 -0
  5. package/dist/backup/export.js +36 -0
  6. package/dist/backup/export.js.map +1 -0
  7. package/dist/core/buffer.d.ts +27 -0
  8. package/dist/core/buffer.d.ts.map +1 -0
  9. package/dist/core/buffer.js +63 -0
  10. package/dist/core/buffer.js.map +1 -0
  11. package/dist/core/cardinality.d.ts +43 -0
  12. package/dist/core/cardinality.d.ts.map +1 -0
  13. package/dist/core/cardinality.js +93 -0
  14. package/dist/core/cardinality.js.map +1 -0
  15. package/dist/core/countex.d.ts +49 -0
  16. package/dist/core/countex.d.ts.map +1 -0
  17. package/dist/core/countex.js +331 -0
  18. package/dist/core/countex.js.map +1 -0
  19. package/dist/core/extraction.d.ts +39 -0
  20. package/dist/core/extraction.d.ts.map +1 -0
  21. package/dist/core/extraction.js +176 -0
  22. package/dist/core/extraction.js.map +1 -0
  23. package/dist/core/mapping.d.ts +8 -0
  24. package/dist/core/mapping.d.ts.map +1 -0
  25. package/dist/core/mapping.js +90 -0
  26. package/dist/core/mapping.js.map +1 -0
  27. package/dist/core/normalization.d.ts +15 -0
  28. package/dist/core/normalization.d.ts.map +1 -0
  29. package/dist/core/normalization.js +43 -0
  30. package/dist/core/normalization.js.map +1 -0
  31. package/dist/core/timeframe.d.ts +30 -0
  32. package/dist/core/timeframe.d.ts.map +1 -0
  33. package/dist/core/timeframe.js +151 -0
  34. package/dist/core/timeframe.js.map +1 -0
  35. package/dist/index.d.ts +17 -0
  36. package/dist/index.d.ts.map +1 -0
  37. package/dist/index.js +28 -0
  38. package/dist/index.js.map +1 -0
  39. package/dist/query/formatter.d.ts +9 -0
  40. package/dist/query/formatter.d.ts.map +1 -0
  41. package/dist/query/formatter.js +119 -0
  42. package/dist/query/formatter.js.map +1 -0
  43. package/dist/query/planner.d.ts +14 -0
  44. package/dist/query/planner.d.ts.map +1 -0
  45. package/dist/query/planner.js +32 -0
  46. package/dist/query/planner.js.map +1 -0
  47. package/dist/storage/memory.d.ts +36 -0
  48. package/dist/storage/memory.d.ts.map +1 -0
  49. package/dist/storage/memory.js +150 -0
  50. package/dist/storage/memory.js.map +1 -0
  51. package/dist/storage/mongo.d.ts +56 -0
  52. package/dist/storage/mongo.d.ts.map +1 -0
  53. package/dist/storage/mongo.js +323 -0
  54. package/dist/storage/mongo.js.map +1 -0
  55. package/dist/storage/redis.d.ts +44 -0
  56. package/dist/storage/redis.d.ts.map +1 -0
  57. package/dist/storage/redis.js +193 -0
  58. package/dist/storage/redis.js.map +1 -0
  59. package/dist/types/index.d.ts +317 -0
  60. package/dist/types/index.d.ts.map +1 -0
  61. package/dist/types/index.js +9 -0
  62. package/dist/types/index.js.map +1 -0
  63. package/dist/utils/errors.d.ts +20 -0
  64. package/dist/utils/errors.d.ts.map +1 -0
  65. package/dist/utils/errors.js +44 -0
  66. package/dist/utils/errors.js.map +1 -0
  67. package/dist/utils/hash.d.ts +30 -0
  68. package/dist/utils/hash.d.ts.map +1 -0
  69. package/dist/utils/hash.js +61 -0
  70. package/dist/utils/hash.js.map +1 -0
  71. package/dist/utils/jsonpath.d.ts +40 -0
  72. package/dist/utils/jsonpath.d.ts.map +1 -0
  73. package/dist/utils/jsonpath.js +162 -0
  74. package/dist/utils/jsonpath.js.map +1 -0
  75. package/dist/utils/logger.d.ts +7 -0
  76. package/dist/utils/logger.d.ts.map +1 -0
  77. package/dist/utils/logger.js +24 -0
  78. package/dist/utils/logger.js.map +1 -0
  79. package/docs/DEPLOYMENT.md +244 -0
  80. package/docs/END_TO_END_EXAMPLE.md +316 -0
  81. package/package.json +91 -0
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 x12i
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md ADDED
@@ -0,0 +1,520 @@
1
+ # @x12i/countex
2
+
3
+ > Generic sparse counter matrices for JSON events.
4
+ > Dynamic counting, aggregation, and retrieval from mapped JSON inputs.
5
+
6
+ `@x12i/countex` is a source-agnostic counting engine. You give it any JSON event, a declarative mapping config, and storage adapters; it projects events into sparse counter cells, batches deltas locally, and persists them atomically to a durable store (MongoDB) and a hot cache (Redis).
7
+
8
+ It does **not** know about users, organizations, documents, labels, or events. It only knows generic primitives: **scope, subject, metric, dimensions, timeframe, delta**. Whatever your domain calls those things — Countex doesn't care.
9
+
10
+ ```
11
+ event JSON ──► map ──► raw increments ──► buffer ──► flush ──► Mongo (durable)
12
+ └───► Redis (hot cache)
13
+
14
+ ◄── query ◄── Redis (full coverage) or ◄── Mongo (fallback) ──◄
15
+ ```
16
+
17
+ ---
18
+
19
+ ## Install
20
+
21
+ ```bash
22
+ npm install @x12i/countex
23
+ # Bring your own clients — peer deps, both optional:
24
+ npm install mongodb ioredis
25
+ ```
26
+
27
+ Node 18+. Zero runtime dependencies in the core package.
28
+
29
+ ### Documentation shipped with the package
30
+
31
+ These guides are included in the published tarball under `docs/` and exposed as stable import subpaths (resolve to `.md` files for editors and tooling):
32
+
33
+ | Topic | Package subpath | File |
34
+ |-------|-----------------|------|
35
+ | Release & production deployment | `@x12i/countex/docs/deployment` | `docs/DEPLOYMENT.md` |
36
+ | Tutorial — install → map → ingest → query | `@x12i/countex/docs/end-to-end-example` | `docs/END_TO_END_EXAMPLE.md` |
37
+
38
+ After install: `node_modules/@x12i/countex/docs/`.
39
+
40
+ ---
41
+
42
+ ## The mental model
43
+
44
+ Countex turns mapped JSON events into sparse aggregate cells.
45
+
46
+ A counter cell is uniquely identified by:
47
+
48
+ ```
49
+ (scope, counterId, timeframeType, timeframeKey, subjectType, subjectId, metricId, dimensionsHash)
50
+ ```
51
+
52
+ Each event can produce one or many cell increments. A counter increment is defined by:
53
+
54
+ - **scope** — your tenant / org / account boundary
55
+ - **counterId** — which counter family this is (e.g. `access_by_subject_metric`)
56
+ - **timeframe** — `day` / `week` / `month` / `quarter` / `year` / `all`
57
+ - **subject** — the thing being counted against (e.g. an actor or a target)
58
+ - **metric** — the count category / label
59
+ - **dimensions** — arbitrary additional breakdowns
60
+ - **delta** — `+1`, `-1`, or any integer
61
+
62
+ Countex buffers increments locally, flushes atomic deltas to durable + cache, and serves queries from cache with durable fallback.
63
+
64
+ ---
65
+
66
+ ## Quick start
67
+
68
+ ```ts
69
+ import { createCountex } from "@x12i/countex";
70
+ import { MemoryDurable, MemoryCache } from "@x12i/countex/adapters/memory";
71
+
72
+ const accessMap = {
73
+ counterId: "access_by_subject_metric",
74
+
75
+ scope: { path: "$.tenant.id" },
76
+ eventTime: { path: "$.event.createdAt" },
77
+
78
+ subjects: [
79
+ { type: "actor", path: "$.actor.id", required: true },
80
+ { type: "target", path: "$.target.id", required: false },
81
+ ],
82
+
83
+ metrics: [
84
+ { path: "$.labels[*]", required: true },
85
+ ],
86
+
87
+ dimensions: [
88
+ { name: "eventType", path: "$.event.type" },
89
+ { name: "situation", path: "$.event.situation" },
90
+ { name: "targetType", path: "$.target.type" },
91
+ ],
92
+
93
+ delta: { default: 1 },
94
+ timeframes: ["day", "week", "month", "quarter", "year"],
95
+ } as const;
96
+
97
+ const countex = createCountex({
98
+ maps: [accessMap],
99
+ durable: new MemoryDurable(),
100
+ cache: new MemoryCache(),
101
+ });
102
+
103
+ await countex.ingest({
104
+ tenant: { id: "org_123" },
105
+ event: { type: "share", situation: "external", createdAt: "2026-05-08T14:33:00Z" },
106
+ actor: { id: "user_42" },
107
+ target: { id: "doc_99", type: "document" },
108
+ labels: ["PCI", "PII"],
109
+ });
110
+
111
+ await countex.flush();
112
+
113
+ const result = await countex.query({
114
+ scope: "org_123",
115
+ counterId: "access_by_subject_metric",
116
+ timeframe: { type: "day", from: "2026-05-01", to: "2026-05-08" },
117
+ groupBy: ["subject", "metric", "dim:eventType"],
118
+ format: "table",
119
+ orderBy: "-count",
120
+ limit: 100,
121
+ });
122
+
123
+ console.log(result.rows);
124
+
125
+ await countex.shutdown();
126
+ ```
127
+
128
+ That single ingest produces:
129
+
130
+ - 1 actor + 1 target = **2 subjects**
131
+ - 2 labels = **2 metrics**
132
+ - 5 timeframes
133
+ - → **2 × 2 × 5 = 20 cell increments**
134
+
135
+ Each cell uses an atomic `$inc` on durable and `HINCRBY` on the cache, so 20 ingest workers can safely run in parallel.
136
+
137
+ ---
138
+
139
+ ## Production setup (Mongo + Redis)
140
+
141
+ ```ts
142
+ import { MongoClient } from "mongodb";
143
+ import IORedis from "ioredis";
144
+ import { createCountex } from "@x12i/countex";
145
+ import { MongoDurable } from "@x12i/countex/adapters/mongo";
146
+ import { RedisCache } from "@x12i/countex/adapters/redis";
147
+
148
+ const mongo = new MongoClient(process.env.MONGO_URI!);
149
+ await mongo.connect();
150
+ const redis = new IORedis(process.env.REDIS_URL!);
151
+
152
+ const countex = createCountex({
153
+ maps: [accessMap],
154
+ durable: new MongoDurable({
155
+ client: mongo,
156
+ dbResolver: ({ scope }) => `countex_${scope}`,
157
+ }),
158
+ cache: new RedisCache({
159
+ client: redis,
160
+ keyPrefix: "countex",
161
+ ttlSeconds: 92 * 24 * 3600, // ~3 months hot window
162
+ }),
163
+ flush: {
164
+ everyMs: 60_000,
165
+ maxBufferedCells: 100_000,
166
+ maxBufferedEvents: 50_000,
167
+ flushOnShutdown: true,
168
+ },
169
+ cardinality: {
170
+ maxDimensionValuesPerName: 10_000,
171
+ onLimit: "overflow",
172
+ overflowValue: "__OTHER__",
173
+ },
174
+ });
175
+
176
+ await countex.init();
177
+
178
+ process.on("SIGTERM", async () => { await countex.shutdown(); });
179
+ ```
180
+
181
+ ---
182
+
183
+ ## Mapping reference
184
+
185
+ A `CounterMap` is the heart of the package. Everything Countex knows about your data is in here.
186
+
187
+ | Field | Type | Notes |
188
+ |---------------|----------------------------|-------|
189
+ | `counterId` | `string` | Stable identifier for this counter family. |
190
+ | `scope` | `{ path }` | Path into the event for the tenant/scope id. Required. |
191
+ | `eventTime` | `{ path }` | Path to a Date / ISO string / epoch ms. Drives bucket selection. |
192
+ | `subjects` | `SubjectMapping[]` | One or many. Each has `type` and `path`; multi-value paths fan out. |
193
+ | `metrics` | `MetricMapping[]` | Same fanout semantics. `$.labels[*]` is common. |
194
+ | `dimensions` | `DimensionMapping[]` | Named extra breakdowns. Optional by default. |
195
+ | `delta` | `{ path?, default? }` | Numeric value at runtime. Default `1`. Negative deltas are fine. |
196
+ | `timeframes` | `TimeframeType[]` | Subset of `day` / `week` / `month` / `quarter` / `year` / `all`. |
197
+ | `idempotency` | `IdempotencyPolicy` | Optional; needs a cache adapter that implements `markEventSeen`. |
198
+ | `cardinality` | `CardinalityPolicy` | Optional; per-map override of engine-level defaults. |
199
+ | `options` | `MappingOptions` | `skipMissingOptionalDimensions` (default true), `normalizeValues` (default true). |
200
+
201
+ ### Path syntax
202
+
203
+ A small JSONPath subset:
204
+
205
+ | Syntax | Meaning |
206
+ |-------------------|--------------------------------------------------|
207
+ | `$` | Root. |
208
+ | `.name` | Property. |
209
+ | `['name']` | Property (bracket form, allows arbitrary chars). |
210
+ | `[N]` | Numeric array index (negatives allowed). |
211
+ | `[*]` | Wildcard — fan out across array elems / object values. |
212
+
213
+ Examples:
214
+
215
+ ```
216
+ $.tenant.id
217
+ $.actor.id
218
+ $.labels[*]
219
+ $.items[*].id
220
+ $['weird.key.with.dots']
221
+ ```
222
+
223
+ ### Fanout
224
+
225
+ A single event with 2 subjects and 3 labels (metrics) and 5 selected timeframes produces **2 × 3 × 5 = 30** atomic increments. This is intentional — read your mapping carefully. Countex emits a `dropped` event when cardinality protection bites, so you'll know.
226
+
227
+ ---
228
+
229
+ ## Storage adapters
230
+
231
+ Countex defines two contracts and ships three adapters.
232
+
233
+ ```ts
234
+ interface DurableStorage {
235
+ applyDeltas(deltas: PendingDelta[]): Promise<void>;
236
+ readCells(spec: QuerySpec): Promise<CounterCell[]>;
237
+ resolveIds(lookups: DictionaryLookup[]): Promise<Map<string, number>>;
238
+ streamCells(filter): AsyncIterable<CounterCell>;
239
+ init?(): Promise<void>;
240
+ close?(): Promise<void>;
241
+ }
242
+
243
+ interface CacheStorage {
244
+ applyDeltas(deltas: PendingDelta[]): Promise<void>;
245
+ readCells(spec: QuerySpec): Promise<{ cells: CounterCell[]; coverage: "full" | "partial" | "miss" }>;
246
+ warm(cells: CounterCell[]): Promise<void>;
247
+ markEventSeen?(scopeId, eventId, ttlSeconds): Promise<boolean>;
248
+ init?(): Promise<void>;
249
+ close?(): Promise<void>;
250
+ }
251
+ ```
252
+
253
+ ### Built-in adapters
254
+
255
+ | Module | Purpose |
256
+ |-----------------------------------|--------------------------------------|
257
+ | `@x12i/countex/adapters/memory` | In-process; tests / local dev only. |
258
+ | `@x12i/countex/adapters/mongo` | Durable. Requires `mongodb` peer. |
259
+ | `@x12i/countex/adapters/redis` | Hot cache. Requires `ioredis` peer. |
260
+
261
+ Bring your own clients — Countex doesn't open or close them. That keeps lifecycle in the host app's hands.
262
+
263
+ ### Mongo schema
264
+
265
+ `countex_cells`:
266
+
267
+ ```json
268
+ {
269
+ "scopeId": "org_123",
270
+ "counterId": "access_by_subject_metric",
271
+ "timeframeType": "day",
272
+ "timeframeKey": "2026-05-08",
273
+ "subjectType": "actor",
274
+ "subjectId": 10001,
275
+ "metricId": 41,
276
+ "dimensionsHash": "a8f21c0b1d2e",
277
+ "dimensions": { "eventType": "share", "situation": "external" },
278
+ "count": 42,
279
+ "createdAt": "...",
280
+ "updatedAt": "..."
281
+ }
282
+ ```
283
+
284
+ Indexes (created automatically on first write):
285
+
286
+ - `uniq_cell_identity` (unique, on the full identity tuple)
287
+ - `by_timeframe` — for range scans
288
+ - `by_subject` — for subject filters
289
+ - `by_metric` — for metric filters
290
+
291
+ `countex_dict` — the per-scope subject/metric dictionary, with monotonic ids allocated via `countex_seq`.
292
+
293
+ ### Redis layout
294
+
295
+ ```
296
+ countex:{scope}:{counterId}:{timeframeType}:{timeframeKey} → HASH
297
+ field = "{subjectType}|{subjectId}|{metricId}|{dimensionsHash}"
298
+ value = count
299
+
300
+ countex:{scope}:{counterId}:...:{timeframeKey}:meta:{field} → JSON of dimensions
301
+
302
+ countex:{scope}:dedupe:{eventId} → "1" with TTL
303
+ ```
304
+
305
+ One hash per `(scope, counterId, bucket)` keeps Redis key counts small and per-bucket reads to a single `HGETALL`.
306
+
307
+ ---
308
+
309
+ ## Query API
310
+
311
+ ```ts
312
+ const result = await countex.query({
313
+ scope: "org_123",
314
+ counterId: "access_by_subject_metric",
315
+
316
+ timeframe: {
317
+ type: "day",
318
+ from: "2026-05-01",
319
+ to: "2026-05-08",
320
+ },
321
+
322
+ // Filters — all optional
323
+ subject: { type: "actor", ids: [10001, 10002] },
324
+ metrics: [41, 42],
325
+ dimensions: { eventType: ["share", "download"], situation: ["external"] },
326
+
327
+ // Shape
328
+ groupBy: ["subject", "metric", "dim:eventType"],
329
+ format: "table", // "cells" | "table" | "matrix" | "timeseries"
330
+ orderBy: "-count",
331
+ limit: 100,
332
+
333
+ // Diagnostics
334
+ bypassCache: false,
335
+ signal: AbortSignal.timeout(5_000),
336
+ });
337
+ ```
338
+
339
+ Result:
340
+
341
+ ```ts
342
+ {
343
+ format: "table",
344
+ rows: [
345
+ { keys: { subject: 10001, metric: 41, "dim:eventType": "share" }, count: 142 },
346
+ ...
347
+ ],
348
+ total: 1234,
349
+ source: "cache" | "durable" | "merged",
350
+ }
351
+ ```
352
+
353
+ `groupBy` accepts:
354
+ - `"subject"`, `"metric"`, `"timeframeKey"`
355
+ - `"dim:NAME"` — group by a dimension value
356
+
357
+ ---
358
+
359
+ ## Events
360
+
361
+ The Countex client is an `EventEmitter`. Subscribe to observe what's happening:
362
+
363
+ ```ts
364
+ countex.on("flush", ({ cells, durationMs }) => {});
365
+ countex.on("flush_error", ({ error }) => {});
366
+ countex.on("cardinality_warning", ({ kind, count, limit })=> {});
367
+ countex.on("overflow", ({ kind, sourceKey }) => {});
368
+ countex.on("dropped", ({ reason }) => {});
369
+ countex.on("duplicate", ({ scopeId, eventId }) => {});
370
+ ```
371
+
372
+ All event payloads are typed via `CountexEvents`.
373
+
374
+ ---
375
+
376
+ ## Operational notes
377
+
378
+ ### Buffering and flushing
379
+
380
+ Each Countex instance keeps only a delta buffer in memory:
381
+
382
+ ```
383
+ Map<cellKey, delta>
384
+ ```
385
+
386
+ Flush triggers (any of):
387
+
388
+ - `flush.everyMs` timer (default 60s)
389
+ - `flush.maxBufferedCells` reached (default 100k)
390
+ - `flush.maxBufferedEvents` reached (default 50k)
391
+ - `shutdown()` if `flushOnShutdown` is true (default)
392
+ - explicit `flush()`
393
+
394
+ Flush is **two-phase**: durable first, cache second. If durable fails, the snapshot is merged back into the buffer — counts are not lost. If cache fails after durable succeeded, it's logged and continues; the next read warms cache from durable.
395
+
396
+ At most one flush runs at a time. Concurrent callers receive the same in-flight promise.
397
+
398
+ ### Cardinality protection
399
+
400
+ A bad mapping (e.g. a free-form URL as a dimension) can balloon the dictionary. Countex tracks per-`(scope, kind, qualifier)` how many distinct keys it has minted, and applies a policy when limits hit:
401
+
402
+ - `overflow` (default) — rewrite the source key to `__OTHER__`. Emits `overflow`.
403
+ - `drop` — silently drop the increment. Emits `dropped`.
404
+ - `error` — throw `CardinalityLimitError`.
405
+
406
+ Defaults are conservative; tune them per workload. A warning fires at 80% of the limit (`cardinality_warning`).
407
+
408
+ ### Idempotency
409
+
410
+ Optional and per-map. Requires a cache adapter that implements `markEventSeen` (the included `RedisCache` and `MemoryCache` both do).
411
+
412
+ ```ts
413
+ idempotency: {
414
+ enabled: true,
415
+ eventIdPath: "$.event.id",
416
+ ttlSeconds: 86_400,
417
+ }
418
+ ```
419
+
420
+ Dedupe keys are namespaced by `counterId`, so different maps with their own idempotency don't collide on shared event ids.
421
+
422
+ ### Late events
423
+
424
+ Bucket selection always uses the **event time**, never the ingestion time. Late-arriving events go into the correct historical bucket — `$inc` doesn't care about ordering.
425
+
426
+ ### Concurrency safety
427
+
428
+ All writes are atomic per cell:
429
+
430
+ - Mongo: `bulkWrite` of `updateOne` with `$inc` and `upsert: true`
431
+ - Redis: `HINCRBY`
432
+
433
+ 20+ Countex workers can safely run in parallel against the same scope. Reads are eventually consistent across the two stores; if Redis is behind durable, the planner will detect partial coverage and read durable.
434
+
435
+ ### Backup / rebuild
436
+
437
+ ```ts
438
+ import { exportCells, rebuildCache } from "@x12i/countex";
439
+
440
+ // Stream all cells in a window — bounded async iteration.
441
+ for await (const cell of exportCells(durable, {
442
+ scope: "org_123",
443
+ counterId: "access_by_subject_metric",
444
+ timeframe: { type: "day", from: "2026-02-01", to: "2026-05-08" },
445
+ })) {
446
+ // ndjson, S3, whatever you want
447
+ }
448
+
449
+ // Re-warm cache from durable.
450
+ await rebuildCache(durable, cache, {
451
+ scope: "org_123",
452
+ counterId: "access_by_subject_metric",
453
+ timeframe: { type: "day", from: "2026-02-01", to: "2026-05-08" },
454
+ });
455
+ ```
456
+
457
+ ---
458
+
459
+ ## Errors
460
+
461
+ All errors thrown by Countex extend `CountexError`:
462
+
463
+ | Class | When |
464
+ |--------------------------|------|
465
+ | `MappingError` | Invalid mapping config (caught at engine startup). |
466
+ | `ExtractionError` | Malformed path or value during extraction. |
467
+ | `CardinalityLimitError` | Cardinality limit hit when policy is `error`. |
468
+ | `StorageError` | Wrapping a storage-adapter failure. |
469
+
470
+ ---
471
+
472
+ ## Versioning & scope
473
+
474
+ `v0.x` is for stabilizing the public API. Once it's hardened:
475
+
476
+ - exact unique counts (HLL / sets) are an opt-in extension, not core
477
+ - complex expression deltas are not in core
478
+ - distributed locks / exactly-once: not Countex's job; durable storage is the source of truth and `$inc` makes worker-count irrelevant
479
+
480
+ ---
481
+
482
+ ## Repository layout
483
+
484
+ | Path | Role |
485
+ |------|------|
486
+ | `src/index.ts` | Public exports (`createCountex`, types, errors). |
487
+ | `src/core/` | Engine: ingest pipeline, buffer, extraction, mapping validation, time buckets, cardinality. |
488
+ | `src/query/` | Query planner and result formatting. |
489
+ | `src/storage/` | Adapters: `memory` (in-process), `mongo` (durable), `redis` (cache). |
490
+ | `src/utils/` | JSONPath subset, hashing, logging. |
491
+ | `src/backup/` | Export / cache rebuild helpers. |
492
+ | `examples/` | Runnable demos (`npx tsx examples/basic.ts`). |
493
+ | `docs/` | Deployment guide and end-to-end tutorial (published in the npm package). |
494
+ | `tests/unit/` | Fast Node test runner tests (`npm test`). |
495
+ | `tests/integration/` | Opt-in live tests against real services (`npm run test:integration`). |
496
+
497
+ ---
498
+
499
+ ## Publishing (npm)
500
+
501
+ This package is scoped as **`@x12i/countex`** and is configured for **public** registry access (`publishConfig.access` in `package.json`).
502
+
503
+ 1. Ensure you are logged in and allowed to publish under the **`@x12i`** scope on [npm](https://www.npmjs.com/).
504
+ 2. Put auth tokens only in **`~/.npmrc`** — do not commit project `.npmrc` with credentials (it is listed in `.gitignore`).
505
+ 3. From the repo root:
506
+
507
+ ```bash
508
+ npm test
509
+ npm publish
510
+ ```
511
+
512
+ `prepublishOnly` runs a clean build so the tarball always contains a fresh `dist/`.
513
+
514
+ The **`docs/`** directory (`DEPLOYMENT.md`, `END_TO_END_EXAMPLE.md`) is part of the published package (`files` in `package.json`).
515
+
516
+ ---
517
+
518
+ ## License
519
+
520
+ MIT — see `LICENSE`.
@@ -0,0 +1,19 @@
1
+ /**
2
+ * Backup helpers.
3
+ *
4
+ * exportCells(filter) async iterable of cells from durable
5
+ * rebuildCache(filter) re-warm the cache from durable for a window
6
+ *
7
+ * Both are bounded by the (scope, counterId, timeframe) filter the caller
8
+ * provides; we never iterate the entire collection.
9
+ */
10
+ import type { CacheStorage, CounterCell, DurableStorage, QuerySpec } from "../types/index.js";
11
+ export type ExportFilter = Pick<QuerySpec, "scope" | "counterId" | "timeframe">;
12
+ export declare function exportCells(durable: DurableStorage, filter: ExportFilter, signal?: AbortSignal): AsyncIterable<CounterCell>;
13
+ export declare function rebuildCache(durable: DurableStorage, cache: CacheStorage, filter: ExportFilter, options?: {
14
+ batchSize?: number;
15
+ signal?: AbortSignal;
16
+ }): Promise<{
17
+ cellsWarmed: number;
18
+ }>;
19
+ //# sourceMappingURL=export.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"export.d.ts","sourceRoot":"","sources":["../../src/backup/export.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EACV,YAAY,EACZ,WAAW,EACX,cAAc,EACd,SAAS,EACV,MAAM,mBAAmB,CAAC;AAE3B,MAAM,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,EAAE,OAAO,GAAG,WAAW,GAAG,WAAW,CAAC,CAAC;AAEhF,wBAAgB,WAAW,CACzB,OAAO,EAAE,cAAc,EACvB,MAAM,EAAE,YAAY,EACpB,MAAM,CAAC,EAAE,WAAW,GACnB,aAAa,CAAC,WAAW,CAAC,CAE5B;AAED,wBAAsB,YAAY,CAChC,OAAO,EAAE,cAAc,EACvB,KAAK,EAAE,YAAY,EACnB,MAAM,EAAE,YAAY,EACpB,OAAO,GAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,WAAW,CAAA;CAAO,GACzD,OAAO,CAAC;IAAE,WAAW,EAAE,MAAM,CAAA;CAAE,CAAC,CAoBlC"}
@@ -0,0 +1,36 @@
1
+ "use strict";
2
+ /**
3
+ * Backup helpers.
4
+ *
5
+ * exportCells(filter) async iterable of cells from durable
6
+ * rebuildCache(filter) re-warm the cache from durable for a window
7
+ *
8
+ * Both are bounded by the (scope, counterId, timeframe) filter the caller
9
+ * provides; we never iterate the entire collection.
10
+ */
11
+ Object.defineProperty(exports, "__esModule", { value: true });
12
+ exports.exportCells = exportCells;
13
+ exports.rebuildCache = rebuildCache;
14
+ function exportCells(durable, filter, signal) {
15
+ return durable.streamCells(filter, signal ? { signal } : undefined);
16
+ }
17
+ async function rebuildCache(durable, cache, filter, options = {}) {
18
+ const batchSize = options.batchSize ?? 1000;
19
+ let warmed = 0;
20
+ let batch = [];
21
+ const opts = options.signal ? { signal: options.signal } : undefined;
22
+ for await (const cell of durable.streamCells(filter, opts)) {
23
+ batch.push(cell);
24
+ if (batch.length >= batchSize) {
25
+ await cache.warm(batch, opts);
26
+ warmed += batch.length;
27
+ batch = [];
28
+ }
29
+ }
30
+ if (batch.length > 0) {
31
+ await cache.warm(batch, opts);
32
+ warmed += batch.length;
33
+ }
34
+ return { cellsWarmed: warmed };
35
+ }
36
+ //# sourceMappingURL=export.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"export.js","sourceRoot":"","sources":["../../src/backup/export.ts"],"names":[],"mappings":";AAAA;;;;;;;;GAQG;;AAWH,kCAMC;AAED,oCAyBC;AAjCD,SAAgB,WAAW,CACzB,OAAuB,EACvB,MAAoB,EACpB,MAAoB;IAEpB,OAAO,OAAO,CAAC,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;AACtE,CAAC;AAEM,KAAK,UAAU,YAAY,CAChC,OAAuB,EACvB,KAAmB,EACnB,MAAoB,EACpB,UAAwD,EAAE;IAE1D,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC;IAC5C,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,KAAK,GAAkB,EAAE,CAAC;IAE9B,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IAErE,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,OAAO,CAAC,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC;QAC3D,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjB,IAAI,KAAK,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;YAC9B,MAAM,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC;YACvB,KAAK,GAAG,EAAE,CAAC;QACb,CAAC;IACH,CAAC;IACD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,MAAM,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC;IACzB,CAAC;IACD,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC;AACjC,CAAC"}
@@ -0,0 +1,27 @@
1
+ /**
2
+ * Local delta buffer.
3
+ *
4
+ * Each app instance keeps just a Map<cellKey, delta>, not full state. On flush
5
+ * we drain the buffer into a snapshot, send it to durable storage, and only
6
+ * then forget those deltas. If the durable write fails, we merge the snapshot
7
+ * back into the buffer so no counts are lost.
8
+ */
9
+ import type { CellIdentity, PendingDelta } from "../types/index.js";
10
+ export declare class DeltaBuffer {
11
+ private readonly map;
12
+ /** Number of source events accumulated since last flush (for flush triggers). */
13
+ private events;
14
+ size(): number;
15
+ eventCount(): number;
16
+ noteEvent(): void;
17
+ /** Add a delta to the cell identified by `identity`. */
18
+ add(identity: CellIdentity, delta: number): void;
19
+ /**
20
+ * Take the current contents as a flush snapshot. The buffer is cleared
21
+ * atomically — if you need to re-merge on failure, call `mergeBack`.
22
+ */
23
+ drain(): PendingDelta[];
24
+ /** Re-merge a previously drained snapshot back into the buffer (failure recovery). */
25
+ mergeBack(snapshot: PendingDelta[]): void;
26
+ }
27
+ //# sourceMappingURL=buffer.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"buffer.d.ts","sourceRoot":"","sources":["../../src/core/buffer.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAQpE,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAyC;IAC7D,iFAAiF;IACjF,OAAO,CAAC,MAAM,CAAK;IAEnB,IAAI,IAAI,MAAM;IAId,UAAU,IAAI,MAAM;IAIpB,SAAS,IAAI,IAAI;IAIjB,wDAAwD;IACxD,GAAG,CAAC,QAAQ,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAahD;;;OAGG;IACH,KAAK,IAAI,YAAY,EAAE;IAUvB,sFAAsF;IACtF,SAAS,CAAC,QAAQ,EAAE,YAAY,EAAE,GAAG,IAAI;CAK1C"}