@x12i/countex 1.6.0 → 1.6.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +81 -4
- package/package.json +14 -2
package/README.md
CHANGED
|
@@ -309,13 +309,20 @@ Use `countex.preview(event, { counterId? })` to inspect what one event would pro
|
|
|
309
309
|
|
|
310
310
|
## Production setup (Mongo + Redis)
|
|
311
311
|
|
|
312
|
+
Do **not** derive Mongo database names directly from raw tenant strings (length, `$`, `/`, and Unicode can break DB naming rules or leak structure). Hash or normalize first:
|
|
313
|
+
|
|
312
314
|
```ts
|
|
315
|
+
import { createHash } from "node:crypto";
|
|
313
316
|
import { MongoClient } from "mongodb";
|
|
314
317
|
import IORedis from "ioredis";
|
|
315
318
|
import { createCountex } from "@x12i/countex";
|
|
316
319
|
import { MongoDurable } from "@x12i/countex/adapters/mongo";
|
|
317
320
|
import { RedisCache } from "@x12i/countex/adapters/redis";
|
|
318
321
|
|
|
322
|
+
function hashScopeId(scope: string): string {
|
|
323
|
+
return createHash("sha256").update(scope, "utf8").digest("hex").slice(0, 16);
|
|
324
|
+
}
|
|
325
|
+
|
|
319
326
|
const mongo = new MongoClient(process.env.MONGO_URI!);
|
|
320
327
|
await mongo.connect();
|
|
321
328
|
const redis = new IORedis(process.env.REDIS_URL!);
|
|
@@ -324,13 +331,30 @@ const countex = createCountex({
|
|
|
324
331
|
maps: [accessMap],
|
|
325
332
|
durable: new MongoDurable({
|
|
326
333
|
client: mongo,
|
|
327
|
-
dbResolver: ({ scope }) => `countex_${scope}`,
|
|
334
|
+
dbResolver: ({ scope }) => `countex_${hashScopeId(scope)}`,
|
|
328
335
|
}),
|
|
329
336
|
cache: new RedisCache({
|
|
330
337
|
client: redis,
|
|
331
338
|
keyPrefix: "countex",
|
|
332
|
-
|
|
339
|
+
// Optional adapter default TTLs; when `retention.redis` is set below, the
|
|
340
|
+
// engine resolves those strings to seconds and forwards them via
|
|
341
|
+
// `setRetentionPolicy`, overriding adapter defaults for matching timeframes.
|
|
333
342
|
}),
|
|
343
|
+
retention: {
|
|
344
|
+
redis: {
|
|
345
|
+
minute: "48h",
|
|
346
|
+
hour: "14d",
|
|
347
|
+
day: "92d",
|
|
348
|
+
week: "52w",
|
|
349
|
+
month: "24mo",
|
|
350
|
+
quarter: "24mo",
|
|
351
|
+
year: "forever",
|
|
352
|
+
all: "forever",
|
|
353
|
+
},
|
|
354
|
+
// Durable TTL / prune is not enforced by the engine — operators apply Mongo
|
|
355
|
+
// TTL indexes or scheduled jobs. `retention.mongo` is optional per-timeframe
|
|
356
|
+
// documentation (same duration grammar as redis); see types / DEPLOYMENT.
|
|
357
|
+
},
|
|
334
358
|
flush: {
|
|
335
359
|
everyMs: 60_000,
|
|
336
360
|
maxBufferedCells: 100_000,
|
|
@@ -471,12 +495,19 @@ countex:{scope}:{counterId}:{timeframeType}:{timeframeKey} → HASH
|
|
|
471
495
|
field = "{subjectType}|{subjectId}|{metricId}|{dimensionsHash}"
|
|
472
496
|
value = count
|
|
473
497
|
|
|
498
|
+
countex:{scope}:{counterId}:{timeframeType}:{timeframeKey}:cov → HASH (coverage meta)
|
|
499
|
+
status = complete | partial | rebuilding (see BucketCoverageMeta; reads may infer missing)
|
|
500
|
+
lastWarmAt = epoch ms (string)
|
|
501
|
+
mapVersion = optional opaque integer (string)
|
|
502
|
+
schemaVersion = cache layout version (string)
|
|
503
|
+
cellCount = cells last warmed into this bucket (string, when known)
|
|
504
|
+
|
|
474
505
|
countex:{scope}:{counterId}:...:{timeframeKey}:meta:{field} → JSON of dimensions
|
|
475
506
|
|
|
476
507
|
countex:{scope}:dedupe:{eventId} → "1" with TTL
|
|
477
508
|
```
|
|
478
509
|
|
|
479
|
-
One hash per `(scope, counterId, bucket)` keeps Redis key counts small and per-bucket reads to a single `HGETALL`.
|
|
510
|
+
One hash per `(scope, counterId, bucket)` keeps Redis key counts small and per-bucket reads to a single `HGETALL`. **Do not** treat “cell hash is non-empty” as full cache coverage — use the **`:cov`** hash (`status === complete` after `warm`, `partial` after incremental `applyDeltas`, `rebuilding` during cache rebuild). See [`docs/guides/adapter-redis.md`](docs/guides/adapter-redis.md) and [`docs/internals/internal-storage-redis.md`](docs/internals/internal-storage-redis.md).
|
|
480
511
|
|
|
481
512
|
---
|
|
482
513
|
|
|
@@ -510,6 +541,18 @@ const result = await countex.query({
|
|
|
510
541
|
});
|
|
511
542
|
```
|
|
512
543
|
|
|
544
|
+
`explainQuery` runs the planner **without** materializing cells. It returns a **`QueryPlan`**: expanded `bucketsRequested`, `cacheEligible` / `cacheReason`, predicted `durableFallbackPredicted`, `cacheBucketsExpectedHit` (when the cache adapter supports `probeBucketCoverage`), heuristic `plannedCellsEstimate`, and cost hints (`expensive`, `groupByCost`). Use it to preflight dashboards and expensive `groupBy` shapes.
|
|
545
|
+
|
|
546
|
+
```ts
|
|
547
|
+
const plan = await countex.explainQuery({
|
|
548
|
+
scope: "org_123",
|
|
549
|
+
counterId: "access_by_subject_metric",
|
|
550
|
+
timeframe: { type: "day", from: "2026-05-01", to: "2026-05-08" },
|
|
551
|
+
groupBy: ["subject", "metric"],
|
|
552
|
+
format: "table",
|
|
553
|
+
});
|
|
554
|
+
```
|
|
555
|
+
|
|
513
556
|
Result:
|
|
514
557
|
|
|
515
558
|
```ts
|
|
@@ -567,7 +610,7 @@ const topCostDrivers = await insights.topPercent({
|
|
|
567
610
|
});
|
|
568
611
|
```
|
|
569
612
|
|
|
570
|
-
Further reading: [docs/
|
|
613
|
+
Further reading: [`docs/guides/insights-overview.md`](docs/guides/insights-overview.md), [`docs/guides/insights-ranking.md`](docs/guides/insights-ranking.md), [`docs/guides/insights-comparison-and-trends.md`](docs/guides/insights-comparison-and-trends.md), [`docs/guides/insights-anomalies-and-rules.md`](docs/guides/insights-anomalies-and-rules.md), [`docs/guides/insights-forecasting-and-budget.md`](docs/guides/insights-forecasting-and-budget.md), [`docs/guides/insights-advisors-and-optimization.md`](docs/guides/insights-advisors-and-optimization.md). Deeper topic notes also live under `docs/INSIGHTS*.md` for maintainers.
|
|
571
614
|
|
|
572
615
|
CLI (after `npm run build` / install):
|
|
573
616
|
|
|
@@ -702,6 +745,40 @@ See [`docs/guides/backup-export.md`](docs/guides/backup-export.md) for the full
|
|
|
702
745
|
|
|
703
746
|
---
|
|
704
747
|
|
|
748
|
+
## Docker integration testing
|
|
749
|
+
|
|
750
|
+
Countex ships a Docker-based stack (MongoDB, Redis, MinIO) and integration tests under `tests/integration/docker/` that exercise real adapters: dictionary allocation under concurrency, multi-worker ingestion, Redis cache coverage and durable fallback, `rebuildCache`, simulation fingerprints against a memory reference, and backup / manifest / verify / restore to S3-compatible storage (via the AWS SDK in devDependencies).
|
|
751
|
+
|
|
752
|
+
**One-shot local run** (starts services, runs tests with default localhost URLs, tears volumes down):
|
|
753
|
+
|
|
754
|
+
```bash
|
|
755
|
+
npm run test:e2e
|
|
756
|
+
```
|
|
757
|
+
|
|
758
|
+
**Host-runner** (services in Docker, tests on the host):
|
|
759
|
+
|
|
760
|
+
```bash
|
|
761
|
+
npm run docker:up
|
|
762
|
+
export COUNTEX_MONGO_URI='mongodb://countex:countex@127.0.0.1:27017/admin'
|
|
763
|
+
export COUNTEX_REDIS_URL='redis://127.0.0.1:6379'
|
|
764
|
+
export COUNTEX_S3_ENDPOINT='http://127.0.0.1:9000'
|
|
765
|
+
export COUNTEX_S3_REGION='us-east-1'
|
|
766
|
+
export COUNTEX_S3_ACCESS_KEY_ID='countex'
|
|
767
|
+
export COUNTEX_S3_SECRET_ACCESS_KEY='countex-password'
|
|
768
|
+
export COUNTEX_S3_BUCKET='countex-backups'
|
|
769
|
+
export COUNTEX_INTEGRATION=1
|
|
770
|
+
npm run test:integration
|
|
771
|
+
npm run docker:down
|
|
772
|
+
```
|
|
773
|
+
|
|
774
|
+
Optional tuning: `COUNTEX_TEST_PROFILE`, `COUNTEX_TEST_EVENTS`, `COUNTEX_TEST_WORKERS`. Stress-style full pipeline (larger simulation + S3 round-trip): set `COUNTEX_TEST_FULL=1` (requires S3 env as above).
|
|
775
|
+
|
|
776
|
+
**Container-runner** (CI parity): `npm run test:integration:docker` builds `docker/test-runner.Dockerfile` and runs `npm run test:integration:exec` with service hostnames injected by Compose.
|
|
777
|
+
|
|
778
|
+
Reports are written to `artifacts/integration/report.json` and `artifacts/integration/report.md` after `npm run test:integration`.
|
|
779
|
+
|
|
780
|
+
---
|
|
781
|
+
|
|
705
782
|
## Errors
|
|
706
783
|
|
|
707
784
|
All errors thrown by Countex extend `CountexError`:
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@x12i/countex",
|
|
3
|
-
"version": "1.6.
|
|
3
|
+
"version": "1.6.5",
|
|
4
4
|
"description": "Generic sparse counter matrices for JSON events. Dynamic counting, aggregation, and retrieval from mapped JSON inputs.",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -122,7 +122,18 @@
|
|
|
122
122
|
"build": "tsc",
|
|
123
123
|
"clean": "rm -rf dist",
|
|
124
124
|
"test": "node --test --import=tsx tests/unit/*.test.ts",
|
|
125
|
-
"
|
|
125
|
+
"docker:up": "docker compose -f docker/docker-compose.integration.yml up -d mongo redis minio minio-setup",
|
|
126
|
+
"docker:down": "docker compose -f docker/docker-compose.integration.yml down",
|
|
127
|
+
"docker:reset": "docker compose -f docker/docker-compose.integration.yml down -v && docker compose -f docker/docker-compose.integration.yml up -d mongo redis minio minio-setup",
|
|
128
|
+
"test:integration": "tsx tests/integration/run.ts",
|
|
129
|
+
"test:integration:exec": "tsx tests/integration/exec-integration-tests.ts",
|
|
130
|
+
"integration:up": "tsx scripts/integration-up.ts",
|
|
131
|
+
"integration:down": "tsx scripts/integration-down.ts",
|
|
132
|
+
"integration:reset": "tsx scripts/integration-reset.ts",
|
|
133
|
+
"integration:run": "tsx scripts/integration-run.ts",
|
|
134
|
+
"integration:report": "tsx scripts/integration-report.ts",
|
|
135
|
+
"test:integration:docker": "docker compose -f docker/docker-compose.integration.yml run --rm test-runner",
|
|
136
|
+
"test:e2e": "tsx scripts/integration-e2e.ts",
|
|
126
137
|
"prepublishOnly": "npm run clean && npm run build"
|
|
127
138
|
},
|
|
128
139
|
"peerDependencies": {
|
|
@@ -138,6 +149,7 @@
|
|
|
138
149
|
}
|
|
139
150
|
},
|
|
140
151
|
"devDependencies": {
|
|
152
|
+
"@aws-sdk/client-s3": "^3.787.0",
|
|
141
153
|
"@types/node": "^20.19.40",
|
|
142
154
|
"ioredis": "^5.10.1",
|
|
143
155
|
"mongodb": "^7.2.0",
|