@x12i/countex 1.6.0 → 1.6.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. package/README.md +81 -4
  2. package/package.json +14 -2
package/README.md CHANGED
@@ -309,13 +309,20 @@ Use `countex.preview(event, { counterId? })` to inspect what one event would pro
309
309
 
310
310
  ## Production setup (Mongo + Redis)
311
311
 
312
+ Do **not** derive Mongo database names directly from raw tenant strings (length, `$`, `/`, and Unicode can break DB naming rules or leak structure). Hash or normalize first:
313
+
312
314
  ```ts
315
+ import { createHash } from "node:crypto";
313
316
  import { MongoClient } from "mongodb";
314
317
  import IORedis from "ioredis";
315
318
  import { createCountex } from "@x12i/countex";
316
319
  import { MongoDurable } from "@x12i/countex/adapters/mongo";
317
320
  import { RedisCache } from "@x12i/countex/adapters/redis";
318
321
 
322
+ function hashScopeId(scope: string): string {
323
+ return createHash("sha256").update(scope, "utf8").digest("hex").slice(0, 16);
324
+ }
325
+
319
326
  const mongo = new MongoClient(process.env.MONGO_URI!);
320
327
  await mongo.connect();
321
328
  const redis = new IORedis(process.env.REDIS_URL!);
@@ -324,13 +331,30 @@ const countex = createCountex({
324
331
  maps: [accessMap],
325
332
  durable: new MongoDurable({
326
333
  client: mongo,
327
- dbResolver: ({ scope }) => `countex_${scope}`,
334
+ dbResolver: ({ scope }) => `countex_${hashScopeId(scope)}`,
328
335
  }),
329
336
  cache: new RedisCache({
330
337
  client: redis,
331
338
  keyPrefix: "countex",
332
- ttlSeconds: 92 * 24 * 3600, // ~3 months hot window
339
+ // Optional adapter default TTLs; when `retention.redis` is set below, the
340
+ // engine resolves those strings to seconds and forwards them via
341
+ // `setRetentionPolicy`, overriding adapter defaults for matching timeframes.
333
342
  }),
343
+ retention: {
344
+ redis: {
345
+ minute: "48h",
346
+ hour: "14d",
347
+ day: "92d",
348
+ week: "52w",
349
+ month: "24mo",
350
+ quarter: "24mo",
351
+ year: "forever",
352
+ all: "forever",
353
+ },
354
+ // Durable TTL / prune is not enforced by the engine — operators apply Mongo
355
+ // TTL indexes or scheduled jobs. `retention.mongo` is optional per-timeframe
356
+ // documentation (same duration grammar as redis); see types / DEPLOYMENT.
357
+ },
334
358
  flush: {
335
359
  everyMs: 60_000,
336
360
  maxBufferedCells: 100_000,
@@ -471,12 +495,19 @@ countex:{scope}:{counterId}:{timeframeType}:{timeframeKey} → HASH
471
495
  field = "{subjectType}|{subjectId}|{metricId}|{dimensionsHash}"
472
496
  value = count
473
497
 
498
+ countex:{scope}:{counterId}:{timeframeType}:{timeframeKey}:cov → HASH (coverage meta)
499
+ status = complete | partial | rebuilding (see BucketCoverageMeta; reads may infer missing)
500
+ lastWarmAt = epoch ms (string)
501
+ mapVersion = optional opaque integer (string)
502
+ schemaVersion = cache layout version (string)
503
+ cellCount = cells last warmed into this bucket (string, when known)
504
+
474
505
  countex:{scope}:{counterId}:...:{timeframeKey}:meta:{field} → JSON of dimensions
475
506
 
476
507
  countex:{scope}:dedupe:{eventId} → "1" with TTL
477
508
  ```
478
509
 
479
- One hash per `(scope, counterId, bucket)` keeps Redis key counts small and per-bucket reads to a single `HGETALL`.
510
+ One hash per `(scope, counterId, bucket)` keeps Redis key counts small and per-bucket reads to a single `HGETALL`. **Do not** treat “cell hash is non-empty” as full cache coverage — use the **`:cov`** hash (`status === complete` after `warm`, `partial` after incremental `applyDeltas`, `rebuilding` during cache rebuild). See [`docs/guides/adapter-redis.md`](docs/guides/adapter-redis.md) and [`docs/internals/internal-storage-redis.md`](docs/internals/internal-storage-redis.md).
480
511
 
481
512
  ---
482
513
 
@@ -510,6 +541,18 @@ const result = await countex.query({
510
541
  });
511
542
  ```
512
543
 
544
+ `explainQuery` runs the planner **without** materializing cells. It returns a **`QueryPlan`**: expanded `bucketsRequested`, `cacheEligible` / `cacheReason`, predicted `durableFallbackPredicted`, `cacheBucketsExpectedHit` (when the cache adapter supports `probeBucketCoverage`), heuristic `plannedCellsEstimate`, and cost hints (`expensive`, `groupByCost`). Use it to preflight dashboards and expensive `groupBy` shapes.
545
+
546
+ ```ts
547
+ const plan = await countex.explainQuery({
548
+ scope: "org_123",
549
+ counterId: "access_by_subject_metric",
550
+ timeframe: { type: "day", from: "2026-05-01", to: "2026-05-08" },
551
+ groupBy: ["subject", "metric"],
552
+ format: "table",
553
+ });
554
+ ```
555
+
513
556
  Result:
514
557
 
515
558
  ```ts
@@ -567,7 +610,7 @@ const topCostDrivers = await insights.topPercent({
567
610
  });
568
611
  ```
569
612
 
570
- Further reading: [docs/INSIGHTS.md](docs/INSIGHTS.md), [docs/INSIGHTS_ANOMALY_DETECTION.md](docs/INSIGHTS_ANOMALY_DETECTION.md), [docs/INSIGHTS_LLM_COST_AND_TOKENS.md](docs/INSIGHTS_LLM_COST_AND_TOKENS.md), [docs/INSIGHTS_RESOURCE_ADVISOR.md](docs/INSIGHTS_RESOURCE_ADVISOR.md).
613
+ Further reading: [`docs/guides/insights-overview.md`](docs/guides/insights-overview.md), [`docs/guides/insights-ranking.md`](docs/guides/insights-ranking.md), [`docs/guides/insights-comparison-and-trends.md`](docs/guides/insights-comparison-and-trends.md), [`docs/guides/insights-anomalies-and-rules.md`](docs/guides/insights-anomalies-and-rules.md), [`docs/guides/insights-forecasting-and-budget.md`](docs/guides/insights-forecasting-and-budget.md), [`docs/guides/insights-advisors-and-optimization.md`](docs/guides/insights-advisors-and-optimization.md). Deeper topic notes also live under `docs/INSIGHTS*.md` for maintainers.
571
614
 
572
615
  CLI (after `npm run build` / install):
573
616
 
@@ -702,6 +745,40 @@ See [`docs/guides/backup-export.md`](docs/guides/backup-export.md) for the full
702
745
 
703
746
  ---
704
747
 
748
+ ## Docker integration testing
749
+
750
+ Countex ships a Docker-based stack (MongoDB, Redis, MinIO) and integration tests under `tests/integration/docker/` that exercise real adapters: dictionary allocation under concurrency, multi-worker ingestion, Redis cache coverage and durable fallback, `rebuildCache`, simulation fingerprints against a memory reference, and backup / manifest / verify / restore to S3-compatible storage (via the AWS SDK in devDependencies).
751
+
752
+ **One-shot local run** (starts services, runs tests with default localhost URLs, tears volumes down):
753
+
754
+ ```bash
755
+ npm run test:e2e
756
+ ```
757
+
758
+ **Host-runner** (services in Docker, tests on the host):
759
+
760
+ ```bash
761
+ npm run docker:up
762
+ export COUNTEX_MONGO_URI='mongodb://countex:countex@127.0.0.1:27017/admin'
763
+ export COUNTEX_REDIS_URL='redis://127.0.0.1:6379'
764
+ export COUNTEX_S3_ENDPOINT='http://127.0.0.1:9000'
765
+ export COUNTEX_S3_REGION='us-east-1'
766
+ export COUNTEX_S3_ACCESS_KEY_ID='countex'
767
+ export COUNTEX_S3_SECRET_ACCESS_KEY='countex-password'
768
+ export COUNTEX_S3_BUCKET='countex-backups'
769
+ export COUNTEX_INTEGRATION=1
770
+ npm run test:integration
771
+ npm run docker:down
772
+ ```
773
+
774
+ Optional tuning: `COUNTEX_TEST_PROFILE`, `COUNTEX_TEST_EVENTS`, `COUNTEX_TEST_WORKERS`. Stress-style full pipeline (larger simulation + S3 round-trip): set `COUNTEX_TEST_FULL=1` (requires S3 env as above).
775
+
776
+ **Container-runner** (CI parity): `npm run test:integration:docker` builds `docker/test-runner.Dockerfile` and runs `npm run test:integration:exec` with service hostnames injected by Compose.
777
+
778
+ Reports are written to `artifacts/integration/report.json` and `artifacts/integration/report.md` after `npm run test:integration`.
779
+
780
+ ---
781
+
705
782
  ## Errors
706
783
 
707
784
  All errors thrown by Countex extend `CountexError`:
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@x12i/countex",
3
- "version": "1.6.0",
3
+ "version": "1.6.5",
4
4
  "description": "Generic sparse counter matrices for JSON events. Dynamic counting, aggregation, and retrieval from mapped JSON inputs.",
5
5
  "repository": {
6
6
  "type": "git",
@@ -122,7 +122,18 @@
122
122
  "build": "tsc",
123
123
  "clean": "rm -rf dist",
124
124
  "test": "node --test --import=tsx tests/unit/*.test.ts",
125
- "test:integration": "node --test --import=tsx tests/integration/*.test.ts",
125
+ "docker:up": "docker compose -f docker/docker-compose.integration.yml up -d mongo redis minio minio-setup",
126
+ "docker:down": "docker compose -f docker/docker-compose.integration.yml down",
127
+ "docker:reset": "docker compose -f docker/docker-compose.integration.yml down -v && docker compose -f docker/docker-compose.integration.yml up -d mongo redis minio minio-setup",
128
+ "test:integration": "tsx tests/integration/run.ts",
129
+ "test:integration:exec": "tsx tests/integration/exec-integration-tests.ts",
130
+ "integration:up": "tsx scripts/integration-up.ts",
131
+ "integration:down": "tsx scripts/integration-down.ts",
132
+ "integration:reset": "tsx scripts/integration-reset.ts",
133
+ "integration:run": "tsx scripts/integration-run.ts",
134
+ "integration:report": "tsx scripts/integration-report.ts",
135
+ "test:integration:docker": "docker compose -f docker/docker-compose.integration.yml run --rm test-runner",
136
+ "test:e2e": "tsx scripts/integration-e2e.ts",
126
137
  "prepublishOnly": "npm run clean && npm run build"
127
138
  },
128
139
  "peerDependencies": {
@@ -138,6 +149,7 @@
138
149
  }
139
150
  },
140
151
  "devDependencies": {
152
+ "@aws-sdk/client-s3": "^3.787.0",
141
153
  "@types/node": "^20.19.40",
142
154
  "ioredis": "^5.10.1",
143
155
  "mongodb": "^7.2.0",