@x12i/countex 1.8.5 → 1.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +867 -860
- package/dist/cli.js +6 -6
- package/dist/core/countex.d.ts +61 -1
- package/dist/core/countex.d.ts.map +1 -1
- package/dist/core/countex.js +241 -0
- package/dist/core/countex.js.map +1 -1
- package/dist/core/managed.d.ts +200 -0
- package/dist/core/managed.d.ts.map +1 -0
- package/dist/core/managed.js +431 -0
- package/dist/core/managed.js.map +1 -0
- package/dist/core/timeframe.d.ts +5 -0
- package/dist/core/timeframe.d.ts.map +1 -1
- package/dist/core/timeframe.js +36 -0
- package/dist/core/timeframe.js.map +1 -1
- package/dist/index.d.ts +6 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +9 -1
- package/dist/index.js.map +1 -1
- package/dist/insights/advisors/resource-advisor.d.ts.map +1 -1
- package/dist/insights/advisors/resource-advisor.js +5 -1
- package/dist/insights/advisors/resource-advisor.js.map +1 -1
- package/dist/insights/types.d.ts +4 -0
- package/dist/insights/types.d.ts.map +1 -1
- package/dist/storage/managed-memory.d.ts +33 -0
- package/dist/storage/managed-memory.d.ts.map +1 -0
- package/dist/storage/managed-memory.js +211 -0
- package/dist/storage/managed-memory.js.map +1 -0
- package/dist/storage/memory.d.ts +7 -4
- package/dist/storage/memory.d.ts.map +1 -1
- package/dist/storage/memory.js.map +1 -1
- package/dist/storage/postgres.d.ts +123 -0
- package/dist/storage/postgres.d.ts.map +1 -0
- package/dist/storage/postgres.js +1325 -0
- package/dist/storage/postgres.js.map +1 -0
- package/dist/storage/redis.d.ts +6 -0
- package/dist/storage/redis.d.ts.map +1 -1
- package/dist/storage/redis.js +21 -0
- package/dist/storage/redis.js.map +1 -1
- package/dist/types/index.d.ts +63 -5
- package/dist/types/index.d.ts.map +1 -1
- package/dist/utils/errors.d.ts +12 -0
- package/dist/utils/errors.d.ts.map +1 -1
- package/dist/utils/errors.js +22 -1
- package/dist/utils/errors.js.map +1 -1
- package/docs/DEPLOYMENT.md +399 -375
- package/docs/END_TO_END_EXAMPLE.md +316 -316
- package/docs/INSIGHTS.md +59 -59
- package/docs/INSIGHTS_ANOMALY_DETECTION.md +33 -33
- package/docs/INSIGHTS_LLM_COST_AND_TOKENS.md +26 -26
- package/docs/INSIGHTS_RESOURCE_ADVISOR.md +33 -31
- package/docs/countex_gap_analysis_complete_checklist.md +1340 -1340
- package/docs/guides/README.md +37 -34
- package/docs/guides/adapter-memory.md +60 -60
- package/docs/guides/adapter-mongo.md +64 -62
- package/docs/guides/adapter-postgres.md +92 -0
- package/docs/guides/adapter-redis.md +89 -89
- package/docs/guides/backup-export.md +135 -135
- package/docs/guides/cardinality.md +50 -50
- package/docs/guides/cli.md +53 -53
- package/docs/guides/client-events.md +65 -65
- package/docs/guides/counter-maps.md +160 -160
- package/docs/guides/idempotency.md +39 -39
- package/docs/guides/ingestion-and-flush.md +67 -67
- package/docs/guides/insights-advisors-and-optimization.md +39 -39
- package/docs/guides/insights-anomalies-and-rules.md +48 -48
- package/docs/guides/insights-comparison-and-trends.md +47 -47
- package/docs/guides/insights-forecasting-and-budget.md +61 -61
- package/docs/guides/insights-overview.md +82 -82
- package/docs/guides/insights-ranking.md +81 -81
- package/docs/guides/managed-commit.md +119 -0
- package/docs/guides/measures.md +72 -71
- package/docs/guides/migrating-mongo-to-postgres.md +127 -0
- package/docs/guides/preview.md +92 -92
- package/docs/guides/querying.md +178 -178
- package/docs/guides/simulation-profiles.md +33 -33
- package/docs/guides/simulation-resource-estimation.md +31 -30
- package/docs/guides/simulation.md +36 -36
- package/docs/guides/timeframes-and-bucketing.md +55 -55
- package/docs/internals/README.md +21 -21
- package/docs/internals/internal-architecture-overview.md +65 -65
- package/docs/internals/internal-backup-stream.md +23 -23
- package/docs/internals/internal-buffer-merge-and-flush.md +51 -51
- package/docs/internals/internal-cardinality.md +30 -30
- package/docs/internals/internal-dimension-hash.md +34 -34
- package/docs/internals/internal-idempotency.md +36 -36
- package/docs/internals/internal-ingest-pipeline.md +52 -52
- package/docs/internals/internal-insights-layer.md +36 -36
- package/docs/internals/internal-mapping-validation-and-extraction.md +59 -59
- package/docs/internals/internal-measures-and-scales.md +35 -35
- package/docs/internals/internal-query-path.md +46 -46
- package/docs/internals/internal-storage-memory.md +13 -13
- package/docs/internals/internal-storage-mongo.md +44 -44
- package/docs/internals/internal-storage-postgres.md +32 -0
- package/docs/internals/internal-storage-redis.md +63 -63
- package/docs/internals/internal-timeframe-bucketing.md +33 -33
- package/docs/use-cases/README.md +56 -56
- package/docs/use-cases/audit-security-log-counts.md +550 -550
- package/docs/use-cases/llm-cost-and-token-metering.md +255 -255
- package/docs/use-cases/multi-tenant-isolation.md +203 -203
- package/docs/use-cases/ops-dashboard-daily-rollups.md +255 -255
- package/docs/use-cases/processing-workflow-activity.md +165 -165
- package/docs/use-cases/product-feature-usage.md +240 -240
- package/docs/use-cases/webhook-idempotent-ingestion.md +188 -188
- package/package.json +190 -170
|
@@ -1,135 +1,135 @@
|
|
|
1
|
-
# Guide: Backup, manifest, verify, restore, and cache rebuild
|
|
2
|
-
|
|
3
|
-
`@x12i/countex` ships export, fingerprint, manifest, verify, restore, and cache-rebuild helpers. Use them as free functions or via the `countex.backup` namespace bound to your engine.
|
|
4
|
-
|
|
5
|
-
## Step 1: Export cells (stream)
|
|
6
|
-
|
|
7
|
-
Free function:
|
|
8
|
-
|
|
9
|
-
```ts
|
|
10
|
-
import { exportCells } from "@x12i/countex";
|
|
11
|
-
|
|
12
|
-
for await (const cell of exportCells(durable, {
|
|
13
|
-
scope: "org_1",
|
|
14
|
-
counterId: "events",
|
|
15
|
-
timeframe: { type: "day", from: "2026-05-01", to: "2026-05-07" },
|
|
16
|
-
})) {
|
|
17
|
-
// serialize to NDJSON / S3 / cold storage…
|
|
18
|
-
}
|
|
19
|
-
```
|
|
20
|
-
|
|
21
|
-
Namespace:
|
|
22
|
-
|
|
23
|
-
```ts
|
|
24
|
-
for await (const cell of countex.backup.export({
|
|
25
|
-
scope: "org_1",
|
|
26
|
-
counterId: "events",
|
|
27
|
-
timeframe: { type: "day", from: "2026-05-01", to: "2026-05-07" },
|
|
28
|
-
})) {
|
|
29
|
-
// …
|
|
30
|
-
}
|
|
31
|
-
```
|
|
32
|
-
|
|
33
|
-
Streaming is bounded by the `(scope, counterId, timeframe)` filter you provide.
|
|
34
|
-
|
|
35
|
-
## Step 2: Fingerprint + manifest
|
|
36
|
-
|
|
37
|
-
The fingerprint sorts canonical JSON lines in memory and computes:
|
|
38
|
-
|
|
39
|
-
- `cellCount` — number of cells in the stream.
|
|
40
|
-
- `contentSha256` — SHA-256 of the sorted canonical lines.
|
|
41
|
-
- `countSum` — sum of `count` across all cells (sanity-check field).
|
|
42
|
-
|
|
43
|
-
```ts
|
|
44
|
-
import { computeCellsFingerprint, createBackupManifest, COUNTEX_CELL_SCHEMA_VERSION } from "@x12i/countex";
|
|
45
|
-
|
|
46
|
-
const filter = {
|
|
47
|
-
scope: "org_1",
|
|
48
|
-
counterId: "events",
|
|
49
|
-
timeframe: { type: "day", from: "2026-05-01", to: "2026-05-07" },
|
|
50
|
-
};
|
|
51
|
-
|
|
52
|
-
const fp = await computeCellsFingerprint(exportCells(durable, filter));
|
|
53
|
-
const manifest = createBackupManifest(filter, fp, COUNTEX_CELL_SCHEMA_VERSION);
|
|
54
|
-
// persist manifest JSON next to the NDJSON dump…
|
|
55
|
-
```
|
|
56
|
-
|
|
57
|
-
Resulting manifest:
|
|
58
|
-
|
|
59
|
-
```json
|
|
60
|
-
{
|
|
61
|
-
"manifestVersion": 1,
|
|
62
|
-
"schemaVersion": 1,
|
|
63
|
-
"createdAtIso": "2026-05-09T08:00:00.000Z",
|
|
64
|
-
"filter": { "scope": "org_1", "counterId": "events", "timeframe": { "type": "day", "from": "2026-05-01", "to": "2026-05-07" } },
|
|
65
|
-
"cellCount": 1234,
|
|
66
|
-
"contentSha256": "…",
|
|
67
|
-
"countSum": 98765
|
|
68
|
-
}
|
|
69
|
-
```
|
|
70
|
-
|
|
71
|
-
## Step 3: Verify
|
|
72
|
-
|
|
73
|
-
```ts
|
|
74
|
-
import { verifyBackupAgainstManifest } from "@x12i/countex";
|
|
75
|
-
|
|
76
|
-
const check = await verifyBackupAgainstManifest(manifest, exportCells(durable, manifest.filter));
|
|
77
|
-
if (!check.ok) throw new Error(check.detail);
|
|
78
|
-
```
|
|
79
|
-
|
|
80
|
-
Or via the namespace, which uses the engine's durable adapter:
|
|
81
|
-
|
|
82
|
-
```ts
|
|
83
|
-
const check = await countex.backup.verify(manifest);
|
|
84
|
-
```
|
|
85
|
-
|
|
86
|
-
`verify` checks `cellCount`, `contentSha256`, and (when present) `countSum`. Any mismatch returns `{ ok: false, detail }`.
|
|
87
|
-
|
|
88
|
-
## Step 4: Restore (modes)
|
|
89
|
-
|
|
90
|
-
```ts
|
|
91
|
-
import { restoreSnapshotCells } from "@x12i/countex";
|
|
92
|
-
|
|
93
|
-
const { restored, mode } = await restoreSnapshotCells(
|
|
94
|
-
targetDurable,
|
|
95
|
-
exportCells(sourceDurable, filter),
|
|
96
|
-
{ mode: "replace-window", batchSize: 500 }
|
|
97
|
-
);
|
|
98
|
-
```
|
|
99
|
-
|
|
100
|
-
Or via the namespace:
|
|
101
|
-
|
|
102
|
-
```ts
|
|
103
|
-
await countex.backup.restore(countex.backup.export(filter), { mode: "replace-window" });
|
|
104
|
-
```
|
|
105
|
-
|
|
106
|
-
Modes:
|
|
107
|
-
|
|
108
|
-
| Mode | Behavior |
|
|
109
|
-
|---|---|
|
|
110
|
-
| `validate-only` | Count cells without writing. Equivalent to legacy `dryRun: true`. |
|
|
111
|
-
| `replace-window` | Replace each cell identity in the stream via `importSnapshotCells`. Default. |
|
|
112
|
-
| `merge-deltas` | **Reserved.** Throws `NotImplementedError` today. |
|
|
113
|
-
|
|
114
|
-
After a `replace-window` restore, **rebuild Redis** for the affected windows so coverage flips back to `complete`.
|
|
115
|
-
|
|
116
|
-
## Step 5: Cache rebuild
|
|
117
|
-
|
|
118
|
-
```ts
|
|
119
|
-
const { cellsWarmed } = await countex.backup.rebuildCache({
|
|
120
|
-
scope: "org_1",
|
|
121
|
-
counterId: "events",
|
|
122
|
-
timeframe: { type: "day", from: "2026-05-01", to: "2026-05-07" },
|
|
123
|
-
});
|
|
124
|
-
```
|
|
125
|
-
|
|
126
|
-
`rebuildCache` streams `streamCells` from durable in batches and calls `cache.warm(batch)`. Each warmed bucket's `:cov` hash flips to `status=complete`.
|
|
127
|
-
|
|
128
|
-
## Cancellation
|
|
129
|
-
|
|
130
|
-
Pass `AbortSignal` where supported (`streamCells` / `warm` / `restoreSnapshotCells` options propagate through adapters).
|
|
131
|
-
|
|
132
|
-
## See also
|
|
133
|
-
|
|
134
|
-
- Internal: [Backup streaming](../internals/internal-backup-stream.md)
|
|
135
|
-
- Internal: [Mongo durable](../internals/internal-storage-mongo.md) — `importSnapshotCells` semantics.
|
|
1
|
+
# Guide: Backup, manifest, verify, restore, and cache rebuild
|
|
2
|
+
|
|
3
|
+
`@x12i/countex` ships export, fingerprint, manifest, verify, restore, and cache-rebuild helpers. Use them as free functions or via the `countex.backup` namespace bound to your engine.
|
|
4
|
+
|
|
5
|
+
## Step 1: Export cells (stream)
|
|
6
|
+
|
|
7
|
+
Free function:
|
|
8
|
+
|
|
9
|
+
```ts
|
|
10
|
+
import { exportCells } from "@x12i/countex";
|
|
11
|
+
|
|
12
|
+
for await (const cell of exportCells(durable, {
|
|
13
|
+
scope: "org_1",
|
|
14
|
+
counterId: "events",
|
|
15
|
+
timeframe: { type: "day", from: "2026-05-01", to: "2026-05-07" },
|
|
16
|
+
})) {
|
|
17
|
+
// serialize to NDJSON / S3 / cold storage…
|
|
18
|
+
}
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
Namespace:
|
|
22
|
+
|
|
23
|
+
```ts
|
|
24
|
+
for await (const cell of countex.backup.export({
|
|
25
|
+
scope: "org_1",
|
|
26
|
+
counterId: "events",
|
|
27
|
+
timeframe: { type: "day", from: "2026-05-01", to: "2026-05-07" },
|
|
28
|
+
})) {
|
|
29
|
+
// …
|
|
30
|
+
}
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
Streaming is bounded by the `(scope, counterId, timeframe)` filter you provide.
|
|
34
|
+
|
|
35
|
+
## Step 2: Fingerprint + manifest
|
|
36
|
+
|
|
37
|
+
The fingerprint sorts canonical JSON lines in memory and computes:
|
|
38
|
+
|
|
39
|
+
- `cellCount` — number of cells in the stream.
|
|
40
|
+
- `contentSha256` — SHA-256 of the sorted canonical lines.
|
|
41
|
+
- `countSum` — sum of `count` across all cells (sanity-check field).
|
|
42
|
+
|
|
43
|
+
```ts
|
|
44
|
+
import { computeCellsFingerprint, createBackupManifest, COUNTEX_CELL_SCHEMA_VERSION } from "@x12i/countex";
|
|
45
|
+
|
|
46
|
+
const filter = {
|
|
47
|
+
scope: "org_1",
|
|
48
|
+
counterId: "events",
|
|
49
|
+
timeframe: { type: "day", from: "2026-05-01", to: "2026-05-07" },
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
const fp = await computeCellsFingerprint(exportCells(durable, filter));
|
|
53
|
+
const manifest = createBackupManifest(filter, fp, COUNTEX_CELL_SCHEMA_VERSION);
|
|
54
|
+
// persist manifest JSON next to the NDJSON dump…
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
Resulting manifest:
|
|
58
|
+
|
|
59
|
+
```json
|
|
60
|
+
{
|
|
61
|
+
"manifestVersion": 1,
|
|
62
|
+
"schemaVersion": 1,
|
|
63
|
+
"createdAtIso": "2026-05-09T08:00:00.000Z",
|
|
64
|
+
"filter": { "scope": "org_1", "counterId": "events", "timeframe": { "type": "day", "from": "2026-05-01", "to": "2026-05-07" } },
|
|
65
|
+
"cellCount": 1234,
|
|
66
|
+
"contentSha256": "…",
|
|
67
|
+
"countSum": 98765
|
|
68
|
+
}
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
## Step 3: Verify
|
|
72
|
+
|
|
73
|
+
```ts
|
|
74
|
+
import { verifyBackupAgainstManifest } from "@x12i/countex";
|
|
75
|
+
|
|
76
|
+
const check = await verifyBackupAgainstManifest(manifest, exportCells(durable, manifest.filter));
|
|
77
|
+
if (!check.ok) throw new Error(check.detail);
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
Or via the namespace, which uses the engine's durable adapter:
|
|
81
|
+
|
|
82
|
+
```ts
|
|
83
|
+
const check = await countex.backup.verify(manifest);
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
`verify` checks `cellCount`, `contentSha256`, and (when present) `countSum`. Any mismatch returns `{ ok: false, detail }`.
|
|
87
|
+
|
|
88
|
+
## Step 4: Restore (modes)
|
|
89
|
+
|
|
90
|
+
```ts
|
|
91
|
+
import { restoreSnapshotCells } from "@x12i/countex";
|
|
92
|
+
|
|
93
|
+
const { restored, mode } = await restoreSnapshotCells(
|
|
94
|
+
targetDurable,
|
|
95
|
+
exportCells(sourceDurable, filter),
|
|
96
|
+
{ mode: "replace-window", batchSize: 500 }
|
|
97
|
+
);
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
Or via the namespace:
|
|
101
|
+
|
|
102
|
+
```ts
|
|
103
|
+
await countex.backup.restore(countex.backup.export(filter), { mode: "replace-window" });
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
Modes:
|
|
107
|
+
|
|
108
|
+
| Mode | Behavior |
|
|
109
|
+
|---|---|
|
|
110
|
+
| `validate-only` | Count cells without writing. Equivalent to legacy `dryRun: true`. |
|
|
111
|
+
| `replace-window` | Replace each cell identity in the stream via `importSnapshotCells`. Default. |
|
|
112
|
+
| `merge-deltas` | **Reserved.** Throws `NotImplementedError` today. |
|
|
113
|
+
|
|
114
|
+
After a `replace-window` restore, **rebuild Redis** for the affected windows so coverage flips back to `complete`.
|
|
115
|
+
|
|
116
|
+
## Step 5: Cache rebuild
|
|
117
|
+
|
|
118
|
+
```ts
|
|
119
|
+
const { cellsWarmed } = await countex.backup.rebuildCache({
|
|
120
|
+
scope: "org_1",
|
|
121
|
+
counterId: "events",
|
|
122
|
+
timeframe: { type: "day", from: "2026-05-01", to: "2026-05-07" },
|
|
123
|
+
});
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
`rebuildCache` streams `streamCells` from durable in batches and calls `cache.warm(batch)`. Each warmed bucket's `:cov` hash flips to `status=complete`.
|
|
127
|
+
|
|
128
|
+
## Cancellation
|
|
129
|
+
|
|
130
|
+
Pass `AbortSignal` where supported (`streamCells` / `warm` / `restoreSnapshotCells` options propagate through adapters).
|
|
131
|
+
|
|
132
|
+
## See also
|
|
133
|
+
|
|
134
|
+
- Internal: [Backup streaming](../internals/internal-backup-stream.md)
|
|
135
|
+
- Internal: [Mongo durable](../internals/internal-storage-mongo.md) — `importSnapshotCells` semantics.
|
|
@@ -1,50 +1,50 @@
|
|
|
1
|
-
# Guide: Cardinality protection
|
|
2
|
-
|
|
3
|
-
Cardinality limits stop runaway distinct **subjects**, **metrics**, or **dimension values** from exploding storage when a map accidentally points at high-cardinality fields (URLs, UUIDs everywhere, etc.).
|
|
4
|
-
|
|
5
|
-
## Step 1: Configure engine defaults
|
|
6
|
-
|
|
7
|
-
```ts
|
|
8
|
-
const countex = createCountex({
|
|
9
|
-
maps: [myMap],
|
|
10
|
-
durable,
|
|
11
|
-
cache,
|
|
12
|
-
cardinality: {
|
|
13
|
-
maxDimensionValuesPerName: 10_000,
|
|
14
|
-
maxSubjectsPerScope: 10_000_000,
|
|
15
|
-
onLimit: "overflow",
|
|
16
|
-
overflowValue: "__OTHER__",
|
|
17
|
-
highCardinalityWarnings: true,
|
|
18
|
-
},
|
|
19
|
-
});
|
|
20
|
-
```
|
|
21
|
-
|
|
22
|
-
## Step 2: Override per map (optional)
|
|
23
|
-
|
|
24
|
-
Add `cardinality` on a `CounterMap` for stricter rules on sensitive projections.
|
|
25
|
-
|
|
26
|
-
## Step 3: Understand behaviors (`onLimit`)
|
|
27
|
-
|
|
28
|
-
| Mode | Behavior |
|
|
29
|
-
|------|-----------|
|
|
30
|
-
| `overflow` | Rewrite additional distinct keys to `overflowValue` |
|
|
31
|
-
| `drop` | Skip increments that would exceed limits |
|
|
32
|
-
| `error` | Throw (surrounding ingest handles via logging; prefer rare use) |
|
|
33
|
-
|
|
34
|
-
## Step 4: Listen for signals
|
|
35
|
-
|
|
36
|
-
```ts
|
|
37
|
-
countex.on("cardinality_warning", (w) => { /* nearing limit */ });
|
|
38
|
-
countex.on("overflow", (o) => { /* key rewritten */ });
|
|
39
|
-
countex.on("dropped", (d) => { /* increment skipped */ });
|
|
40
|
-
```
|
|
41
|
-
|
|
42
|
-
## Practical tips
|
|
43
|
-
|
|
44
|
-
- Prefer bounded dimensions (enums, categories).
|
|
45
|
-
- Use `overflow` in production pipelines so one bad event shape does not halt ingestion.
|
|
46
|
-
|
|
47
|
-
## See also
|
|
48
|
-
|
|
49
|
-
- [Counter maps](./counter-maps.md)
|
|
50
|
-
- Internal: [Cardinality guard](../internals/internal-cardinality.md)
|
|
1
|
+
# Guide: Cardinality protection
|
|
2
|
+
|
|
3
|
+
Cardinality limits stop runaway distinct **subjects**, **metrics**, or **dimension values** from exploding storage when a map accidentally points at high-cardinality fields (URLs, UUIDs everywhere, etc.).
|
|
4
|
+
|
|
5
|
+
## Step 1: Configure engine defaults
|
|
6
|
+
|
|
7
|
+
```ts
|
|
8
|
+
const countex = createCountex({
|
|
9
|
+
maps: [myMap],
|
|
10
|
+
durable,
|
|
11
|
+
cache,
|
|
12
|
+
cardinality: {
|
|
13
|
+
maxDimensionValuesPerName: 10_000,
|
|
14
|
+
maxSubjectsPerScope: 10_000_000,
|
|
15
|
+
onLimit: "overflow",
|
|
16
|
+
overflowValue: "__OTHER__",
|
|
17
|
+
highCardinalityWarnings: true,
|
|
18
|
+
},
|
|
19
|
+
});
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
## Step 2: Override per map (optional)
|
|
23
|
+
|
|
24
|
+
Add `cardinality` on a `CounterMap` for stricter rules on sensitive projections.
|
|
25
|
+
|
|
26
|
+
## Step 3: Understand behaviors (`onLimit`)
|
|
27
|
+
|
|
28
|
+
| Mode | Behavior |
|
|
29
|
+
|------|-----------|
|
|
30
|
+
| `overflow` | Rewrite additional distinct keys to `overflowValue` |
|
|
31
|
+
| `drop` | Skip increments that would exceed limits |
|
|
32
|
+
| `error` | Throw (surrounding ingest handles via logging; prefer rare use) |
|
|
33
|
+
|
|
34
|
+
## Step 4: Listen for signals
|
|
35
|
+
|
|
36
|
+
```ts
|
|
37
|
+
countex.on("cardinality_warning", (w) => { /* nearing limit */ });
|
|
38
|
+
countex.on("overflow", (o) => { /* key rewritten */ });
|
|
39
|
+
countex.on("dropped", (d) => { /* increment skipped */ });
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
## Practical tips
|
|
43
|
+
|
|
44
|
+
- Prefer bounded dimensions (enums, categories).
|
|
45
|
+
- Use `overflow` in production pipelines so one bad event shape does not halt ingestion.
|
|
46
|
+
|
|
47
|
+
## See also
|
|
48
|
+
|
|
49
|
+
- [Counter maps](./counter-maps.md)
|
|
50
|
+
- Internal: [Cardinality guard](../internals/internal-cardinality.md)
|
package/docs/guides/cli.md
CHANGED
|
@@ -1,53 +1,53 @@
|
|
|
1
|
-
# Guide: CLI (`countex`)
|
|
2
|
-
|
|
3
|
-
The package exposes a minimal **`countex`** binary for running selected insights against a **config module** you provide.
|
|
4
|
-
|
|
5
|
-
## Step 1: Build or install
|
|
6
|
-
|
|
7
|
-
After install, the binary resolves to `dist/cli.js`. For local development:
|
|
8
|
-
|
|
9
|
-
```bash
|
|
10
|
-
npm run build
|
|
11
|
-
npx countex insights:top ./my-context.mjs --scope org_1 --counter my_counter
|
|
12
|
-
```
|
|
13
|
-
|
|
14
|
-
## Step 2: Author a config module
|
|
15
|
-
|
|
16
|
-
Export one of:
|
|
17
|
-
|
|
18
|
-
- `export async function createContext()` returning `{ insights: CountexInsights }`
|
|
19
|
-
- `export default { insights }`
|
|
20
|
-
- `export const insights`
|
|
21
|
-
|
|
22
|
-
## Step 3: Run a command
|
|
23
|
-
|
|
24
|
-
```bash
|
|
25
|
-
countex insights:top ./ctx.mjs --scope org_1 --counter usage
|
|
26
|
-
countex insights:pareto ./ctx.mjs --format markdown
|
|
27
|
-
countex insights:anomalies ./ctx.mjs
|
|
28
|
-
countex insights:cost ./ctx.mjs
|
|
29
|
-
countex insights:tokens ./ctx.mjs
|
|
30
|
-
countex insights:map-health ./ctx.mjs
|
|
31
|
-
countex insights:resources ./ctx.mjs
|
|
32
|
-
```
|
|
33
|
-
|
|
34
|
-
Use `--format json|markdown` for output shaping.
|
|
35
|
-
|
|
36
|
-
## Simulation demo (no config file)
|
|
37
|
-
|
|
38
|
-
```bash
|
|
39
|
-
countex simulate:demo --events 200 --seed 1
|
|
40
|
-
```
|
|
41
|
-
|
|
42
|
-
Emits JSON **`SimulationReport`** from a dry-run against the bundled demo map (`audit-log` profile). No Mongo/Redis required.
|
|
43
|
-
|
|
44
|
-
## Step 4: TypeScript configs
|
|
45
|
-
|
|
46
|
-
```bash
|
|
47
|
-
node --import=tsx node_modules/@x12i/countex/dist/cli.js insights:top ./ctx.ts --scope org_1
|
|
48
|
-
```
|
|
49
|
-
|
|
50
|
-
## See also
|
|
51
|
-
|
|
52
|
-
- [Insights overview](./insights-overview.md)
|
|
53
|
-
- `src/cli.ts` for the authoritative command list
|
|
1
|
+
# Guide: CLI (`countex`)
|
|
2
|
+
|
|
3
|
+
The package exposes a minimal **`countex`** binary for running selected insights against a **config module** you provide.
|
|
4
|
+
|
|
5
|
+
## Step 1: Build or install
|
|
6
|
+
|
|
7
|
+
After install, the binary resolves to `dist/cli.js`. For local development:
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
npm run build
|
|
11
|
+
npx countex insights:top ./my-context.mjs --scope org_1 --counter my_counter
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
## Step 2: Author a config module
|
|
15
|
+
|
|
16
|
+
Export one of:
|
|
17
|
+
|
|
18
|
+
- `export async function createContext()` returning `{ insights: CountexInsights }`
|
|
19
|
+
- `export default { insights }`
|
|
20
|
+
- `export const insights`
|
|
21
|
+
|
|
22
|
+
## Step 3: Run a command
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
countex insights:top ./ctx.mjs --scope org_1 --counter usage
|
|
26
|
+
countex insights:pareto ./ctx.mjs --format markdown
|
|
27
|
+
countex insights:anomalies ./ctx.mjs
|
|
28
|
+
countex insights:cost ./ctx.mjs
|
|
29
|
+
countex insights:tokens ./ctx.mjs
|
|
30
|
+
countex insights:map-health ./ctx.mjs
|
|
31
|
+
countex insights:resources ./ctx.mjs
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
Use `--format json|markdown` for output shaping.
|
|
35
|
+
|
|
36
|
+
## Simulation demo (no config file)
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
countex simulate:demo --events 200 --seed 1
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
Emits JSON **`SimulationReport`** from a dry-run against the bundled demo map (`audit-log` profile). No Mongo/Redis required.
|
|
43
|
+
|
|
44
|
+
## Step 4: TypeScript configs
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
node --import=tsx node_modules/@x12i/countex/dist/cli.js insights:top ./ctx.ts --scope org_1
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
## See also
|
|
51
|
+
|
|
52
|
+
- [Insights overview](./insights-overview.md)
|
|
53
|
+
- `src/cli.ts` for the authoritative command list
|
|
@@ -1,65 +1,65 @@
|
|
|
1
|
-
# Guide: Client events (`Countex`)
|
|
2
|
-
|
|
3
|
-
`Countex` extends `EventEmitter`. Subscribe for operational visibility.
|
|
4
|
-
|
|
5
|
-
## Step 1: Subscribe
|
|
6
|
-
|
|
7
|
-
```ts
|
|
8
|
-
countex.on("flush", ({ cells, durationMs }) => {
|
|
9
|
-
console.log("flushed", cells, durationMs);
|
|
10
|
-
});
|
|
11
|
-
|
|
12
|
-
countex.on("flush_error", ({ error }) => {
|
|
13
|
-
console.error(error);
|
|
14
|
-
});
|
|
15
|
-
|
|
16
|
-
countex.on("cardinality_warning", (w) => {
|
|
17
|
-
console.warn("cardinality_warning", w);
|
|
18
|
-
});
|
|
19
|
-
|
|
20
|
-
countex.on("overflow", (o) => {
|
|
21
|
-
console.warn("overflow", o);
|
|
22
|
-
});
|
|
23
|
-
|
|
24
|
-
countex.on("dropped", (d) => {
|
|
25
|
-
console.warn("dropped", d);
|
|
26
|
-
});
|
|
27
|
-
|
|
28
|
-
countex.on("duplicate", ({ scopeId, eventId }) => {
|
|
29
|
-
console.info("duplicate", scopeId, eventId);
|
|
30
|
-
});
|
|
31
|
-
|
|
32
|
-
countex.on("measure_invalid", (m) => {
|
|
33
|
-
console.warn("measure_invalid", m);
|
|
34
|
-
});
|
|
35
|
-
|
|
36
|
-
countex.on("evidence_flush_error", ({ error }) => {
|
|
37
|
-
console.error("evidence_flush_error", error);
|
|
38
|
-
});
|
|
39
|
-
|
|
40
|
-
countex.on("triage_skipped", (t) => {
|
|
41
|
-
console.info("triage_skipped", t.counterId, t.reason);
|
|
42
|
-
});
|
|
43
|
-
|
|
44
|
-
countex.on("triage_error", (t) => {
|
|
45
|
-
console.error("triage_error", t.counterId, t.error);
|
|
46
|
-
});
|
|
47
|
-
|
|
48
|
-
countex.on("triage_latency", (t) => {
|
|
49
|
-
console.debug("triage_latency", t.counterId, t.durationMs);
|
|
50
|
-
});
|
|
51
|
-
```
|
|
52
|
-
|
|
53
|
-
## Step 2: Type payloads
|
|
54
|
-
|
|
55
|
-
Import `CountexEvents` from types if you want compile-time checks when wiring handlers.
|
|
56
|
-
|
|
57
|
-
## Step 3: Unsubscribe
|
|
58
|
-
|
|
59
|
-
Use `off` with the same listener reference when tearing down tests.
|
|
60
|
-
|
|
61
|
-
## See also
|
|
62
|
-
|
|
63
|
-
- [Ingestion and flush](./ingestion-and-flush.md)
|
|
64
|
-
- [Cardinality](./cardinality.md)
|
|
65
|
-
- [Measures](./measures.md)
|
|
1
|
+
# Guide: Client events (`Countex`)
|
|
2
|
+
|
|
3
|
+
`Countex` extends `EventEmitter`. Subscribe for operational visibility.
|
|
4
|
+
|
|
5
|
+
## Step 1: Subscribe
|
|
6
|
+
|
|
7
|
+
```ts
|
|
8
|
+
countex.on("flush", ({ cells, durationMs }) => {
|
|
9
|
+
console.log("flushed", cells, durationMs);
|
|
10
|
+
});
|
|
11
|
+
|
|
12
|
+
countex.on("flush_error", ({ error }) => {
|
|
13
|
+
console.error(error);
|
|
14
|
+
});
|
|
15
|
+
|
|
16
|
+
countex.on("cardinality_warning", (w) => {
|
|
17
|
+
console.warn("cardinality_warning", w);
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
countex.on("overflow", (o) => {
|
|
21
|
+
console.warn("overflow", o);
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
countex.on("dropped", (d) => {
|
|
25
|
+
console.warn("dropped", d);
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
countex.on("duplicate", ({ scopeId, eventId }) => {
|
|
29
|
+
console.info("duplicate", scopeId, eventId);
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
countex.on("measure_invalid", (m) => {
|
|
33
|
+
console.warn("measure_invalid", m);
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
countex.on("evidence_flush_error", ({ error }) => {
|
|
37
|
+
console.error("evidence_flush_error", error);
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
countex.on("triage_skipped", (t) => {
|
|
41
|
+
console.info("triage_skipped", t.counterId, t.reason);
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
countex.on("triage_error", (t) => {
|
|
45
|
+
console.error("triage_error", t.counterId, t.error);
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
countex.on("triage_latency", (t) => {
|
|
49
|
+
console.debug("triage_latency", t.counterId, t.durationMs);
|
|
50
|
+
});
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
## Step 2: Type payloads
|
|
54
|
+
|
|
55
|
+
Import `CountexEvents` from types if you want compile-time checks when wiring handlers.
|
|
56
|
+
|
|
57
|
+
## Step 3: Unsubscribe
|
|
58
|
+
|
|
59
|
+
Use `off` with the same listener reference when tearing down tests.
|
|
60
|
+
|
|
61
|
+
## See also
|
|
62
|
+
|
|
63
|
+
- [Ingestion and flush](./ingestion-and-flush.md)
|
|
64
|
+
- [Cardinality](./cardinality.md)
|
|
65
|
+
- [Measures](./measures.md)
|