@kici-dev/shared 0.0.0 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +1 -6
- package/dist/chunk-gOLHoazu.js +4 -0
- package/dist/cold-store/bucket.d.ts +26 -0
- package/dist/cold-store/bucket.js +40 -0
- package/dist/cold-store/bucket.test.d.ts +2 -0
- package/dist/cold-store/chunk-encoder.d.ts +63 -0
- package/dist/cold-store/chunk-encoder.js +94 -0
- package/dist/cold-store/chunk-encoder.test.d.ts +2 -0
- package/dist/cold-store/chunk-id.d.ts +10 -0
- package/dist/cold-store/chunk-id.js +29 -0
- package/dist/cold-store/chunk-id.test.d.ts +2 -0
- package/dist/cold-store/cold-store.d.ts +345 -0
- package/dist/cold-store/cold-store.js +1078 -0
- package/dist/cold-store/cold-store.test.d.ts +2 -0
- package/dist/cold-store/config.d.ts +62 -0
- package/dist/cold-store/config.js +28 -0
- package/dist/cold-store/index.d.ts +20 -0
- package/dist/cold-store/index.js +11 -0
- package/dist/cold-store/key.d.ts +89 -0
- package/dist/cold-store/key.js +88 -0
- package/dist/cold-store/key.test.d.ts +2 -0
- package/dist/cold-store/lru.d.ts +33 -0
- package/dist/cold-store/lru.js +59 -0
- package/dist/cold-store/lru.test.d.ts +2 -0
- package/dist/cold-store/manifest.d.ts +11 -0
- package/dist/cold-store/manifest.js +57 -0
- package/dist/cold-store/manifest.test.d.ts +2 -0
- package/dist/cold-store/metrics.d.ts +103 -0
- package/dist/cold-store/metrics.js +179 -0
- package/dist/cold-store/table-adapter.d.ts +229 -0
- package/dist/cold-store/table-adapter.js +2 -0
- package/dist/cold-store/types.d.ts +111 -0
- package/dist/cold-store/types.js +2 -0
- package/dist/crypto.d.ts +33 -0
- package/dist/crypto.js +67 -0
- package/dist/db-admin.d.ts +1422 -0
- package/dist/db-admin.js +2525 -0
- package/dist/db-admin.test.d.ts +2 -0
- package/dist/db.d.ts +14 -0
- package/dist/db.js +23 -0
- package/dist/env/allowlist.d.ts +79 -0
- package/dist/env/allowlist.js +86 -0
- package/dist/env/define-env.d.ts +172 -0
- package/dist/env/define-env.js +295 -0
- package/dist/env/define-env.test.d.ts +2 -0
- package/dist/env/env-rule-allowlist.test.d.ts +2 -0
- package/dist/env/index.d.ts +11 -0
- package/dist/env/index.js +4 -0
- package/dist/env/logger-env.d.ts +30 -0
- package/dist/env/logger-env.js +95 -0
- package/dist/error.d.ts +16 -0
- package/dist/error.js +58 -0
- package/dist/error.test.d.ts +2 -0
- package/dist/format-bytes.d.ts +5 -0
- package/dist/format-bytes.js +15 -0
- package/dist/format-bytes.test.d.ts +2 -0
- package/dist/format-duration.d.ts +11 -0
- package/dist/format-duration.js +32 -0
- package/dist/format-duration.test.d.ts +2 -0
- package/dist/graceful-shutdown.d.ts +58 -0
- package/dist/graceful-shutdown.js +71 -0
- package/dist/graceful-shutdown.test.d.ts +2 -0
- package/dist/idempotency-files.d.ts +113 -0
- package/dist/idempotency-files.js +190 -0
- package/dist/idempotency-files.test.d.ts +2 -0
- package/dist/idempotency.d.ts +46 -0
- package/dist/idempotency.js +45 -0
- package/dist/idempotency.test.d.ts +2 -0
- package/dist/index.d.ts +19 -0
- package/dist/index.js +31 -0
- package/dist/logger.d.ts +57 -0
- package/dist/logger.js +175 -0
- package/dist/logger.test.d.ts +2 -0
- package/dist/reconnect-delay.d.ts +11 -0
- package/dist/reconnect-delay.js +22 -0
- package/dist/request-context.d.ts +42 -0
- package/dist/request-context.js +37 -0
- package/dist/ring-buffer.d.ts +35 -0
- package/dist/ring-buffer.js +62 -0
- package/dist/ring-buffer.test.d.ts +2 -0
- package/dist/routes/health.d.ts +26 -0
- package/dist/routes/health.js +45 -0
- package/dist/routes/health.test.d.ts +2 -0
- package/dist/routes/metrics.d.ts +20 -0
- package/dist/routes/metrics.js +38 -0
- package/dist/s3-client.d.ts +42 -0
- package/dist/s3-client.js +25 -0
- package/dist/telemetry/index.d.ts +3 -0
- package/dist/telemetry/index.js +4 -0
- package/dist/telemetry/init.d.ts +20 -0
- package/dist/telemetry/init.js +42 -0
- package/dist/telemetry/init.test.d.ts +2 -0
- package/dist/telemetry/metrics.d.ts +8 -0
- package/dist/telemetry/metrics.js +16 -0
- package/dist/tool-check.d.ts +33 -0
- package/dist/tool-check.js +60 -0
- package/dist/tool-check.test.d.ts +2 -0
- package/dist/ts-loader-hook.d.ts +26 -0
- package/dist/ts-loader-hook.js +48 -0
- package/dist/zx.d.ts +8 -0
- package/dist/zx.js +78 -0
- package/package.json +69 -5
- package/sbom.spdx.json +8775 -0
- package/index.js +0 -3
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
import "../chunk-gOLHoazu.js";
|
|
2
|
+
import { createMeter } from "../telemetry/metrics.js";
|
|
3
|
+
//#region src/cold-store/metrics.ts
|
|
4
|
+
let _meter;
|
|
5
|
+
function meter() {
|
|
6
|
+
if (!_meter) _meter = createMeter("kici-cold-store");
|
|
7
|
+
return _meter;
|
|
8
|
+
}
|
|
9
|
+
let _archiveCyclesTotal;
|
|
10
|
+
let _archiveRowsTotal;
|
|
11
|
+
let _archiveBytesTotal;
|
|
12
|
+
let _archiveDurationSeconds;
|
|
13
|
+
let _rehydrateRequestsTotal;
|
|
14
|
+
let _rehydrateBytesTotal;
|
|
15
|
+
let _rehydrateDurationSeconds;
|
|
16
|
+
let _verifyFailuresTotal;
|
|
17
|
+
let _replayRowsTotal;
|
|
18
|
+
let _replayDurationSeconds;
|
|
19
|
+
let _purgeChunksTotal;
|
|
20
|
+
let _purgeBytesTotal;
|
|
21
|
+
let _purgeDurationSeconds;
|
|
22
|
+
/**
|
|
23
|
+
* Cycles that `runArchiveCycle()` has completed, labeled by outcome.
|
|
24
|
+
* Always incremented by 1 on every tick — guaranteed-visible proof
|
|
25
|
+
* that the cold-store subsystem is registered and running.
|
|
26
|
+
*
|
|
27
|
+
* `result` ∈ `no_tables` | `disabled` | `success` | `failure`.
|
|
28
|
+
*/
|
|
29
|
+
function coldStoreArchiveCyclesTotal() {
|
|
30
|
+
if (!_archiveCyclesTotal) _archiveCyclesTotal = meter().createCounter("cold_store_archive_cycles_total", { description: "Completed archive cycles by db and outcome" });
|
|
31
|
+
return _archiveCyclesTotal;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Total rows archived, by db / table / outcome.
|
|
35
|
+
* `result` ∈ success | failure | skipped_min_chunk | skipped_min_warm.
|
|
36
|
+
*
|
|
37
|
+
* Hidden from /metrics until the first Phase B+ TableAdapter moves
|
|
38
|
+
* a row; the visible proof that the subsystem is wired is
|
|
39
|
+
* `cold_store_archive_cycles_total`, not this counter.
|
|
40
|
+
*/
|
|
41
|
+
function coldStoreArchiveRowsTotal() {
|
|
42
|
+
if (!_archiveRowsTotal) _archiveRowsTotal = meter().createCounter("cold_store_archive_rows_total", { description: "Rows archived into cold storage by db, table, and outcome" });
|
|
43
|
+
return _archiveRowsTotal;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Total bytes archived, by kind ∈ raw | gzipped. Lets us compute the
|
|
47
|
+
* ongoing compression ratio.
|
|
48
|
+
*/
|
|
49
|
+
function coldStoreArchiveBytesTotal() {
|
|
50
|
+
if (!_archiveBytesTotal) _archiveBytesTotal = meter().createCounter("cold_store_archive_bytes_total", { description: "Bytes archived (raw vs gzipped) by db and table" });
|
|
51
|
+
return _archiveBytesTotal;
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Per-chunk archive duration histogram.
|
|
55
|
+
* Buckets: 10ms, 100ms, 500ms, 1s, 5s, 30s, 120s.
|
|
56
|
+
*/
|
|
57
|
+
function coldStoreArchiveDurationSeconds() {
|
|
58
|
+
if (!_archiveDurationSeconds) _archiveDurationSeconds = meter().createHistogram("cold_store_archive_duration_seconds", {
|
|
59
|
+
description: "Archive duration per chunk, seconds",
|
|
60
|
+
advice: { explicitBucketBoundaries: [
|
|
61
|
+
.01,
|
|
62
|
+
.1,
|
|
63
|
+
.5,
|
|
64
|
+
1,
|
|
65
|
+
5,
|
|
66
|
+
30,
|
|
67
|
+
120
|
|
68
|
+
] }
|
|
69
|
+
});
|
|
70
|
+
return _archiveDurationSeconds;
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Rehydrate requests (cache hit vs miss). Incremented by the
|
|
74
|
+
* read-through layer (Phase B+) when it serves a range query from
|
|
75
|
+
* cold storage.
|
|
76
|
+
*/
|
|
77
|
+
function coldStoreRehydrateRequestsTotal() {
|
|
78
|
+
if (!_rehydrateRequestsTotal) _rehydrateRequestsTotal = meter().createCounter("cold_store_rehydrate_requests_total", { description: "Cold-store rehydrate requests by cache outcome" });
|
|
79
|
+
return _rehydrateRequestsTotal;
|
|
80
|
+
}
|
|
81
|
+
/** Bytes read from S3 on rehydrate cache miss. */
|
|
82
|
+
function coldStoreRehydrateBytesTotal() {
|
|
83
|
+
if (!_rehydrateBytesTotal) _rehydrateBytesTotal = meter().createCounter("cold_store_rehydrate_bytes_total", { description: "Bytes read from S3 on rehydrate cache miss, by db and table" });
|
|
84
|
+
return _rehydrateBytesTotal;
|
|
85
|
+
}
|
|
86
|
+
/** Rehydrate duration (S3 fetch + decode), seconds. */
|
|
87
|
+
function coldStoreRehydrateDurationSeconds() {
|
|
88
|
+
if (!_rehydrateDurationSeconds) _rehydrateDurationSeconds = meter().createHistogram("cold_store_rehydrate_duration_seconds", {
|
|
89
|
+
description: "Rehydrate duration per request, seconds",
|
|
90
|
+
advice: { explicitBucketBoundaries: [
|
|
91
|
+
.01,
|
|
92
|
+
.1,
|
|
93
|
+
.5,
|
|
94
|
+
1,
|
|
95
|
+
5,
|
|
96
|
+
30
|
|
97
|
+
] }
|
|
98
|
+
});
|
|
99
|
+
return _rehydrateDurationSeconds;
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* Verify-failure counter — bumped on the rare `contentHash` mismatch
|
|
103
|
+
* path during post-write verification.
|
|
104
|
+
*/
|
|
105
|
+
function coldStoreVerifyFailuresTotal() {
|
|
106
|
+
if (!_verifyFailuresTotal) _verifyFailuresTotal = meter().createCounter("cold_store_verify_failures_total", { description: "Post-write contentHash mismatch count, by db and table" });
|
|
107
|
+
return _verifyFailuresTotal;
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Phase F — rows promoted back into PG via `replayChunk` / `replayRow`.
|
|
111
|
+
* `result` ∈ `success` | `failure` | `idempotent_skip`. The
|
|
112
|
+
* `idempotent_skip` bucket counts rows already present in PG (chunk
|
|
113
|
+
* replayed twice — no-op via ON CONFLICT). Hidden until the first
|
|
114
|
+
* replay runs; the visible proof of the subsystem is
|
|
115
|
+
* `cold_store_archive_cycles_total`.
|
|
116
|
+
*/
|
|
117
|
+
function coldStoreReplayRowsTotal() {
|
|
118
|
+
if (!_replayRowsTotal) _replayRowsTotal = meter().createCounter("cold_store_replay_rows_total", { description: "Rows promoted back into PG by db, table, and outcome" });
|
|
119
|
+
return _replayRowsTotal;
|
|
120
|
+
}
|
|
121
|
+
/**
|
|
122
|
+
* Phase F — replay duration (S3 GET + manifest scan + decode + INSERT),
|
|
123
|
+
* seconds. Buckets mirror archive duration: 10ms, 100ms, 500ms, 1s, 5s,
|
|
124
|
+
* 30s, 120s.
|
|
125
|
+
*/
|
|
126
|
+
function coldStoreReplayDurationSeconds() {
|
|
127
|
+
if (!_replayDurationSeconds) _replayDurationSeconds = meter().createHistogram("cold_store_replay_duration_seconds", {
|
|
128
|
+
description: "Replay duration per chunk, seconds",
|
|
129
|
+
advice: { explicitBucketBoundaries: [
|
|
130
|
+
.01,
|
|
131
|
+
.1,
|
|
132
|
+
.5,
|
|
133
|
+
1,
|
|
134
|
+
5,
|
|
135
|
+
30,
|
|
136
|
+
120
|
|
137
|
+
] }
|
|
138
|
+
});
|
|
139
|
+
return _replayDurationSeconds;
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Phase 2 — chunks acted on by `purgeExpiredChunks`, labeled by db,
|
|
143
|
+
* table, and outcome. `result` ∈ `purged` | `dry_run` | `skipped_locked`
|
|
144
|
+
* | `failure`. Hidden until the first GC sweep finds candidates;
|
|
145
|
+
* `cold_store_archive_cycles_total` remains the always-non-zero
|
|
146
|
+
* heartbeat for the subsystem.
|
|
147
|
+
*/
|
|
148
|
+
function coldStorePurgeChunksTotal() {
|
|
149
|
+
if (!_purgeChunksTotal) _purgeChunksTotal = meter().createCounter("cold_store_purge_chunks_total", { description: "Chunks processed by the purge sweep, by db, table, and outcome" });
|
|
150
|
+
return _purgeChunksTotal;
|
|
151
|
+
}
|
|
152
|
+
/** Phase 2 — gzipped bytes deleted from S3 by the purge sweep. */
|
|
153
|
+
function coldStorePurgeBytesTotal() {
|
|
154
|
+
if (!_purgeBytesTotal) _purgeBytesTotal = meter().createCounter("cold_store_purge_bytes_total", { description: "Gzipped bytes deleted by the purge sweep, by db and table" });
|
|
155
|
+
return _purgeBytesTotal;
|
|
156
|
+
}
|
|
157
|
+
/**
|
|
158
|
+
* Phase 2 — purge duration per sweep, seconds. Buckets mirror the
|
|
159
|
+
* archive duration histogram: 10ms, 100ms, 500ms, 1s, 5s, 30s, 120s.
|
|
160
|
+
*/
|
|
161
|
+
function coldStorePurgeDurationSeconds() {
|
|
162
|
+
if (!_purgeDurationSeconds) _purgeDurationSeconds = meter().createHistogram("cold_store_purge_duration_seconds", {
|
|
163
|
+
description: "Purge sweep duration per cycle, seconds",
|
|
164
|
+
advice: { explicitBucketBoundaries: [
|
|
165
|
+
.01,
|
|
166
|
+
.1,
|
|
167
|
+
.5,
|
|
168
|
+
1,
|
|
169
|
+
5,
|
|
170
|
+
30,
|
|
171
|
+
120
|
|
172
|
+
] }
|
|
173
|
+
});
|
|
174
|
+
return _purgeDurationSeconds;
|
|
175
|
+
}
|
|
176
|
+
//#endregion
|
|
177
|
+
export { coldStoreArchiveBytesTotal, coldStoreArchiveCyclesTotal, coldStoreArchiveDurationSeconds, coldStoreArchiveRowsTotal, coldStorePurgeBytesTotal, coldStorePurgeChunksTotal, coldStorePurgeDurationSeconds, coldStoreRehydrateBytesTotal, coldStoreRehydrateDurationSeconds, coldStoreRehydrateRequestsTotal, coldStoreReplayDurationSeconds, coldStoreReplayRowsTotal, coldStoreVerifyFailuresTotal };
|
|
178
|
+
|
|
179
|
+
//# sourceMappingURL=metrics.js.map
|
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* TableAdapter interface — the concrete per-table contract.
|
|
3
|
+
*
|
|
4
|
+
* Phase A shipped this interface with ZERO implementations. Phase B adds
|
|
5
|
+
* `run_events`; Phase C adds `execution_runs` / `execution_jobs` /
|
|
6
|
+
* `execution_steps`; etc. Each adapter lives next to its owning
|
|
7
|
+
* package (`packages/{platform,orchestrator}/src/cold-store/tables/`)
|
|
8
|
+
* and is registered on the respective `BaseColdStore` subclass.
|
|
9
|
+
*/
|
|
10
|
+
import type { ColdStoreTableConfig } from './config.js';
|
|
11
|
+
import type { DbKind } from './key.js';
|
|
12
|
+
import type { ColdRetention } from './types.js';
|
|
13
|
+
/**
|
|
14
|
+
* Discovered (tenant, partition-day) tuple eligible for archival.
|
|
15
|
+
* Yielded by `listEligiblePartitions` in PK-ordered streams.
|
|
16
|
+
*/
|
|
17
|
+
export interface EligiblePartition {
|
|
18
|
+
tenantId: string;
|
|
19
|
+
/** `YYYY-MM-DD`. */
|
|
20
|
+
partitionDate: string;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Arguments to `writeAuditAndChunkCount`. The adapter's implementation
|
|
24
|
+
* is responsible for writing a row into whichever audit surface its DB
|
|
25
|
+
* has (Platform → `audit_log`, Orchestrator → `access_log` in later
|
|
26
|
+
* phases) AND upserting the per-(db, table, tenant) row in
|
|
27
|
+
* `cold_store_chunk_counts` — both inside the same transaction as the
|
|
28
|
+
* data-mutation in `markArchivedAndDelete`.
|
|
29
|
+
*/
|
|
30
|
+
export interface ChunkCommitMetadata {
|
|
31
|
+
chunkId: string;
|
|
32
|
+
tenantId: string;
|
|
33
|
+
partitionDate: string;
|
|
34
|
+
rowCount: number;
|
|
35
|
+
byteCount: number;
|
|
36
|
+
gzipByteCount: number;
|
|
37
|
+
objectKey: string;
|
|
38
|
+
/**
|
|
39
|
+
* Phase 2 — present on chunks written by adapters that implement
|
|
40
|
+
* `coldTtlDays(row)` (the per-bucket layout). Adapters use this to
|
|
41
|
+
* INSERT a corresponding row into `cold_store_chunks` inside the same
|
|
42
|
+
* transaction as the data delete + audit + rollup updates.
|
|
43
|
+
*
|
|
44
|
+
* Pre-Phase-2 (v1) chunks omit both fields — adapters skip the
|
|
45
|
+
* `cold_store_chunks` insert in that case and the GC sweep treats
|
|
46
|
+
* the chunk as `'forever'`.
|
|
47
|
+
*/
|
|
48
|
+
bucket?: string;
|
|
49
|
+
/** Phase 2 — see `bucket`. Numeric day-count or `'forever'`. */
|
|
50
|
+
maxColdDays?: ColdRetention;
|
|
51
|
+
}
|
|
52
|
+
export interface TableAdapter<TRow> {
|
|
53
|
+
readonly db: DbKind;
|
|
54
|
+
/** Postgres table name the adapter archives. */
|
|
55
|
+
readonly table: string;
|
|
56
|
+
/** Column used to shard S3 keys (e.g. `org_id`, `routing_key`). */
|
|
57
|
+
readonly tenantColumn: string;
|
|
58
|
+
/** Column used for day-partitioning (e.g. `created_at`). */
|
|
59
|
+
readonly partitionColumn: string;
|
|
60
|
+
/** Effective per-table config after merging defaults + overrides. */
|
|
61
|
+
readonly config: ColdStoreTableConfig;
|
|
62
|
+
/**
|
|
63
|
+
* Stream the distinct (tenant, partitionDate) combinations that carry
|
|
64
|
+
* rows older than `warmCutoff`. The framework iterates these and,
|
|
65
|
+
* per combination, calls `selectEligible` inside `withPartitionLock`.
|
|
66
|
+
*
|
|
67
|
+
* Order: `(tenantId, partitionDate)` ascending. This lets the
|
|
68
|
+
* framework pause mid-iteration at a byte / row cap and resume on
|
|
69
|
+
* the next cycle without skipping partitions.
|
|
70
|
+
*/
|
|
71
|
+
listEligiblePartitions(args: {
|
|
72
|
+
warmCutoff: Date;
|
|
73
|
+
}): AsyncIterable<EligiblePartition>;
|
|
74
|
+
/**
|
|
75
|
+
* Approximate size-in-bytes of warm rows for a single tenant.
|
|
76
|
+
* Used to enforce `minWarmTenantBytes` — we skip archival for
|
|
77
|
+
* low-traffic tenants where the chunk would be tiny.
|
|
78
|
+
*
|
|
79
|
+
* Implementations MAY approximate via `pg_column_size` sum + row
|
|
80
|
+
* count; accuracy within ~20% is fine.
|
|
81
|
+
*/
|
|
82
|
+
countTenantWarmBytes(args: {
|
|
83
|
+
tenantId: string;
|
|
84
|
+
warmCutoff: Date;
|
|
85
|
+
}): Promise<number>;
|
|
86
|
+
/**
|
|
87
|
+
* Run `fn` while holding a Postgres advisory lock keyed by
|
|
88
|
+
* `hashtext('cold-store|<db>|<table>|<tenantId>|<partitionDate>')`.
|
|
89
|
+
*
|
|
90
|
+
* Returns the `fn` result on successful lock acquisition, or `null`
|
|
91
|
+
* when another archiver replica already holds the lock. The adapter
|
|
92
|
+
* owns this because the lock lives on its Kysely instance.
|
|
93
|
+
*
|
|
94
|
+
* Implementations SHOULD use `pg_try_advisory_xact_lock` inside a
|
|
95
|
+
* short-lived transaction so the lock is auto-released on commit
|
|
96
|
+
* or rollback.
|
|
97
|
+
*/
|
|
98
|
+
withPartitionLock<T>(args: {
|
|
99
|
+
tenantId: string;
|
|
100
|
+
partitionDate: string;
|
|
101
|
+
}, fn: () => Promise<T>): Promise<T | null>;
|
|
102
|
+
/**
|
|
103
|
+
* Stream the eligible rows for one (tenant, partitionDate) combination,
|
|
104
|
+
* up to the adapter's byte/row cap. Rows yielded in PK order so min/max
|
|
105
|
+
* bounds are computed correctly.
|
|
106
|
+
*/
|
|
107
|
+
selectEligible(args: {
|
|
108
|
+
tenantId: string;
|
|
109
|
+
partitionDate: string;
|
|
110
|
+
limit: number;
|
|
111
|
+
}): AsyncIterable<TRow>;
|
|
112
|
+
/** Stringify a row for the JSONL body. */
|
|
113
|
+
encodeRow(row: TRow): string;
|
|
114
|
+
/** Parse a row back from a JSONL line. */
|
|
115
|
+
decodeRow(line: string): TRow;
|
|
116
|
+
/** Extract the primary key; used for chunk-id bounds. */
|
|
117
|
+
rowId(row: TRow): string | number;
|
|
118
|
+
/** Extract the partition-column timestamp; used for manifest bounds. */
|
|
119
|
+
rowTimestamp(row: TRow): Date | string;
|
|
120
|
+
/**
|
|
121
|
+
* Phase F — optional natural-key extractor used by
|
|
122
|
+
* `BaseColdStore.replayRow()` to find a chunk by an externally
|
|
123
|
+
* meaningful identifier (e.g. UUID `run_id`) rather than the internal
|
|
124
|
+
* SERIAL `id` returned by `rowId()`. Adapters that implement this
|
|
125
|
+
* method opt the table into single-row replay-into-PG: every chunk
|
|
126
|
+
* persisted afterwards carries a `replayLookupKeys` array on its
|
|
127
|
+
* manifest. Adapters whose primary key already IS the natural key
|
|
128
|
+
* (e.g. `event_log` keyed by UUID) can leave this undefined — the
|
|
129
|
+
* `minRowId`/`maxRowId` range comparison covers them.
|
|
130
|
+
*/
|
|
131
|
+
replayLookupKey?(row: TRow): string;
|
|
132
|
+
/**
|
|
133
|
+
* Transactionally:
|
|
134
|
+
* 1. `UPDATE <table> SET archived_at = now(), archive_object_key = <objectKey> WHERE id IN (<rowIds>)`.
|
|
135
|
+
* 2. `DELETE FROM <table> WHERE id IN (<rowIds>)`.
|
|
136
|
+
* 3. Call `writeAuditAndChunkCount` with `chunkMeta` on the same
|
|
137
|
+
* transaction handle (adapter-internal wiring).
|
|
138
|
+
*
|
|
139
|
+
* Implemented inside each adapter because the row-ID column name
|
|
140
|
+
* varies per table.
|
|
141
|
+
*/
|
|
142
|
+
markArchivedAndDelete(args: {
|
|
143
|
+
rowIds: ReadonlyArray<string | number>;
|
|
144
|
+
chunkMeta: ChunkCommitMetadata;
|
|
145
|
+
}): Promise<void>;
|
|
146
|
+
/**
|
|
147
|
+
* Phase F — replay a chunk's rows back into PG.
|
|
148
|
+
*
|
|
149
|
+
* Optional. Adapters that do NOT implement this method cannot be
|
|
150
|
+
* promoted back into PG via `BaseColdStore.replayChunk()` /
|
|
151
|
+
* `replayRow()` — only those tables whose rerun / rehydrate path
|
|
152
|
+
* genuinely needs read-after-write semantics on PG (currently only
|
|
153
|
+
* `execution_runs` on both DBs) opt in.
|
|
154
|
+
*
|
|
155
|
+
* Contract:
|
|
156
|
+
* 1. INSERT each row, clearing `archived_at` / `archive_object_key`,
|
|
157
|
+
* `ON CONFLICT ON the natural unique key DO NOTHING` so a duplicate
|
|
158
|
+
* replay is a no-op. Exact unique key varies per table (e.g.
|
|
159
|
+
* `run_id` for `execution_runs`).
|
|
160
|
+
* 2. Write a "replay" audit row to whichever audit surface the DB has
|
|
161
|
+
* (`audit_log` for Platform, `access_log` for Orchestrator).
|
|
162
|
+
* `actor_id='cold-store-replay:<instanceId>'`,
|
|
163
|
+
* `action='replay_chunk'`, `target_id=<chunkId>`.
|
|
164
|
+
* 3. Decrement `cold_store_chunk_counts` for the (db, table, tenant):
|
|
165
|
+
* `chunk_count = chunk_count - 1`,
|
|
166
|
+
* `total_rows = total_rows - <inserted>`,
|
|
167
|
+
* `total_bytes = total_bytes - <gzipByteCount>`. Floors at 0 so a
|
|
168
|
+
* replay-then-rearchive cycle doesn't underflow.
|
|
169
|
+
*
|
|
170
|
+
* All three steps run in one transaction — either everything commits
|
|
171
|
+
* or nothing does.
|
|
172
|
+
*
|
|
173
|
+
* Returns `{ inserted, skipped }`. `skipped` covers rows whose unique
|
|
174
|
+
* key already exists in PG (idempotent replays — fine).
|
|
175
|
+
*/
|
|
176
|
+
replayInsert?(args: {
|
|
177
|
+
rows: ReadonlyArray<TRow>;
|
|
178
|
+
chunkMeta: ChunkCommitMetadata;
|
|
179
|
+
}): Promise<{
|
|
180
|
+
inserted: number;
|
|
181
|
+
skipped: number;
|
|
182
|
+
}>;
|
|
183
|
+
/**
|
|
184
|
+
* Phase 2 — per-row cold-retention TTL.
|
|
185
|
+
*
|
|
186
|
+
* Returns the number of days the row's chunk should live in S3 after
|
|
187
|
+
* archival, or `'forever'` to never purge. The framework groups rows by
|
|
188
|
+
* `coldDaysToBucket(coldTtlDays(row))` at archive time so each chunk
|
|
189
|
+
* spans a single bucket subprefix; the chunk's manifest carries the
|
|
190
|
+
* MAXIMUM `coldTtlDays` of any row in the chunk so the GC sweep keys
|
|
191
|
+
* off row-level retention rather than the bucket name.
|
|
192
|
+
*
|
|
193
|
+
* Adapters that don't implement this method opt out of the per-bucket
|
|
194
|
+
* layout: every chunk is written to the day-prefix root with the
|
|
195
|
+
* legacy v1 manifest and is treated as `'forever'` by the GC sweep.
|
|
196
|
+
* Audit-style adapters (`audit_log`, `access_log`, `secret_audit_log`)
|
|
197
|
+
* implement this; the high-volume execution / event tables retain
|
|
198
|
+
* their existing layout until their own retention policy is finalized.
|
|
199
|
+
*/
|
|
200
|
+
coldTtlDays?(row: TRow): ColdRetention;
|
|
201
|
+
/**
|
|
202
|
+
* Phase 2 — transactionally purge a chunk's PG bookkeeping after the
|
|
203
|
+
* S3 objects have been deleted.
|
|
204
|
+
*
|
|
205
|
+
* Implementations MUST run all three steps in one transaction so a
|
|
206
|
+
* partial failure doesn't leave inconsistent state:
|
|
207
|
+
* 1. `DELETE FROM cold_store_chunks WHERE chunk_id = $1`.
|
|
208
|
+
* 2. Decrement `cold_store_chunk_counts` (`chunk_count -= 1`,
|
|
209
|
+
* `total_bytes -= gzipBytes`, `total_rows -= rowCount`,
|
|
210
|
+
* floored at 0 so a purge-then-rearchive doesn't underflow).
|
|
211
|
+
* 3. INSERT a `'purge_chunk'` audit row into the adapter's audit
|
|
212
|
+
* surface (Platform → `audit_log`, Orchestrator → `access_log`)
|
|
213
|
+
* with `actor_id='cold-store-purge:<instanceId>'` and details
|
|
214
|
+
* `{ bucket, maxColdDays, gzipBytes, rowCount, objectKey }`.
|
|
215
|
+
*
|
|
216
|
+
* Optional — adapters that don't opt into the per-bucket layout via
|
|
217
|
+
* `coldTtlDays` also can't be purged, so they don't need this hook.
|
|
218
|
+
*/
|
|
219
|
+
purgeChunkRecord?(args: {
|
|
220
|
+
tenantId: string;
|
|
221
|
+
chunkId: string;
|
|
222
|
+
gzipBytes: number;
|
|
223
|
+
rowCount: number;
|
|
224
|
+
bucket: string;
|
|
225
|
+
maxColdDays: number;
|
|
226
|
+
objectKey: string;
|
|
227
|
+
}): Promise<void>;
|
|
228
|
+
}
|
|
229
|
+
//# sourceMappingURL=table-adapter.d.ts.map
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Core types for the cold-storage archival framework.
|
|
3
|
+
*
|
|
4
|
+
* for
|
|
5
|
+
* the full design. This module defines the shapes that the DB-agnostic
|
|
6
|
+
* cold-store framework exchanges with its two concrete consumers
|
|
7
|
+
* (Platform, Orchestrator).
|
|
8
|
+
*/
|
|
9
|
+
import type { DbKind } from './key.js';
|
|
10
|
+
export type { DbKind } from './key.js';
|
|
11
|
+
/**
|
|
12
|
+
* Sidecar manifest written next to every archived chunk.
|
|
13
|
+
*
|
|
14
|
+
* The manifest is the authoritative description of a chunk: callers
|
|
15
|
+
* MUST rely on it (not on file listings) to decide whether to rehydrate
|
|
16
|
+
* a given chunk for a range query. The `contentHash` field lets the
|
|
17
|
+
* `verify-chunk` CLI detect bit-rot or mutation without re-reading the
|
|
18
|
+
* body at ingestion time.
|
|
19
|
+
*/
|
|
20
|
+
/**
|
|
21
|
+
* Cold-retention horizon for a chunk: number of days the chunk lives in S3
|
|
22
|
+
* after archival before being purged, or `'forever'` to never purge.
|
|
23
|
+
*/
|
|
24
|
+
export type ColdRetention = number | 'forever';
|
|
25
|
+
export interface ChunkManifest {
|
|
26
|
+
/**
|
|
27
|
+
* Manifest schema version.
|
|
28
|
+
*
|
|
29
|
+
* - `1`: Phase A–F. No bucket, no cold-TTL metadata. The framework
|
|
30
|
+
* treats v1 chunks as `bucket='forever'` / `maxColdDays='forever'` so
|
|
31
|
+
* the GC sweep never purges pre-Phase-2 chunks.
|
|
32
|
+
* - `2`: Phase 2 (cold-store purge). Adds `bucket` (S3-segment string)
|
|
33
|
+
* and `maxColdDays` (numeric or `'forever'`). The GC sweep keys off
|
|
34
|
+
* `archivedAt + maxColdDays`, NOT the bucket name — multiple actions
|
|
35
|
+
* with different per-row TTLs can share a bucket.
|
|
36
|
+
*/
|
|
37
|
+
schemaVersion: 1 | 2;
|
|
38
|
+
db: DbKind;
|
|
39
|
+
/** Postgres table name the chunk was sourced from. */
|
|
40
|
+
table: string;
|
|
41
|
+
/** `org_id` for Platform tables, `routing_key` for Orchestrator tables. */
|
|
42
|
+
tenantId: string;
|
|
43
|
+
/** `YYYY-MM-DD`. Derived from the table's partition column. */
|
|
44
|
+
partitionDate: string;
|
|
45
|
+
rowCount: number;
|
|
46
|
+
/** Size of the uncompressed JSONL body. */
|
|
47
|
+
byteCount: number;
|
|
48
|
+
/** Size of the gzipped data file. */
|
|
49
|
+
gzipByteCount: number;
|
|
50
|
+
/** Min of the partition column (ISO) across rows in this chunk. */
|
|
51
|
+
minTimestamp: string;
|
|
52
|
+
/** Max of the partition column (ISO) across rows in this chunk. */
|
|
53
|
+
maxTimestamp: string;
|
|
54
|
+
/** Min PK in the chunk (string or numeric, matches adapter output). */
|
|
55
|
+
minRowId: string | number;
|
|
56
|
+
/** Max PK in the chunk. */
|
|
57
|
+
maxRowId: string | number;
|
|
58
|
+
/** sha256 hex of the gzipped body. */
|
|
59
|
+
contentHash: string;
|
|
60
|
+
/** Filename-stem of the chunk; matches the deterministic ID. */
|
|
61
|
+
chunkId: string;
|
|
62
|
+
/** ISO wall clock at archive time. */
|
|
63
|
+
createdAt: string;
|
|
64
|
+
/** Instance ID of the archiver process, for forensics. */
|
|
65
|
+
archiverInstanceId: string;
|
|
66
|
+
/**
|
|
67
|
+
* Phase F — optional list of natural-key lookup tokens (e.g. UUID
|
|
68
|
+
* `run_id`) for every row in the chunk, populated when the adapter
|
|
69
|
+
* implements `replayLookupKey()`. Used by `BaseColdStore.replayRow()`
|
|
70
|
+
* to find the chunk holding a specific natural key when the manifest's
|
|
71
|
+
* `minRowId`/`maxRowId` are an internal SERIAL `id` that callers don't
|
|
72
|
+
* have. Older chunks (Phase C–E or any adapter without
|
|
73
|
+
* `replayLookupKey()`) omit this field; replayRow falls back to the
|
|
74
|
+
* `minRowId`/`maxRowId` range comparison in that case.
|
|
75
|
+
*/
|
|
76
|
+
replayLookupKeys?: string[];
|
|
77
|
+
/**
|
|
78
|
+
* Phase 2 — S3 prefix segment under the `<tenant>/<YYYY>/<MM>/<DD>/`
|
|
79
|
+
* day directory. Examples: `'30d'`, `'180d'`, `'1y'`, `'2y'`, `'forever'`.
|
|
80
|
+
* V1 manifests omit this; the framework treats them as `'forever'`.
|
|
81
|
+
*/
|
|
82
|
+
bucket?: string;
|
|
83
|
+
/**
|
|
84
|
+
* Phase 2 — maximum per-row cold TTL (days, or `'forever'`) across all
|
|
85
|
+
* rows in the chunk. The GC sweep purges a chunk when
|
|
86
|
+
* `now > archivedAt + maxColdDays * 86_400_000`. Multiple actions with
|
|
87
|
+
* different per-row TTLs can share a bucket, so the sweep MUST use this
|
|
88
|
+
* field (not the bucket name) for correctness.
|
|
89
|
+
*/
|
|
90
|
+
maxColdDays?: ColdRetention;
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Return value of `ColdStore.runArchiveCycle()`.
|
|
94
|
+
*
|
|
95
|
+
* Phase A's no-op path always reports zero work with
|
|
96
|
+
* `skipped.no_tables = 1` to signal "framework alive, no adapters
|
|
97
|
+
* registered". Phase B onward will report real values.
|
|
98
|
+
*/
|
|
99
|
+
export interface ArchiveCycleSummary {
|
|
100
|
+
tablesProcessed: number;
|
|
101
|
+
chunksWritten: number;
|
|
102
|
+
rowsArchived: number;
|
|
103
|
+
rowsFailed: number;
|
|
104
|
+
skipped: {
|
|
105
|
+
disabled: number;
|
|
106
|
+
min_chunk: number;
|
|
107
|
+
min_warm: number;
|
|
108
|
+
no_tables: number;
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
//# sourceMappingURL=types.d.ts.map
|
package/dist/crypto.d.ts
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/** Compute SHA-256 hex digest of a string or Buffer. */
|
|
2
|
+
export declare function sha256(input: string | Buffer): string;
|
|
3
|
+
/** Compute SHA-256 hex digest of a file's contents. */
|
|
4
|
+
export declare function sha256File(filePath: string): Promise<string>;
|
|
5
|
+
/**
|
|
6
|
+
* Normalize line endings to LF so hashes computed on different platforms agree.
|
|
7
|
+
*
|
|
8
|
+
* Git for Windows ships with `core.autocrlf=true` in the system gitconfig, so a
|
|
9
|
+
* `git clone` of a Linux-authored repo on a Windows host checks out text files
|
|
10
|
+
* with CRLF in the working tree. The compiler hashed the LF source on Linux,
|
|
11
|
+
* but the agent on Windows reads CRLF and computes a different hash — every
|
|
12
|
+
* dispatch fails with a "lock file is out of date" error even though the
|
|
13
|
+
* semantic content is identical.
|
|
14
|
+
*
|
|
15
|
+
* Applied at the boundaries where source / asset content enters the hash:
|
|
16
|
+
* - raw workflow source (`.kici/workflows/*.ts`) at hash time, in both the
|
|
17
|
+
* compiler (lockfile generation) and the agent (drift verification).
|
|
18
|
+
* - file content portions of the asset digest (`hashFiles` resolution) on
|
|
19
|
+
* both sides.
|
|
20
|
+
*
|
|
21
|
+
* Standalone `\r` is also collapsed to `\n` for safety against legacy
|
|
22
|
+
* Mac-style endings, though TypeScript source files essentially never carry
|
|
23
|
+
* those in practice.
|
|
24
|
+
*/
|
|
25
|
+
export declare function normalizeLineEndings(input: string): string;
|
|
26
|
+
/**
|
|
27
|
+
* Derive an AES-256 key from an X25519 ECDH shared secret using HKDF.
|
|
28
|
+
*
|
|
29
|
+
* Used by the compiler (encrypt) and agent (decrypt) for tarball uploads.
|
|
30
|
+
* Keys must be DER-encoded (PKCS8 for private, SPKI for public).
|
|
31
|
+
*/
|
|
32
|
+
export declare function deriveSharedSecret(ourPrivateKey: Buffer, theirPublicKey: Buffer): Buffer;
|
|
33
|
+
//# sourceMappingURL=crypto.d.ts.map
|
package/dist/crypto.js
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import "./chunk-gOLHoazu.js";
|
|
2
|
+
import crypto, { createHash } from "node:crypto";
|
|
3
|
+
import { readFile } from "node:fs/promises";
|
|
4
|
+
//#region src/crypto.ts
|
|
5
|
+
/** Compute SHA-256 hex digest of a string or Buffer. */
|
|
6
|
+
function sha256(input) {
|
|
7
|
+
return createHash("sha256").update(input).digest("hex");
|
|
8
|
+
}
|
|
9
|
+
/** Compute SHA-256 hex digest of a file's contents. */
|
|
10
|
+
async function sha256File(filePath) {
|
|
11
|
+
return sha256(await readFile(filePath));
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Normalize line endings to LF so hashes computed on different platforms agree.
|
|
15
|
+
*
|
|
16
|
+
* Git for Windows ships with `core.autocrlf=true` in the system gitconfig, so a
|
|
17
|
+
* `git clone` of a Linux-authored repo on a Windows host checks out text files
|
|
18
|
+
* with CRLF in the working tree. The compiler hashed the LF source on Linux,
|
|
19
|
+
* but the agent on Windows reads CRLF and computes a different hash — every
|
|
20
|
+
* dispatch fails with a "lock file is out of date" error even though the
|
|
21
|
+
* semantic content is identical.
|
|
22
|
+
*
|
|
23
|
+
* Applied at the boundaries where source / asset content enters the hash:
|
|
24
|
+
* - raw workflow source (`.kici/workflows/*.ts`) at hash time, in both the
|
|
25
|
+
* compiler (lockfile generation) and the agent (drift verification).
|
|
26
|
+
* - file content portions of the asset digest (`hashFiles` resolution) on
|
|
27
|
+
* both sides.
|
|
28
|
+
*
|
|
29
|
+
* Standalone `\r` is also collapsed to `\n` for safety against legacy
|
|
30
|
+
* Mac-style endings, though TypeScript source files essentially never carry
|
|
31
|
+
* those in practice.
|
|
32
|
+
*/
|
|
33
|
+
function normalizeLineEndings(input) {
|
|
34
|
+
return input.replace(/\r\n?/g, "\n");
|
|
35
|
+
}
|
|
36
|
+
/** HKDF info string for ECDH-derived AES keys (upload encryption). */
|
|
37
|
+
const HKDF_INFO = "kici-upload-encryption";
|
|
38
|
+
/** Empty salt — ECDH output is already high entropy. */
|
|
39
|
+
const HKDF_SALT = Buffer.alloc(0);
|
|
40
|
+
/**
|
|
41
|
+
* Derive an AES-256 key from an X25519 ECDH shared secret using HKDF.
|
|
42
|
+
*
|
|
43
|
+
* Used by the compiler (encrypt) and agent (decrypt) for tarball uploads.
|
|
44
|
+
* Keys must be DER-encoded (PKCS8 for private, SPKI for public).
|
|
45
|
+
*/
|
|
46
|
+
function deriveSharedSecret(ourPrivateKey, theirPublicKey) {
|
|
47
|
+
const ourKeyObj = crypto.createPrivateKey({
|
|
48
|
+
key: ourPrivateKey,
|
|
49
|
+
format: "der",
|
|
50
|
+
type: "pkcs8"
|
|
51
|
+
});
|
|
52
|
+
const theirKeyObj = crypto.createPublicKey({
|
|
53
|
+
key: theirPublicKey,
|
|
54
|
+
format: "der",
|
|
55
|
+
type: "spki"
|
|
56
|
+
});
|
|
57
|
+
const sharedSecret = crypto.diffieHellman({
|
|
58
|
+
publicKey: theirKeyObj,
|
|
59
|
+
privateKey: ourKeyObj
|
|
60
|
+
});
|
|
61
|
+
const derivedKey = crypto.hkdfSync("sha256", sharedSecret, HKDF_SALT, HKDF_INFO, 32);
|
|
62
|
+
return Buffer.from(derivedKey);
|
|
63
|
+
}
|
|
64
|
+
//#endregion
|
|
65
|
+
export { deriveSharedSecret, normalizeLineEndings, sha256, sha256File };
|
|
66
|
+
|
|
67
|
+
//# sourceMappingURL=crypto.js.map
|