@kici-dev/shared 0.0.0 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +1 -6
- package/dist/chunk-gOLHoazu.js +4 -0
- package/dist/cold-store/bucket.d.ts +26 -0
- package/dist/cold-store/bucket.js +40 -0
- package/dist/cold-store/bucket.test.d.ts +2 -0
- package/dist/cold-store/chunk-encoder.d.ts +63 -0
- package/dist/cold-store/chunk-encoder.js +94 -0
- package/dist/cold-store/chunk-encoder.test.d.ts +2 -0
- package/dist/cold-store/chunk-id.d.ts +10 -0
- package/dist/cold-store/chunk-id.js +29 -0
- package/dist/cold-store/chunk-id.test.d.ts +2 -0
- package/dist/cold-store/cold-store.d.ts +345 -0
- package/dist/cold-store/cold-store.js +1078 -0
- package/dist/cold-store/cold-store.test.d.ts +2 -0
- package/dist/cold-store/config.d.ts +62 -0
- package/dist/cold-store/config.js +28 -0
- package/dist/cold-store/index.d.ts +20 -0
- package/dist/cold-store/index.js +11 -0
- package/dist/cold-store/key.d.ts +89 -0
- package/dist/cold-store/key.js +88 -0
- package/dist/cold-store/key.test.d.ts +2 -0
- package/dist/cold-store/lru.d.ts +33 -0
- package/dist/cold-store/lru.js +59 -0
- package/dist/cold-store/lru.test.d.ts +2 -0
- package/dist/cold-store/manifest.d.ts +11 -0
- package/dist/cold-store/manifest.js +57 -0
- package/dist/cold-store/manifest.test.d.ts +2 -0
- package/dist/cold-store/metrics.d.ts +103 -0
- package/dist/cold-store/metrics.js +179 -0
- package/dist/cold-store/table-adapter.d.ts +229 -0
- package/dist/cold-store/table-adapter.js +2 -0
- package/dist/cold-store/types.d.ts +111 -0
- package/dist/cold-store/types.js +2 -0
- package/dist/crypto.d.ts +33 -0
- package/dist/crypto.js +67 -0
- package/dist/db-admin.d.ts +1422 -0
- package/dist/db-admin.js +2525 -0
- package/dist/db-admin.test.d.ts +2 -0
- package/dist/db.d.ts +14 -0
- package/dist/db.js +23 -0
- package/dist/env/allowlist.d.ts +79 -0
- package/dist/env/allowlist.js +86 -0
- package/dist/env/define-env.d.ts +172 -0
- package/dist/env/define-env.js +295 -0
- package/dist/env/define-env.test.d.ts +2 -0
- package/dist/env/env-rule-allowlist.test.d.ts +2 -0
- package/dist/env/index.d.ts +11 -0
- package/dist/env/index.js +4 -0
- package/dist/env/logger-env.d.ts +30 -0
- package/dist/env/logger-env.js +95 -0
- package/dist/error.d.ts +16 -0
- package/dist/error.js +58 -0
- package/dist/error.test.d.ts +2 -0
- package/dist/format-bytes.d.ts +5 -0
- package/dist/format-bytes.js +15 -0
- package/dist/format-bytes.test.d.ts +2 -0
- package/dist/format-duration.d.ts +11 -0
- package/dist/format-duration.js +32 -0
- package/dist/format-duration.test.d.ts +2 -0
- package/dist/graceful-shutdown.d.ts +58 -0
- package/dist/graceful-shutdown.js +71 -0
- package/dist/graceful-shutdown.test.d.ts +2 -0
- package/dist/idempotency-files.d.ts +113 -0
- package/dist/idempotency-files.js +190 -0
- package/dist/idempotency-files.test.d.ts +2 -0
- package/dist/idempotency.d.ts +46 -0
- package/dist/idempotency.js +45 -0
- package/dist/idempotency.test.d.ts +2 -0
- package/dist/index.d.ts +19 -0
- package/dist/index.js +31 -0
- package/dist/logger.d.ts +57 -0
- package/dist/logger.js +175 -0
- package/dist/logger.test.d.ts +2 -0
- package/dist/reconnect-delay.d.ts +11 -0
- package/dist/reconnect-delay.js +22 -0
- package/dist/request-context.d.ts +42 -0
- package/dist/request-context.js +37 -0
- package/dist/ring-buffer.d.ts +35 -0
- package/dist/ring-buffer.js +62 -0
- package/dist/ring-buffer.test.d.ts +2 -0
- package/dist/routes/health.d.ts +26 -0
- package/dist/routes/health.js +45 -0
- package/dist/routes/health.test.d.ts +2 -0
- package/dist/routes/metrics.d.ts +20 -0
- package/dist/routes/metrics.js +38 -0
- package/dist/s3-client.d.ts +42 -0
- package/dist/s3-client.js +25 -0
- package/dist/telemetry/index.d.ts +3 -0
- package/dist/telemetry/index.js +4 -0
- package/dist/telemetry/init.d.ts +20 -0
- package/dist/telemetry/init.js +42 -0
- package/dist/telemetry/init.test.d.ts +2 -0
- package/dist/telemetry/metrics.d.ts +8 -0
- package/dist/telemetry/metrics.js +16 -0
- package/dist/tool-check.d.ts +33 -0
- package/dist/tool-check.js +60 -0
- package/dist/tool-check.test.d.ts +2 -0
- package/dist/ts-loader-hook.d.ts +26 -0
- package/dist/ts-loader-hook.js +48 -0
- package/dist/zx.d.ts +8 -0
- package/dist/zx.js +78 -0
- package/package.json +69 -5
- package/sbom.spdx.json +8775 -0
- package/index.js +0 -3
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cold-store configuration types.
|
|
3
|
+
*
|
|
4
|
+
* section 5. Per-table defaults in DEFAULT_TABLE_CONFIG mirror the
|
|
5
|
+
* design doc's table.
|
|
6
|
+
*/
|
|
7
|
+
import type { SharedS3Config } from '../s3-client.js';
|
|
8
|
+
/**
|
|
9
|
+
* Per-table archival tuning parameters.
|
|
10
|
+
*
|
|
11
|
+
* A table's effective config is the global default, overridden by any
|
|
12
|
+
* values from the YAML/env config, overridden by any per-org values
|
|
13
|
+
* (Platform only) from `cold_store_org_overrides`.
|
|
14
|
+
*/
|
|
15
|
+
export interface ColdStoreTableConfig {
|
|
16
|
+
/** Minimum age (in days) before a row is eligible to archive. */
|
|
17
|
+
warmTtlDays: number;
|
|
18
|
+
/**
|
|
19
|
+
* Floor for total warm bytes per tenant per table. Below this, no
|
|
20
|
+
* archival happens — even if rows are old. Prevents thrashing on
|
|
21
|
+
* low-traffic tenants.
|
|
22
|
+
*/
|
|
23
|
+
minWarmTenantBytes: number;
|
|
24
|
+
/** Floor for chunk size. Smaller eligible sets are deferred. */
|
|
25
|
+
minChunkBytes: number;
|
|
26
|
+
/** Ceiling for chunk size. Larger eligible sets are split. */
|
|
27
|
+
maxChunkBytes: number;
|
|
28
|
+
/** Hard cap on rows archived per (table, tenant) per cycle. */
|
|
29
|
+
maxRowsPerCycle: number;
|
|
30
|
+
/** Whether the table's archival is enabled (kill switch). */
|
|
31
|
+
enabled: boolean;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Reasonable defaults — matches the design doc's section 5 table.
|
|
35
|
+
* Concrete adapters can override any subset of these.
|
|
36
|
+
*/
|
|
37
|
+
export declare const DEFAULT_TABLE_CONFIG: ColdStoreTableConfig;
|
|
38
|
+
/**
|
|
39
|
+
* Top-level cold-store configuration passed to `BaseColdStore`.
|
|
40
|
+
*/
|
|
41
|
+
export interface ColdStoreConfig {
|
|
42
|
+
/**
|
|
43
|
+
* S3-compatible storage config. The `prefix` defaults to
|
|
44
|
+
* `'cold-store/'` when unset.
|
|
45
|
+
*/
|
|
46
|
+
storage: SharedS3Config;
|
|
47
|
+
/**
|
|
48
|
+
* Per-table overrides keyed by table name; unset tables use
|
|
49
|
+
* DEFAULT_TABLE_CONFIG.
|
|
50
|
+
*/
|
|
51
|
+
tables: Record<string, Partial<ColdStoreTableConfig>>;
|
|
52
|
+
/** Global concurrency cap on in-flight S3 PUTs. Default 4. */
|
|
53
|
+
s3Concurrency: number;
|
|
54
|
+
/** Master feature toggle; when false, cycles no-op fast. */
|
|
55
|
+
enabled: boolean;
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Merge DEFAULT_TABLE_CONFIG with overrides to produce the effective
|
|
59
|
+
* per-table config.
|
|
60
|
+
*/
|
|
61
|
+
export declare function resolveTableConfig(overrides: Partial<ColdStoreTableConfig> | undefined): ColdStoreTableConfig;
|
|
62
|
+
//# sourceMappingURL=config.d.ts.map
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import "../chunk-gOLHoazu.js";
|
|
2
|
+
//#region src/cold-store/config.ts
|
|
3
|
+
/**
|
|
4
|
+
* Reasonable defaults — matches the design doc's section 5 table.
|
|
5
|
+
* Concrete adapters can override any subset of these.
|
|
6
|
+
*/
|
|
7
|
+
const DEFAULT_TABLE_CONFIG = {
|
|
8
|
+
warmTtlDays: 30,
|
|
9
|
+
minWarmTenantBytes: 5 * 1024 * 1024,
|
|
10
|
+
minChunkBytes: 1 * 1024 * 1024,
|
|
11
|
+
maxChunkBytes: 50 * 1024 * 1024,
|
|
12
|
+
maxRowsPerCycle: 5e4,
|
|
13
|
+
enabled: true
|
|
14
|
+
};
|
|
15
|
+
/**
|
|
16
|
+
* Merge DEFAULT_TABLE_CONFIG with overrides to produce the effective
|
|
17
|
+
* per-table config.
|
|
18
|
+
*/
|
|
19
|
+
function resolveTableConfig(overrides) {
|
|
20
|
+
return {
|
|
21
|
+
...DEFAULT_TABLE_CONFIG,
|
|
22
|
+
...overrides ?? {}
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
//#endregion
|
|
26
|
+
export { DEFAULT_TABLE_CONFIG, resolveTableConfig };
|
|
27
|
+
|
|
28
|
+
//# sourceMappingURL=config.js.map
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cold-store framework — public surface for @kici-dev/shared consumers.
|
|
3
|
+
*
|
|
4
|
+
* Multiple implementations extend `BaseColdStore` with their own
|
|
5
|
+
* DB-specific adapters. Phase A ships framework types + the abstract
|
|
6
|
+
* class with a no-op archive cycle; Phase B+ adds concrete
|
|
7
|
+
* `TableAdapter` implementations.
|
|
8
|
+
*/
|
|
9
|
+
export { chunkObjectKey, encodeKeySegment, tablePrefix, tenantDayBucketPrefix, tenantDayPrefix, type DbKind, } from './key.js';
|
|
10
|
+
export { COLD_BUCKET_NAMES, coldDaysToBucket, isLongerColdRetention, type ColdBucketName, } from './bucket.js';
|
|
11
|
+
export { BaseColdStore, type BaseColdStoreDeps, type ColdStore, type ColdStoreFetchRangeArgs, type ColdStoreReplayChunkArgs, type ColdStoreReplayResult, type ColdStoreReplayRowArgs, type PurgeableChunk, type PurgeChunkResult, type PurgeExpiredChunksOpts, type PurgeExpiredChunksSummary, } from './cold-store.js';
|
|
12
|
+
export { ChunkLru, type ChunkLruOptions } from './lru.js';
|
|
13
|
+
export { computeChunkId } from './chunk-id.js';
|
|
14
|
+
export { decodeChunk, encodeChunk, type DecodeChunkArgs, type EncodeChunkArgs, type EncodedChunk, } from './chunk-encoder.js';
|
|
15
|
+
export { parseManifest, serializeManifest } from './manifest.js';
|
|
16
|
+
export { DEFAULT_TABLE_CONFIG, resolveTableConfig, type ColdStoreConfig, type ColdStoreTableConfig, } from './config.js';
|
|
17
|
+
export { coldStoreArchiveBytesTotal, coldStoreArchiveCyclesTotal, coldStoreArchiveDurationSeconds, coldStoreArchiveRowsTotal, coldStorePurgeBytesTotal, coldStorePurgeChunksTotal, coldStorePurgeDurationSeconds, coldStoreRehydrateBytesTotal, coldStoreRehydrateDurationSeconds, coldStoreRehydrateRequestsTotal, coldStoreReplayDurationSeconds, coldStoreReplayRowsTotal, coldStoreVerifyFailuresTotal, } from './metrics.js';
|
|
18
|
+
export { type ChunkCommitMetadata, type EligiblePartition, type TableAdapter, } from './table-adapter.js';
|
|
19
|
+
export { type ArchiveCycleSummary, type ChunkManifest, type ColdRetention } from './types.js';
|
|
20
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import "../chunk-gOLHoazu.js";
|
|
2
|
+
import { chunkObjectKey, encodeKeySegment, tablePrefix, tenantDayBucketPrefix, tenantDayPrefix } from "./key.js";
|
|
3
|
+
import { COLD_BUCKET_NAMES, coldDaysToBucket, isLongerColdRetention } from "./bucket.js";
|
|
4
|
+
import { computeChunkId } from "./chunk-id.js";
|
|
5
|
+
import { decodeChunk, encodeChunk } from "./chunk-encoder.js";
|
|
6
|
+
import { parseManifest, serializeManifest } from "./manifest.js";
|
|
7
|
+
import { coldStoreArchiveBytesTotal, coldStoreArchiveCyclesTotal, coldStoreArchiveDurationSeconds, coldStoreArchiveRowsTotal, coldStorePurgeBytesTotal, coldStorePurgeChunksTotal, coldStorePurgeDurationSeconds, coldStoreRehydrateBytesTotal, coldStoreRehydrateDurationSeconds, coldStoreRehydrateRequestsTotal, coldStoreReplayDurationSeconds, coldStoreReplayRowsTotal, coldStoreVerifyFailuresTotal } from "./metrics.js";
|
|
8
|
+
import { BaseColdStore } from "./cold-store.js";
|
|
9
|
+
import { ChunkLru } from "./lru.js";
|
|
10
|
+
import { DEFAULT_TABLE_CONFIG, resolveTableConfig } from "./config.js";
|
|
11
|
+
export { BaseColdStore, COLD_BUCKET_NAMES, ChunkLru, DEFAULT_TABLE_CONFIG, chunkObjectKey, coldDaysToBucket, coldStoreArchiveBytesTotal, coldStoreArchiveCyclesTotal, coldStoreArchiveDurationSeconds, coldStoreArchiveRowsTotal, coldStorePurgeBytesTotal, coldStorePurgeChunksTotal, coldStorePurgeDurationSeconds, coldStoreRehydrateBytesTotal, coldStoreRehydrateDurationSeconds, coldStoreRehydrateRequestsTotal, coldStoreReplayDurationSeconds, coldStoreReplayRowsTotal, coldStoreVerifyFailuresTotal, computeChunkId, decodeChunk, encodeChunk, encodeKeySegment, isLongerColdRetention, parseManifest, resolveTableConfig, serializeManifest, tablePrefix, tenantDayBucketPrefix, tenantDayPrefix };
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Key helpers for S3 object layouts.
|
|
3
|
+
*
|
|
4
|
+
* `encodeKeySegment` escapes non-safe characters in a single path segment
|
|
5
|
+
* so that values coming from user-controlled sources (delivery IDs,
|
|
6
|
+
* routing keys, org IDs, partition values) can be embedded in an object
|
|
7
|
+
* key without breaking the key structure.
|
|
8
|
+
*
|
|
9
|
+
* This function was originally defined in
|
|
10
|
+
* packages/orchestrator/src/webhook/event-log.ts and is now shared with
|
|
11
|
+
* the cold-store framework. The encoding algorithm MUST NOT change —
|
|
12
|
+
* previously-written chunk and event-log keys depend on this mapping
|
|
13
|
+
* remaining stable forever. A fixture test in key.test.ts locks the
|
|
14
|
+
* behavior.
|
|
15
|
+
*/
|
|
16
|
+
/**
|
|
17
|
+
* Encode a single S3 path segment: anything outside `[A-Za-z0-9._:-]`
|
|
18
|
+
* becomes `_<hex>` where `<hex>` is the lowercase UTF-16 code unit of
|
|
19
|
+
* the replaced character (same as `ch.charCodeAt(0).toString(16)`).
|
|
20
|
+
*/
|
|
21
|
+
export declare function encodeKeySegment(s: string): string;
|
|
22
|
+
/**
|
|
23
|
+
* DB identifier for cold-store chunk keys.
|
|
24
|
+
*/
|
|
25
|
+
export type DbKind = 'platform' | 'orchestrator';
|
|
26
|
+
/**
|
|
27
|
+
* Compose the prefix for a specific table's cold-store objects.
|
|
28
|
+
*
|
|
29
|
+
* Layout: `<prefix>/<db>/<table>`.
|
|
30
|
+
*/
|
|
31
|
+
export declare function tablePrefix(args: {
|
|
32
|
+
prefix: string;
|
|
33
|
+
db: DbKind;
|
|
34
|
+
table: string;
|
|
35
|
+
}): string;
|
|
36
|
+
/**
|
|
37
|
+
* Compose the prefix for a specific tenant's day-partitioned objects.
|
|
38
|
+
*
|
|
39
|
+
* Layout: `<prefix>/<db>/<table>/<tenantId>/<YYYY>/<MM>/<DD>`.
|
|
40
|
+
* `tenantId` is encoded via `encodeKeySegment`.
|
|
41
|
+
* `partitionDate` must be `YYYY-MM-DD` (no other format is accepted).
|
|
42
|
+
*/
|
|
43
|
+
export declare function tenantDayPrefix(args: {
|
|
44
|
+
prefix: string;
|
|
45
|
+
db: DbKind;
|
|
46
|
+
table: string;
|
|
47
|
+
tenantId: string;
|
|
48
|
+
partitionDate: string;
|
|
49
|
+
}): string;
|
|
50
|
+
/**
|
|
51
|
+
* Compose a tenant-day prefix that includes the cold-bucket segment
|
|
52
|
+
* introduced in Phase 2. Layout:
|
|
53
|
+
*
|
|
54
|
+
* `<prefix>/<db>/<table>/<tenantId>/<YYYY>/<MM>/<DD>/<bucket>`
|
|
55
|
+
*
|
|
56
|
+
* Phase-1 (v1 manifest) chunks live at the day-prefix root and are
|
|
57
|
+
* addressed via `tenantDayPrefix` directly — those legacy chunks are
|
|
58
|
+
* treated as the `'forever'` bucket by `parseManifest` but DO NOT carry
|
|
59
|
+
* a `forever` segment in their key (the chunk-purge sweep keys off the
|
|
60
|
+
* manifest's `bucket` / `maxColdDays`, not the path).
|
|
61
|
+
*/
|
|
62
|
+
export declare function tenantDayBucketPrefix(args: {
|
|
63
|
+
prefix: string;
|
|
64
|
+
db: DbKind;
|
|
65
|
+
table: string;
|
|
66
|
+
tenantId: string;
|
|
67
|
+
partitionDate: string;
|
|
68
|
+
bucket: string;
|
|
69
|
+
}): string;
|
|
70
|
+
/**
|
|
71
|
+
* Compose a full object key for a cold-store chunk or its manifest.
|
|
72
|
+
*
|
|
73
|
+
* - V1 (no `bucket` arg): `<tenant-day-prefix>/<chunkId>.<ext>` — original
|
|
74
|
+
* layout, preserved for read-back compatibility with V1 chunks.
|
|
75
|
+
* - V2 (with `bucket`): `<tenant-day-prefix>/<bucket>/<chunkId>.<ext>` —
|
|
76
|
+
* the GC sweep can list a single bucket subprefix to find purge
|
|
77
|
+
* candidates without scanning the whole day.
|
|
78
|
+
*/
|
|
79
|
+
export declare function chunkObjectKey(args: {
|
|
80
|
+
prefix: string;
|
|
81
|
+
db: DbKind;
|
|
82
|
+
table: string;
|
|
83
|
+
tenantId: string;
|
|
84
|
+
partitionDate: string;
|
|
85
|
+
chunkId: string;
|
|
86
|
+
kind: 'data' | 'manifest';
|
|
87
|
+
bucket?: string;
|
|
88
|
+
}): string;
|
|
89
|
+
//# sourceMappingURL=key.d.ts.map
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import "../chunk-gOLHoazu.js";
|
|
2
|
+
//#region src/cold-store/key.ts
|
|
3
|
+
/**
|
|
4
|
+
* Key helpers for S3 object layouts.
|
|
5
|
+
*
|
|
6
|
+
* `encodeKeySegment` escapes non-safe characters in a single path segment
|
|
7
|
+
* so that values coming from user-controlled sources (delivery IDs,
|
|
8
|
+
* routing keys, org IDs, partition values) can be embedded in an object
|
|
9
|
+
* key without breaking the key structure.
|
|
10
|
+
*
|
|
11
|
+
* This function was originally defined in
|
|
12
|
+
* packages/orchestrator/src/webhook/event-log.ts and is now shared with
|
|
13
|
+
* the cold-store framework. The encoding algorithm MUST NOT change —
|
|
14
|
+
* previously-written chunk and event-log keys depend on this mapping
|
|
15
|
+
* remaining stable forever. A fixture test in key.test.ts locks the
|
|
16
|
+
* behavior.
|
|
17
|
+
*/
|
|
18
|
+
/**
|
|
19
|
+
* Encode a single S3 path segment: anything outside `[A-Za-z0-9._:-]`
|
|
20
|
+
* becomes `_<hex>` where `<hex>` is the lowercase UTF-16 code unit of
|
|
21
|
+
* the replaced character (same as `ch.charCodeAt(0).toString(16)`).
|
|
22
|
+
*/
|
|
23
|
+
function encodeKeySegment(s) {
|
|
24
|
+
return s.replace(/[^A-Za-z0-9._:-]/g, (ch) => `_${ch.charCodeAt(0).toString(16)}`);
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Compose the prefix for a specific table's cold-store objects.
|
|
28
|
+
*
|
|
29
|
+
* Layout: `<prefix>/<db>/<table>`.
|
|
30
|
+
*/
|
|
31
|
+
function tablePrefix(args) {
|
|
32
|
+
return `${args.prefix.replace(/\/+$/, "")}/${args.db}/${args.table}`;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Compose the prefix for a specific tenant's day-partitioned objects.
|
|
36
|
+
*
|
|
37
|
+
* Layout: `<prefix>/<db>/<table>/<tenantId>/<YYYY>/<MM>/<DD>`.
|
|
38
|
+
* `tenantId` is encoded via `encodeKeySegment`.
|
|
39
|
+
* `partitionDate` must be `YYYY-MM-DD` (no other format is accepted).
|
|
40
|
+
*/
|
|
41
|
+
function tenantDayPrefix(args) {
|
|
42
|
+
if (!/^\d{4}-\d{2}-\d{2}$/.test(args.partitionDate)) throw new Error(`partitionDate must be YYYY-MM-DD, got ${JSON.stringify(args.partitionDate)}`);
|
|
43
|
+
const [yyyy, mm, dd] = args.partitionDate.split("-");
|
|
44
|
+
const encodedTenant = encodeKeySegment(args.tenantId);
|
|
45
|
+
return `${tablePrefix(args)}/${encodedTenant}/${yyyy}/${mm}/${dd}`;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Allowed bucket-segment shape: lowercase letters, digits, the literal
|
|
49
|
+
* `forever`, e.g. `30d` / `180d` / `1y` / `2y` / `forever`. Validated to
|
|
50
|
+
* keep the S3 path safe (no `/`, no `.`, no path traversal).
|
|
51
|
+
*/
|
|
52
|
+
const BUCKET_SEGMENT_RE = /^[a-z0-9]+$/;
|
|
53
|
+
/**
|
|
54
|
+
* Compose a tenant-day prefix that includes the cold-bucket segment
|
|
55
|
+
* introduced in Phase 2. Layout:
|
|
56
|
+
*
|
|
57
|
+
* `<prefix>/<db>/<table>/<tenantId>/<YYYY>/<MM>/<DD>/<bucket>`
|
|
58
|
+
*
|
|
59
|
+
* Phase-1 (v1 manifest) chunks live at the day-prefix root and are
|
|
60
|
+
* addressed via `tenantDayPrefix` directly — those legacy chunks are
|
|
61
|
+
* treated as the `'forever'` bucket by `parseManifest` but DO NOT carry
|
|
62
|
+
* a `forever` segment in their key (the chunk-purge sweep keys off the
|
|
63
|
+
* manifest's `bucket` / `maxColdDays`, not the path).
|
|
64
|
+
*/
|
|
65
|
+
function tenantDayBucketPrefix(args) {
|
|
66
|
+
if (!BUCKET_SEGMENT_RE.test(args.bucket)) throw new Error(`bucket must match ${BUCKET_SEGMENT_RE.source}, got ${JSON.stringify(args.bucket)}`);
|
|
67
|
+
return `${tenantDayPrefix(args)}/${args.bucket}`;
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Compose a full object key for a cold-store chunk or its manifest.
|
|
71
|
+
*
|
|
72
|
+
* - V1 (no `bucket` arg): `<tenant-day-prefix>/<chunkId>.<ext>` — original
|
|
73
|
+
* layout, preserved for read-back compatibility with V1 chunks.
|
|
74
|
+
* - V2 (with `bucket`): `<tenant-day-prefix>/<bucket>/<chunkId>.<ext>` —
|
|
75
|
+
* the GC sweep can list a single bucket subprefix to find purge
|
|
76
|
+
* candidates without scanning the whole day.
|
|
77
|
+
*/
|
|
78
|
+
function chunkObjectKey(args) {
|
|
79
|
+
const suffix = args.kind === "data" ? ".jsonl.gz" : ".manifest.json";
|
|
80
|
+
return `${args.bucket ? tenantDayBucketPrefix({
|
|
81
|
+
...args,
|
|
82
|
+
bucket: args.bucket
|
|
83
|
+
}) : tenantDayPrefix(args)}/${args.chunkId}${suffix}`;
|
|
84
|
+
}
|
|
85
|
+
//#endregion
|
|
86
|
+
export { chunkObjectKey, encodeKeySegment, tablePrefix, tenantDayBucketPrefix, tenantDayPrefix };
|
|
87
|
+
|
|
88
|
+
//# sourceMappingURL=key.js.map
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* In-process size-bounded LRU cache.
|
|
3
|
+
*
|
|
4
|
+
* Used by the cold-store rehydrate path (Phase B+) to keep recently-read
|
|
5
|
+
* chunks hot in memory, so that e.g. a user paginating a 90-day audit
|
|
6
|
+
* window doesn't re-fetch the same S3 chunks for each page. Phase A
|
|
7
|
+
* ships the primitive so Phase B's first consumer gets a debugged
|
|
8
|
+
* LRU rather than having to write one.
|
|
9
|
+
*
|
|
10
|
+
* Size is tracked in bytes via a caller-supplied `sizeOf` function.
|
|
11
|
+
* Eviction happens on `set()` when the accumulated byte count exceeds
|
|
12
|
+
* `maxBytes`.
|
|
13
|
+
*/
|
|
14
|
+
export interface ChunkLruOptions<V> {
|
|
15
|
+
maxBytes: number;
|
|
16
|
+
sizeOf: (value: V) => number;
|
|
17
|
+
onEvict?: (key: unknown, value: V) => void;
|
|
18
|
+
}
|
|
19
|
+
export declare class ChunkLru<K, V> {
|
|
20
|
+
private readonly maxBytes;
|
|
21
|
+
private readonly sizeOf;
|
|
22
|
+
private readonly onEvict?;
|
|
23
|
+
private readonly map;
|
|
24
|
+
private readonly sizes;
|
|
25
|
+
private totalBytes;
|
|
26
|
+
constructor(opts: ChunkLruOptions<V>);
|
|
27
|
+
get bytes(): number;
|
|
28
|
+
get entries(): number;
|
|
29
|
+
get(key: K): V | undefined;
|
|
30
|
+
set(key: K, value: V): void;
|
|
31
|
+
clear(): void;
|
|
32
|
+
}
|
|
33
|
+
//# sourceMappingURL=lru.d.ts.map
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import "../chunk-gOLHoazu.js";
|
|
2
|
+
//#region src/cold-store/lru.ts
|
|
3
|
+
var ChunkLru = class {
|
|
4
|
+
maxBytes;
|
|
5
|
+
sizeOf;
|
|
6
|
+
onEvict;
|
|
7
|
+
map = /* @__PURE__ */ new Map();
|
|
8
|
+
sizes = /* @__PURE__ */ new Map();
|
|
9
|
+
totalBytes = 0;
|
|
10
|
+
constructor(opts) {
|
|
11
|
+
this.maxBytes = opts.maxBytes;
|
|
12
|
+
this.sizeOf = opts.sizeOf;
|
|
13
|
+
this.onEvict = opts.onEvict;
|
|
14
|
+
}
|
|
15
|
+
get bytes() {
|
|
16
|
+
return this.totalBytes;
|
|
17
|
+
}
|
|
18
|
+
get entries() {
|
|
19
|
+
return this.map.size;
|
|
20
|
+
}
|
|
21
|
+
get(key) {
|
|
22
|
+
const v = this.map.get(key);
|
|
23
|
+
if (v === void 0) return void 0;
|
|
24
|
+
this.map.delete(key);
|
|
25
|
+
this.map.set(key, v);
|
|
26
|
+
return v;
|
|
27
|
+
}
|
|
28
|
+
set(key, value) {
|
|
29
|
+
if (this.map.has(key)) {
|
|
30
|
+
const prevSize = this.sizes.get(key) ?? 0;
|
|
31
|
+
this.totalBytes -= prevSize;
|
|
32
|
+
this.map.delete(key);
|
|
33
|
+
this.sizes.delete(key);
|
|
34
|
+
}
|
|
35
|
+
const size = this.sizeOf(value);
|
|
36
|
+
this.map.set(key, value);
|
|
37
|
+
this.sizes.set(key, size);
|
|
38
|
+
this.totalBytes += size;
|
|
39
|
+
while (this.totalBytes > this.maxBytes && this.map.size > 0) {
|
|
40
|
+
const firstKey = this.map.keys().next().value;
|
|
41
|
+
const evicted = this.map.get(firstKey);
|
|
42
|
+
const evictedSize = this.sizes.get(firstKey) ?? 0;
|
|
43
|
+
this.map.delete(firstKey);
|
|
44
|
+
this.sizes.delete(firstKey);
|
|
45
|
+
this.totalBytes -= evictedSize;
|
|
46
|
+
this.onEvict?.(firstKey, evicted);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
clear() {
|
|
50
|
+
if (this.onEvict) for (const [key, value] of this.map.entries()) this.onEvict(key, value);
|
|
51
|
+
this.map.clear();
|
|
52
|
+
this.sizes.clear();
|
|
53
|
+
this.totalBytes = 0;
|
|
54
|
+
}
|
|
55
|
+
};
|
|
56
|
+
//#endregion
|
|
57
|
+
export { ChunkLru };
|
|
58
|
+
|
|
59
|
+
//# sourceMappingURL=lru.js.map
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Manifest serialization / parsing.
|
|
3
|
+
*
|
|
4
|
+
* Manifests are small JSON sidecar files next to each `.jsonl.gz`
|
|
5
|
+
* chunk. Serialized with stable key ordering so byte-diffs across
|
|
6
|
+
* archive runs are meaningful.
|
|
7
|
+
*/
|
|
8
|
+
import type { ChunkManifest } from './types.js';
|
|
9
|
+
export declare function serializeManifest(m: ChunkManifest): string;
|
|
10
|
+
export declare function parseManifest(raw: string | Buffer): ChunkManifest;
|
|
11
|
+
//# sourceMappingURL=manifest.d.ts.map
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import "../chunk-gOLHoazu.js";
|
|
2
|
+
//#region src/cold-store/manifest.ts
|
|
3
|
+
/**
|
|
4
|
+
* Required fields shared by v1 and v2 manifests. Order is the on-disk
|
|
5
|
+
* key order — keep stable so byte-diffs across archive runs remain
|
|
6
|
+
* meaningful.
|
|
7
|
+
*/
|
|
8
|
+
const COMMON_REQUIRED_KEYS = [
|
|
9
|
+
"schemaVersion",
|
|
10
|
+
"db",
|
|
11
|
+
"table",
|
|
12
|
+
"tenantId",
|
|
13
|
+
"partitionDate",
|
|
14
|
+
"rowCount",
|
|
15
|
+
"byteCount",
|
|
16
|
+
"gzipByteCount",
|
|
17
|
+
"minTimestamp",
|
|
18
|
+
"maxTimestamp",
|
|
19
|
+
"minRowId",
|
|
20
|
+
"maxRowId",
|
|
21
|
+
"contentHash",
|
|
22
|
+
"chunkId",
|
|
23
|
+
"createdAt",
|
|
24
|
+
"archiverInstanceId"
|
|
25
|
+
];
|
|
26
|
+
/** v2-only required fields — always emitted by `serializeManifest`. */
|
|
27
|
+
const V2_REQUIRED_KEYS = ["bucket", "maxColdDays"];
|
|
28
|
+
const OPTIONAL_MANIFEST_KEYS = ["replayLookupKeys"];
|
|
29
|
+
function serializeManifest(m) {
|
|
30
|
+
const ordered = {};
|
|
31
|
+
for (const k of COMMON_REQUIRED_KEYS) ordered[k] = m[k];
|
|
32
|
+
if (m.schemaVersion === 2) for (const k of V2_REQUIRED_KEYS) {
|
|
33
|
+
if (m[k] === void 0) throw new Error(`serializeManifest: schemaVersion=2 requires field "${k}"`);
|
|
34
|
+
ordered[k] = m[k];
|
|
35
|
+
}
|
|
36
|
+
for (const k of OPTIONAL_MANIFEST_KEYS) if (m[k] !== void 0) ordered[k] = m[k];
|
|
37
|
+
return JSON.stringify(ordered, null, 2);
|
|
38
|
+
}
|
|
39
|
+
function parseManifest(raw) {
|
|
40
|
+
const text = typeof raw === "string" ? raw : raw.toString("utf-8");
|
|
41
|
+
const parsed = JSON.parse(text);
|
|
42
|
+
if (!parsed || typeof parsed !== "object") throw new Error("parseManifest: not a JSON object");
|
|
43
|
+
const obj = parsed;
|
|
44
|
+
if (obj.schemaVersion !== 1 && obj.schemaVersion !== 2) throw new Error(`parseManifest: unsupported schemaVersion ${String(obj.schemaVersion)}`);
|
|
45
|
+
for (const k of COMMON_REQUIRED_KEYS) if (!(k in obj)) throw new Error(`parseManifest: missing field ${k}`);
|
|
46
|
+
if (obj.schemaVersion === 2) {
|
|
47
|
+
for (const k of V2_REQUIRED_KEYS) if (!(k in obj)) throw new Error(`parseManifest: schemaVersion=2 missing field ${k}`);
|
|
48
|
+
} else {
|
|
49
|
+
if (obj.bucket === void 0) obj.bucket = "forever";
|
|
50
|
+
if (obj.maxColdDays === void 0) obj.maxColdDays = "forever";
|
|
51
|
+
}
|
|
52
|
+
return obj;
|
|
53
|
+
}
|
|
54
|
+
//#endregion
|
|
55
|
+
export { parseManifest, serializeManifest };
|
|
56
|
+
|
|
57
|
+
//# sourceMappingURL=manifest.js.map
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cold-store domain metrics.
|
|
3
|
+
*
|
|
4
|
+
* These are the "how much data moved through cold-store" metrics;
|
|
5
|
+
* complements (but does not overlap with) the scheduler-layer
|
|
6
|
+
* metrics (`kici_{platform,orch}_job_*`) owned by the scheduled-job
|
|
7
|
+
* wrappers.
|
|
8
|
+
*
|
|
9
|
+
* `cold_store_archive_cycles_total` is always-non-zero — incremented
|
|
10
|
+
* on every `runArchiveCycle()` call with a `result` label — so the
|
|
11
|
+
* meter scope + instrument metadata always surface on /metrics.
|
|
12
|
+
* This is the counterpart to the design doc's original "zero-add
|
|
13
|
+
* trick": `@opentelemetry/exporter-prometheus` 0.213.0 filters out
|
|
14
|
+
* counters whose only observations are `.add(0)`, so we need a
|
|
15
|
+
* guaranteed-non-zero counter to keep the series visible.
|
|
16
|
+
*
|
|
17
|
+
* `cold_store_archive_rows_total` stays dormant (hidden) until a
|
|
18
|
+
* Phase B+ `TableAdapter` actually archives rows — at which point
|
|
19
|
+
* its samples appear naturally.
|
|
20
|
+
*
|
|
21
|
+
* Lazy meter initialization: the `@kici-dev/shared` barrel is imported
|
|
22
|
+
* statically at the top of service entry points (server.ts, worker.ts),
|
|
23
|
+
* which evaluates this module BEFORE `initTelemetry()` sets the global
|
|
24
|
+
* MeterProvider. If we call `createMeter('kici-cold-store')` at module
|
|
25
|
+
* load time, the returned ProxyMeter binds to the no-op provider and
|
|
26
|
+
* its instruments never reach the Prometheus exporter. Resolving the
|
|
27
|
+
* meter + instruments on first access (inside `getCold...()` getters)
|
|
28
|
+
* guarantees we resolve AFTER telemetry is wired up.
|
|
29
|
+
*/
|
|
30
|
+
import type { Counter, Histogram } from '@opentelemetry/api';
|
|
31
|
+
/**
|
|
32
|
+
* Cycles that `runArchiveCycle()` has completed, labeled by outcome.
|
|
33
|
+
* Always incremented by 1 on every tick — guaranteed-visible proof
|
|
34
|
+
* that the cold-store subsystem is registered and running.
|
|
35
|
+
*
|
|
36
|
+
* `result` ∈ `no_tables` | `disabled` | `success` | `failure`.
|
|
37
|
+
*/
|
|
38
|
+
export declare function coldStoreArchiveCyclesTotal(): Counter;
|
|
39
|
+
/**
|
|
40
|
+
* Total rows archived, by db / table / outcome.
|
|
41
|
+
* `result` ∈ success | failure | skipped_min_chunk | skipped_min_warm.
|
|
42
|
+
*
|
|
43
|
+
* Hidden from /metrics until the first Phase B+ TableAdapter moves
|
|
44
|
+
* a row; the visible proof that the subsystem is wired is
|
|
45
|
+
* `cold_store_archive_cycles_total`, not this counter.
|
|
46
|
+
*/
|
|
47
|
+
export declare function coldStoreArchiveRowsTotal(): Counter;
|
|
48
|
+
/**
|
|
49
|
+
* Total bytes archived, by kind ∈ raw | gzipped. Lets us compute the
|
|
50
|
+
* ongoing compression ratio.
|
|
51
|
+
*/
|
|
52
|
+
export declare function coldStoreArchiveBytesTotal(): Counter;
|
|
53
|
+
/**
|
|
54
|
+
* Per-chunk archive duration histogram.
|
|
55
|
+
* Buckets: 10ms, 100ms, 500ms, 1s, 5s, 30s, 120s.
|
|
56
|
+
*/
|
|
57
|
+
export declare function coldStoreArchiveDurationSeconds(): Histogram;
|
|
58
|
+
/**
|
|
59
|
+
* Rehydrate requests (cache hit vs miss). Incremented by the
|
|
60
|
+
* read-through layer (Phase B+) when it serves a range query from
|
|
61
|
+
* cold storage.
|
|
62
|
+
*/
|
|
63
|
+
export declare function coldStoreRehydrateRequestsTotal(): Counter;
|
|
64
|
+
/** Bytes read from S3 on rehydrate cache miss. */
|
|
65
|
+
export declare function coldStoreRehydrateBytesTotal(): Counter;
|
|
66
|
+
/** Rehydrate duration (S3 fetch + decode), seconds. */
|
|
67
|
+
export declare function coldStoreRehydrateDurationSeconds(): Histogram;
|
|
68
|
+
/**
|
|
69
|
+
* Verify-failure counter — bumped on the rare `contentHash` mismatch
|
|
70
|
+
* path during post-write verification.
|
|
71
|
+
*/
|
|
72
|
+
export declare function coldStoreVerifyFailuresTotal(): Counter;
|
|
73
|
+
/**
|
|
74
|
+
* Phase F — rows promoted back into PG via `replayChunk` / `replayRow`.
|
|
75
|
+
* `result` ∈ `success` | `failure` | `idempotent_skip`. The
|
|
76
|
+
* `idempotent_skip` bucket counts rows already present in PG (chunk
|
|
77
|
+
* replayed twice — no-op via ON CONFLICT). Hidden until the first
|
|
78
|
+
* replay runs; the visible proof of the subsystem is
|
|
79
|
+
* `cold_store_archive_cycles_total`.
|
|
80
|
+
*/
|
|
81
|
+
export declare function coldStoreReplayRowsTotal(): Counter;
|
|
82
|
+
/**
|
|
83
|
+
* Phase F — replay duration (S3 GET + manifest scan + decode + INSERT),
|
|
84
|
+
* seconds. Buckets mirror archive duration: 10ms, 100ms, 500ms, 1s, 5s,
|
|
85
|
+
* 30s, 120s.
|
|
86
|
+
*/
|
|
87
|
+
export declare function coldStoreReplayDurationSeconds(): Histogram;
|
|
88
|
+
/**
|
|
89
|
+
* Phase 2 — chunks acted on by `purgeExpiredChunks`, labeled by db,
|
|
90
|
+
* table, and outcome. `result` ∈ `purged` | `dry_run` | `skipped_locked`
|
|
91
|
+
* | `failure`. Hidden until the first GC sweep finds candidates;
|
|
92
|
+
* `cold_store_archive_cycles_total` remains the always-non-zero
|
|
93
|
+
* heartbeat for the subsystem.
|
|
94
|
+
*/
|
|
95
|
+
export declare function coldStorePurgeChunksTotal(): Counter;
|
|
96
|
+
/** Phase 2 — gzipped bytes deleted from S3 by the purge sweep. */
|
|
97
|
+
export declare function coldStorePurgeBytesTotal(): Counter;
|
|
98
|
+
/**
|
|
99
|
+
* Phase 2 — purge duration per sweep, seconds. Buckets mirror the
|
|
100
|
+
* archive duration histogram: 10ms, 100ms, 500ms, 1s, 5s, 30s, 120s.
|
|
101
|
+
*/
|
|
102
|
+
export declare function coldStorePurgeDurationSeconds(): Histogram;
|
|
103
|
+
//# sourceMappingURL=metrics.d.ts.map
|