@kici-dev/shared 0.0.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. package/LICENSE +202 -0
  2. package/README.md +1 -6
  3. package/dist/chunk-gOLHoazu.js +4 -0
  4. package/dist/cold-store/bucket.d.ts +26 -0
  5. package/dist/cold-store/bucket.js +40 -0
  6. package/dist/cold-store/bucket.test.d.ts +2 -0
  7. package/dist/cold-store/chunk-encoder.d.ts +63 -0
  8. package/dist/cold-store/chunk-encoder.js +94 -0
  9. package/dist/cold-store/chunk-encoder.test.d.ts +2 -0
  10. package/dist/cold-store/chunk-id.d.ts +10 -0
  11. package/dist/cold-store/chunk-id.js +29 -0
  12. package/dist/cold-store/chunk-id.test.d.ts +2 -0
  13. package/dist/cold-store/cold-store.d.ts +345 -0
  14. package/dist/cold-store/cold-store.js +1078 -0
  15. package/dist/cold-store/cold-store.test.d.ts +2 -0
  16. package/dist/cold-store/config.d.ts +62 -0
  17. package/dist/cold-store/config.js +28 -0
  18. package/dist/cold-store/index.d.ts +20 -0
  19. package/dist/cold-store/index.js +11 -0
  20. package/dist/cold-store/key.d.ts +89 -0
  21. package/dist/cold-store/key.js +88 -0
  22. package/dist/cold-store/key.test.d.ts +2 -0
  23. package/dist/cold-store/lru.d.ts +33 -0
  24. package/dist/cold-store/lru.js +59 -0
  25. package/dist/cold-store/lru.test.d.ts +2 -0
  26. package/dist/cold-store/manifest.d.ts +11 -0
  27. package/dist/cold-store/manifest.js +57 -0
  28. package/dist/cold-store/manifest.test.d.ts +2 -0
  29. package/dist/cold-store/metrics.d.ts +103 -0
  30. package/dist/cold-store/metrics.js +179 -0
  31. package/dist/cold-store/table-adapter.d.ts +229 -0
  32. package/dist/cold-store/table-adapter.js +2 -0
  33. package/dist/cold-store/types.d.ts +111 -0
  34. package/dist/cold-store/types.js +2 -0
  35. package/dist/crypto.d.ts +33 -0
  36. package/dist/crypto.js +67 -0
  37. package/dist/db-admin.d.ts +1422 -0
  38. package/dist/db-admin.js +2525 -0
  39. package/dist/db-admin.test.d.ts +2 -0
  40. package/dist/db.d.ts +14 -0
  41. package/dist/db.js +23 -0
  42. package/dist/env/allowlist.d.ts +79 -0
  43. package/dist/env/allowlist.js +86 -0
  44. package/dist/env/define-env.d.ts +172 -0
  45. package/dist/env/define-env.js +295 -0
  46. package/dist/env/define-env.test.d.ts +2 -0
  47. package/dist/env/env-rule-allowlist.test.d.ts +2 -0
  48. package/dist/env/index.d.ts +11 -0
  49. package/dist/env/index.js +4 -0
  50. package/dist/env/logger-env.d.ts +30 -0
  51. package/dist/env/logger-env.js +95 -0
  52. package/dist/error.d.ts +16 -0
  53. package/dist/error.js +58 -0
  54. package/dist/error.test.d.ts +2 -0
  55. package/dist/format-bytes.d.ts +5 -0
  56. package/dist/format-bytes.js +15 -0
  57. package/dist/format-bytes.test.d.ts +2 -0
  58. package/dist/format-duration.d.ts +11 -0
  59. package/dist/format-duration.js +32 -0
  60. package/dist/format-duration.test.d.ts +2 -0
  61. package/dist/graceful-shutdown.d.ts +58 -0
  62. package/dist/graceful-shutdown.js +71 -0
  63. package/dist/graceful-shutdown.test.d.ts +2 -0
  64. package/dist/idempotency-files.d.ts +113 -0
  65. package/dist/idempotency-files.js +190 -0
  66. package/dist/idempotency-files.test.d.ts +2 -0
  67. package/dist/idempotency.d.ts +68 -0
  68. package/dist/idempotency.js +49 -0
  69. package/dist/idempotency.test.d.ts +2 -0
  70. package/dist/index.d.ts +19 -0
  71. package/dist/index.js +31 -0
  72. package/dist/logger.d.ts +57 -0
  73. package/dist/logger.js +175 -0
  74. package/dist/logger.test.d.ts +2 -0
  75. package/dist/reconnect-delay.d.ts +11 -0
  76. package/dist/reconnect-delay.js +22 -0
  77. package/dist/request-context.d.ts +42 -0
  78. package/dist/request-context.js +37 -0
  79. package/dist/ring-buffer.d.ts +35 -0
  80. package/dist/ring-buffer.js +62 -0
  81. package/dist/ring-buffer.test.d.ts +2 -0
  82. package/dist/routes/health.d.ts +26 -0
  83. package/dist/routes/health.js +45 -0
  84. package/dist/routes/health.test.d.ts +2 -0
  85. package/dist/routes/metrics.d.ts +20 -0
  86. package/dist/routes/metrics.js +38 -0
  87. package/dist/s3-client.d.ts +42 -0
  88. package/dist/s3-client.js +25 -0
  89. package/dist/telemetry/index.d.ts +3 -0
  90. package/dist/telemetry/index.js +4 -0
  91. package/dist/telemetry/init.d.ts +20 -0
  92. package/dist/telemetry/init.js +42 -0
  93. package/dist/telemetry/init.test.d.ts +2 -0
  94. package/dist/telemetry/metrics.d.ts +8 -0
  95. package/dist/telemetry/metrics.js +16 -0
  96. package/dist/tool-check.d.ts +33 -0
  97. package/dist/tool-check.js +60 -0
  98. package/dist/tool-check.test.d.ts +2 -0
  99. package/dist/ts-loader-hook.d.ts +26 -0
  100. package/dist/ts-loader-hook.js +48 -0
  101. package/dist/zx.d.ts +8 -0
  102. package/dist/zx.js +78 -0
  103. package/package.json +69 -5
  104. package/sbom.spdx.json +8775 -0
  105. package/index.js +0 -3
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=cold-store.test.d.ts.map
@@ -0,0 +1,62 @@
1
+ /**
2
+ * Cold-store configuration types.
3
+ *
4
+ * section 5. Per-table defaults in DEFAULT_TABLE_CONFIG mirror the
5
+ * design doc's table.
6
+ */
7
+ import type { SharedS3Config } from '../s3-client.js';
8
+ /**
9
+ * Per-table archival tuning parameters.
10
+ *
11
+ * A table's effective config is the global default, overridden by any
12
+ * values from the YAML/env config, overridden by any per-org values
13
+ * (Platform only) from `cold_store_org_overrides`.
14
+ */
15
+ export interface ColdStoreTableConfig {
16
+ /** Minimum age (in days) before a row is eligible to archive. */
17
+ warmTtlDays: number;
18
+ /**
19
+ * Floor for total warm bytes per tenant per table. Below this, no
20
+ * archival happens — even if rows are old. Prevents thrashing on
21
+ * low-traffic tenants.
22
+ */
23
+ minWarmTenantBytes: number;
24
+ /** Floor for chunk size. Smaller eligible sets are deferred. */
25
+ minChunkBytes: number;
26
+ /** Ceiling for chunk size. Larger eligible sets are split. */
27
+ maxChunkBytes: number;
28
+ /** Hard cap on rows archived per (table, tenant) per cycle. */
29
+ maxRowsPerCycle: number;
30
+ /** Whether the table's archival is enabled (kill switch). */
31
+ enabled: boolean;
32
+ }
33
+ /**
34
+ * Reasonable defaults — matches the design doc's section 5 table.
35
+ * Concrete adapters can override any subset of these.
36
+ */
37
+ export declare const DEFAULT_TABLE_CONFIG: ColdStoreTableConfig;
38
+ /**
39
+ * Top-level cold-store configuration passed to `BaseColdStore`.
40
+ */
41
+ export interface ColdStoreConfig {
42
+ /**
43
+ * S3-compatible storage config. The `prefix` defaults to
44
+ * `'cold-store/'` when unset.
45
+ */
46
+ storage: SharedS3Config;
47
+ /**
48
+ * Per-table overrides keyed by table name; unset tables use
49
+ * DEFAULT_TABLE_CONFIG.
50
+ */
51
+ tables: Record<string, Partial<ColdStoreTableConfig>>;
52
+ /** Global concurrency cap on in-flight S3 PUTs. Default 4. */
53
+ s3Concurrency: number;
54
+ /** Master feature toggle; when false, cycles no-op fast. */
55
+ enabled: boolean;
56
+ }
57
+ /**
58
+ * Merge DEFAULT_TABLE_CONFIG with overrides to produce the effective
59
+ * per-table config.
60
+ */
61
+ export declare function resolveTableConfig(overrides: Partial<ColdStoreTableConfig> | undefined): ColdStoreTableConfig;
62
+ //# sourceMappingURL=config.d.ts.map
@@ -0,0 +1,28 @@
1
+ import "../chunk-gOLHoazu.js";
2
+ //#region src/cold-store/config.ts
3
+ /**
4
+ * Reasonable defaults — matches the design doc's section 5 table.
5
+ * Concrete adapters can override any subset of these.
6
+ */
7
+ const DEFAULT_TABLE_CONFIG = {
8
+ warmTtlDays: 30,
9
+ minWarmTenantBytes: 5 * 1024 * 1024,
10
+ minChunkBytes: 1 * 1024 * 1024,
11
+ maxChunkBytes: 50 * 1024 * 1024,
12
+ maxRowsPerCycle: 5e4,
13
+ enabled: true
14
+ };
15
+ /**
16
+ * Merge DEFAULT_TABLE_CONFIG with overrides to produce the effective
17
+ * per-table config.
18
+ */
19
+ function resolveTableConfig(overrides) {
20
+ return {
21
+ ...DEFAULT_TABLE_CONFIG,
22
+ ...overrides ?? {}
23
+ };
24
+ }
25
+ //#endregion
26
+ export { DEFAULT_TABLE_CONFIG, resolveTableConfig };
27
+
28
+ //# sourceMappingURL=config.js.map
@@ -0,0 +1,20 @@
1
+ /**
2
+ * Cold-store framework — public surface for @kici-dev/shared consumers.
3
+ *
4
+ * Multiple implementations extend `BaseColdStore` with their own
5
+ * DB-specific adapters. Phase A ships framework types + the abstract
6
+ * class with a no-op archive cycle; Phase B+ adds concrete
7
+ * `TableAdapter` implementations.
8
+ */
9
+ export { chunkObjectKey, encodeKeySegment, tablePrefix, tenantDayBucketPrefix, tenantDayPrefix, type DbKind, } from './key.js';
10
+ export { COLD_BUCKET_NAMES, coldDaysToBucket, isLongerColdRetention, type ColdBucketName, } from './bucket.js';
11
+ export { BaseColdStore, type BaseColdStoreDeps, type ColdStore, type ColdStoreFetchRangeArgs, type ColdStoreReplayChunkArgs, type ColdStoreReplayResult, type ColdStoreReplayRowArgs, type PurgeableChunk, type PurgeChunkResult, type PurgeExpiredChunksOpts, type PurgeExpiredChunksSummary, } from './cold-store.js';
12
+ export { ChunkLru, type ChunkLruOptions } from './lru.js';
13
+ export { computeChunkId } from './chunk-id.js';
14
+ export { decodeChunk, encodeChunk, type DecodeChunkArgs, type EncodeChunkArgs, type EncodedChunk, } from './chunk-encoder.js';
15
+ export { parseManifest, serializeManifest } from './manifest.js';
16
+ export { DEFAULT_TABLE_CONFIG, resolveTableConfig, type ColdStoreConfig, type ColdStoreTableConfig, } from './config.js';
17
+ export { coldStoreArchiveBytesTotal, coldStoreArchiveCyclesTotal, coldStoreArchiveDurationSeconds, coldStoreArchiveRowsTotal, coldStorePurgeBytesTotal, coldStorePurgeChunksTotal, coldStorePurgeDurationSeconds, coldStoreRehydrateBytesTotal, coldStoreRehydrateDurationSeconds, coldStoreRehydrateRequestsTotal, coldStoreReplayDurationSeconds, coldStoreReplayRowsTotal, coldStoreVerifyFailuresTotal, } from './metrics.js';
18
+ export { type ChunkCommitMetadata, type EligiblePartition, type TableAdapter, } from './table-adapter.js';
19
+ export { type ArchiveCycleSummary, type ChunkManifest, type ColdRetention } from './types.js';
20
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1,11 @@
1
+ import "../chunk-gOLHoazu.js";
2
+ import { chunkObjectKey, encodeKeySegment, tablePrefix, tenantDayBucketPrefix, tenantDayPrefix } from "./key.js";
3
+ import { COLD_BUCKET_NAMES, coldDaysToBucket, isLongerColdRetention } from "./bucket.js";
4
+ import { computeChunkId } from "./chunk-id.js";
5
+ import { decodeChunk, encodeChunk } from "./chunk-encoder.js";
6
+ import { parseManifest, serializeManifest } from "./manifest.js";
7
+ import { coldStoreArchiveBytesTotal, coldStoreArchiveCyclesTotal, coldStoreArchiveDurationSeconds, coldStoreArchiveRowsTotal, coldStorePurgeBytesTotal, coldStorePurgeChunksTotal, coldStorePurgeDurationSeconds, coldStoreRehydrateBytesTotal, coldStoreRehydrateDurationSeconds, coldStoreRehydrateRequestsTotal, coldStoreReplayDurationSeconds, coldStoreReplayRowsTotal, coldStoreVerifyFailuresTotal } from "./metrics.js";
8
+ import { BaseColdStore } from "./cold-store.js";
9
+ import { ChunkLru } from "./lru.js";
10
+ import { DEFAULT_TABLE_CONFIG, resolveTableConfig } from "./config.js";
11
+ export { BaseColdStore, COLD_BUCKET_NAMES, ChunkLru, DEFAULT_TABLE_CONFIG, chunkObjectKey, coldDaysToBucket, coldStoreArchiveBytesTotal, coldStoreArchiveCyclesTotal, coldStoreArchiveDurationSeconds, coldStoreArchiveRowsTotal, coldStorePurgeBytesTotal, coldStorePurgeChunksTotal, coldStorePurgeDurationSeconds, coldStoreRehydrateBytesTotal, coldStoreRehydrateDurationSeconds, coldStoreRehydrateRequestsTotal, coldStoreReplayDurationSeconds, coldStoreReplayRowsTotal, coldStoreVerifyFailuresTotal, computeChunkId, decodeChunk, encodeChunk, encodeKeySegment, isLongerColdRetention, parseManifest, resolveTableConfig, serializeManifest, tablePrefix, tenantDayBucketPrefix, tenantDayPrefix };
@@ -0,0 +1,89 @@
1
+ /**
2
+ * Key helpers for S3 object layouts.
3
+ *
4
+ * `encodeKeySegment` escapes non-safe characters in a single path segment
5
+ * so that values coming from user-controlled sources (delivery IDs,
6
+ * routing keys, org IDs, partition values) can be embedded in an object
7
+ * key without breaking the key structure.
8
+ *
9
+ * This function was originally defined in
10
+ * packages/orchestrator/src/webhook/event-log.ts and is now shared with
11
+ * the cold-store framework. The encoding algorithm MUST NOT change —
12
+ * previously-written chunk and event-log keys depend on this mapping
13
+ * remaining stable forever. A fixture test in key.test.ts locks the
14
+ * behavior.
15
+ */
16
+ /**
17
+ * Encode a single S3 path segment: anything outside `[A-Za-z0-9._:-]`
18
+ * becomes `_<hex>` where `<hex>` is the lowercase UTF-16 code unit of
19
+ * the replaced character (same as `ch.charCodeAt(0).toString(16)`).
20
+ */
21
+ export declare function encodeKeySegment(s: string): string;
22
+ /**
23
+ * DB identifier for cold-store chunk keys.
24
+ */
25
+ export type DbKind = 'platform' | 'orchestrator';
26
+ /**
27
+ * Compose the prefix for a specific table's cold-store objects.
28
+ *
29
+ * Layout: `<prefix>/<db>/<table>`.
30
+ */
31
+ export declare function tablePrefix(args: {
32
+ prefix: string;
33
+ db: DbKind;
34
+ table: string;
35
+ }): string;
36
+ /**
37
+ * Compose the prefix for a specific tenant's day-partitioned objects.
38
+ *
39
+ * Layout: `<prefix>/<db>/<table>/<tenantId>/<YYYY>/<MM>/<DD>`.
40
+ * `tenantId` is encoded via `encodeKeySegment`.
41
+ * `partitionDate` must be `YYYY-MM-DD` (no other format is accepted).
42
+ */
43
+ export declare function tenantDayPrefix(args: {
44
+ prefix: string;
45
+ db: DbKind;
46
+ table: string;
47
+ tenantId: string;
48
+ partitionDate: string;
49
+ }): string;
50
+ /**
51
+ * Compose a tenant-day prefix that includes the cold-bucket segment
52
+ * introduced in Phase 2. Layout:
53
+ *
54
+ * `<prefix>/<db>/<table>/<tenantId>/<YYYY>/<MM>/<DD>/<bucket>`
55
+ *
56
+ * Phase-1 (v1 manifest) chunks live at the day-prefix root and are
57
+ * addressed via `tenantDayPrefix` directly — those legacy chunks are
58
+ * treated as the `'forever'` bucket by `parseManifest` but DO NOT carry
59
+ * a `forever` segment in their key (the chunk-purge sweep keys off the
60
+ * manifest's `bucket` / `maxColdDays`, not the path).
61
+ */
62
+ export declare function tenantDayBucketPrefix(args: {
63
+ prefix: string;
64
+ db: DbKind;
65
+ table: string;
66
+ tenantId: string;
67
+ partitionDate: string;
68
+ bucket: string;
69
+ }): string;
70
+ /**
71
+ * Compose a full object key for a cold-store chunk or its manifest.
72
+ *
73
+ * - V1 (no `bucket` arg): `<tenant-day-prefix>/<chunkId>.<ext>` — original
74
+ * layout, preserved for read-back compatibility with V1 chunks.
75
+ * - V2 (with `bucket`): `<tenant-day-prefix>/<bucket>/<chunkId>.<ext>` —
76
+ * the GC sweep can list a single bucket subprefix to find purge
77
+ * candidates without scanning the whole day.
78
+ */
79
+ export declare function chunkObjectKey(args: {
80
+ prefix: string;
81
+ db: DbKind;
82
+ table: string;
83
+ tenantId: string;
84
+ partitionDate: string;
85
+ chunkId: string;
86
+ kind: 'data' | 'manifest';
87
+ bucket?: string;
88
+ }): string;
89
+ //# sourceMappingURL=key.d.ts.map
@@ -0,0 +1,88 @@
1
+ import "../chunk-gOLHoazu.js";
2
+ //#region src/cold-store/key.ts
3
+ /**
4
+ * Key helpers for S3 object layouts.
5
+ *
6
+ * `encodeKeySegment` escapes non-safe characters in a single path segment
7
+ * so that values coming from user-controlled sources (delivery IDs,
8
+ * routing keys, org IDs, partition values) can be embedded in an object
9
+ * key without breaking the key structure.
10
+ *
11
+ * This function was originally defined in
12
+ * packages/orchestrator/src/webhook/event-log.ts and is now shared with
13
+ * the cold-store framework. The encoding algorithm MUST NOT change —
14
+ * previously-written chunk and event-log keys depend on this mapping
15
+ * remaining stable forever. A fixture test in key.test.ts locks the
16
+ * behavior.
17
+ */
18
+ /**
19
+ * Encode a single S3 path segment: anything outside `[A-Za-z0-9._:-]`
20
+ * becomes `_<hex>` where `<hex>` is the lowercase UTF-16 code unit of
21
+ * the replaced character (same as `ch.charCodeAt(0).toString(16)`).
22
+ */
23
+ function encodeKeySegment(s) {
24
+ return s.replace(/[^A-Za-z0-9._:-]/g, (ch) => `_${ch.charCodeAt(0).toString(16)}`);
25
+ }
26
+ /**
27
+ * Compose the prefix for a specific table's cold-store objects.
28
+ *
29
+ * Layout: `<prefix>/<db>/<table>`.
30
+ */
31
+ function tablePrefix(args) {
32
+ return `${args.prefix.replace(/\/+$/, "")}/${args.db}/${args.table}`;
33
+ }
34
+ /**
35
+ * Compose the prefix for a specific tenant's day-partitioned objects.
36
+ *
37
+ * Layout: `<prefix>/<db>/<table>/<tenantId>/<YYYY>/<MM>/<DD>`.
38
+ * `tenantId` is encoded via `encodeKeySegment`.
39
+ * `partitionDate` must be `YYYY-MM-DD` (no other format is accepted).
40
+ */
41
+ function tenantDayPrefix(args) {
42
+ if (!/^\d{4}-\d{2}-\d{2}$/.test(args.partitionDate)) throw new Error(`partitionDate must be YYYY-MM-DD, got ${JSON.stringify(args.partitionDate)}`);
43
+ const [yyyy, mm, dd] = args.partitionDate.split("-");
44
+ const encodedTenant = encodeKeySegment(args.tenantId);
45
+ return `${tablePrefix(args)}/${encodedTenant}/${yyyy}/${mm}/${dd}`;
46
+ }
47
+ /**
48
+ * Allowed bucket-segment shape: lowercase letters, digits, the literal
49
+ * `forever`, e.g. `30d` / `180d` / `1y` / `2y` / `forever`. Validated to
50
+ * keep the S3 path safe (no `/`, no `.`, no path traversal).
51
+ */
52
+ const BUCKET_SEGMENT_RE = /^[a-z0-9]+$/;
53
+ /**
54
+ * Compose a tenant-day prefix that includes the cold-bucket segment
55
+ * introduced in Phase 2. Layout:
56
+ *
57
+ * `<prefix>/<db>/<table>/<tenantId>/<YYYY>/<MM>/<DD>/<bucket>`
58
+ *
59
+ * Phase-1 (v1 manifest) chunks live at the day-prefix root and are
60
+ * addressed via `tenantDayPrefix` directly — those legacy chunks are
61
+ * treated as the `'forever'` bucket by `parseManifest` but DO NOT carry
62
+ * a `forever` segment in their key (the chunk-purge sweep keys off the
63
+ * manifest's `bucket` / `maxColdDays`, not the path).
64
+ */
65
+ function tenantDayBucketPrefix(args) {
66
+ if (!BUCKET_SEGMENT_RE.test(args.bucket)) throw new Error(`bucket must match ${BUCKET_SEGMENT_RE.source}, got ${JSON.stringify(args.bucket)}`);
67
+ return `${tenantDayPrefix(args)}/${args.bucket}`;
68
+ }
69
+ /**
70
+ * Compose a full object key for a cold-store chunk or its manifest.
71
+ *
72
+ * - V1 (no `bucket` arg): `<tenant-day-prefix>/<chunkId>.<ext>` — original
73
+ * layout, preserved for read-back compatibility with V1 chunks.
74
+ * - V2 (with `bucket`): `<tenant-day-prefix>/<bucket>/<chunkId>.<ext>` —
75
+ * the GC sweep can list a single bucket subprefix to find purge
76
+ * candidates without scanning the whole day.
77
+ */
78
+ function chunkObjectKey(args) {
79
+ const suffix = args.kind === "data" ? ".jsonl.gz" : ".manifest.json";
80
+ return `${args.bucket ? tenantDayBucketPrefix({
81
+ ...args,
82
+ bucket: args.bucket
83
+ }) : tenantDayPrefix(args)}/${args.chunkId}${suffix}`;
84
+ }
85
+ //#endregion
86
+ export { chunkObjectKey, encodeKeySegment, tablePrefix, tenantDayBucketPrefix, tenantDayPrefix };
87
+
88
+ //# sourceMappingURL=key.js.map
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=key.test.d.ts.map
@@ -0,0 +1,33 @@
1
+ /**
2
+ * In-process size-bounded LRU cache.
3
+ *
4
+ * Used by the cold-store rehydrate path (Phase B+) to keep recently-read
5
+ * chunks hot in memory, so that e.g. a user paginating a 90-day audit
6
+ * window doesn't re-fetch the same S3 chunks for each page. Phase A
7
+ * ships the primitive so Phase B's first consumer gets a debugged
8
+ * LRU rather than having to write one.
9
+ *
10
+ * Size is tracked in bytes via a caller-supplied `sizeOf` function.
11
+ * Eviction happens on `set()` when the accumulated byte count exceeds
12
+ * `maxBytes`.
13
+ */
14
+ export interface ChunkLruOptions<V> {
15
+ maxBytes: number;
16
+ sizeOf: (value: V) => number;
17
+ onEvict?: (key: unknown, value: V) => void;
18
+ }
19
+ export declare class ChunkLru<K, V> {
20
+ private readonly maxBytes;
21
+ private readonly sizeOf;
22
+ private readonly onEvict?;
23
+ private readonly map;
24
+ private readonly sizes;
25
+ private totalBytes;
26
+ constructor(opts: ChunkLruOptions<V>);
27
+ get bytes(): number;
28
+ get entries(): number;
29
+ get(key: K): V | undefined;
30
+ set(key: K, value: V): void;
31
+ clear(): void;
32
+ }
33
+ //# sourceMappingURL=lru.d.ts.map
@@ -0,0 +1,59 @@
1
+ import "../chunk-gOLHoazu.js";
2
+ //#region src/cold-store/lru.ts
3
+ var ChunkLru = class {
4
+ maxBytes;
5
+ sizeOf;
6
+ onEvict;
7
+ map = /* @__PURE__ */ new Map();
8
+ sizes = /* @__PURE__ */ new Map();
9
+ totalBytes = 0;
10
+ constructor(opts) {
11
+ this.maxBytes = opts.maxBytes;
12
+ this.sizeOf = opts.sizeOf;
13
+ this.onEvict = opts.onEvict;
14
+ }
15
+ get bytes() {
16
+ return this.totalBytes;
17
+ }
18
+ get entries() {
19
+ return this.map.size;
20
+ }
21
+ get(key) {
22
+ const v = this.map.get(key);
23
+ if (v === void 0) return void 0;
24
+ this.map.delete(key);
25
+ this.map.set(key, v);
26
+ return v;
27
+ }
28
+ set(key, value) {
29
+ if (this.map.has(key)) {
30
+ const prevSize = this.sizes.get(key) ?? 0;
31
+ this.totalBytes -= prevSize;
32
+ this.map.delete(key);
33
+ this.sizes.delete(key);
34
+ }
35
+ const size = this.sizeOf(value);
36
+ this.map.set(key, value);
37
+ this.sizes.set(key, size);
38
+ this.totalBytes += size;
39
+ while (this.totalBytes > this.maxBytes && this.map.size > 0) {
40
+ const firstKey = this.map.keys().next().value;
41
+ const evicted = this.map.get(firstKey);
42
+ const evictedSize = this.sizes.get(firstKey) ?? 0;
43
+ this.map.delete(firstKey);
44
+ this.sizes.delete(firstKey);
45
+ this.totalBytes -= evictedSize;
46
+ this.onEvict?.(firstKey, evicted);
47
+ }
48
+ }
49
+ clear() {
50
+ if (this.onEvict) for (const [key, value] of this.map.entries()) this.onEvict(key, value);
51
+ this.map.clear();
52
+ this.sizes.clear();
53
+ this.totalBytes = 0;
54
+ }
55
+ };
56
+ //#endregion
57
+ export { ChunkLru };
58
+
59
+ //# sourceMappingURL=lru.js.map
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=lru.test.d.ts.map
@@ -0,0 +1,11 @@
1
+ /**
2
+ * Manifest serialization / parsing.
3
+ *
4
+ * Manifests are small JSON sidecar files next to each `.jsonl.gz`
5
+ * chunk. Serialized with stable key ordering so byte-diffs across
6
+ * archive runs are meaningful.
7
+ */
8
+ import type { ChunkManifest } from './types.js';
9
+ export declare function serializeManifest(m: ChunkManifest): string;
10
+ export declare function parseManifest(raw: string | Buffer): ChunkManifest;
11
+ //# sourceMappingURL=manifest.d.ts.map
@@ -0,0 +1,57 @@
1
+ import "../chunk-gOLHoazu.js";
2
+ //#region src/cold-store/manifest.ts
3
+ /**
4
+ * Required fields shared by v1 and v2 manifests. Order is the on-disk
5
+ * key order — keep stable so byte-diffs across archive runs remain
6
+ * meaningful.
7
+ */
8
+ const COMMON_REQUIRED_KEYS = [
9
+ "schemaVersion",
10
+ "db",
11
+ "table",
12
+ "tenantId",
13
+ "partitionDate",
14
+ "rowCount",
15
+ "byteCount",
16
+ "gzipByteCount",
17
+ "minTimestamp",
18
+ "maxTimestamp",
19
+ "minRowId",
20
+ "maxRowId",
21
+ "contentHash",
22
+ "chunkId",
23
+ "createdAt",
24
+ "archiverInstanceId"
25
+ ];
26
+ /** v2-only required fields — always emitted by `serializeManifest`. */
27
+ const V2_REQUIRED_KEYS = ["bucket", "maxColdDays"];
28
+ const OPTIONAL_MANIFEST_KEYS = ["replayLookupKeys"];
29
+ function serializeManifest(m) {
30
+ const ordered = {};
31
+ for (const k of COMMON_REQUIRED_KEYS) ordered[k] = m[k];
32
+ if (m.schemaVersion === 2) for (const k of V2_REQUIRED_KEYS) {
33
+ if (m[k] === void 0) throw new Error(`serializeManifest: schemaVersion=2 requires field "${k}"`);
34
+ ordered[k] = m[k];
35
+ }
36
+ for (const k of OPTIONAL_MANIFEST_KEYS) if (m[k] !== void 0) ordered[k] = m[k];
37
+ return JSON.stringify(ordered, null, 2);
38
+ }
39
+ function parseManifest(raw) {
40
+ const text = typeof raw === "string" ? raw : raw.toString("utf-8");
41
+ const parsed = JSON.parse(text);
42
+ if (!parsed || typeof parsed !== "object") throw new Error("parseManifest: not a JSON object");
43
+ const obj = parsed;
44
+ if (obj.schemaVersion !== 1 && obj.schemaVersion !== 2) throw new Error(`parseManifest: unsupported schemaVersion ${String(obj.schemaVersion)}`);
45
+ for (const k of COMMON_REQUIRED_KEYS) if (!(k in obj)) throw new Error(`parseManifest: missing field ${k}`);
46
+ if (obj.schemaVersion === 2) {
47
+ for (const k of V2_REQUIRED_KEYS) if (!(k in obj)) throw new Error(`parseManifest: schemaVersion=2 missing field ${k}`);
48
+ } else {
49
+ if (obj.bucket === void 0) obj.bucket = "forever";
50
+ if (obj.maxColdDays === void 0) obj.maxColdDays = "forever";
51
+ }
52
+ return obj;
53
+ }
54
+ //#endregion
55
+ export { parseManifest, serializeManifest };
56
+
57
+ //# sourceMappingURL=manifest.js.map
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=manifest.test.d.ts.map
@@ -0,0 +1,103 @@
1
+ /**
2
+ * Cold-store domain metrics.
3
+ *
4
+ * These are the "how much data moved through cold-store" metrics;
5
+ * complements (but does not overlap with) the scheduler-layer
6
+ * metrics (`kici_{platform,orch}_job_*`) owned by the scheduled-job
7
+ * wrappers.
8
+ *
9
+ * `cold_store_archive_cycles_total` is always-non-zero — incremented
10
+ * on every `runArchiveCycle()` call with a `result` label — so the
11
+ * meter scope + instrument metadata always surface on /metrics.
12
+ * This is the counterpart to the design doc's original "zero-add
13
+ * trick": `@opentelemetry/exporter-prometheus` 0.213.0 filters out
14
+ * counters whose only observations are `.add(0)`, so we need a
15
+ * guaranteed-non-zero counter to keep the series visible.
16
+ *
17
+ * `cold_store_archive_rows_total` stays dormant (hidden) until a
18
+ * Phase B+ `TableAdapter` actually archives rows — at which point
19
+ * its samples appear naturally.
20
+ *
21
+ * Lazy meter initialization: the `@kici-dev/shared` barrel is imported
22
+ * statically at the top of service entry points (server.ts, worker.ts),
23
+ * which evaluates this module BEFORE `initTelemetry()` sets the global
24
+ * MeterProvider. If we call `createMeter('kici-cold-store')` at module
25
+ * load time, the returned ProxyMeter binds to the no-op provider and
26
+ * its instruments never reach the Prometheus exporter. Resolving the
27
+ * meter + instruments on first access (inside `getCold...()` getters)
28
+ * guarantees we resolve AFTER telemetry is wired up.
29
+ */
30
+ import type { Counter, Histogram } from '@opentelemetry/api';
31
+ /**
32
+ * Cycles that `runArchiveCycle()` has completed, labeled by outcome.
33
+ * Always incremented by 1 on every tick — guaranteed-visible proof
34
+ * that the cold-store subsystem is registered and running.
35
+ *
36
+ * `result` ∈ `no_tables` | `disabled` | `success` | `failure`.
37
+ */
38
+ export declare function coldStoreArchiveCyclesTotal(): Counter;
39
+ /**
40
+ * Total rows archived, by db / table / outcome.
41
+ * `result` ∈ success | failure | skipped_min_chunk | skipped_min_warm.
42
+ *
43
+ * Hidden from /metrics until the first Phase B+ TableAdapter moves
44
+ * a row; the visible proof that the subsystem is wired is
45
+ * `cold_store_archive_cycles_total`, not this counter.
46
+ */
47
+ export declare function coldStoreArchiveRowsTotal(): Counter;
48
+ /**
49
+ * Total bytes archived, by kind ∈ raw | gzipped. Lets us compute the
50
+ * ongoing compression ratio.
51
+ */
52
+ export declare function coldStoreArchiveBytesTotal(): Counter;
53
+ /**
54
+ * Per-chunk archive duration histogram.
55
+ * Buckets: 10ms, 100ms, 500ms, 1s, 5s, 30s, 120s.
56
+ */
57
+ export declare function coldStoreArchiveDurationSeconds(): Histogram;
58
+ /**
59
+ * Rehydrate requests (cache hit vs miss). Incremented by the
60
+ * read-through layer (Phase B+) when it serves a range query from
61
+ * cold storage.
62
+ */
63
+ export declare function coldStoreRehydrateRequestsTotal(): Counter;
64
+ /** Bytes read from S3 on rehydrate cache miss. */
65
+ export declare function coldStoreRehydrateBytesTotal(): Counter;
66
+ /** Rehydrate duration (S3 fetch + decode), seconds. */
67
+ export declare function coldStoreRehydrateDurationSeconds(): Histogram;
68
+ /**
69
+ * Verify-failure counter — bumped on the rare `contentHash` mismatch
70
+ * path during post-write verification.
71
+ */
72
+ export declare function coldStoreVerifyFailuresTotal(): Counter;
73
+ /**
74
+ * Phase F — rows promoted back into PG via `replayChunk` / `replayRow`.
75
+ * `result` ∈ `success` | `failure` | `idempotent_skip`. The
76
+ * `idempotent_skip` bucket counts rows already present in PG (chunk
77
+ * replayed twice — no-op via ON CONFLICT). Hidden until the first
78
+ * replay runs; the visible proof of the subsystem is
79
+ * `cold_store_archive_cycles_total`.
80
+ */
81
+ export declare function coldStoreReplayRowsTotal(): Counter;
82
+ /**
83
+ * Phase F — replay duration (S3 GET + manifest scan + decode + INSERT),
84
+ * seconds. Buckets mirror archive duration: 10ms, 100ms, 500ms, 1s, 5s,
85
+ * 30s, 120s.
86
+ */
87
+ export declare function coldStoreReplayDurationSeconds(): Histogram;
88
+ /**
89
+ * Phase 2 — chunks acted on by `purgeExpiredChunks`, labeled by db,
90
+ * table, and outcome. `result` ∈ `purged` | `dry_run` | `skipped_locked`
91
+ * | `failure`. Hidden until the first GC sweep finds candidates;
92
+ * `cold_store_archive_cycles_total` remains the always-non-zero
93
+ * heartbeat for the subsystem.
94
+ */
95
+ export declare function coldStorePurgeChunksTotal(): Counter;
96
+ /** Phase 2 — gzipped bytes deleted from S3 by the purge sweep. */
97
+ export declare function coldStorePurgeBytesTotal(): Counter;
98
+ /**
99
+ * Phase 2 — purge duration per sweep, seconds. Buckets mirror the
100
+ * archive duration histogram: 10ms, 100ms, 500ms, 1s, 5s, 30s, 120s.
101
+ */
102
+ export declare function coldStorePurgeDurationSeconds(): Histogram;
103
+ //# sourceMappingURL=metrics.d.ts.map