@nathanvale/chatline 0.0.1 → 0.0.2-next.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (213) hide show
  1. package/CHANGELOG.md +6 -0
  2. package/dist/bin/index.js +1 -1
  3. package/dist/index.js +1 -1
  4. package/package.json +1 -1
  5. package/dist/cli/commands/clean.d.ts +0 -17
  6. package/dist/cli/commands/clean.d.ts.map +0 -1
  7. package/dist/cli/commands/clean.js +0 -142
  8. package/dist/cli/commands/clean.js.map +0 -1
  9. package/dist/cli/commands/doctor.d.ts +0 -17
  10. package/dist/cli/commands/doctor.d.ts.map +0 -1
  11. package/dist/cli/commands/doctor.js +0 -202
  12. package/dist/cli/commands/doctor.js.map +0 -1
  13. package/dist/cli/commands/enrich-ai.d.ts +0 -17
  14. package/dist/cli/commands/enrich-ai.d.ts.map +0 -1
  15. package/dist/cli/commands/enrich-ai.js +0 -371
  16. package/dist/cli/commands/enrich-ai.js.map +0 -1
  17. package/dist/cli/commands/index.d.ts +0 -16
  18. package/dist/cli/commands/index.d.ts.map +0 -1
  19. package/dist/cli/commands/index.js +0 -16
  20. package/dist/cli/commands/index.js.map +0 -1
  21. package/dist/cli/commands/ingest-csv.d.ts +0 -17
  22. package/dist/cli/commands/ingest-csv.d.ts.map +0 -1
  23. package/dist/cli/commands/ingest-csv.js +0 -138
  24. package/dist/cli/commands/ingest-csv.js.map +0 -1
  25. package/dist/cli/commands/ingest-db.d.ts +0 -17
  26. package/dist/cli/commands/ingest-db.d.ts.map +0 -1
  27. package/dist/cli/commands/ingest-db.js +0 -159
  28. package/dist/cli/commands/ingest-db.js.map +0 -1
  29. package/dist/cli/commands/init.d.ts +0 -17
  30. package/dist/cli/commands/init.d.ts.map +0 -1
  31. package/dist/cli/commands/init.js +0 -110
  32. package/dist/cli/commands/init.js.map +0 -1
  33. package/dist/cli/commands/normalize-link.d.ts +0 -16
  34. package/dist/cli/commands/normalize-link.d.ts.map +0 -1
  35. package/dist/cli/commands/normalize-link.js +0 -144
  36. package/dist/cli/commands/normalize-link.js.map +0 -1
  37. package/dist/cli/commands/render-markdown.d.ts +0 -17
  38. package/dist/cli/commands/render-markdown.d.ts.map +0 -1
  39. package/dist/cli/commands/render-markdown.js +0 -218
  40. package/dist/cli/commands/render-markdown.js.map +0 -1
  41. package/dist/cli/commands/stats.d.ts +0 -17
  42. package/dist/cli/commands/stats.d.ts.map +0 -1
  43. package/dist/cli/commands/stats.js +0 -175
  44. package/dist/cli/commands/stats.js.map +0 -1
  45. package/dist/cli/commands/validate.d.ts +0 -17
  46. package/dist/cli/commands/validate.d.ts.map +0 -1
  47. package/dist/cli/commands/validate.js +0 -152
  48. package/dist/cli/commands/validate.js.map +0 -1
  49. package/dist/cli/index.d.ts +0 -13
  50. package/dist/cli/index.d.ts.map +0 -1
  51. package/dist/cli/index.js +0 -121
  52. package/dist/cli/index.js.map +0 -1
  53. package/dist/cli/types.d.ts +0 -93
  54. package/dist/cli/types.d.ts.map +0 -1
  55. package/dist/cli/types.js +0 -7
  56. package/dist/cli/types.js.map +0 -1
  57. package/dist/cli/utils.d.ts +0 -29
  58. package/dist/cli/utils.d.ts.map +0 -1
  59. package/dist/cli/utils.js +0 -53
  60. package/dist/cli/utils.js.map +0 -1
  61. package/dist/cli.d.ts +0 -9
  62. package/dist/cli.d.ts.map +0 -1
  63. package/dist/cli.js +0 -1805
  64. package/dist/config/generator.d.ts +0 -90
  65. package/dist/config/generator.d.ts.map +0 -1
  66. package/dist/config/generator.js +0 -320
  67. package/dist/config/generator.js.map +0 -1
  68. package/dist/config/loader.d.ts +0 -107
  69. package/dist/config/loader.d.ts.map +0 -1
  70. package/dist/config/loader.js +0 -251
  71. package/dist/config/loader.js.map +0 -1
  72. package/dist/config/schema.d.ts +0 -107
  73. package/dist/config/schema.d.ts.map +0 -1
  74. package/dist/config/schema.js +0 -169
  75. package/dist/config/schema.js.map +0 -1
  76. package/dist/enrich/audio-transcription.d.ts +0 -77
  77. package/dist/enrich/audio-transcription.d.ts.map +0 -1
  78. package/dist/enrich/audio-transcription.js +0 -370
  79. package/dist/enrich/audio-transcription.js.map +0 -1
  80. package/dist/enrich/checkpoint.d.ts +0 -137
  81. package/dist/enrich/checkpoint.d.ts.map +0 -1
  82. package/dist/enrich/checkpoint.js +0 -205
  83. package/dist/enrich/checkpoint.js.map +0 -1
  84. package/dist/enrich/idempotency.d.ts +0 -90
  85. package/dist/enrich/idempotency.d.ts.map +0 -1
  86. package/dist/enrich/idempotency.js +0 -188
  87. package/dist/enrich/idempotency.js.map +0 -1
  88. package/dist/enrich/image-analysis.d.ts +0 -62
  89. package/dist/enrich/image-analysis.d.ts.map +0 -1
  90. package/dist/enrich/image-analysis.js +0 -264
  91. package/dist/enrich/image-analysis.js.map +0 -1
  92. package/dist/enrich/index.d.ts +0 -60
  93. package/dist/enrich/index.d.ts.map +0 -1
  94. package/dist/enrich/index.js +0 -74
  95. package/dist/enrich/index.js.map +0 -1
  96. package/dist/enrich/link-enrichment.d.ts +0 -37
  97. package/dist/enrich/link-enrichment.d.ts.map +0 -1
  98. package/dist/enrich/link-enrichment.js +0 -202
  99. package/dist/enrich/link-enrichment.js.map +0 -1
  100. package/dist/enrich/pdf-video-handling.d.ts +0 -49
  101. package/dist/enrich/pdf-video-handling.d.ts.map +0 -1
  102. package/dist/enrich/pdf-video-handling.js +0 -325
  103. package/dist/enrich/pdf-video-handling.js.map +0 -1
  104. package/dist/enrich/progress-tracker.d.ts +0 -120
  105. package/dist/enrich/progress-tracker.d.ts.map +0 -1
  106. package/dist/enrich/progress-tracker.js +0 -220
  107. package/dist/enrich/progress-tracker.js.map +0 -1
  108. package/dist/enrich/providers/firecrawl.d.ts +0 -18
  109. package/dist/enrich/providers/firecrawl.d.ts.map +0 -1
  110. package/dist/enrich/providers/firecrawl.js +0 -48
  111. package/dist/enrich/providers/firecrawl.js.map +0 -1
  112. package/dist/enrich/providers/generic.d.ts +0 -16
  113. package/dist/enrich/providers/generic.d.ts.map +0 -1
  114. package/dist/enrich/providers/generic.js +0 -36
  115. package/dist/enrich/providers/generic.js.map +0 -1
  116. package/dist/enrich/providers/index.d.ts +0 -14
  117. package/dist/enrich/providers/index.d.ts.map +0 -1
  118. package/dist/enrich/providers/index.js +0 -13
  119. package/dist/enrich/providers/index.js.map +0 -1
  120. package/dist/enrich/providers/instagram.d.ts +0 -16
  121. package/dist/enrich/providers/instagram.d.ts.map +0 -1
  122. package/dist/enrich/providers/instagram.js +0 -43
  123. package/dist/enrich/providers/instagram.js.map +0 -1
  124. package/dist/enrich/providers/spotify.d.ts +0 -16
  125. package/dist/enrich/providers/spotify.d.ts.map +0 -1
  126. package/dist/enrich/providers/spotify.js +0 -45
  127. package/dist/enrich/providers/spotify.js.map +0 -1
  128. package/dist/enrich/providers/twitter.d.ts +0 -16
  129. package/dist/enrich/providers/twitter.d.ts.map +0 -1
  130. package/dist/enrich/providers/twitter.js +0 -43
  131. package/dist/enrich/providers/twitter.js.map +0 -1
  132. package/dist/enrich/providers/types.d.ts +0 -47
  133. package/dist/enrich/providers/types.d.ts.map +0 -1
  134. package/dist/enrich/providers/types.js +0 -15
  135. package/dist/enrich/providers/types.js.map +0 -1
  136. package/dist/enrich/providers/youtube.d.ts +0 -16
  137. package/dist/enrich/providers/youtube.d.ts.map +0 -1
  138. package/dist/enrich/providers/youtube.js +0 -43
  139. package/dist/enrich/providers/youtube.js.map +0 -1
  140. package/dist/enrich/rate-limiting.d.ts +0 -118
  141. package/dist/enrich/rate-limiting.d.ts.map +0 -1
  142. package/dist/enrich/rate-limiting.js +0 -258
  143. package/dist/enrich/rate-limiting.js.map +0 -1
  144. package/dist/index.d.ts.map +0 -1
  145. package/dist/index.js.map +0 -1
  146. package/dist/ingest/dedup-merge.d.ts +0 -82
  147. package/dist/ingest/dedup-merge.d.ts.map +0 -1
  148. package/dist/ingest/dedup-merge.js +0 -262
  149. package/dist/ingest/dedup-merge.js.map +0 -1
  150. package/dist/ingest/ingest-csv.d.ts +0 -62
  151. package/dist/ingest/ingest-csv.d.ts.map +0 -1
  152. package/dist/ingest/ingest-csv.js +0 -300
  153. package/dist/ingest/ingest-csv.js.map +0 -1
  154. package/dist/ingest/ingest-db.d.ts +0 -64
  155. package/dist/ingest/ingest-db.d.ts.map +0 -1
  156. package/dist/ingest/ingest-db.js +0 -172
  157. package/dist/ingest/ingest-db.js.map +0 -1
  158. package/dist/ingest/link-replies-and-tapbacks.d.ts +0 -53
  159. package/dist/ingest/link-replies-and-tapbacks.d.ts.map +0 -1
  160. package/dist/ingest/link-replies-and-tapbacks.js +0 -381
  161. package/dist/ingest/link-replies-and-tapbacks.js.map +0 -1
  162. package/dist/normalize/date-converters.d.ts +0 -45
  163. package/dist/normalize/date-converters.d.ts.map +0 -1
  164. package/dist/normalize/date-converters.js +0 -166
  165. package/dist/normalize/date-converters.js.map +0 -1
  166. package/dist/normalize/path-validator.d.ts +0 -65
  167. package/dist/normalize/path-validator.d.ts.map +0 -1
  168. package/dist/normalize/path-validator.js +0 -221
  169. package/dist/normalize/path-validator.js.map +0 -1
  170. package/dist/normalize/validate-normalized.d.ts +0 -45
  171. package/dist/normalize/validate-normalized.d.ts.map +0 -1
  172. package/dist/normalize/validate-normalized.js +0 -144
  173. package/dist/normalize/validate-normalized.js.map +0 -1
  174. package/dist/render/embeds-blockquotes.d.ts +0 -84
  175. package/dist/render/embeds-blockquotes.d.ts.map +0 -1
  176. package/dist/render/embeds-blockquotes.js +0 -204
  177. package/dist/render/embeds-blockquotes.js.map +0 -1
  178. package/dist/render/grouping.d.ts +0 -78
  179. package/dist/render/grouping.d.ts.map +0 -1
  180. package/dist/render/grouping.js +0 -134
  181. package/dist/render/grouping.js.map +0 -1
  182. package/dist/render/index.d.ts +0 -47
  183. package/dist/render/index.d.ts.map +0 -1
  184. package/dist/render/index.js +0 -245
  185. package/dist/render/index.js.map +0 -1
  186. package/dist/render/reply-rendering.d.ts +0 -88
  187. package/dist/render/reply-rendering.d.ts.map +0 -1
  188. package/dist/render/reply-rendering.js +0 -196
  189. package/dist/render/reply-rendering.js.map +0 -1
  190. package/dist/schema/message.d.ts +0 -125
  191. package/dist/schema/message.d.ts.map +0 -1
  192. package/dist/schema/message.js +0 -331
  193. package/dist/schema/message.js.map +0 -1
  194. package/dist/utils/delta-detection.d.ts +0 -107
  195. package/dist/utils/delta-detection.d.ts.map +0 -1
  196. package/dist/utils/delta-detection.js +0 -199
  197. package/dist/utils/delta-detection.js.map +0 -1
  198. package/dist/utils/enrichment-merge.d.ts +0 -135
  199. package/dist/utils/enrichment-merge.d.ts.map +0 -1
  200. package/dist/utils/enrichment-merge.js +0 -280
  201. package/dist/utils/enrichment-merge.js.map +0 -1
  202. package/dist/utils/human.d.ts +0 -15
  203. package/dist/utils/human.d.ts.map +0 -1
  204. package/dist/utils/human.js +0 -27
  205. package/dist/utils/human.js.map +0 -1
  206. package/dist/utils/incremental-state.d.ts +0 -133
  207. package/dist/utils/incremental-state.d.ts.map +0 -1
  208. package/dist/utils/incremental-state.js +0 -237
  209. package/dist/utils/incremental-state.js.map +0 -1
  210. package/dist/utils/logger.d.ts +0 -40
  211. package/dist/utils/logger.d.ts.map +0 -1
  212. package/dist/utils/logger.js +0 -176
  213. package/dist/utils/logger.js.map +0 -1
@@ -1,27 +0,0 @@
1
- /**
2
- * Human Output Helper
3
- *
4
- * Centralizes human-facing console output so we can globally toggle it
5
- * (e.g. suppressed when --json or LOG_FORMAT=json-only is active).
6
- * Tests that spy on console.* remain compatible.
7
- */
8
- let humanEnabled = true;
9
- export function setHumanLoggingEnabled(enabled) {
10
- humanEnabled = enabled;
11
- }
12
- function safeConsole(kind, ...args) {
13
- if (!humanEnabled)
14
- return;
15
- const c = globalThis.console;
16
- c?.[kind]?.(...args);
17
- }
18
- export function humanInfo(...args) {
19
- safeConsole('info', ...args);
20
- }
21
- export function humanWarn(...args) {
22
- safeConsole('warn', ...args);
23
- }
24
- export function humanError(...args) {
25
- safeConsole('error', ...args);
26
- }
27
- //# sourceMappingURL=human.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"human.js","sourceRoot":"","sources":["../../src/utils/human.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAMH,IAAI,YAAY,GAAG,IAAI,CAAA;AAEvB,MAAM,UAAU,sBAAsB,CAAC,OAAgB;IACtD,YAAY,GAAG,OAAO,CAAA;AACvB,CAAC;AAED,SAAS,WAAW,CACnB,IAAO,EACP,GAAG,IAAoB;IAEvB,IAAI,CAAC,YAAY;QAAE,OAAM;IACzB,MAAM,CAAC,GACN,UAGA,CAAC,OAAO,CAAA;IACT,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,GAAI,IAAW,CAAC,CAAA;AAC7B,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,GAAG,IAAoB;IAChD,WAAW,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,CAAA;AAC7B,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,GAAG,IAAoB;IAChD,WAAW,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,CAAA;AAC7B,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,GAAG,IAAoB;IACjD,WAAW,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,CAAA;AAC9B,CAAC"}
@@ -1,133 +0,0 @@
1
- /**
2
- * Incremental State Tracking Module (INCREMENTAL--T01)
3
- *
4
- * Implements resumable enrichment by tracking:
5
- * - AC01: State file with last run metadata
6
- * - AC02: Last enriched date, total messages, config hash
7
- * - AC03: GUID delta detection for new messages
8
- * - AC04: Atomic writes with temp + rename pattern
9
- * - AC05: --incremental flag integration
10
- *
11
- * Architecture:
12
- * - IncrementalState: Complete state schema with version, metadata, GUID tracking
13
- * - createIncrementalState: Factory for new state
14
- * - loadIncrementalState: Safe loading from disk with corruption handling
15
- * - saveIncrementalState: Atomic writes with temp file + rename
16
- * - detectNewMessages: O(n) GUID comparison using Set intersection
17
- * - updateStateWithEnrichedGuids: Add new enrichments and update metadata
18
- * - verifyConfigHash: Detect config changes between runs
19
- */
20
- export type EnrichmentStats = {
21
- processedCount: number;
22
- failedCount: number;
23
- startTime: string;
24
- endTime: string;
25
- };
26
- export type PipelineConfig = {
27
- configHash: string;
28
- };
29
- /**
30
- * Complete state for incremental enrichment tracking
31
- * Stored in .imessage-state.json at output directory root
32
- */
33
- export type IncrementalState = {
34
- /** Schema version for backward compatibility */
35
- version: string;
36
- /** ISO 8601 UTC timestamp of last enrichment run */
37
- lastEnrichedAt: string;
38
- /** Total messages as of last run (for progress reporting) */
39
- totalMessages: number;
40
- /** Array of enriched message GUIDs (for delta detection) */
41
- enrichedGuids: string[];
42
- /** Pipeline configuration hash (detects config changes) */
43
- pipelineConfig: PipelineConfig;
44
- /** Optional: Stats from last enrichment run */
45
- enrichmentStats: EnrichmentStats | null;
46
- };
47
- export type CreateStateOptions = {
48
- totalMessages?: number;
49
- enrichedGuids?: string[];
50
- enrichmentStats?: EnrichmentStats | null;
51
- };
52
- /**
53
- * AC01 + AC02: Create new incremental state with current metadata
54
- *
55
- * @param options - Optional initial values
56
- * @returns New IncrementalState with current timestamp and config hash
57
- */
58
- export declare function createIncrementalState(options?: CreateStateOptions): IncrementalState;
59
- /**
60
- * AC03: Detect new messages by comparing GUIDs with state
61
- *
62
- * Performance: O(n) where n = number of current messages
63
- * Using Set for fast O(1) lookup of previously enriched GUIDs
64
- *
65
- * @param currentGuids - Set of message GUIDs from normalized output
66
- * @param state - Previous state with enriched GUIDs
67
- * @returns Array of new GUID strings not in enriched set
68
- */
69
- export declare function detectNewMessages(currentGuids: Set<string>, state: IncrementalState): string[];
70
- /**
71
- * AC04: Save state atomically to disk
72
- *
73
- * Pattern:
74
- * 1. Write to temp file with .tmp suffix
75
- * 2. Atomic rename (temp → final)
76
- * 3. Prevents corruption from power loss or crashes
77
- *
78
- * @param state - IncrementalState to persist
79
- * @param filePath - Target .imessage-state.json path
80
- * @throws Error if write fails (permission, disk full, etc.)
81
- */
82
- export declare function saveIncrementalState(state: IncrementalState, filePath: string): Promise<void>;
83
- /**
84
- * AC05: Load state from disk safely
85
- *
86
- * - Returns null if file doesn't exist (treat as first run)
87
- * - Returns null if JSON is corrupted (ignore stale state)
88
- * - Validates schema version for future compatibility
89
- *
90
- * @param filePath - Path to .imessage-state.json
91
- * @returns IncrementalState if valid, null if missing or corrupted
92
- */
93
- export declare function loadIncrementalState(filePath: string): Promise<IncrementalState | null>;
94
- /**
95
- * AC02: Verify config hash to detect changes
96
- *
97
- * @param currentHash - Hash from current state
98
- * @param expectedHash - Hash to verify against (default: newly generated)
99
- * @returns true if hashes match
100
- */
101
- export declare function verifyConfigHash(currentHash: string, expectedHash?: string): boolean;
102
- /**
103
- * AC02: Check if state is stale (old enrichment run)
104
- *
105
- * Useful for warning about old state that may be out of sync
106
- *
107
- * @param state - IncrementalState to check
108
- * @param daysThreshold - Days before state is considered stale (default: 7)
109
- * @returns true if state is older than threshold
110
- */
111
- export declare function isStateOutdated(state: IncrementalState, daysThreshold?: number): boolean;
112
- /**
113
- * Update state with newly enriched GUIDs
114
- *
115
- * Called after successful enrichment to:
116
- * - Add new enriched GUIDs (avoid duplicates)
117
- * - Update lastEnrichedAt timestamp
118
- * - Record enrichment statistics
119
- *
120
- * @param state - State to update (mutated in place)
121
- * @param newGuids - GUIDs that were just enriched
122
- * @param enrichmentStats - Optional enrichment stats
123
- */
124
- export declare function updateStateWithEnrichedGuids(state: IncrementalState, newGuids: string[], enrichmentStats?: EnrichmentStats): void;
125
- /**
126
- * Create fresh state, discarding incremental tracking
127
- * Used with --force-refresh flag to re-enrich everything
128
- *
129
- * @param totalMessages - Total messages to initialize with
130
- * @returns Fresh IncrementalState with no enriched GUIDs
131
- */
132
- export declare function resetIncrementalState(totalMessages?: number): IncrementalState;
133
- //# sourceMappingURL=incremental-state.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"incremental-state.d.ts","sourceRoot":"","sources":["../../src/utils/incremental-state.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAaH,MAAM,MAAM,eAAe,GAAG;IAC7B,cAAc,EAAE,MAAM,CAAA;IACtB,WAAW,EAAE,MAAM,CAAA;IACnB,SAAS,EAAE,MAAM,CAAA;IACjB,OAAO,EAAE,MAAM,CAAA;CACf,CAAA;AAED,MAAM,MAAM,cAAc,GAAG;IAC5B,UAAU,EAAE,MAAM,CAAA;CAElB,CAAA;AAED;;;GAGG;AACH,MAAM,MAAM,gBAAgB,GAAG;IAC9B,gDAAgD;IAChD,OAAO,EAAE,MAAM,CAAA;IAEf,oDAAoD;IACpD,cAAc,EAAE,MAAM,CAAA;IAEtB,6DAA6D;IAC7D,aAAa,EAAE,MAAM,CAAA;IAErB,4DAA4D;IAC5D,aAAa,EAAE,MAAM,EAAE,CAAA;IAEvB,2DAA2D;IAC3D,cAAc,EAAE,cAAc,CAAA;IAE9B,+CAA+C;IAC/C,eAAe,EAAE,eAAe,GAAG,IAAI,CAAA;CACvC,CAAA;AAMD,MAAM,MAAM,kBAAkB,GAAG;IAChC,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,aAAa,CAAC,EAAE,MAAM,EAAE,CAAA;IACxB,eAAe,CAAC,EAAE,eAAe,GAAG,IAAI,CAAA;CACxC,CAAA;AAED;;;;;GAKG;AACH,wBAAgB,sBAAsB,CACrC,OAAO,GAAE,kBAAuB,GAC9B,gBAAgB,CAWlB;AA0BD;;;;;;;;;GASG;AACH,wBAAgB,iBAAiB,CAChC,YAAY,EAAE,GAAG,CAAC,MAAM,CAAC,EACzB,KAAK,EAAE,gBAAgB,GACrB,MAAM,EAAE,CAWV;AAMD;;;;;;;;;;;GAWG;AACH,wBAAsB,oBAAoB,CACzC,KAAK,EAAE,gBAAgB,EACvB,QAAQ,EAAE,MAAM,GACd,OAAO,CAAC,IAAI,CAAC,CAuBf;AAED;;;;;;;;;GASG;AACH,wBAAsB,oBAAoB,CACzC,QAAQ,EAAE,MAAM,GACd,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAuBlC;AAMD;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAC/B,WAAW,EAAE,MAAM,EACnB,YAAY,GAAE,MAA6B,GACzC,OAAO,CAET;AAED;;;;;;;;GAQG;AACH,wBAAgB,eAAe,CAC9B,KAAK,EAAE,gBAAgB,EACvB,aAAa,SAAI,GACf,OAAO,CAoBT;AAMD;;;;;;;;;;;GAWG;AACH,wBAAgB,4BAA4B,CAC3C,KAAK,EAAE,gBAAgB,EACvB,QAAQ,EAAE,MAAM,EAAE,EAClB,eAAe,CAAC,EAAE,eAAe,GAC/B,IAAI,CAgBN;AAMD;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CAAC,aAAa,SAAI,GAAG,gBAAgB,CAEzE"}
@@ -1,237 +0,0 @@
1
- /**
2
- * Incremental State Tracking Module (INCREMENTAL--T01)
3
- *
4
- * Implements resumable enrichment by tracking:
5
- * - AC01: State file with last run metadata
6
- * - AC02: Last enriched date, total messages, config hash
7
- * - AC03: GUID delta detection for new messages
8
- * - AC04: Atomic writes with temp + rename pattern
9
- * - AC05: --incremental flag integration
10
- *
11
- * Architecture:
12
- * - IncrementalState: Complete state schema with version, metadata, GUID tracking
13
- * - createIncrementalState: Factory for new state
14
- * - loadIncrementalState: Safe loading from disk with corruption handling
15
- * - saveIncrementalState: Atomic writes with temp file + rename
16
- * - detectNewMessages: O(n) GUID comparison using Set intersection
17
- * - updateStateWithEnrichedGuids: Add new enrichments and update metadata
18
- * - verifyConfigHash: Detect config changes between runs
19
- */
20
- import crypto from 'node:crypto';
21
- import { promises as fs } from 'node:fs';
22
- import path from 'node:path';
23
- import { humanWarn } from '#utils/human';
24
- import { createLogger } from '#utils/logger';
25
- /**
26
- * AC01 + AC02: Create new incremental state with current metadata
27
- *
28
- * @param options - Optional initial values
29
- * @returns New IncrementalState with current timestamp and config hash
30
- */
31
- export function createIncrementalState(options = {}) {
32
- return {
33
- version: '1.0',
34
- lastEnrichedAt: new Date().toISOString(),
35
- totalMessages: options.totalMessages ?? 0,
36
- enrichedGuids: options.enrichedGuids ?? [],
37
- pipelineConfig: {
38
- configHash: generateConfigHash(),
39
- },
40
- enrichmentStats: options.enrichmentStats ?? null,
41
- };
42
- }
43
- /**
44
- * AC02: Generate hash of pipeline configuration
45
- * Used to detect when settings change (API keys, flags, etc.)
46
- *
47
- * @returns SHA-256 hex digest of current config
48
- */
49
- function generateConfigHash() {
50
- // Hash config including API key presence (not the actual key)
51
- const hasGeminiKey = process.env.GEMINI_API_KEY !== undefined;
52
- const hasFirecrawlKey = process.env.FIRECRAWL_API_KEY !== undefined;
53
- const config = JSON.stringify({
54
- version: '1.0',
55
- // Add API key presence (not the actual key)
56
- hasGeminiKey,
57
- hasFirecrawlKey,
58
- });
59
- return crypto.createHash('sha256').update(config).digest('hex');
60
- }
61
- // ============================================================================
62
- // AC03: GUID Delta Detection
63
- // ============================================================================
64
- /**
65
- * AC03: Detect new messages by comparing GUIDs with state
66
- *
67
- * Performance: O(n) where n = number of current messages
68
- * Using Set for fast O(1) lookup of previously enriched GUIDs
69
- *
70
- * @param currentGuids - Set of message GUIDs from normalized output
71
- * @param state - Previous state with enriched GUIDs
72
- * @returns Array of new GUID strings not in enriched set
73
- */
74
- export function detectNewMessages(currentGuids, state) {
75
- const enrichedSet = new Set(state.enrichedGuids);
76
- const newGuids = [];
77
- for (const guid of currentGuids) {
78
- if (!enrichedSet.has(guid)) {
79
- newGuids.push(guid);
80
- }
81
- }
82
- return newGuids;
83
- }
84
- // ============================================================================
85
- // AC04: Atomic Writes (Temp + Rename Pattern)
86
- // ============================================================================
87
- /**
88
- * AC04: Save state atomically to disk
89
- *
90
- * Pattern:
91
- * 1. Write to temp file with .tmp suffix
92
- * 2. Atomic rename (temp → final)
93
- * 3. Prevents corruption from power loss or crashes
94
- *
95
- * @param state - IncrementalState to persist
96
- * @param filePath - Target .imessage-state.json path
97
- * @throws Error if write fails (permission, disk full, etc.)
98
- */
99
- export async function saveIncrementalState(state, filePath) {
100
- const dir = path.dirname(filePath);
101
- const tempFile = `${filePath}.${Date.now()}.tmp`;
102
- try {
103
- // Ensure parent directory exists
104
- await fs.mkdir(dir, { recursive: true });
105
- // Write to temp file
106
- const content = JSON.stringify(state, null, 2);
107
- await fs.writeFile(tempFile, content, 'utf-8');
108
- // Atomic rename
109
- await fs.rename(tempFile, filePath);
110
- }
111
- catch (error) {
112
- // Clean up temp file if rename failed
113
- try {
114
- await fs.unlink(tempFile);
115
- }
116
- catch {
117
- // ignore cleanup errors
118
- }
119
- throw error;
120
- }
121
- }
122
- /**
123
- * AC05: Load state from disk safely
124
- *
125
- * - Returns null if file doesn't exist (treat as first run)
126
- * - Returns null if JSON is corrupted (ignore stale state)
127
- * - Validates schema version for future compatibility
128
- *
129
- * @param filePath - Path to .imessage-state.json
130
- * @returns IncrementalState if valid, null if missing or corrupted
131
- */
132
- export async function loadIncrementalState(filePath) {
133
- try {
134
- const content = await fs.readFile(filePath, 'utf-8');
135
- const parsed = JSON.parse(content);
136
- // Validate schema version
137
- if (parsed.version !== '1.0') {
138
- const logger = createLogger('utils:incremental-state');
139
- logger.warn('Unknown state version. Ignoring.', {
140
- version: parsed.version,
141
- });
142
- return null;
143
- }
144
- return parsed;
145
- }
146
- catch (error) {
147
- if (error instanceof Error && error.message.includes('ENOENT')) {
148
- // File doesn't exist - first run
149
- return null;
150
- }
151
- // Corrupted JSON or other read error - ignore
152
- return null;
153
- }
154
- }
155
- // ============================================================================
156
- // Config Hash Verification
157
- // ============================================================================
158
- /**
159
- * AC02: Verify config hash to detect changes
160
- *
161
- * @param currentHash - Hash from current state
162
- * @param expectedHash - Hash to verify against (default: newly generated)
163
- * @returns true if hashes match
164
- */
165
- export function verifyConfigHash(currentHash, expectedHash = generateConfigHash()) {
166
- return currentHash === expectedHash;
167
- }
168
- /**
169
- * AC02: Check if state is stale (old enrichment run)
170
- *
171
- * Useful for warning about old state that may be out of sync
172
- *
173
- * @param state - IncrementalState to check
174
- * @param daysThreshold - Days before state is considered stale (default: 7)
175
- * @returns true if state is older than threshold
176
- */
177
- export function isStateOutdated(state, daysThreshold = 7) {
178
- const lastEnrichedTime = new Date(state.lastEnrichedAt).getTime();
179
- const now = Date.now();
180
- const ageMs = now - lastEnrichedTime;
181
- const ageDays = ageMs / (1000 * 60 * 60 * 24);
182
- if (ageDays > daysThreshold) {
183
- const logger = createLogger('utils:incremental-state');
184
- logger.warn('State file is old. Consider full re-enrichment.', {
185
- ageDays: Math.floor(ageDays),
186
- daysThreshold,
187
- });
188
- // Human-visible warning for tests & interactive CLI (console.warn spied in tests)
189
- humanWarn(`State file is old (ageDays=${Math.floor(ageDays)}, threshold=${daysThreshold}). Consider full re-enrichment.`);
190
- return true;
191
- }
192
- return false;
193
- }
194
- // ============================================================================
195
- // Update State with Enrichment Results
196
- // ============================================================================
197
- /**
198
- * Update state with newly enriched GUIDs
199
- *
200
- * Called after successful enrichment to:
201
- * - Add new enriched GUIDs (avoid duplicates)
202
- * - Update lastEnrichedAt timestamp
203
- * - Record enrichment statistics
204
- *
205
- * @param state - State to update (mutated in place)
206
- * @param newGuids - GUIDs that were just enriched
207
- * @param enrichmentStats - Optional enrichment stats
208
- */
209
- export function updateStateWithEnrichedGuids(state, newGuids, enrichmentStats) {
210
- // Add new GUIDs, avoiding duplicates
211
- const existingSet = new Set(state.enrichedGuids);
212
- for (const guid of newGuids) {
213
- if (!existingSet.has(guid)) {
214
- state.enrichedGuids.push(guid);
215
- }
216
- }
217
- // Update timestamp
218
- state.lastEnrichedAt = new Date().toISOString();
219
- // Update stats if provided
220
- if (enrichmentStats) {
221
- state.enrichmentStats = enrichmentStats;
222
- }
223
- }
224
- // ============================================================================
225
- // Helper: Reset state (for testing or --force-refresh)
226
- // ============================================================================
227
- /**
228
- * Create fresh state, discarding incremental tracking
229
- * Used with --force-refresh flag to re-enrich everything
230
- *
231
- * @param totalMessages - Total messages to initialize with
232
- * @returns Fresh IncrementalState with no enriched GUIDs
233
- */
234
- export function resetIncrementalState(totalMessages = 0) {
235
- return createIncrementalState({ totalMessages });
236
- }
237
- //# sourceMappingURL=incremental-state.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"incremental-state.js","sourceRoot":"","sources":["../../src/utils/incremental-state.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,MAAM,MAAM,aAAa,CAAA;AAChC,OAAO,EAAE,QAAQ,IAAI,EAAE,EAAE,MAAM,SAAS,CAAA;AACxC,OAAO,IAAI,MAAM,WAAW,CAAA;AAE5B,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAA;AACxC,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAoD5C;;;;;GAKG;AACH,MAAM,UAAU,sBAAsB,CACrC,UAA8B,EAAE;IAEhC,OAAO;QACN,OAAO,EAAE,KAAK;QACd,cAAc,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACxC,aAAa,EAAE,OAAO,CAAC,aAAa,IAAI,CAAC;QACzC,aAAa,EAAE,OAAO,CAAC,aAAa,IAAI,EAAE;QAC1C,cAAc,EAAE;YACf,UAAU,EAAE,kBAAkB,EAAE;SAChC;QACD,eAAe,EAAE,OAAO,CAAC,eAAe,IAAI,IAAI;KAChD,CAAA;AACF,CAAC;AAED;;;;;GAKG;AACH,SAAS,kBAAkB;IAC1B,8DAA8D;IAC9D,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,KAAK,SAAS,CAAA;IAC7D,MAAM,eAAe,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,KAAK,SAAS,CAAA;IAEnE,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC;QAC7B,OAAO,EAAE,KAAK;QACd,4CAA4C;QAC5C,YAAY;QACZ,eAAe;KACf,CAAC,CAAA;IACF,OAAO,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;AAChE,CAAC;AAED,+EAA+E;AAC/E,6BAA6B;AAC7B,+EAA+E;AAE/E;;;;;;;;;GASG;AACH,MAAM,UAAU,iBAAiB,CAChC,YAAyB,EACzB,KAAuB;IAEvB,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,CAAA;IAChD,MAAM,QAAQ,GAAa,EAAE,CAAA;IAE7B,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;QACjC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5B,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpB,CAAC;IACF,CAAC;IAED,OAAO,QAAQ,CAAA;AAChB,CAAC;AAED,+EAA+E;AAC/E,8CAA8C;AAC9C,+EAA+E;AAE/E;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACzC,KAAuB,EACvB,QAAgB;IAEhB,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;IAClC,MAAM,QAAQ,GAAG,GAAG,QAAQ,IAAI,IAAI,CAAC,GAAG,EAAE,MAAM,CAAA;IAEhD,IAAI,CAAC;QACJ,iCAAiC;QACjC,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;QAExC,qBAAqB;QACrB,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAA;QAC9C,MAAM,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,CAAA;QAE9C,gBAAgB;QAChB,MAAM,EAAE,CAAC,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAA;IACpC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,sCAAsC;QACtC,IAAI,CAAC;YACJ,MAAM,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;QAC1B,CAAC;QAAC,MAAM,CAAC;YACR,wBAAwB;QACzB,CAAC;QACD,MAAM,KAAK,CAAA;IACZ,CAAC;AACF,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACzC,QAAgB;IAEhB,IAAI,CAAC;QACJ,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAA;QACpD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAqB,CAAA;QAEtD,0BAA0B;QAC1B,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,EAAE,CAAC;YAC9B,MAAM,MAAM,GAAG,YAAY,CAAC,yBAAyB,CAAC,CAAA;YACtD,MAAM,CAAC,IAAI,CAAC,kCAAkC,EAAE;gBAC/C,OAAO,EAAE,MAAM,CAAC,OAAO;aACvB,CAAC,CAAA;YACF,OAAO,IAAI,CAAA;QACZ,CAAC;QAED,OAAO,MAAM,CAAA;IACd,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;YAChE,iCAAiC;YACjC,OAAO,IAAI,CAAA;QACZ,CAAC;QACD,8CAA8C;QAC9C,OAAO,IAAI,CAAA;IACZ,CAAC;AACF,CAAC;AAED,+EAA+E;AAC/E,2BAA2B;AAC3B,+EAA+E;AAE/E;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAC/B,WAAmB,EACnB,eAAuB,kBAAkB,EAAE;IAE3C,OAAO,WAAW,KAAK,YAAY,CAAA;AACpC,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,eAAe,CAC9B,KAAuB,EACvB,aAAa,GAAG,CAAC;IAEjB,MAAM,gBAAgB,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,OAAO,EAAE,CAAA;IACjE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;IACtB,MAAM,KAAK,GAAG,GAAG,GAAG,gBAAgB,CAAA;IACpC,MAAM,OAAO,GAAG,KAAK,GAAG,CAAC,IAAI,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAA;IAE7C,IAAI,OAAO,GAAG,aAAa,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAG,YAAY,CAAC,yBAAyB,CAAC,CAAA;QACtD,MAAM,CAAC,IAAI,CAAC,iDAAiD,EAAE;YAC9D,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC;YAC5B,aAAa;SACb,CAAC,CAAA;QACF,kFAAkF;QAClF,SAAS,CACR,8BAA8B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,eAAe,aAAa,iCAAiC,CAC9G,CAAA;QACD,OAAO,IAAI,CAAA;IACZ,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED,+EAA+E;AAC/E,uCAAuC;AACvC,+EAA+E;AAE/E;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,4BAA4B,CAC3C,KAAuB,EACvB,QAAkB,EAClB,eAAiC;IAEjC,qCAAqC;IACrC,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,CAAA;IAChD,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAC7B,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5B,KAAK,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC/B,CAAC;IACF,CAAC;IAED,mBAAmB;IACnB,KAAK,CAAC,cAAc,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;IAE/C,2BAA2B;IAC3B,IAAI,eAAe,EAAE,CAAC;QACrB,KAAK,CAAC,eAAe,GAAG,eAAe,CAAA;IACxC,CAAC;AACF,CAAC;AAED,+EAA+E;AAC/E,uDAAuD;AACvD,+EAA+E;AAE/E;;;;;;GAMG;AACH,MAAM,UAAU,qBAAqB,CAAC,aAAa,GAAG,CAAC;IACtD,OAAO,sBAAsB,CAAC,EAAE,aAAa,EAAE,CAAC,CAAA;AACjD,CAAC"}
@@ -1,40 +0,0 @@
1
- /**
2
- * Structured JSON Logger (Pino + JSONL)
3
- *
4
- * - Uses Pino for fast JSON logging to stdout
5
- * - Writes JSON Lines to ./logs/YYYY-MM-DD.jsonl when enabled
6
- * - Stable keys for Pinot ingestion (ts, level, component, msg, context, pid, ver, seq)
7
- *
8
- * Environment variables:
9
- * LOG_LEVEL=debug|info|warn|error minimum level (default info)
10
- * LOG_FORMAT=json|pretty pretty prints to stdout when set to pretty
11
- * LOG_TO_FILE=true|false write JSONL to ./logs (default true except during tests)
12
- */
13
- export type LogLevel = 'debug' | 'info' | 'warn' | 'error';
14
- export type LogEntry = {
15
- ts: string;
16
- level: LogLevel;
17
- component: string;
18
- msg: string;
19
- context?: Record<string, unknown>;
20
- pid: number;
21
- ver?: string;
22
- seq: number;
23
- correlationId?: string;
24
- };
25
- export declare function log(component: string, level: LogLevel, msg: string, context?: Record<string, unknown>): void;
26
- export type ComponentLogger = {
27
- debug: (msg: string, context?: Record<string, unknown>) => void;
28
- info: (msg: string, context?: Record<string, unknown>) => void;
29
- warn: (msg: string, context?: Record<string, unknown>) => void;
30
- error: (msg: string, context?: Record<string, unknown>) => void;
31
- };
32
- export declare function createLogger(component: string): ComponentLogger;
33
- export type LogSink = (entry: LogEntry) => void;
34
- export declare function registerSink(sink: LogSink): void;
35
- export declare function clearSinks(): void;
36
- export declare function setCorrelationId(id: string | undefined): void;
37
- export declare function getCorrelationId(): string | undefined;
38
- export declare function withCorrelationId<T>(id: string, fn: () => Promise<T> | T): Promise<T>;
39
- export declare function setLogLevel(level: LogLevel): void;
40
- //# sourceMappingURL=logger.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../../src/utils/logger.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,MAAM,MAAM,QAAQ,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAA;AAE1D,MAAM,MAAM,QAAQ,GAAG;IACtB,EAAE,EAAE,MAAM,CAAA;IACV,KAAK,EAAE,QAAQ,CAAA;IACf,SAAS,EAAE,MAAM,CAAA;IACjB,GAAG,EAAE,MAAM,CAAA;IACX,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IACjC,GAAG,EAAE,MAAM,CAAA;IACX,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,CAAA;IACX,aAAa,CAAC,EAAE,MAAM,CAAA;CACtB,CAAA;AAyED,wBAAgB,GAAG,CAClB,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,QAAQ,EACf,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC/B,IAAI,CA2CN;AAED,MAAM,MAAM,eAAe,GAAG;IAC7B,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAA;IAC/D,IAAI,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAA;IAC9D,IAAI,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAA;IAC9D,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAA;CAC/D,CAAA;AAED,wBAAgB,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,eAAe,CAO/D;AAKD,MAAM,MAAM,OAAO,GAAG,CAAC,KAAK,EAAE,QAAQ,KAAK,IAAI,CAAA;AAE/C,wBAAgB,YAAY,CAAC,IAAI,EAAE,OAAO,GAAG,IAAI,CAEhD;AACD,wBAAgB,UAAU,IAAI,IAAI,CAEjC;AAGD,wBAAgB,gBAAgB,CAAC,EAAE,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,CAE7D;AAED,wBAAgB,gBAAgB,IAAI,MAAM,GAAG,SAAS,CAErD;AAED,wBAAsB,iBAAiB,CAAC,CAAC,EACxC,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,GACtB,OAAO,CAAC,CAAC,CAAC,CAUZ;AAID,wBAAgB,WAAW,CAAC,KAAK,EAAE,QAAQ,GAAG,IAAI,CAQjD"}
@@ -1,176 +0,0 @@
1
- /**
2
- * Structured JSON Logger (Pino + JSONL)
3
- *
4
- * - Uses Pino for fast JSON logging to stdout
5
- * - Writes JSON Lines to ./logs/YYYY-MM-DD.jsonl when enabled
6
- * - Stable keys for Pinot ingestion (ts, level, component, msg, context, pid, ver, seq)
7
- *
8
- * Environment variables:
9
- * LOG_LEVEL=debug|info|warn|error minimum level (default info)
10
- * LOG_FORMAT=json|pretty pretty prints to stdout when set to pretty
11
- * LOG_TO_FILE=true|false write JSONL to ./logs (default true except during tests)
12
- */
13
- const LEVEL_ORDER = {
14
- debug: 10,
15
- info: 20,
16
- warn: 30,
17
- error: 40,
18
- };
19
- import { AsyncLocalStorage } from 'node:async_hooks';
20
- import fs from 'node:fs';
21
- import path from 'node:path';
22
- import pino, {} from 'pino';
23
- import pkgJson from '../../package.json' with { type: 'json' };
24
- let sequenceCounter = 0;
25
- let cachedVersion;
26
- let currentCorrelationId;
27
- const correlationStore = new AsyncLocalStorage();
28
- function loadVersion() {
29
- if (cachedVersion)
30
- return cachedVersion;
31
- const version = pkgJson.version || '0.0.0';
32
- cachedVersion = version;
33
- return version;
34
- }
35
- const isTestEnv = process.env.VITEST === 'true' || process.env.NODE_ENV === 'test';
36
- const shouldWriteFile = (process.env.LOG_TO_FILE ?? (isTestEnv ? 'false' : 'true')) === 'true';
37
- // stdout logger via Pino
38
- const baseLevel = (process.env.LOG_LEVEL || 'info');
39
- const pinoTransport = process.env.LOG_FORMAT === 'pretty'
40
- ? pino.transport({
41
- target: 'pino-pretty',
42
- options: { colorize: true, singleLine: true },
43
- })
44
- : undefined;
45
- const pinoStdout = pino({ level: baseLevel, base: null }, pinoTransport);
46
- // JSONL file stream (lazy)
47
- let fileStream;
48
- let fileDate;
49
- function ensureFileStream() {
50
- if (!shouldWriteFile)
51
- return;
52
- const nowDate = new Date().toISOString().slice(0, 10);
53
- if (fileStream && fileDate === nowDate)
54
- return;
55
- try {
56
- const logsDir = path.resolve(process.cwd(), 'logs');
57
- fs.mkdirSync(logsDir, { recursive: true });
58
- const filePath = path.join(logsDir, `${nowDate}.jsonl`);
59
- if (fileStream)
60
- fileStream.end();
61
- fileStream = fs.createWriteStream(filePath, { flags: 'a' });
62
- fileDate = nowDate;
63
- }
64
- catch {
65
- // swallow file sink errors to avoid impacting app
66
- fileStream = undefined;
67
- fileDate = undefined;
68
- }
69
- }
70
- // will be overridden with dynamic version later in file
71
- // initial implementation kept for reference; dynamicShouldLog is used instead
72
- // kept for backward compatibility in case other modules import it later
73
- function _format(entry) {
74
- return JSON.stringify(entry);
75
- }
76
- export function log(component, level, msg, context) {
77
- if (!dynamicShouldLog(level))
78
- return;
79
- const effectiveCorrelationId = correlationStore.getStore() ?? currentCorrelationId;
80
- const entryBase = {
81
- ts: new Date().toISOString(),
82
- level,
83
- component,
84
- msg,
85
- pid: process.pid,
86
- ver: loadVersion(),
87
- seq: ++sequenceCounter,
88
- };
89
- const withCorr = effectiveCorrelationId ? { correlationId: effectiveCorrelationId } : {};
90
- const entryNoCtx = { ...entryBase, ...withCorr };
91
- const entry = context ? { ...entryNoCtx, context } : entryNoCtx;
92
- // 1) stdout via Pino (keeps human-friendly output and fast JSON)
93
- const bindings = {
94
- component,
95
- ver: entry.ver,
96
- seq: entry.seq,
97
- };
98
- if (entry.correlationId)
99
- bindings.correlationId = entry.correlationId;
100
- const logger = pinoStdout.child(bindings);
101
- // Pino expects object then msg for structured logs
102
- logger[level](context ?? {}, msg);
103
- // 2) JSONL file sink (Pinot-friendly schema)
104
- if (shouldWriteFile) {
105
- ensureFileStream();
106
- if (fileStream)
107
- fileStream.write(`${JSON.stringify(entry)}\n`);
108
- }
109
- // 3) Custom sinks (if any registered)
110
- if (sinks.length > 0) {
111
- try {
112
- for (const s of sinks)
113
- s(entry);
114
- }
115
- catch {
116
- // ignore sink errors
117
- }
118
- }
119
- }
120
- export function createLogger(component) {
121
- return {
122
- debug: (msg, context) => log(component, 'debug', msg, context),
123
- info: (msg, context) => log(component, 'info', msg, context),
124
- warn: (msg, context) => log(component, 'warn', msg, context),
125
- error: (msg, context) => log(component, 'error', msg, context),
126
- };
127
- }
128
- let sinks = [];
129
- export function registerSink(sink) {
130
- sinks.push(sink);
131
- }
132
- export function clearSinks() {
133
- sinks = [];
134
- }
135
- // Correlation ID management
136
- export function setCorrelationId(id) {
137
- currentCorrelationId = id;
138
- }
139
- export function getCorrelationId() {
140
- return correlationStore.getStore() ?? currentCorrelationId;
141
- }
142
- export async function withCorrelationId(id, fn) {
143
- const prev = currentCorrelationId;
144
- currentCorrelationId = id;
145
- return await correlationStore.run(id, async () => {
146
- try {
147
- return await fn();
148
- }
149
- finally {
150
- currentCorrelationId = prev;
151
- }
152
- });
153
- }
154
- // Dynamic log level control
155
- let dynamicLevel;
156
- export function setLogLevel(level) {
157
- dynamicLevel = level;
158
- try {
159
- // biome-ignore lint/suspicious/noExplicitAny: pino internal API
160
- ;
161
- pinoStdout.level = level;
162
- }
163
- catch {
164
- // ignore if pinoStdout not ready
165
- }
166
- }
167
- function getEffectiveLevel() {
168
- return dynamicLevel || 'info';
169
- }
170
- // Override original shouldLog: replace earlier implementation by assigning new function
171
- // Use a const to avoid duplicate declarations
172
- const dynamicShouldLog = (level) => {
173
- const envLevel = getEffectiveLevel();
174
- return LEVEL_ORDER[level] >= LEVEL_ORDER[envLevel];
175
- };
176
- //# sourceMappingURL=logger.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"logger.js","sourceRoot":"","sources":["../../src/utils/logger.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAgBH,MAAM,WAAW,GAA6B;IAC7C,KAAK,EAAE,EAAE;IACT,IAAI,EAAE,EAAE;IACR,IAAI,EAAE,EAAE;IACR,KAAK,EAAE,EAAE;CACT,CAAA;AAED,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAA;AACpD,OAAO,EAAE,MAAM,SAAS,CAAA;AACxB,OAAO,IAAI,MAAM,WAAW,CAAA;AAE5B,OAAO,IAAI,EAAE,EAA6B,MAAM,MAAM,CAAA;AAEtD,OAAO,OAAO,MAAM,oBAAoB,CAAC,OAAO,IAAI,EAAE,MAAM,EAAE,CAAA;AAE9D,IAAI,eAAe,GAAG,CAAC,CAAA;AACvB,IAAI,aAAiC,CAAA;AACrC,IAAI,oBAAwC,CAAA;AAC5C,MAAM,gBAAgB,GAAG,IAAI,iBAAiB,EAAU,CAAA;AACxD,SAAS,WAAW;IACnB,IAAI,aAAa;QAAE,OAAO,aAAc,CAAA;IACxC,MAAM,OAAO,GAAI,OAAgC,CAAC,OAAO,IAAI,OAAO,CAAA;IACpE,aAAa,GAAG,OAAO,CAAA;IACvB,OAAO,OAAO,CAAA;AACf,CAAC;AAED,MAAM,SAAS,GACd,OAAO,CAAC,GAAG,CAAC,MAAM,KAAK,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,MAAM,CAAA;AACjE,MAAM,eAAe,GACpB,CAAC,OAAO,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,MAAM,CAAA;AAEvE,yBAAyB;AACzB,MAAM,SAAS,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,MAAM,CAAa,CAAA;AAC/D,MAAM,aAAa,GAClB,OAAO,CAAC,GAAG,CAAC,UAAU,KAAK,QAAQ;IAClC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC;QACf,MAAM,EAAE,aAAa;QACrB,OAAO,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE;KAC7C,CAAC;IACH,CAAC,CAAC,SAAS,CAAA;AACb,MAAM,UAAU,GAAG,IAAI,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,aAAa,CAAC,CAAA;AAExE,2BAA2B;AAC3B,IAAI,UAAsC,CAAA;AAC1C,IAAI,QAA4B,CAAA;AAChC,SAAS,gBAAgB;IACxB,IAAI,CAAC,eAAe;QAAE,OAAM;IAC5B,MAAM,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;IACrD,IAAI,UAAU,IAAI,QAAQ,KAAK,OAAO;QAAE,OAAM;IAC9C,IAAI,CAAC;QACJ,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,CAAC,CAAA;QACnD,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;QAC1C,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,OAAO,QAAQ,CAAC,CAAA;QACvD,IAAI,UAAU;YAAE,UAAU,CAAC,GAAG,EAAE,CAAA;QAChC,UAAU,GAAG,EAAE,CAAC,iBAAiB,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAA;QAC3D,QAAQ,GAAG,OAAO,CAAA;IACnB,CAAC;IAAC,MAAM,CAAC;QACR,kDAAkD;QAClD,UAAU,GAAG,SAAS,CAAA;QACtB,QAAQ,GAAG,SAAS,CAAA;IACrB,CAAC;AACF,CAAC;AAED,wDAAwD;AACxD,8EAA8E;AAE9E,wEAAwE;AACxE,SAAS,OAAO,CAAC,KAAe;IAC/B,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;AAC7B,CAAC;AAED,MAAM,UAAU,GAAG,CAClB,SAAiB,EACjB,KAAe,EACf,GAAW,EACX,OAAiC;IAEjC,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC;QAAE,OAAM;IACpC,MAAM,sBAAsB,GAC3B,gBAAgB,CAAC,QAAQ,EAAE,IAAI,oBAAoB,CAAA;IACpD,MAAM,SAAS,GAAgD;QAC9D,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QAC5B,KAAK;QACL,SAAS;QACT,GAAG;QACH,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,GAAG,EAAE,WAAW,EAAE;QAClB,GAAG,EAAE,EAAE,eAAe;KACtB,CAAA;IACD,MAAM,QAAQ,GACb,sBAAsB,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,sBAAsB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAA;IACxE,MAAM,UAAU,GAAa,EAAE,GAAI,SAAsB,EAAE,GAAG,QAAQ,EAAE,CAAA;IACxE,MAAM,KAAK,GAAa,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,UAAU,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,UAAU,CAAA;IAEzE,iEAAiE;IACjE,MAAM,QAAQ,GAA4B;QACzC,SAAS;QACT,GAAG,EAAE,KAAK,CAAC,GAAG;QACd,GAAG,EAAE,KAAK,CAAC,GAAG;KACd,CAAA;IACD,IAAI,KAAK,CAAC,aAAa;QAAE,QAAQ,CAAC,aAAa,GAAG,KAAK,CAAC,aAAa,CAAA;IACrE,MAAM,MAAM,GAAe,UAAU,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;IACrD,mDAAmD;IACnD,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,IAAI,EAAE,EAAE,GAAG,CAAC,CAAA;IAEjC,6CAA6C;IAC7C,IAAI,eAAe,EAAE,CAAC;QACrB,gBAAgB,EAAE,CAAA;QAClB,IAAI,UAAU;YAAE,UAAU,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAC/D,CAAC;IAED,sCAAsC;IACtC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,IAAI,CAAC;YACJ,KAAK,MAAM,CAAC,IAAI,KAAK;gBAAE,CAAC,CAAC,KAAK,CAAC,CAAA;QAChC,CAAC;QAAC,MAAM,CAAC;YACR,qBAAqB;QACtB,CAAC;IACF,CAAC;AACF,CAAC;AASD,MAAM,UAAU,YAAY,CAAC,SAAiB;IAC7C,OAAO;QACN,KAAK,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,EAAE,GAAG,EAAE,OAAO,CAAC;QAC9D,IAAI,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,CAAC;QAC5D,IAAI,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,CAAC;QAC5D,KAAK,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,EAAE,GAAG,EAAE,OAAO,CAAC;KAC9D,CAAA;AACF,CAAC;AAMD,IAAI,KAAK,GAAc,EAAE,CAAA;AACzB,MAAM,UAAU,YAAY,CAAC,IAAa;IACzC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACjB,CAAC;AACD,MAAM,UAAU,UAAU;IACzB,KAAK,GAAG,EAAE,CAAA;AACX,CAAC;AAED,4BAA4B;AAC5B,MAAM,UAAU,gBAAgB,CAAC,EAAsB;IACtD,oBAAoB,GAAG,EAAE,CAAA;AAC1B,CAAC;AAED,MAAM,UAAU,gBAAgB;IAC/B,OAAO,gBAAgB,CAAC,QAAQ,EAAE,IAAI,oBAAoB,CAAA;AAC3D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACtC,EAAU,EACV,EAAwB;IAExB,MAAM,IAAI,GAAG,oBAAoB,CAAA;IACjC,oBAAoB,GAAG,EAAE,CAAA;IACzB,OAAO,MAAM,gBAAgB,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,IAAI,EAAE;QAChD,IAAI,CAAC;YACJ,OAAO,MAAM,EAAE,EAAE,CAAA;QAClB,CAAC;gBAAS,CAAC;YACV,oBAAoB,GAAG,IAAI,CAAA;QAC5B,CAAC;IACF,CAAC,CAAC,CAAA;AACH,CAAC;AAED,4BAA4B;AAC5B,IAAI,YAAkC,CAAA;AACtC,MAAM,UAAU,WAAW,CAAC,KAAe;IAC1C,YAAY,GAAG,KAAK,CAAA;IACpB,IAAI,CAAC;QACJ,gEAAgE;QAChE,CAAC;QAAC,UAAkB,CAAC,KAAK,GAAG,KAAK,CAAA;IACnC,CAAC;IAAC,MAAM,CAAC;QACR,iCAAiC;IAClC,CAAC;AACF,CAAC;AAED,SAAS,iBAAiB;IACzB,OAAO,YAAY,IAAI,MAAM,CAAA;AAC9B,CAAC;AAED,wFAAwF;AACxF,8CAA8C;AAC9C,MAAM,gBAAgB,GAAG,CAAC,KAAe,EAAW,EAAE;IACrD,MAAM,QAAQ,GAAG,iBAAiB,EAAE,CAAA;IACpC,OAAO,WAAW,CAAC,KAAK,CAAC,IAAI,WAAW,CAAC,QAAQ,CAAC,CAAA;AACnD,CAAC,CAAA"}