@nathanvale/chatline 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (216) hide show
  1. package/CHANGELOG.md +1 -0
  2. package/LICENSE +21 -0
  3. package/README.md +1535 -0
  4. package/dist/bin/index.js +5121 -0
  5. package/dist/cli/commands/clean.d.ts +17 -0
  6. package/dist/cli/commands/clean.d.ts.map +1 -0
  7. package/dist/cli/commands/clean.js +142 -0
  8. package/dist/cli/commands/clean.js.map +1 -0
  9. package/dist/cli/commands/doctor.d.ts +17 -0
  10. package/dist/cli/commands/doctor.d.ts.map +1 -0
  11. package/dist/cli/commands/doctor.js +202 -0
  12. package/dist/cli/commands/doctor.js.map +1 -0
  13. package/dist/cli/commands/enrich-ai.d.ts +17 -0
  14. package/dist/cli/commands/enrich-ai.d.ts.map +1 -0
  15. package/dist/cli/commands/enrich-ai.js +371 -0
  16. package/dist/cli/commands/enrich-ai.js.map +1 -0
  17. package/dist/cli/commands/index.d.ts +16 -0
  18. package/dist/cli/commands/index.d.ts.map +1 -0
  19. package/dist/cli/commands/index.js +16 -0
  20. package/dist/cli/commands/index.js.map +1 -0
  21. package/dist/cli/commands/ingest-csv.d.ts +17 -0
  22. package/dist/cli/commands/ingest-csv.d.ts.map +1 -0
  23. package/dist/cli/commands/ingest-csv.js +138 -0
  24. package/dist/cli/commands/ingest-csv.js.map +1 -0
  25. package/dist/cli/commands/ingest-db.d.ts +17 -0
  26. package/dist/cli/commands/ingest-db.d.ts.map +1 -0
  27. package/dist/cli/commands/ingest-db.js +159 -0
  28. package/dist/cli/commands/ingest-db.js.map +1 -0
  29. package/dist/cli/commands/init.d.ts +17 -0
  30. package/dist/cli/commands/init.d.ts.map +1 -0
  31. package/dist/cli/commands/init.js +110 -0
  32. package/dist/cli/commands/init.js.map +1 -0
  33. package/dist/cli/commands/normalize-link.d.ts +16 -0
  34. package/dist/cli/commands/normalize-link.d.ts.map +1 -0
  35. package/dist/cli/commands/normalize-link.js +144 -0
  36. package/dist/cli/commands/normalize-link.js.map +1 -0
  37. package/dist/cli/commands/render-markdown.d.ts +17 -0
  38. package/dist/cli/commands/render-markdown.d.ts.map +1 -0
  39. package/dist/cli/commands/render-markdown.js +218 -0
  40. package/dist/cli/commands/render-markdown.js.map +1 -0
  41. package/dist/cli/commands/stats.d.ts +17 -0
  42. package/dist/cli/commands/stats.d.ts.map +1 -0
  43. package/dist/cli/commands/stats.js +175 -0
  44. package/dist/cli/commands/stats.js.map +1 -0
  45. package/dist/cli/commands/validate.d.ts +17 -0
  46. package/dist/cli/commands/validate.d.ts.map +1 -0
  47. package/dist/cli/commands/validate.js +152 -0
  48. package/dist/cli/commands/validate.js.map +1 -0
  49. package/dist/cli/index.d.ts +13 -0
  50. package/dist/cli/index.d.ts.map +1 -0
  51. package/dist/cli/index.js +121 -0
  52. package/dist/cli/index.js.map +1 -0
  53. package/dist/cli/types.d.ts +93 -0
  54. package/dist/cli/types.d.ts.map +1 -0
  55. package/dist/cli/types.js +7 -0
  56. package/dist/cli/types.js.map +1 -0
  57. package/dist/cli/utils.d.ts +29 -0
  58. package/dist/cli/utils.d.ts.map +1 -0
  59. package/dist/cli/utils.js +53 -0
  60. package/dist/cli/utils.js.map +1 -0
  61. package/dist/cli.d.ts +9 -0
  62. package/dist/cli.d.ts.map +1 -0
  63. package/dist/cli.js +1805 -0
  64. package/dist/config/generator.d.ts +90 -0
  65. package/dist/config/generator.d.ts.map +1 -0
  66. package/dist/config/generator.js +320 -0
  67. package/dist/config/generator.js.map +1 -0
  68. package/dist/config/loader.d.ts +107 -0
  69. package/dist/config/loader.d.ts.map +1 -0
  70. package/dist/config/loader.js +251 -0
  71. package/dist/config/loader.js.map +1 -0
  72. package/dist/config/schema.d.ts +107 -0
  73. package/dist/config/schema.d.ts.map +1 -0
  74. package/dist/config/schema.js +169 -0
  75. package/dist/config/schema.js.map +1 -0
  76. package/dist/enrich/audio-transcription.d.ts +77 -0
  77. package/dist/enrich/audio-transcription.d.ts.map +1 -0
  78. package/dist/enrich/audio-transcription.js +370 -0
  79. package/dist/enrich/audio-transcription.js.map +1 -0
  80. package/dist/enrich/checkpoint.d.ts +137 -0
  81. package/dist/enrich/checkpoint.d.ts.map +1 -0
  82. package/dist/enrich/checkpoint.js +205 -0
  83. package/dist/enrich/checkpoint.js.map +1 -0
  84. package/dist/enrich/idempotency.d.ts +90 -0
  85. package/dist/enrich/idempotency.d.ts.map +1 -0
  86. package/dist/enrich/idempotency.js +188 -0
  87. package/dist/enrich/idempotency.js.map +1 -0
  88. package/dist/enrich/image-analysis.d.ts +62 -0
  89. package/dist/enrich/image-analysis.d.ts.map +1 -0
  90. package/dist/enrich/image-analysis.js +264 -0
  91. package/dist/enrich/image-analysis.js.map +1 -0
  92. package/dist/enrich/index.d.ts +60 -0
  93. package/dist/enrich/index.d.ts.map +1 -0
  94. package/dist/enrich/index.js +74 -0
  95. package/dist/enrich/index.js.map +1 -0
  96. package/dist/enrich/link-enrichment.d.ts +37 -0
  97. package/dist/enrich/link-enrichment.d.ts.map +1 -0
  98. package/dist/enrich/link-enrichment.js +202 -0
  99. package/dist/enrich/link-enrichment.js.map +1 -0
  100. package/dist/enrich/pdf-video-handling.d.ts +49 -0
  101. package/dist/enrich/pdf-video-handling.d.ts.map +1 -0
  102. package/dist/enrich/pdf-video-handling.js +325 -0
  103. package/dist/enrich/pdf-video-handling.js.map +1 -0
  104. package/dist/enrich/progress-tracker.d.ts +120 -0
  105. package/dist/enrich/progress-tracker.d.ts.map +1 -0
  106. package/dist/enrich/progress-tracker.js +220 -0
  107. package/dist/enrich/progress-tracker.js.map +1 -0
  108. package/dist/enrich/providers/firecrawl.d.ts +18 -0
  109. package/dist/enrich/providers/firecrawl.d.ts.map +1 -0
  110. package/dist/enrich/providers/firecrawl.js +48 -0
  111. package/dist/enrich/providers/firecrawl.js.map +1 -0
  112. package/dist/enrich/providers/generic.d.ts +16 -0
  113. package/dist/enrich/providers/generic.d.ts.map +1 -0
  114. package/dist/enrich/providers/generic.js +36 -0
  115. package/dist/enrich/providers/generic.js.map +1 -0
  116. package/dist/enrich/providers/index.d.ts +14 -0
  117. package/dist/enrich/providers/index.d.ts.map +1 -0
  118. package/dist/enrich/providers/index.js +13 -0
  119. package/dist/enrich/providers/index.js.map +1 -0
  120. package/dist/enrich/providers/instagram.d.ts +16 -0
  121. package/dist/enrich/providers/instagram.d.ts.map +1 -0
  122. package/dist/enrich/providers/instagram.js +43 -0
  123. package/dist/enrich/providers/instagram.js.map +1 -0
  124. package/dist/enrich/providers/spotify.d.ts +16 -0
  125. package/dist/enrich/providers/spotify.d.ts.map +1 -0
  126. package/dist/enrich/providers/spotify.js +45 -0
  127. package/dist/enrich/providers/spotify.js.map +1 -0
  128. package/dist/enrich/providers/twitter.d.ts +16 -0
  129. package/dist/enrich/providers/twitter.d.ts.map +1 -0
  130. package/dist/enrich/providers/twitter.js +43 -0
  131. package/dist/enrich/providers/twitter.js.map +1 -0
  132. package/dist/enrich/providers/types.d.ts +47 -0
  133. package/dist/enrich/providers/types.d.ts.map +1 -0
  134. package/dist/enrich/providers/types.js +15 -0
  135. package/dist/enrich/providers/types.js.map +1 -0
  136. package/dist/enrich/providers/youtube.d.ts +16 -0
  137. package/dist/enrich/providers/youtube.d.ts.map +1 -0
  138. package/dist/enrich/providers/youtube.js +43 -0
  139. package/dist/enrich/providers/youtube.js.map +1 -0
  140. package/dist/enrich/rate-limiting.d.ts +118 -0
  141. package/dist/enrich/rate-limiting.d.ts.map +1 -0
  142. package/dist/enrich/rate-limiting.js +258 -0
  143. package/dist/enrich/rate-limiting.js.map +1 -0
  144. package/dist/index.d.ts +688 -0
  145. package/dist/index.d.ts.map +1 -0
  146. package/dist/index.js +1729 -0
  147. package/dist/index.js.map +1 -0
  148. package/dist/ingest/dedup-merge.d.ts +82 -0
  149. package/dist/ingest/dedup-merge.d.ts.map +1 -0
  150. package/dist/ingest/dedup-merge.js +262 -0
  151. package/dist/ingest/dedup-merge.js.map +1 -0
  152. package/dist/ingest/ingest-csv.d.ts +62 -0
  153. package/dist/ingest/ingest-csv.d.ts.map +1 -0
  154. package/dist/ingest/ingest-csv.js +300 -0
  155. package/dist/ingest/ingest-csv.js.map +1 -0
  156. package/dist/ingest/ingest-db.d.ts +64 -0
  157. package/dist/ingest/ingest-db.d.ts.map +1 -0
  158. package/dist/ingest/ingest-db.js +172 -0
  159. package/dist/ingest/ingest-db.js.map +1 -0
  160. package/dist/ingest/link-replies-and-tapbacks.d.ts +53 -0
  161. package/dist/ingest/link-replies-and-tapbacks.d.ts.map +1 -0
  162. package/dist/ingest/link-replies-and-tapbacks.js +381 -0
  163. package/dist/ingest/link-replies-and-tapbacks.js.map +1 -0
  164. package/dist/normalize/date-converters.d.ts +45 -0
  165. package/dist/normalize/date-converters.d.ts.map +1 -0
  166. package/dist/normalize/date-converters.js +166 -0
  167. package/dist/normalize/date-converters.js.map +1 -0
  168. package/dist/normalize/path-validator.d.ts +65 -0
  169. package/dist/normalize/path-validator.d.ts.map +1 -0
  170. package/dist/normalize/path-validator.js +221 -0
  171. package/dist/normalize/path-validator.js.map +1 -0
  172. package/dist/normalize/validate-normalized.d.ts +45 -0
  173. package/dist/normalize/validate-normalized.d.ts.map +1 -0
  174. package/dist/normalize/validate-normalized.js +144 -0
  175. package/dist/normalize/validate-normalized.js.map +1 -0
  176. package/dist/render/embeds-blockquotes.d.ts +84 -0
  177. package/dist/render/embeds-blockquotes.d.ts.map +1 -0
  178. package/dist/render/embeds-blockquotes.js +204 -0
  179. package/dist/render/embeds-blockquotes.js.map +1 -0
  180. package/dist/render/grouping.d.ts +78 -0
  181. package/dist/render/grouping.d.ts.map +1 -0
  182. package/dist/render/grouping.js +134 -0
  183. package/dist/render/grouping.js.map +1 -0
  184. package/dist/render/index.d.ts +47 -0
  185. package/dist/render/index.d.ts.map +1 -0
  186. package/dist/render/index.js +245 -0
  187. package/dist/render/index.js.map +1 -0
  188. package/dist/render/reply-rendering.d.ts +88 -0
  189. package/dist/render/reply-rendering.d.ts.map +1 -0
  190. package/dist/render/reply-rendering.js +196 -0
  191. package/dist/render/reply-rendering.js.map +1 -0
  192. package/dist/schema/message.d.ts +125 -0
  193. package/dist/schema/message.d.ts.map +1 -0
  194. package/dist/schema/message.js +331 -0
  195. package/dist/schema/message.js.map +1 -0
  196. package/dist/utils/delta-detection.d.ts +107 -0
  197. package/dist/utils/delta-detection.d.ts.map +1 -0
  198. package/dist/utils/delta-detection.js +199 -0
  199. package/dist/utils/delta-detection.js.map +1 -0
  200. package/dist/utils/enrichment-merge.d.ts +135 -0
  201. package/dist/utils/enrichment-merge.d.ts.map +1 -0
  202. package/dist/utils/enrichment-merge.js +280 -0
  203. package/dist/utils/enrichment-merge.js.map +1 -0
  204. package/dist/utils/human.d.ts +15 -0
  205. package/dist/utils/human.d.ts.map +1 -0
  206. package/dist/utils/human.js +27 -0
  207. package/dist/utils/human.js.map +1 -0
  208. package/dist/utils/incremental-state.d.ts +133 -0
  209. package/dist/utils/incremental-state.d.ts.map +1 -0
  210. package/dist/utils/incremental-state.js +237 -0
  211. package/dist/utils/incremental-state.js.map +1 -0
  212. package/dist/utils/logger.d.ts +40 -0
  213. package/dist/utils/logger.d.ts.map +1 -0
  214. package/dist/utils/logger.js +176 -0
  215. package/dist/utils/logger.js.map +1 -0
  216. package/package.json +165 -0
@@ -0,0 +1 @@
1
+ {"version":3,"file":"delta-detection.js","sourceRoot":"","sources":["../../src/utils/delta-detection.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAGH,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAA;AACxC,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAE5C,OAAO,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAA;AAqC7E,+EAA+E;AAC/E,0CAA0C;AAC1C,+EAA+E;AAE/E;;;;;;;;;;GAUG;AACH,KAAK,UAAU,iBAAiB,CAC/B,aAAqB;IAErB,OAAO,oBAAoB,CAAC,aAAa,CAAC,CAAA;AAC3C,CAAC;AAED,+EAA+E;AAC/E,+CAA+C;AAC/C,+EAA+E;AAE/E;;;;;;;GAOG;AACH,MAAM,UAAU,wBAAwB,CAAC,QAAmB;IAC3D,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAA;IAC/B,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;QAC5B,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IACpB,CAAC;IACD,OAAO,KAAK,CAAA;AACb,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,SAAS,YAAY,CACpB,YAAyB,EACzB,aAAsC;IAEtC,IAAI,CAAC,aAAa,EAAE,CAAC;QACpB,4CAA4C;QAC5C,OAAO,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;IAChC,CAAC;IAED,6DAA6D;IAC7D,OAAO,iBAAiB,CAAC,YAAY,EAAE,aAAa,CAAC,CAAA;AACtD,CAAC;AAED,+EAA+E;AAC/E,8BAA8B;AAC9B,+EAA+E;AAE/E;;;;;;;;;;GAUG;AACH,MAAM,UAAU,eAAe,CAAC,MAAmB;IAClD,MAAM,MAAM,GAAG,YAAY,CAAC,uBAAuB,CAAC,CAAA;IACpD,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAA;IAChC,MAAM,UAAU,GAAG,MAAM,CAAC,aAAa,CAAA;IACvC,MAAM,aAAa,GAAG,MAAM,CAAC,qBAAqB,CAAA;IAElD,2BAA2B;IAC3B,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QACvB,uCAAuC;QACvC,MAAM,CAAC,IAAI,CAAC,sBAAsB,EAAE,EAAE,aAAa,EAAE,UAAU,EAAE,CAAC,CAAA;QAClE,4DAA4D;QAC5D,oFAAoF;QACpF,SAAS,CAAC,yBAAyB,UAAU,WAAW,CAAC,CAAA;IAC1D,CAAC;SAAM,CAAC;QACP,MAAM,UAAU,GAAG,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;QACrE,iBAAiB;QACjB,MAAM,CAAC,IAAI,CAAC,gBAAgB,EAAE;YAC7B,WAAW,EAAE,QAAQ;YACrB,UAAU;YACV,aAAa,EAAE,UAAU;YACzB,kBAAkB,EAAE,aAAa;SACjC,CAAC,CAAA;QACF,yCAAyC;QACzC,kGAAkG;QAClG,SAAS,CACR,mBAAmB,QAAQ,kBAAkB,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,UAAU,iBAAiB,CACtG,CAAA;QACD,SAAS,CAAC,wBAAwB,aAAa,EAAE,CAAC,CAAA;IACnD,CAAC;AACF,CAAC;AAED,+EAA+E;AAC/E,kDAAkD;AAClD,+EAA+E;AAE/E;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAChC,QAAmB,EACnB,aAAqB;IAErB,qDAAqD;IACrD,MAAM,aAAa,GAAG,MAAM,iBAAiB,CAAC,aAAa,CAAC,CAAA;IAC5D,MAAM,UAAU,GAAG,aAAa,KAAK,IAAI,CAAA;IAEzC,uDAAuD;IACvD,MAAM,YAAY,GAAG,wBAAwB,CAAC,QAAQ,CAAC,CAAA;IACvD,MAAM,QAAQ,GAAG,YAAY,CAAC,YAAY,EAAE,aAAa,CAAC,CAAA;IAE1D,eAAe;IACf,MAAM,MAAM,GAAgB;QAC3B,QAAQ;QACR,aAAa,EAAE,QAAQ,CAAC,MAAM;QAC9B,qBAAqB,EAAE,aAAa,EAAE,aAAa,CAAC,MAAM,IAAI,CAAC;QAC/D,QAAQ,EAAE,QAAQ,CAAC,MAAM;QACzB,UAAU;QACV,KAAK,EAAE,aAAa,IAAI;YACvB,OAAO,EAAE,KAAK;YACd,cAAc,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACxC,aAAa,EAAE,QAAQ,CAAC,MAAM;YAC9B,aAAa,EAAE,EAAE;YACjB,cAAc,EAAE;gBACf,UAAU,EAAE,EAAE;aACd;YACD,eAAe,EAAE,IAAI;SACrB;KACD,CAAA;IAED,oBAAoB;IACpB,eAAe,CAAC,MAAM,CAAC,CAAA;IAEvB,OAAO,MAAM,CAAA;AACd,CAAC;AAED,+EAA+E;AAC/E,uDAAuD;AACvD,+EAA+E;AAE/E;;;;;;;;;;GAUG;AACH,MAAM,UAAU,aAAa,CAAC,MAAmB;IAOhD,MAAM,KAAK,GAAG,MAAM,CAAC,aAAa,CAAA;IAClC,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAA;IAChC,MAAM,aAAa,GAAG,MAAM,CAAC,qBAAqB,CAAA;IAElD,OAAO;QACN,KAAK;QACL,GAAG,EAAE,QAAQ;QACb,QAAQ,EAAE,aAAa;QACvB,UAAU,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QACpD,eAAe,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;KAC9D,CAAA;AACF,CAAC"}
@@ -0,0 +1,135 @@
1
+ /**
2
+ * Enrichment Merge Module (INCREMENTAL--T03)
3
+ *
4
+ * Implements merge strategy for incremental enrichment results into existing
5
+ * enriched JSON, preserving prior work and handling new enrichments.
6
+ *
7
+ * Implements:
8
+ * - AC01: Load existing enriched.json if exists
9
+ * - AC02: Merge new enrichments by GUID (new messages + updates)
10
+ * - AC03: Preserve existing enrichments (no overwrites unless --force-refresh)
11
+ * - AC04: Update statistics in state file
12
+ * - AC05: Backup previous enriched.json as enriched.json.backup
13
+ *
14
+ * Architecture:
15
+ * - MergeResult: Complete merge operation output
16
+ * - MergeOptions: Configuration for merge behavior
17
+ * - loadExistingEnriched: Load prior enriched.json safely
18
+ * - mergeEnrichments: Core merge logic with preservation
19
+ * - backupEnrichedJson: Atomic backup creation
20
+ * - updateMergeStatistics: Statistics calculation
21
+ */
22
+ import type { ExportEnvelope, Message } from '#schema/message';
23
+ /**
24
+ * Options for merge behavior
25
+ */
26
+ export type MergeOptions = {
27
+ /** Force refresh overwrites existing enrichments (default: false) */
28
+ forceRefresh?: boolean;
29
+ };
30
+ /**
31
+ * Statistics from merge operation
32
+ */
33
+ export type MergeStatistics = {
34
+ /** Number of messages merged (existing GUIDs updated) */
35
+ mergedCount: number;
36
+ /** Number of messages added (new GUIDs) */
37
+ addedCount: number;
38
+ /** Number of messages with preserved enrichments */
39
+ preservedCount: number;
40
+ /** Total messages in result */
41
+ totalMessages: number;
42
+ /** Percentage of messages that were merged */
43
+ mergedPercentage?: number;
44
+ /** Percentage of messages that were added */
45
+ addedPercentage?: number;
46
+ };
47
+ /**
48
+ * Result of merge operation
49
+ */
50
+ export type MergeResult = {
51
+ /** Merged messages with enrichments preserved/updated */
52
+ messages: Message[];
53
+ /** Statistics about the merge operation */
54
+ statistics: MergeStatistics;
55
+ /** Count of messages merged */
56
+ mergedCount: number;
57
+ /** Count of messages added */
58
+ addedCount: number;
59
+ /** Count of messages with preserved enrichments */
60
+ preservedCount: number;
61
+ };
62
+ /**
63
+ * AC01: Load existing enriched JSON safely
64
+ *
65
+ * Handles:
66
+ * - File doesn't exist (returns null)
67
+ * - JSON is corrupted (returns null)
68
+ * - Schema version mismatch (returns null or handles gracefully)
69
+ * - Preserves all enrichment data
70
+ *
71
+ * @param filePath - Path to existing enriched.json
72
+ * @returns ExportEnvelope if valid, null if missing or corrupted
73
+ */
74
+ export declare function loadExistingEnriched(filePath: string): Promise<ExportEnvelope | null>;
75
+ /**
76
+ * AC02 + AC03: Merge new enrichments with existing messages
77
+ *
78
+ * Strategy:
79
+ * 1. Build index of existing messages by GUID
80
+ * 2. For each new message:
81
+ * - If GUID exists: merge enrichments (preserve existing unless --force-refresh)
82
+ * - If GUID is new: add message
83
+ * 3. Preserve enrichment order and structure
84
+ *
85
+ * @param existingMessages - Messages from prior enriched.json
86
+ * @param newMessages - New/updated messages from enrichment
87
+ * @param options - Merge options (forceRefresh, etc.)
88
+ * @returns MergeResult with merged messages and statistics
89
+ */
90
+ export declare function mergeEnrichments(existingMessages: Message[], newMessages: Message[], options?: MergeOptions): MergeResult;
91
+ /**
92
+ * AC04: Calculate and update merge statistics
93
+ *
94
+ * Computes merge percentages and summary information
95
+ *
96
+ * @param stats - Raw statistics
97
+ * @returns Updated statistics with percentages
98
+ */
99
+ export declare function updateMergeStatistics(stats: MergeStatistics): MergeStatistics;
100
+ /**
101
+ * AC05: Create atomic backup of existing enriched.json
102
+ *
103
+ * Pattern:
104
+ * 1. Read existing file
105
+ * 2. Write to enriched.json.backup (overwrite if exists)
106
+ * 3. Ensures no data loss on merge
107
+ *
108
+ * @param filePath - Path to enriched.json to backup
109
+ * @throws Error if source file can't be read
110
+ */
111
+ export declare function backupEnrichedJson(filePath: string): Promise<void>;
112
+ /**
113
+ * Create a new MergeResult with given parameters
114
+ *
115
+ * @param messages - Merged messages
116
+ * @param options - Statistics options
117
+ * @returns Complete MergeResult
118
+ */
119
+ export declare function createEnrichmentMergeResult(messages: Message[], options: {
120
+ mergedCount: number;
121
+ addedCount: number;
122
+ preservedCount: number;
123
+ }): MergeResult;
124
+ /**
125
+ * AC05: Log human-readable merge summary
126
+ *
127
+ * Outputs lines like:
128
+ * - "Merged 50 existing messages with new enrichments"
129
+ * - "Added 10 new messages to enriched.json"
130
+ * - "Preserved 60 enrichments from prior run"
131
+ *
132
+ * @param result - MergeResult to summarize
133
+ */
134
+ export declare function logMergeSummary(result: MergeResult): void;
135
+ //# sourceMappingURL=enrichment-merge.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"enrichment-merge.d.ts","sourceRoot":"","sources":["../../src/utils/enrichment-merge.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAIH,OAAO,KAAK,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAA;AAU9D;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG;IAC1B,qEAAqE;IACrE,YAAY,CAAC,EAAE,OAAO,CAAA;CACtB,CAAA;AAED;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG;IAC7B,yDAAyD;IACzD,WAAW,EAAE,MAAM,CAAA;IAEnB,2CAA2C;IAC3C,UAAU,EAAE,MAAM,CAAA;IAElB,oDAAoD;IACpD,cAAc,EAAE,MAAM,CAAA;IAEtB,+BAA+B;IAC/B,aAAa,EAAE,MAAM,CAAA;IAErB,8CAA8C;IAC9C,gBAAgB,CAAC,EAAE,MAAM,CAAA;IAEzB,6CAA6C;IAC7C,eAAe,CAAC,EAAE,MAAM,CAAA;CACxB,CAAA;AAED;;GAEG;AACH,MAAM,MAAM,WAAW,GAAG;IACzB,yDAAyD;IACzD,QAAQ,EAAE,OAAO,EAAE,CAAA;IAEnB,2CAA2C;IAC3C,UAAU,EAAE,eAAe,CAAA;IAE3B,+BAA+B;IAC/B,WAAW,EAAE,MAAM,CAAA;IAEnB,8BAA8B;IAC9B,UAAU,EAAE,MAAM,CAAA;IAElB,mDAAmD;IACnD,cAAc,EAAE,MAAM,CAAA;CACtB,CAAA;AAMD;;;;;;;;;;;GAWG;AACH,wBAAsB,oBAAoB,CACzC,QAAQ,EAAE,MAAM,GACd,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,CAmBhC;AAMD;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,gBAAgB,CAC/B,gBAAgB,EAAE,OAAO,EAAE,EAC3B,WAAW,EAAE,OAAO,EAAE,EACtB,OAAO,GAAE,YAAiB,GACxB,WAAW,CAoEb;AAqED;;;;;;;GAOG;AACH,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,eAAe,GAAG,eAAe,CAQ7E;AAMD;;;;;;;;;;GAUG;AACH,wBAAsB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAIxE;AAMD;;;;;;GAMG;AACH,wBAAgB,2BAA2B,CAC1C,QAAQ,EAAE,OAAO,EAAE,EACnB,OAAO,EAAE;IACR,WAAW,EAAE,MAAM,CAAA;IACnB,UAAU,EAAE,MAAM,CAAA;IAClB,cAAc,EAAE,MAAM,CAAA;CACtB,GACC,WAAW,CAiBb;AAMD;;;;;;;;;GASG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,WAAW,GAAG,IAAI,CAqBzD"}
@@ -0,0 +1,280 @@
1
+ /**
2
+ * Enrichment Merge Module (INCREMENTAL--T03)
3
+ *
4
+ * Implements merge strategy for incremental enrichment results into existing
5
+ * enriched JSON, preserving prior work and handling new enrichments.
6
+ *
7
+ * Implements:
8
+ * - AC01: Load existing enriched.json if exists
9
+ * - AC02: Merge new enrichments by GUID (new messages + updates)
10
+ * - AC03: Preserve existing enrichments (no overwrites unless --force-refresh)
11
+ * - AC04: Update statistics in state file
12
+ * - AC05: Backup previous enriched.json as enriched.json.backup
13
+ *
14
+ * Architecture:
15
+ * - MergeResult: Complete merge operation output
16
+ * - MergeOptions: Configuration for merge behavior
17
+ * - loadExistingEnriched: Load prior enriched.json safely
18
+ * - mergeEnrichments: Core merge logic with preservation
19
+ * - backupEnrichedJson: Atomic backup creation
20
+ * - updateMergeStatistics: Statistics calculation
21
+ */
22
+ import { promises as fs } from 'node:fs';
23
+ import { createLogger } from '#utils/logger';
24
+ // ============================================================================
25
+ // AC01: Load existing enriched.json
26
+ // ============================================================================
27
+ /**
28
+ * AC01: Load existing enriched JSON safely
29
+ *
30
+ * Handles:
31
+ * - File doesn't exist (returns null)
32
+ * - JSON is corrupted (returns null)
33
+ * - Schema version mismatch (returns null or handles gracefully)
34
+ * - Preserves all enrichment data
35
+ *
36
+ * @param filePath - Path to existing enriched.json
37
+ * @returns ExportEnvelope if valid, null if missing or corrupted
38
+ */
39
+ export async function loadExistingEnriched(filePath) {
40
+ try {
41
+ const content = await fs.readFile(filePath, 'utf-8');
42
+ const parsed = JSON.parse(content);
43
+ // Validate basic structure
44
+ if (!parsed.messages || !Array.isArray(parsed.messages)) {
45
+ return null;
46
+ }
47
+ return parsed;
48
+ }
49
+ catch (error) {
50
+ // File doesn't exist or is corrupted
51
+ if (error instanceof Error && error.message.includes('ENOENT')) {
52
+ return null;
53
+ }
54
+ // JSON parse error or other read error
55
+ return null;
56
+ }
57
+ }
58
+ // ============================================================================
59
+ // AC02 + AC03: Merge enrichments by GUID
60
+ // ============================================================================
61
+ /**
62
+ * AC02 + AC03: Merge new enrichments with existing messages
63
+ *
64
+ * Strategy:
65
+ * 1. Build index of existing messages by GUID
66
+ * 2. For each new message:
67
+ * - If GUID exists: merge enrichments (preserve existing unless --force-refresh)
68
+ * - If GUID is new: add message
69
+ * 3. Preserve enrichment order and structure
70
+ *
71
+ * @param existingMessages - Messages from prior enriched.json
72
+ * @param newMessages - New/updated messages from enrichment
73
+ * @param options - Merge options (forceRefresh, etc.)
74
+ * @returns MergeResult with merged messages and statistics
75
+ */
76
+ export function mergeEnrichments(existingMessages, newMessages, options = {}) {
77
+ // Build index of existing messages by GUID
78
+ const existingByGuid = new Map();
79
+ for (const msg of existingMessages) {
80
+ existingByGuid.set(msg.guid, msg);
81
+ }
82
+ // Track merge statistics
83
+ let mergedCount = 0;
84
+ let addedCount = 0;
85
+ let preservedCount = 0;
86
+ // Track GUIDs we've processed to avoid duplicates
87
+ const processedGuids = new Set();
88
+ // Result messages with merged enrichments
89
+ const resultMessages = [];
90
+ // Process new messages
91
+ for (const newMsg of newMessages) {
92
+ const existing = existingByGuid.get(newMsg.guid);
93
+ if (existing) {
94
+ // Message exists: merge enrichments
95
+ if (!processedGuids.has(newMsg.guid)) {
96
+ const merged = mergeMessageEnrichments(existing, newMsg, options);
97
+ resultMessages.push(merged);
98
+ mergedCount++;
99
+ // Count preserved enrichments
100
+ if (merged.messageKind === 'media' &&
101
+ merged.media?.enrichment &&
102
+ merged.media.enrichment.length > 0) {
103
+ preservedCount++;
104
+ }
105
+ processedGuids.add(newMsg.guid);
106
+ }
107
+ }
108
+ else {
109
+ // New message: add to result
110
+ if (!processedGuids.has(newMsg.guid)) {
111
+ resultMessages.push(newMsg);
112
+ addedCount++;
113
+ processedGuids.add(newMsg.guid);
114
+ }
115
+ }
116
+ }
117
+ const totalMessages = resultMessages.length;
118
+ return {
119
+ messages: resultMessages,
120
+ statistics: {
121
+ mergedCount,
122
+ addedCount,
123
+ preservedCount,
124
+ totalMessages,
125
+ mergedPercentage: totalMessages > 0 ? (mergedCount / totalMessages) * 100 : 0,
126
+ addedPercentage: totalMessages > 0 ? (addedCount / totalMessages) * 100 : 0,
127
+ },
128
+ mergedCount,
129
+ addedCount,
130
+ preservedCount,
131
+ };
132
+ }
133
+ /**
134
+ * AC02 + AC03: Merge enrichments for a single message
135
+ *
136
+ * Preservation logic:
137
+ * - If forceRefresh: use new enrichments
138
+ * - Otherwise:
139
+ * - Preserve existing enrichments by kind
140
+ * - Append new enrichments for different kinds
141
+ * - Skip enrichments for kinds that already exist (unless force)
142
+ *
143
+ * @param existing - Existing message with enrichments
144
+ * @param newMsg - New message with enrichments
145
+ * @param options - Merge options
146
+ * @returns Merged message with preserved enrichments
147
+ */
148
+ function mergeMessageEnrichments(existing, newMsg, options) {
149
+ // If not media message, just return existing (no enrichment to merge)
150
+ if (existing.messageKind !== 'media' || !existing.media) {
151
+ return existing;
152
+ }
153
+ // If no new enrichments, keep existing as-is
154
+ if (!newMsg.media?.enrichment || newMsg.media.enrichment.length === 0) {
155
+ return existing;
156
+ }
157
+ // If forceRefresh, use all new enrichments
158
+ if (options.forceRefresh) {
159
+ return {
160
+ ...existing,
161
+ media: {
162
+ ...existing.media,
163
+ enrichment: newMsg.media.enrichment,
164
+ },
165
+ };
166
+ }
167
+ // Otherwise: preserve existing, append new for different kinds
168
+ const existingEnrichment = existing.media.enrichment ?? [];
169
+ const newEnrichment = newMsg.media.enrichment ?? [];
170
+ // Build set of existing enrichment kinds
171
+ const existingKinds = new Set(existingEnrichment.map((e) => e.kind));
172
+ // Only add new enrichments for kinds not already present
173
+ const mergedEnrichment = [
174
+ ...existingEnrichment,
175
+ ...newEnrichment.filter((e) => !existingKinds.has(e.kind)),
176
+ ];
177
+ return {
178
+ ...existing,
179
+ media: {
180
+ ...existing.media,
181
+ enrichment: mergedEnrichment,
182
+ },
183
+ };
184
+ }
185
+ // ============================================================================
186
+ // AC04: Update merge statistics
187
+ // ============================================================================
188
+ /**
189
+ * AC04: Calculate and update merge statistics
190
+ *
191
+ * Computes merge percentages and summary information
192
+ *
193
+ * @param stats - Raw statistics
194
+ * @returns Updated statistics with percentages
195
+ */
196
+ export function updateMergeStatistics(stats) {
197
+ const total = stats.totalMessages;
198
+ return {
199
+ ...stats,
200
+ mergedPercentage: total > 0 ? (stats.mergedCount / total) * 100 : 0,
201
+ addedPercentage: total > 0 ? (stats.addedCount / total) * 100 : 0,
202
+ };
203
+ }
204
+ // ============================================================================
205
+ // AC05: Backup existing enriched.json
206
+ // ============================================================================
207
+ /**
208
+ * AC05: Create atomic backup of existing enriched.json
209
+ *
210
+ * Pattern:
211
+ * 1. Read existing file
212
+ * 2. Write to enriched.json.backup (overwrite if exists)
213
+ * 3. Ensures no data loss on merge
214
+ *
215
+ * @param filePath - Path to enriched.json to backup
216
+ * @throws Error if source file can't be read
217
+ */
218
+ export async function backupEnrichedJson(filePath) {
219
+ const backupPath = `${filePath}.backup`;
220
+ const content = await fs.readFile(filePath, 'utf-8');
221
+ await fs.writeFile(backupPath, content, 'utf-8');
222
+ }
223
+ // ============================================================================
224
+ // Helper: Create enrichment merge result
225
+ // ============================================================================
226
+ /**
227
+ * Create a new MergeResult with given parameters
228
+ *
229
+ * @param messages - Merged messages
230
+ * @param options - Statistics options
231
+ * @returns Complete MergeResult
232
+ */
233
+ export function createEnrichmentMergeResult(messages, options) {
234
+ return {
235
+ messages,
236
+ statistics: {
237
+ mergedCount: options.mergedCount,
238
+ addedCount: options.addedCount,
239
+ preservedCount: options.preservedCount,
240
+ totalMessages: messages.length,
241
+ mergedPercentage: messages.length > 0 ? (options.mergedCount / messages.length) * 100 : 0,
242
+ addedPercentage: messages.length > 0 ? (options.addedCount / messages.length) * 100 : 0,
243
+ },
244
+ mergedCount: options.mergedCount,
245
+ addedCount: options.addedCount,
246
+ preservedCount: options.preservedCount,
247
+ };
248
+ }
249
+ // ============================================================================
250
+ // Helper: Log merge summary
251
+ // ============================================================================
252
+ /**
253
+ * AC05: Log human-readable merge summary
254
+ *
255
+ * Outputs lines like:
256
+ * - "Merged 50 existing messages with new enrichments"
257
+ * - "Added 10 new messages to enriched.json"
258
+ * - "Preserved 60 enrichments from prior run"
259
+ *
260
+ * @param result - MergeResult to summarize
261
+ */
262
+ export function logMergeSummary(result) {
263
+ const { mergedCount, addedCount, preservedCount } = result;
264
+ const logger = createLogger('utils:enrichment-merge');
265
+ if (mergedCount > 0) {
266
+ logger.info('Merged existing messages with new enrichments', {
267
+ mergedCount,
268
+ });
269
+ }
270
+ if (addedCount > 0) {
271
+ logger.info('Added new messages to enriched.json', { addedCount });
272
+ }
273
+ if (preservedCount > 0) {
274
+ logger.info('Preserved enrichments from prior run', { preservedCount });
275
+ }
276
+ if (mergedCount === 0 && addedCount === 0) {
277
+ logger.info('No new enrichments to merge');
278
+ }
279
+ }
280
+ //# sourceMappingURL=enrichment-merge.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"enrichment-merge.js","sourceRoot":"","sources":["../../src/utils/enrichment-merge.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EAAE,QAAQ,IAAI,EAAE,EAAE,MAAM,SAAS,CAAA;AAIxC,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AA2D5C,+EAA+E;AAC/E,oCAAoC;AACpC,+EAA+E;AAE/E;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACzC,QAAgB;IAEhB,IAAI,CAAC;QACJ,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAA;QACpD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAmB,CAAA;QAEpD,2BAA2B;QAC3B,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;YACzD,OAAO,IAAI,CAAA;QACZ,CAAC;QAED,OAAO,MAAM,CAAA;IACd,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,qCAAqC;QACrC,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;YAChE,OAAO,IAAI,CAAA;QACZ,CAAC;QACD,uCAAuC;QACvC,OAAO,IAAI,CAAA;IACZ,CAAC;AACF,CAAC;AAED,+EAA+E;AAC/E,yCAAyC;AACzC,+EAA+E;AAE/E;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,gBAAgB,CAC/B,gBAA2B,EAC3B,WAAsB,EACtB,UAAwB,EAAE;IAE1B,2CAA2C;IAC3C,MAAM,cAAc,GAAG,IAAI,GAAG,EAAmB,CAAA;IACjD,KAAK,MAAM,GAAG,IAAI,gBAAgB,EAAE,CAAC;QACpC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;IAClC,CAAC;IAED,yBAAyB;IACzB,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,IAAI,UAAU,GAAG,CAAC,CAAA;IAClB,IAAI,cAAc,GAAG,CAAC,CAAA;IAEtB,kDAAkD;IAClD,MAAM,cAAc,GAAG,IAAI,GAAG,EAAU,CAAA;IAExC,0CAA0C;IAC1C,MAAM,cAAc,GAAc,EAAE,CAAA;IAEpC,uBAAuB;IACvB,KAAK,MAAM,MAAM,IAAI,WAAW,EAAE,CAAC;QAClC,MAAM,QAAQ,GAAG,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;QAEhD,IAAI,QAAQ,EAAE,CAAC;YACd,oCAAoC;YACpC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBACtC,MAAM,MAAM,GAAG,uBAAuB,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,CAAA;gBACjE,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;gBAC3B,WAAW,EAAE,CAAA;gBAEb,8BAA8B;gBAC9B,IACC,MAAM,CAAC,WAAW,KAAK,OAAO;oBAC9B,MAAM,CAAC,KAAK,EAAE,UAAU;oBACxB,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EACjC,CAAC;oBACF,cAAc,EAAE,CAAA;gBACjB,CAAC;gBAED,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YAChC,CAAC;QACF,CAAC;aAAM,CAAC;YACP,6BAA6B;YAC7B,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBACtC,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;gBAC3B,UAAU,EAAE,CAAA;gBACZ,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YAChC,CAAC;QACF,CAAC;IACF,CAAC;IAED,MAAM,aAAa,GAAG,cAAc,CAAC,MAAM,CAAA;IAE3C,OAAO;QACN,QAAQ,EAAE,cAAc;QACxB,UAAU,EAAE;YACX,WAAW;YACX,UAAU;YACV,cAAc;YACd,aAAa;YACb,gBAAgB,EACf,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,aAAa,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;YAC5D,eAAe,EACd,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,aAAa,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;SAC3D;QACD,WAAW;QACX,UAAU;QACV,cAAc;KACd,CAAA;AACF,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,SAAS,uBAAuB,CAC/B,QAAiB,EACjB,MAAe,EACf,OAAqB;IAErB,sEAAsE;IACtE,IAAI,QAAQ,CAAC,WAAW,KAAK,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;QACzD,OAAO,QAAQ,CAAA;IAChB,CAAC;IAED,6CAA6C;IAC7C,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,UAAU,IAAI,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvE,OAAO,QAAQ,CAAA;IAChB,CAAC;IAED,2CAA2C;IAC3C,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;QAC1B,OAAO;YACN,GAAG,QAAQ;YACX,KAAK,EAAE;gBACN,GAAG,QAAQ,CAAC,KAAK;gBACjB,UAAU,EAAE,MAAM,CAAC,KAAK,CAAC,UAAU;aACnC;SACD,CAAA;IACF,CAAC;IAED,+DAA+D;IAC/D,MAAM,kBAAkB,GAAG,QAAQ,CAAC,KAAK,CAAC,UAAU,IAAI,EAAE,CAAA;IAC1D,MAAM,aAAa,GAAG,MAAM,CAAC,KAAK,CAAC,UAAU,IAAI,EAAE,CAAA;IAEnD,yCAAyC;IACzC,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAA;IAEpE,yDAAyD;IACzD,MAAM,gBAAgB,GAAG;QACxB,GAAG,kBAAkB;QACrB,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;KAC1D,CAAA;IAED,OAAO;QACN,GAAG,QAAQ;QACX,KAAK,EAAE;YACN,GAAG,QAAQ,CAAC,KAAK;YACjB,UAAU,EAAE,gBAAgB;SAC5B;KACD,CAAA;AACF,CAAC;AAED,+EAA+E;AAC/E,gCAAgC;AAChC,+EAA+E;AAE/E;;;;;;;GAOG;AACH,MAAM,UAAU,qBAAqB,CAAC,KAAsB;IAC3D,MAAM,KAAK,GAAG,KAAK,CAAC,aAAa,CAAA;IAEjC,OAAO;QACN,GAAG,KAAK;QACR,gBAAgB,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QACnE,eAAe,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;KACjE,CAAA;AACF,CAAC;AAED,+EAA+E;AAC/E,sCAAsC;AACtC,+EAA+E;AAE/E;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,QAAgB;IACxD,MAAM,UAAU,GAAG,GAAG,QAAQ,SAAS,CAAA;IACvC,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAA;IACpD,MAAM,EAAE,CAAC,SAAS,CAAC,UAAU,EAAE,OAAO,EAAE,OAAO,CAAC,CAAA;AACjD,CAAC;AAED,+EAA+E;AAC/E,yCAAyC;AACzC,+EAA+E;AAE/E;;;;;;GAMG;AACH,MAAM,UAAU,2BAA2B,CAC1C,QAAmB,EACnB,OAIC;IAED,OAAO;QACN,QAAQ;QACR,UAAU,EAAE;YACX,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,cAAc,EAAE,OAAO,CAAC,cAAc;YACtC,aAAa,EAAE,QAAQ,CAAC,MAAM;YAC9B,gBAAgB,EACf,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,GAAG,QAAQ,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;YACxE,eAAe,EACd,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;SACvE;QACD,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,cAAc,EAAE,OAAO,CAAC,cAAc;KACtC,CAAA;AACF,CAAC;AAED,+EAA+E;AAC/E,4BAA4B;AAC5B,+EAA+E;AAE/E;;;;;;;;;GASG;AACH,MAAM,UAAU,eAAe,CAAC,MAAmB;IAClD,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,cAAc,EAAE,GAAG,MAAM,CAAA;IAC1D,MAAM,MAAM,GAAG,YAAY,CAAC,wBAAwB,CAAC,CAAA;IAErD,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;QACrB,MAAM,CAAC,IAAI,CAAC,+CAA+C,EAAE;YAC5D,WAAW;SACX,CAAC,CAAA;IACH,CAAC;IAED,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;QACpB,MAAM,CAAC,IAAI,CAAC,qCAAqC,EAAE,EAAE,UAAU,EAAE,CAAC,CAAA;IACnE,CAAC;IAED,IAAI,cAAc,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,CAAC,IAAI,CAAC,sCAAsC,EAAE,EAAE,cAAc,EAAE,CAAC,CAAA;IACxE,CAAC;IAED,IAAI,WAAW,KAAK,CAAC,IAAI,UAAU,KAAK,CAAC,EAAE,CAAC;QAC3C,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAA;IAC3C,CAAC;AACF,CAAC"}
@@ -0,0 +1,15 @@
1
+ /**
2
+ * Human Output Helper
3
+ *
4
+ * Centralizes human-facing console output so we can globally toggle it
5
+ * (e.g. suppressed when --json or LOG_FORMAT=json-only is active).
6
+ * Tests that spy on console.* remain compatible.
7
+ */
8
+ export type HumanLoggerOptions = {
9
+ enabled?: boolean;
10
+ };
11
+ export declare function setHumanLoggingEnabled(enabled: boolean): void;
12
+ export declare function humanInfo(...args: Array<unknown>): void;
13
+ export declare function humanWarn(...args: Array<unknown>): void;
14
+ export declare function humanError(...args: Array<unknown>): void;
15
+ //# sourceMappingURL=human.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"human.d.ts","sourceRoot":"","sources":["../../src/utils/human.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,MAAM,MAAM,kBAAkB,GAAG;IAChC,OAAO,CAAC,EAAE,OAAO,CAAA;CACjB,CAAA;AAID,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAE7D;AAeD,wBAAgB,SAAS,CAAC,GAAG,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,GAAG,IAAI,CAEvD;AAED,wBAAgB,SAAS,CAAC,GAAG,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,GAAG,IAAI,CAEvD;AAED,wBAAgB,UAAU,CAAC,GAAG,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,GAAG,IAAI,CAExD"}
@@ -0,0 +1,27 @@
1
+ /**
2
+ * Human Output Helper
3
+ *
4
+ * Centralizes human-facing console output so we can globally toggle it
5
+ * (e.g. suppressed when --json or LOG_FORMAT=json-only is active).
6
+ * Tests that spy on console.* remain compatible.
7
+ */
8
+ let humanEnabled = true;
9
+ export function setHumanLoggingEnabled(enabled) {
10
+ humanEnabled = enabled;
11
+ }
12
+ function safeConsole(kind, ...args) {
13
+ if (!humanEnabled)
14
+ return;
15
+ const c = globalThis.console;
16
+ c?.[kind]?.(...args);
17
+ }
18
+ export function humanInfo(...args) {
19
+ safeConsole('info', ...args);
20
+ }
21
+ export function humanWarn(...args) {
22
+ safeConsole('warn', ...args);
23
+ }
24
+ export function humanError(...args) {
25
+ safeConsole('error', ...args);
26
+ }
27
+ //# sourceMappingURL=human.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"human.js","sourceRoot":"","sources":["../../src/utils/human.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAMH,IAAI,YAAY,GAAG,IAAI,CAAA;AAEvB,MAAM,UAAU,sBAAsB,CAAC,OAAgB;IACtD,YAAY,GAAG,OAAO,CAAA;AACvB,CAAC;AAED,SAAS,WAAW,CACnB,IAAO,EACP,GAAG,IAAoB;IAEvB,IAAI,CAAC,YAAY;QAAE,OAAM;IACzB,MAAM,CAAC,GACN,UAGA,CAAC,OAAO,CAAA;IACT,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,GAAI,IAAW,CAAC,CAAA;AAC7B,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,GAAG,IAAoB;IAChD,WAAW,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,CAAA;AAC7B,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,GAAG,IAAoB;IAChD,WAAW,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,CAAA;AAC7B,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,GAAG,IAAoB;IACjD,WAAW,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,CAAA;AAC9B,CAAC"}
@@ -0,0 +1,133 @@
1
+ /**
2
+ * Incremental State Tracking Module (INCREMENTAL--T01)
3
+ *
4
+ * Implements resumable enrichment by tracking:
5
+ * - AC01: State file with last run metadata
6
+ * - AC02: Last enriched date, total messages, config hash
7
+ * - AC03: GUID delta detection for new messages
8
+ * - AC04: Atomic writes with temp + rename pattern
9
+ * - AC05: --incremental flag integration
10
+ *
11
+ * Architecture:
12
+ * - IncrementalState: Complete state schema with version, metadata, GUID tracking
13
+ * - createIncrementalState: Factory for new state
14
+ * - loadIncrementalState: Safe loading from disk with corruption handling
15
+ * - saveIncrementalState: Atomic writes with temp file + rename
16
+ * - detectNewMessages: O(n) GUID comparison using Set intersection
17
+ * - updateStateWithEnrichedGuids: Add new enrichments and update metadata
18
+ * - verifyConfigHash: Detect config changes between runs
19
+ */
20
+ export type EnrichmentStats = {
21
+ processedCount: number;
22
+ failedCount: number;
23
+ startTime: string;
24
+ endTime: string;
25
+ };
26
+ export type PipelineConfig = {
27
+ configHash: string;
28
+ };
29
+ /**
30
+ * Complete state for incremental enrichment tracking
31
+ * Stored in .imessage-state.json at output directory root
32
+ */
33
+ export type IncrementalState = {
34
+ /** Schema version for backward compatibility */
35
+ version: string;
36
+ /** ISO 8601 UTC timestamp of last enrichment run */
37
+ lastEnrichedAt: string;
38
+ /** Total messages as of last run (for progress reporting) */
39
+ totalMessages: number;
40
+ /** Array of enriched message GUIDs (for delta detection) */
41
+ enrichedGuids: string[];
42
+ /** Pipeline configuration hash (detects config changes) */
43
+ pipelineConfig: PipelineConfig;
44
+ /** Optional: Stats from last enrichment run */
45
+ enrichmentStats: EnrichmentStats | null;
46
+ };
47
+ export type CreateStateOptions = {
48
+ totalMessages?: number;
49
+ enrichedGuids?: string[];
50
+ enrichmentStats?: EnrichmentStats | null;
51
+ };
52
+ /**
53
+ * AC01 + AC02: Create new incremental state with current metadata
54
+ *
55
+ * @param options - Optional initial values
56
+ * @returns New IncrementalState with current timestamp and config hash
57
+ */
58
+ export declare function createIncrementalState(options?: CreateStateOptions): IncrementalState;
59
+ /**
60
+ * AC03: Detect new messages by comparing GUIDs with state
61
+ *
62
+ * Performance: O(n) where n = number of current messages
63
+ * Using Set for fast O(1) lookup of previously enriched GUIDs
64
+ *
65
+ * @param currentGuids - Set of message GUIDs from normalized output
66
+ * @param state - Previous state with enriched GUIDs
67
+ * @returns Array of new GUID strings not in enriched set
68
+ */
69
+ export declare function detectNewMessages(currentGuids: Set<string>, state: IncrementalState): string[];
70
+ /**
71
+ * AC04: Save state atomically to disk
72
+ *
73
+ * Pattern:
74
+ * 1. Write to temp file with .tmp suffix
75
+ * 2. Atomic rename (temp → final)
76
+ * 3. Prevents corruption from power loss or crashes
77
+ *
78
+ * @param state - IncrementalState to persist
79
+ * @param filePath - Target .imessage-state.json path
80
+ * @throws Error if write fails (permission, disk full, etc.)
81
+ */
82
+ export declare function saveIncrementalState(state: IncrementalState, filePath: string): Promise<void>;
83
+ /**
84
+ * AC05: Load state from disk safely
85
+ *
86
+ * - Returns null if file doesn't exist (treat as first run)
87
+ * - Returns null if JSON is corrupted (ignore stale state)
88
+ * - Validates schema version for future compatibility
89
+ *
90
+ * @param filePath - Path to .imessage-state.json
91
+ * @returns IncrementalState if valid, null if missing or corrupted
92
+ */
93
+ export declare function loadIncrementalState(filePath: string): Promise<IncrementalState | null>;
94
+ /**
95
+ * AC02: Verify config hash to detect changes
96
+ *
97
+ * @param currentHash - Hash from current state
98
+ * @param expectedHash - Hash to verify against (default: newly generated)
99
+ * @returns true if hashes match
100
+ */
101
+ export declare function verifyConfigHash(currentHash: string, expectedHash?: string): boolean;
102
+ /**
103
+ * AC02: Check if state is stale (old enrichment run)
104
+ *
105
+ * Useful for warning about old state that may be out of sync
106
+ *
107
+ * @param state - IncrementalState to check
108
+ * @param daysThreshold - Days before state is considered stale (default: 7)
109
+ * @returns true if state is older than threshold
110
+ */
111
+ export declare function isStateOutdated(state: IncrementalState, daysThreshold?: number): boolean;
112
+ /**
113
+ * Update state with newly enriched GUIDs
114
+ *
115
+ * Called after successful enrichment to:
116
+ * - Add new enriched GUIDs (avoid duplicates)
117
+ * - Update lastEnrichedAt timestamp
118
+ * - Record enrichment statistics
119
+ *
120
+ * @param state - State to update (mutated in place)
121
+ * @param newGuids - GUIDs that were just enriched
122
+ * @param enrichmentStats - Optional enrichment stats
123
+ */
124
+ export declare function updateStateWithEnrichedGuids(state: IncrementalState, newGuids: string[], enrichmentStats?: EnrichmentStats): void;
125
+ /**
126
+ * Create fresh state, discarding incremental tracking
127
+ * Used with --force-refresh flag to re-enrich everything
128
+ *
129
+ * @param totalMessages - Total messages to initialize with
130
+ * @returns Fresh IncrementalState with no enriched GUIDs
131
+ */
132
+ export declare function resetIncrementalState(totalMessages?: number): IncrementalState;
133
+ //# sourceMappingURL=incremental-state.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"incremental-state.d.ts","sourceRoot":"","sources":["../../src/utils/incremental-state.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAaH,MAAM,MAAM,eAAe,GAAG;IAC7B,cAAc,EAAE,MAAM,CAAA;IACtB,WAAW,EAAE,MAAM,CAAA;IACnB,SAAS,EAAE,MAAM,CAAA;IACjB,OAAO,EAAE,MAAM,CAAA;CACf,CAAA;AAED,MAAM,MAAM,cAAc,GAAG;IAC5B,UAAU,EAAE,MAAM,CAAA;CAElB,CAAA;AAED;;;GAGG;AACH,MAAM,MAAM,gBAAgB,GAAG;IAC9B,gDAAgD;IAChD,OAAO,EAAE,MAAM,CAAA;IAEf,oDAAoD;IACpD,cAAc,EAAE,MAAM,CAAA;IAEtB,6DAA6D;IAC7D,aAAa,EAAE,MAAM,CAAA;IAErB,4DAA4D;IAC5D,aAAa,EAAE,MAAM,EAAE,CAAA;IAEvB,2DAA2D;IAC3D,cAAc,EAAE,cAAc,CAAA;IAE9B,+CAA+C;IAC/C,eAAe,EAAE,eAAe,GAAG,IAAI,CAAA;CACvC,CAAA;AAMD,MAAM,MAAM,kBAAkB,GAAG;IAChC,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,aAAa,CAAC,EAAE,MAAM,EAAE,CAAA;IACxB,eAAe,CAAC,EAAE,eAAe,GAAG,IAAI,CAAA;CACxC,CAAA;AAED;;;;;GAKG;AACH,wBAAgB,sBAAsB,CACrC,OAAO,GAAE,kBAAuB,GAC9B,gBAAgB,CAWlB;AA0BD;;;;;;;;;GASG;AACH,wBAAgB,iBAAiB,CAChC,YAAY,EAAE,GAAG,CAAC,MAAM,CAAC,EACzB,KAAK,EAAE,gBAAgB,GACrB,MAAM,EAAE,CAWV;AAMD;;;;;;;;;;;GAWG;AACH,wBAAsB,oBAAoB,CACzC,KAAK,EAAE,gBAAgB,EACvB,QAAQ,EAAE,MAAM,GACd,OAAO,CAAC,IAAI,CAAC,CAuBf;AAED;;;;;;;;;GASG;AACH,wBAAsB,oBAAoB,CACzC,QAAQ,EAAE,MAAM,GACd,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAuBlC;AAMD;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAC/B,WAAW,EAAE,MAAM,EACnB,YAAY,GAAE,MAA6B,GACzC,OAAO,CAET;AAED;;;;;;;;GAQG;AACH,wBAAgB,eAAe,CAC9B,KAAK,EAAE,gBAAgB,EACvB,aAAa,SAAI,GACf,OAAO,CAoBT;AAMD;;;;;;;;;;;GAWG;AACH,wBAAgB,4BAA4B,CAC3C,KAAK,EAAE,gBAAgB,EACvB,QAAQ,EAAE,MAAM,EAAE,EAClB,eAAe,CAAC,EAAE,eAAe,GAC/B,IAAI,CAgBN;AAMD;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CAAC,aAAa,SAAI,GAAG,gBAAgB,CAEzE"}