@nathanvale/chatline 0.0.1 → 0.0.2-next.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (213) hide show
  1. package/CHANGELOG.md +12 -0
  2. package/dist/bin/index.js +1 -1
  3. package/dist/index.js +1 -1
  4. package/package.json +1 -1
  5. package/dist/cli/commands/clean.d.ts +0 -17
  6. package/dist/cli/commands/clean.d.ts.map +0 -1
  7. package/dist/cli/commands/clean.js +0 -142
  8. package/dist/cli/commands/clean.js.map +0 -1
  9. package/dist/cli/commands/doctor.d.ts +0 -17
  10. package/dist/cli/commands/doctor.d.ts.map +0 -1
  11. package/dist/cli/commands/doctor.js +0 -202
  12. package/dist/cli/commands/doctor.js.map +0 -1
  13. package/dist/cli/commands/enrich-ai.d.ts +0 -17
  14. package/dist/cli/commands/enrich-ai.d.ts.map +0 -1
  15. package/dist/cli/commands/enrich-ai.js +0 -371
  16. package/dist/cli/commands/enrich-ai.js.map +0 -1
  17. package/dist/cli/commands/index.d.ts +0 -16
  18. package/dist/cli/commands/index.d.ts.map +0 -1
  19. package/dist/cli/commands/index.js +0 -16
  20. package/dist/cli/commands/index.js.map +0 -1
  21. package/dist/cli/commands/ingest-csv.d.ts +0 -17
  22. package/dist/cli/commands/ingest-csv.d.ts.map +0 -1
  23. package/dist/cli/commands/ingest-csv.js +0 -138
  24. package/dist/cli/commands/ingest-csv.js.map +0 -1
  25. package/dist/cli/commands/ingest-db.d.ts +0 -17
  26. package/dist/cli/commands/ingest-db.d.ts.map +0 -1
  27. package/dist/cli/commands/ingest-db.js +0 -159
  28. package/dist/cli/commands/ingest-db.js.map +0 -1
  29. package/dist/cli/commands/init.d.ts +0 -17
  30. package/dist/cli/commands/init.d.ts.map +0 -1
  31. package/dist/cli/commands/init.js +0 -110
  32. package/dist/cli/commands/init.js.map +0 -1
  33. package/dist/cli/commands/normalize-link.d.ts +0 -16
  34. package/dist/cli/commands/normalize-link.d.ts.map +0 -1
  35. package/dist/cli/commands/normalize-link.js +0 -144
  36. package/dist/cli/commands/normalize-link.js.map +0 -1
  37. package/dist/cli/commands/render-markdown.d.ts +0 -17
  38. package/dist/cli/commands/render-markdown.d.ts.map +0 -1
  39. package/dist/cli/commands/render-markdown.js +0 -218
  40. package/dist/cli/commands/render-markdown.js.map +0 -1
  41. package/dist/cli/commands/stats.d.ts +0 -17
  42. package/dist/cli/commands/stats.d.ts.map +0 -1
  43. package/dist/cli/commands/stats.js +0 -175
  44. package/dist/cli/commands/stats.js.map +0 -1
  45. package/dist/cli/commands/validate.d.ts +0 -17
  46. package/dist/cli/commands/validate.d.ts.map +0 -1
  47. package/dist/cli/commands/validate.js +0 -152
  48. package/dist/cli/commands/validate.js.map +0 -1
  49. package/dist/cli/index.d.ts +0 -13
  50. package/dist/cli/index.d.ts.map +0 -1
  51. package/dist/cli/index.js +0 -121
  52. package/dist/cli/index.js.map +0 -1
  53. package/dist/cli/types.d.ts +0 -93
  54. package/dist/cli/types.d.ts.map +0 -1
  55. package/dist/cli/types.js +0 -7
  56. package/dist/cli/types.js.map +0 -1
  57. package/dist/cli/utils.d.ts +0 -29
  58. package/dist/cli/utils.d.ts.map +0 -1
  59. package/dist/cli/utils.js +0 -53
  60. package/dist/cli/utils.js.map +0 -1
  61. package/dist/cli.d.ts +0 -9
  62. package/dist/cli.d.ts.map +0 -1
  63. package/dist/cli.js +0 -1805
  64. package/dist/config/generator.d.ts +0 -90
  65. package/dist/config/generator.d.ts.map +0 -1
  66. package/dist/config/generator.js +0 -320
  67. package/dist/config/generator.js.map +0 -1
  68. package/dist/config/loader.d.ts +0 -107
  69. package/dist/config/loader.d.ts.map +0 -1
  70. package/dist/config/loader.js +0 -251
  71. package/dist/config/loader.js.map +0 -1
  72. package/dist/config/schema.d.ts +0 -107
  73. package/dist/config/schema.d.ts.map +0 -1
  74. package/dist/config/schema.js +0 -169
  75. package/dist/config/schema.js.map +0 -1
  76. package/dist/enrich/audio-transcription.d.ts +0 -77
  77. package/dist/enrich/audio-transcription.d.ts.map +0 -1
  78. package/dist/enrich/audio-transcription.js +0 -370
  79. package/dist/enrich/audio-transcription.js.map +0 -1
  80. package/dist/enrich/checkpoint.d.ts +0 -137
  81. package/dist/enrich/checkpoint.d.ts.map +0 -1
  82. package/dist/enrich/checkpoint.js +0 -205
  83. package/dist/enrich/checkpoint.js.map +0 -1
  84. package/dist/enrich/idempotency.d.ts +0 -90
  85. package/dist/enrich/idempotency.d.ts.map +0 -1
  86. package/dist/enrich/idempotency.js +0 -188
  87. package/dist/enrich/idempotency.js.map +0 -1
  88. package/dist/enrich/image-analysis.d.ts +0 -62
  89. package/dist/enrich/image-analysis.d.ts.map +0 -1
  90. package/dist/enrich/image-analysis.js +0 -264
  91. package/dist/enrich/image-analysis.js.map +0 -1
  92. package/dist/enrich/index.d.ts +0 -60
  93. package/dist/enrich/index.d.ts.map +0 -1
  94. package/dist/enrich/index.js +0 -74
  95. package/dist/enrich/index.js.map +0 -1
  96. package/dist/enrich/link-enrichment.d.ts +0 -37
  97. package/dist/enrich/link-enrichment.d.ts.map +0 -1
  98. package/dist/enrich/link-enrichment.js +0 -202
  99. package/dist/enrich/link-enrichment.js.map +0 -1
  100. package/dist/enrich/pdf-video-handling.d.ts +0 -49
  101. package/dist/enrich/pdf-video-handling.d.ts.map +0 -1
  102. package/dist/enrich/pdf-video-handling.js +0 -325
  103. package/dist/enrich/pdf-video-handling.js.map +0 -1
  104. package/dist/enrich/progress-tracker.d.ts +0 -120
  105. package/dist/enrich/progress-tracker.d.ts.map +0 -1
  106. package/dist/enrich/progress-tracker.js +0 -220
  107. package/dist/enrich/progress-tracker.js.map +0 -1
  108. package/dist/enrich/providers/firecrawl.d.ts +0 -18
  109. package/dist/enrich/providers/firecrawl.d.ts.map +0 -1
  110. package/dist/enrich/providers/firecrawl.js +0 -48
  111. package/dist/enrich/providers/firecrawl.js.map +0 -1
  112. package/dist/enrich/providers/generic.d.ts +0 -16
  113. package/dist/enrich/providers/generic.d.ts.map +0 -1
  114. package/dist/enrich/providers/generic.js +0 -36
  115. package/dist/enrich/providers/generic.js.map +0 -1
  116. package/dist/enrich/providers/index.d.ts +0 -14
  117. package/dist/enrich/providers/index.d.ts.map +0 -1
  118. package/dist/enrich/providers/index.js +0 -13
  119. package/dist/enrich/providers/index.js.map +0 -1
  120. package/dist/enrich/providers/instagram.d.ts +0 -16
  121. package/dist/enrich/providers/instagram.d.ts.map +0 -1
  122. package/dist/enrich/providers/instagram.js +0 -43
  123. package/dist/enrich/providers/instagram.js.map +0 -1
  124. package/dist/enrich/providers/spotify.d.ts +0 -16
  125. package/dist/enrich/providers/spotify.d.ts.map +0 -1
  126. package/dist/enrich/providers/spotify.js +0 -45
  127. package/dist/enrich/providers/spotify.js.map +0 -1
  128. package/dist/enrich/providers/twitter.d.ts +0 -16
  129. package/dist/enrich/providers/twitter.d.ts.map +0 -1
  130. package/dist/enrich/providers/twitter.js +0 -43
  131. package/dist/enrich/providers/twitter.js.map +0 -1
  132. package/dist/enrich/providers/types.d.ts +0 -47
  133. package/dist/enrich/providers/types.d.ts.map +0 -1
  134. package/dist/enrich/providers/types.js +0 -15
  135. package/dist/enrich/providers/types.js.map +0 -1
  136. package/dist/enrich/providers/youtube.d.ts +0 -16
  137. package/dist/enrich/providers/youtube.d.ts.map +0 -1
  138. package/dist/enrich/providers/youtube.js +0 -43
  139. package/dist/enrich/providers/youtube.js.map +0 -1
  140. package/dist/enrich/rate-limiting.d.ts +0 -118
  141. package/dist/enrich/rate-limiting.d.ts.map +0 -1
  142. package/dist/enrich/rate-limiting.js +0 -258
  143. package/dist/enrich/rate-limiting.js.map +0 -1
  144. package/dist/index.d.ts.map +0 -1
  145. package/dist/index.js.map +0 -1
  146. package/dist/ingest/dedup-merge.d.ts +0 -82
  147. package/dist/ingest/dedup-merge.d.ts.map +0 -1
  148. package/dist/ingest/dedup-merge.js +0 -262
  149. package/dist/ingest/dedup-merge.js.map +0 -1
  150. package/dist/ingest/ingest-csv.d.ts +0 -62
  151. package/dist/ingest/ingest-csv.d.ts.map +0 -1
  152. package/dist/ingest/ingest-csv.js +0 -300
  153. package/dist/ingest/ingest-csv.js.map +0 -1
  154. package/dist/ingest/ingest-db.d.ts +0 -64
  155. package/dist/ingest/ingest-db.d.ts.map +0 -1
  156. package/dist/ingest/ingest-db.js +0 -172
  157. package/dist/ingest/ingest-db.js.map +0 -1
  158. package/dist/ingest/link-replies-and-tapbacks.d.ts +0 -53
  159. package/dist/ingest/link-replies-and-tapbacks.d.ts.map +0 -1
  160. package/dist/ingest/link-replies-and-tapbacks.js +0 -381
  161. package/dist/ingest/link-replies-and-tapbacks.js.map +0 -1
  162. package/dist/normalize/date-converters.d.ts +0 -45
  163. package/dist/normalize/date-converters.d.ts.map +0 -1
  164. package/dist/normalize/date-converters.js +0 -166
  165. package/dist/normalize/date-converters.js.map +0 -1
  166. package/dist/normalize/path-validator.d.ts +0 -65
  167. package/dist/normalize/path-validator.d.ts.map +0 -1
  168. package/dist/normalize/path-validator.js +0 -221
  169. package/dist/normalize/path-validator.js.map +0 -1
  170. package/dist/normalize/validate-normalized.d.ts +0 -45
  171. package/dist/normalize/validate-normalized.d.ts.map +0 -1
  172. package/dist/normalize/validate-normalized.js +0 -144
  173. package/dist/normalize/validate-normalized.js.map +0 -1
  174. package/dist/render/embeds-blockquotes.d.ts +0 -84
  175. package/dist/render/embeds-blockquotes.d.ts.map +0 -1
  176. package/dist/render/embeds-blockquotes.js +0 -204
  177. package/dist/render/embeds-blockquotes.js.map +0 -1
  178. package/dist/render/grouping.d.ts +0 -78
  179. package/dist/render/grouping.d.ts.map +0 -1
  180. package/dist/render/grouping.js +0 -134
  181. package/dist/render/grouping.js.map +0 -1
  182. package/dist/render/index.d.ts +0 -47
  183. package/dist/render/index.d.ts.map +0 -1
  184. package/dist/render/index.js +0 -245
  185. package/dist/render/index.js.map +0 -1
  186. package/dist/render/reply-rendering.d.ts +0 -88
  187. package/dist/render/reply-rendering.d.ts.map +0 -1
  188. package/dist/render/reply-rendering.js +0 -196
  189. package/dist/render/reply-rendering.js.map +0 -1
  190. package/dist/schema/message.d.ts +0 -125
  191. package/dist/schema/message.d.ts.map +0 -1
  192. package/dist/schema/message.js +0 -331
  193. package/dist/schema/message.js.map +0 -1
  194. package/dist/utils/delta-detection.d.ts +0 -107
  195. package/dist/utils/delta-detection.d.ts.map +0 -1
  196. package/dist/utils/delta-detection.js +0 -199
  197. package/dist/utils/delta-detection.js.map +0 -1
  198. package/dist/utils/enrichment-merge.d.ts +0 -135
  199. package/dist/utils/enrichment-merge.d.ts.map +0 -1
  200. package/dist/utils/enrichment-merge.js +0 -280
  201. package/dist/utils/enrichment-merge.js.map +0 -1
  202. package/dist/utils/human.d.ts +0 -15
  203. package/dist/utils/human.d.ts.map +0 -1
  204. package/dist/utils/human.js +0 -27
  205. package/dist/utils/human.js.map +0 -1
  206. package/dist/utils/incremental-state.d.ts +0 -133
  207. package/dist/utils/incremental-state.d.ts.map +0 -1
  208. package/dist/utils/incremental-state.js +0 -237
  209. package/dist/utils/incremental-state.js.map +0 -1
  210. package/dist/utils/logger.d.ts +0 -40
  211. package/dist/utils/logger.d.ts.map +0 -1
  212. package/dist/utils/logger.js +0 -176
  213. package/dist/utils/logger.js.map +0 -1
@@ -1,47 +0,0 @@
1
- /**
2
- * Link Enrichment Provider Types
3
- *
4
- * Shared types for link enrichment providers.
5
- */
6
- /**
7
- * Link context extracted from a URL
8
- */
9
- export type LinkContext = {
10
- url: string;
11
- title?: string;
12
- description?: string;
13
- provider: 'gemini' | 'firecrawl' | 'local' | 'youtube' | 'spotify' | 'twitter' | 'instagram' | 'generic';
14
- usedFallback?: boolean;
15
- failedProviders?: string[];
16
- };
17
- /**
18
- * Configuration for link enrichment
19
- */
20
- export type LinkEnrichmentConfig = {
21
- enableLinkAnalysis: boolean;
22
- firecrawlApiKey?: string;
23
- youtubeApiKey?: string;
24
- spotifyClientId?: string;
25
- spotifyClientSecret?: string;
26
- rateLimitDelay?: number;
27
- maxRetries?: number;
28
- };
29
- /**
30
- * Provider interface for cascading fallbacks
31
- */
32
- export interface Provider {
33
- name: string;
34
- priority: number;
35
- detect: (url: string) => boolean;
36
- extract: (url: string) => Promise<LinkContext>;
37
- }
38
- /**
39
- * URL detection patterns for different provider types
40
- */
41
- export declare const URL_PATTERNS: {
42
- youtube: RegExp;
43
- spotify: RegExp;
44
- twitter: RegExp;
45
- instagram: RegExp;
46
- };
47
- //# sourceMappingURL=types.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/enrich/providers/types.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH;;GAEG;AACH,MAAM,MAAM,WAAW,GAAG;IACzB,GAAG,EAAE,MAAM,CAAA;IACX,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,QAAQ,EACL,QAAQ,GACR,WAAW,GACX,OAAO,GACP,SAAS,GACT,SAAS,GACT,SAAS,GACT,WAAW,GACX,SAAS,CAAA;IACZ,YAAY,CAAC,EAAE,OAAO,CAAA;IACtB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAA;CAC1B,CAAA;AAED;;GAEG;AACH,MAAM,MAAM,oBAAoB,GAAG;IAClC,kBAAkB,EAAE,OAAO,CAAA;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,UAAU,CAAC,EAAE,MAAM,CAAA;CACnB,CAAA;AAED;;GAEG;AACH,MAAM,WAAW,QAAQ;IACxB,IAAI,EAAE,MAAM,CAAA;IACZ,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAA;IAChC,OAAO,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,WAAW,CAAC,CAAA;CAC9C;AAED;;GAEG;AACH,eAAO,MAAM,YAAY;;;;;CAMxB,CAAA"}
@@ -1,15 +0,0 @@
1
- /**
2
- * Link Enrichment Provider Types
3
- *
4
- * Shared types for link enrichment providers.
5
- */
6
- /**
7
- * URL detection patterns for different provider types
8
- */
9
- export const URL_PATTERNS = {
10
- youtube: /^https?:\/\/(www\.)?(youtube\.com|youtu\.be|youtube-nocookie\.com)/i,
11
- spotify: /^https?:\/\/open\.spotify\.com\/(track|album|playlist|artist)/i,
12
- twitter: /^https?:\/\/(www\.)?(twitter\.com|x\.com)/i,
13
- instagram: /^https?:\/\/(www\.)?instagram\.com/i,
14
- };
15
- //# sourceMappingURL=types.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/enrich/providers/types.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AA6CH;;GAEG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG;IAC3B,OAAO,EACN,qEAAqE;IACtE,OAAO,EAAE,gEAAgE;IACzE,OAAO,EAAE,4CAA4C;IACrD,SAAS,EAAE,qCAAqC;CAChD,CAAA"}
@@ -1,16 +0,0 @@
1
- /**
2
- * YouTube Provider (ENRICH--T04-AC02)
3
- *
4
- * Fallback provider for YouTube video metadata extraction.
5
- */
6
- import type { LinkContext, Provider } from './types.js';
7
- /**
8
- * AC02: YouTube Provider (fallback)
9
- */
10
- export declare class YouTubeProvider implements Provider {
11
- name: string;
12
- priority: number;
13
- detect(url: string): boolean;
14
- extract(url: string): Promise<LinkContext>;
15
- }
16
- //# sourceMappingURL=youtube.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"youtube.d.ts","sourceRoot":"","sources":["../../../src/enrich/providers/youtube.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAA;AAKvD;;GAEG;AACH,qBAAa,eAAgB,YAAW,QAAQ;IAC/C,IAAI,SAAY;IAChB,QAAQ,SAAI;IAEZ,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO;IAItB,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;CA4BhD"}
@@ -1,43 +0,0 @@
1
- /**
2
- * YouTube Provider (ENRICH--T04-AC02)
3
- *
4
- * Fallback provider for YouTube video metadata extraction.
5
- */
6
- import { createLogger } from '#utils/logger';
7
- import { URL_PATTERNS } from './types.js';
8
- const logger = createLogger('enrich:youtube-provider');
9
- /**
10
- * AC02: YouTube Provider (fallback)
11
- */
12
- export class YouTubeProvider {
13
- name = 'youtube';
14
- priority = 1;
15
- detect(url) {
16
- return URL_PATTERNS.youtube.test(url);
17
- }
18
- async extract(url) {
19
- try {
20
- // Extract video ID from URL
21
- const videoIdMatch = url.match(/(?:youtube\.com\/watch\?v=|youtu\.be\/)([a-zA-Z0-9_-]{11})/);
22
- const videoId = videoIdMatch?.[1];
23
- if (!videoId) {
24
- throw new Error('Could not extract YouTube video ID');
25
- }
26
- // In production, would fetch from YouTube API or parse HTML
27
- const title = `YouTube Video: ${videoId}`;
28
- logger.debug(`YouTube extracted metadata for ${url}`);
29
- return {
30
- url,
31
- title,
32
- description: 'YouTube video',
33
- provider: 'youtube',
34
- usedFallback: true,
35
- };
36
- }
37
- catch (error) {
38
- logger.error(`YouTube extraction failed for ${url}`, { error });
39
- throw error;
40
- }
41
- }
42
- }
43
- //# sourceMappingURL=youtube.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"youtube.js","sourceRoot":"","sources":["../../../src/enrich/providers/youtube.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAE5C,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AAEzC,MAAM,MAAM,GAAG,YAAY,CAAC,yBAAyB,CAAC,CAAA;AAEtD;;GAEG;AACH,MAAM,OAAO,eAAe;IAC3B,IAAI,GAAG,SAAS,CAAA;IAChB,QAAQ,GAAG,CAAC,CAAA;IAEZ,MAAM,CAAC,GAAW;QACjB,OAAO,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IACtC,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,GAAW;QACxB,IAAI,CAAC;YACJ,4BAA4B;YAC5B,MAAM,YAAY,GAAG,GAAG,CAAC,KAAK,CAC7B,4DAA4D,CAC5D,CAAA;YACD,MAAM,OAAO,GAAG,YAAY,EAAE,CAAC,CAAC,CAAC,CAAA;YAEjC,IAAI,CAAC,OAAO,EAAE,CAAC;gBACd,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAA;YACtD,CAAC;YAED,4DAA4D;YAC5D,MAAM,KAAK,GAAG,kBAAkB,OAAO,EAAE,CAAA;YAEzC,MAAM,CAAC,KAAK,CAAC,kCAAkC,GAAG,EAAE,CAAC,CAAA;YACrD,OAAO;gBACN,GAAG;gBACH,KAAK;gBACL,WAAW,EAAE,eAAe;gBAC5B,QAAQ,EAAE,SAAS;gBACnB,YAAY,EAAE,IAAI;aAClB,CAAA;QACF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,CAAC,KAAK,CAAC,iCAAiC,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CAAA;YAC/D,MAAM,KAAK,CAAA;QACZ,CAAC;IACF,CAAC;CACD"}
@@ -1,118 +0,0 @@
1
- /**
2
- * Rate Limiting and Retry Logic Module (ENRICH--T07)
3
- *
4
- * Implements comprehensive rate limiting with:
5
- * - AC01: Configurable delays between API calls (default 1000ms)
6
- * - AC02: Exponential backoff for 429 responses with ±25% jitter
7
- * - AC03: Retry 5xx errors with maxRetries limit (default 3)
8
- * - AC04: Respect Retry-After header for 429/503 responses
9
- * - AC05: Circuit breaker after N consecutive failures (default 5)
10
- *
11
- * Architecture:
12
- * - RateLimiter: Main class managing all rate limiting logic
13
- * - Configuration with sensible defaults
14
- * - Per-provider state isolation
15
- * - Deterministic jitter calculation
16
- */
17
- export type RateLimitConfig = {
18
- rateLimitDelay: number;
19
- maxRetries: number;
20
- circuitBreakerThreshold: number;
21
- circuitBreakerResetMs: number;
22
- };
23
- export type RateLimitState = {
24
- consecutiveFailures: number;
25
- circuitOpen: boolean;
26
- circuitOpenedAt: number | null;
27
- lastCallTime: number | null;
28
- };
29
- export type ApiResponse = {
30
- status: number;
31
- headers?: Record<string, string | number | undefined>;
32
- };
33
- export declare class RateLimiter {
34
- private config;
35
- private state;
36
- constructor(partialConfig?: Partial<RateLimitConfig>);
37
- private validateConfig;
38
- /**
39
- * Check if rate limiting should delay the next call
40
- * @returns delay in ms, or 0 if no delay needed
41
- */
42
- shouldRateLimit(): number;
43
- /**
44
- * Record a successful API call for rate limiting tracking
45
- */
46
- recordCall(): void;
47
- /**
48
- * Calculate exponential backoff with ±25% jitter
49
- * Formula: 2^n seconds ± 25%
50
- * @param attemptNumber - retry attempt number (1-based)
51
- * @returns delay in ms
52
- */
53
- private calculateExponentialBackoff;
54
- /**
55
- * Parse Retry-After header (can be integer seconds or HTTP date)
56
- * @param retryAfterValue - header value
57
- * @returns delay in ms, or null if invalid
58
- */
59
- private parseRetryAfterHeader;
60
- /**
61
- * Determine if response should be retried and calculate delay
62
- * @param response - API response with status and optional headers
63
- * @param attemptNumber - current attempt (1-based)
64
- * @returns { shouldRetry: boolean, delayMs: number }
65
- */
66
- getRetryStrategy(response: ApiResponse, attemptNumber: number): {
67
- shouldRetry: boolean;
68
- delayMs: number;
69
- };
70
- /**
71
- * Check if we should retry based on attempt count
72
- * @param attemptNumber - current attempt (1-based)
73
- * @returns true if we should retry
74
- */
75
- shouldRetryAttempt(attemptNumber: number): boolean;
76
- /**
77
- * Check if circuit breaker is open
78
- * @returns true if circuit is open (should fail fast)
79
- */
80
- isCircuitOpen(): boolean;
81
- /**
82
- * Record a failure for circuit breaker tracking
83
- */
84
- recordFailure(): void;
85
- /**
86
- * Record a success to reset failure counter
87
- */
88
- recordSuccess(): void;
89
- /**
90
- * Manually reset circuit breaker
91
- */
92
- resetCircuitBreaker(): void;
93
- /**
94
- * Get current state for inspection
95
- */
96
- getState(): RateLimitState;
97
- /**
98
- * Get configuration
99
- */
100
- getConfig(): RateLimitConfig;
101
- /**
102
- * Reset all state (for testing)
103
- */
104
- reset(): void;
105
- }
106
- /**
107
- * Create a new rate limiter with default configuration
108
- */
109
- export declare function createRateLimiter(config?: Partial<RateLimitConfig>): RateLimiter;
110
- /**
111
- * Determine if a status code is a 5xx error (server error)
112
- */
113
- export declare function is5xx(status: number): boolean;
114
- /**
115
- * Determine if a status code should trigger retry
116
- */
117
- export declare function isRetryableStatus(status: number): boolean;
118
- //# sourceMappingURL=rate-limiting.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"rate-limiting.d.ts","sourceRoot":"","sources":["../../src/enrich/rate-limiting.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAMH,MAAM,MAAM,eAAe,GAAG;IAC7B,cAAc,EAAE,MAAM,CAAA;IACtB,UAAU,EAAE,MAAM,CAAA;IAClB,uBAAuB,EAAE,MAAM,CAAA;IAC/B,qBAAqB,EAAE,MAAM,CAAA;CAC7B,CAAA;AAED,MAAM,MAAM,cAAc,GAAG;IAC5B,mBAAmB,EAAE,MAAM,CAAA;IAC3B,WAAW,EAAE,OAAO,CAAA;IACpB,eAAe,EAAE,MAAM,GAAG,IAAI,CAAA;IAC9B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAA;CAC3B,CAAA;AAED,MAAM,MAAM,WAAW,GAAG;IACzB,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC,CAAA;CACrD,CAAA;AAMD,qBAAa,WAAW;IACvB,OAAO,CAAC,MAAM,CAAiB;IAC/B,OAAO,CAAC,KAAK,CAAgB;gBAEjB,aAAa,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC;IAqBpD,OAAO,CAAC,cAAc;IAetB;;;OAGG;IACI,eAAe,IAAI,MAAM;IAgBhC;;OAEG;IACI,UAAU,IAAI,IAAI;IAQzB;;;;;OAKG;IACH,OAAO,CAAC,2BAA2B;IAgBnC;;;;OAIG;IACH,OAAO,CAAC,qBAAqB;IAuC7B;;;;;OAKG;IACI,gBAAgB,CACtB,QAAQ,EAAE,WAAW,EACrB,aAAa,EAAE,MAAM,GACnB;QAAE,WAAW,EAAE,OAAO,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE;IAyB5C;;;;OAIG;IACI,kBAAkB,CAAC,aAAa,EAAE,MAAM,GAAG,OAAO;IAQzD;;;OAGG;IACI,aAAa,IAAI,OAAO;IAgB/B;;OAEG;IACI,aAAa,IAAI,IAAI;IAS5B;;OAEG;IACI,aAAa,IAAI,IAAI;IAM5B;;OAEG;IACI,mBAAmB,IAAI,IAAI;IAMlC;;OAEG;IACI,QAAQ,IAAI,cAAc;IAIjC;;OAEG;IACI,SAAS,IAAI,eAAe;IAInC;;OAEG;IACI,KAAK,IAAI,IAAI;CAQpB;AAMD;;GAEG;AACH,wBAAgB,iBAAiB,CAChC,MAAM,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,GAC/B,WAAW,CAEb;AAED;;GAEG;AACH,wBAAgB,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAE7C;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAEzD"}
@@ -1,258 +0,0 @@
1
- /**
2
- * Rate Limiting and Retry Logic Module (ENRICH--T07)
3
- *
4
- * Implements comprehensive rate limiting with:
5
- * - AC01: Configurable delays between API calls (default 1000ms)
6
- * - AC02: Exponential backoff for 429 responses with ±25% jitter
7
- * - AC03: Retry 5xx errors with maxRetries limit (default 3)
8
- * - AC04: Respect Retry-After header for 429/503 responses
9
- * - AC05: Circuit breaker after N consecutive failures (default 5)
10
- *
11
- * Architecture:
12
- * - RateLimiter: Main class managing all rate limiting logic
13
- * - Configuration with sensible defaults
14
- * - Per-provider state isolation
15
- * - Deterministic jitter calculation
16
- */
17
- // ============================================================================
18
- // AC01 + AC02 + AC03 + AC04 + AC05: Main RateLimiter Class
19
- // ============================================================================
20
- export class RateLimiter {
21
- config;
22
- state;
23
- constructor(partialConfig) {
24
- // Set defaults and merge with provided config
25
- this.config = {
26
- rateLimitDelay: partialConfig?.rateLimitDelay ?? 1000,
27
- maxRetries: partialConfig?.maxRetries ?? 3,
28
- circuitBreakerThreshold: partialConfig?.circuitBreakerThreshold ?? 5,
29
- circuitBreakerResetMs: partialConfig?.circuitBreakerResetMs ?? 60000,
30
- };
31
- // Validate config
32
- this.validateConfig(this.config);
33
- // Initialize state
34
- this.state = {
35
- consecutiveFailures: 0,
36
- circuitOpen: false,
37
- circuitOpenedAt: null,
38
- lastCallTime: null,
39
- };
40
- }
41
- validateConfig(config) {
42
- if (config.rateLimitDelay < 0)
43
- throw new Error('rateLimitDelay must be non-negative');
44
- if (config.maxRetries < 0)
45
- throw new Error('maxRetries must be non-negative');
46
- if (config.circuitBreakerThreshold < 1)
47
- throw new Error('circuitBreakerThreshold must be >= 1');
48
- if (config.circuitBreakerResetMs < 0)
49
- throw new Error('circuitBreakerResetMs must be non-negative');
50
- }
51
- // ============================================================================
52
- // AC01: Configurable Rate Limit Delays
53
- // ============================================================================
54
- /**
55
- * Check if rate limiting should delay the next call
56
- * @returns delay in ms, or 0 if no delay needed
57
- */
58
- shouldRateLimit() {
59
- // No delay for first call
60
- if (this.state.lastCallTime === null) {
61
- return 0;
62
- }
63
- const timeSinceLastCall = Date.now() - this.state.lastCallTime;
64
- const requiredDelay = this.config.rateLimitDelay;
65
- if (timeSinceLastCall < requiredDelay) {
66
- return requiredDelay - timeSinceLastCall;
67
- }
68
- return 0;
69
- }
70
- /**
71
- * Record a successful API call for rate limiting tracking
72
- */
73
- recordCall() {
74
- this.state.lastCallTime = Date.now();
75
- }
76
- // ============================================================================
77
- // AC02: Exponential Backoff with Jitter
78
- // ============================================================================
79
- /**
80
- * Calculate exponential backoff with ±25% jitter
81
- * Formula: 2^n seconds ± 25%
82
- * @param attemptNumber - retry attempt number (1-based)
83
- * @returns delay in ms
84
- */
85
- calculateExponentialBackoff(attemptNumber) {
86
- // Base calculation: 2^n seconds
87
- const baseDelaySeconds = 2 ** attemptNumber;
88
- const baseDelayMs = baseDelaySeconds * 1000;
89
- // Apply ±25% jitter
90
- const jitterAmount = baseDelayMs * 0.25;
91
- const jitter = (Math.random() - 0.5) * 2 * jitterAmount;
92
- return baseDelayMs + jitter;
93
- }
94
- // ============================================================================
95
- // AC04: Retry-After Header Parsing
96
- // ============================================================================
97
- /**
98
- * Parse Retry-After header (can be integer seconds or HTTP date)
99
- * @param retryAfterValue - header value
100
- * @returns delay in ms, or null if invalid
101
- */
102
- parseRetryAfterHeader(retryAfterValue) {
103
- if (retryAfterValue === undefined || retryAfterValue === null) {
104
- return null;
105
- }
106
- // Handle numeric (seconds)
107
- if (typeof retryAfterValue === 'number') {
108
- return retryAfterValue * 1000;
109
- }
110
- const strValue = String(retryAfterValue).trim();
111
- // Try parsing as integer seconds
112
- const seconds = Number.parseInt(strValue, 10);
113
- if (!Number.isNaN(seconds) && seconds >= 0) {
114
- return seconds * 1000;
115
- }
116
- // Try parsing as HTTP date
117
- try {
118
- const date = new Date(strValue);
119
- if (!Number.isNaN(date.getTime())) {
120
- const delayMs = date.getTime() - Date.now();
121
- return Math.max(0, delayMs);
122
- }
123
- }
124
- catch {
125
- // Invalid date format
126
- }
127
- // Invalid format - return null to use calculated backoff
128
- return null;
129
- }
130
- // ============================================================================
131
- // AC03: Retry 5xx Errors + AC04: Retry-After Integration
132
- // ============================================================================
133
- /**
134
- * Determine if response should be retried and calculate delay
135
- * @param response - API response with status and optional headers
136
- * @param attemptNumber - current attempt (1-based)
137
- * @returns { shouldRetry: boolean, delayMs: number }
138
- */
139
- getRetryStrategy(response, attemptNumber) {
140
- const { status, headers } = response;
141
- // Don't retry on success (2xx)
142
- if (status >= 200 && status < 300) {
143
- return { shouldRetry: false, delayMs: 0 };
144
- }
145
- // Retry on 429 or 5xx
146
- const isRetryableStatus = status === 429 || (status >= 500 && status < 600);
147
- if (!isRetryableStatus) {
148
- return { shouldRetry: false, delayMs: 0 };
149
- }
150
- // AC04: Check for Retry-After header (takes precedence)
151
- const retryAfterMs = this.parseRetryAfterHeader(headers?.['Retry-After']);
152
- if (retryAfterMs !== null) {
153
- return { shouldRetry: true, delayMs: retryAfterMs };
154
- }
155
- // AC02: Use exponential backoff with jitter
156
- const backoffMs = this.calculateExponentialBackoff(attemptNumber);
157
- return { shouldRetry: true, delayMs: backoffMs };
158
- }
159
- /**
160
- * Check if we should retry based on attempt count
161
- * @param attemptNumber - current attempt (1-based)
162
- * @returns true if we should retry
163
- */
164
- shouldRetryAttempt(attemptNumber) {
165
- return attemptNumber <= this.config.maxRetries;
166
- }
167
- // ============================================================================
168
- // AC05: Circuit Breaker
169
- // ============================================================================
170
- /**
171
- * Check if circuit breaker is open
172
- * @returns true if circuit is open (should fail fast)
173
- */
174
- isCircuitOpen() {
175
- if (!this.state.circuitOpen) {
176
- return false;
177
- }
178
- // Check if circuit breaker timeout has elapsed
179
- const timeSinceOpened = Date.now() - (this.state.circuitOpenedAt ?? 0);
180
- if (timeSinceOpened >= this.config.circuitBreakerResetMs) {
181
- // Reset circuit breaker
182
- this.resetCircuitBreaker();
183
- return false;
184
- }
185
- return true;
186
- }
187
- /**
188
- * Record a failure for circuit breaker tracking
189
- */
190
- recordFailure() {
191
- this.state.consecutiveFailures += 1;
192
- if (this.state.consecutiveFailures >= this.config.circuitBreakerThreshold) {
193
- this.state.circuitOpen = true;
194
- this.state.circuitOpenedAt = Date.now();
195
- }
196
- }
197
- /**
198
- * Record a success to reset failure counter
199
- */
200
- recordSuccess() {
201
- this.state.consecutiveFailures = 0;
202
- this.state.circuitOpen = false;
203
- this.state.circuitOpenedAt = null;
204
- }
205
- /**
206
- * Manually reset circuit breaker
207
- */
208
- resetCircuitBreaker() {
209
- this.state.consecutiveFailures = 0;
210
- this.state.circuitOpen = false;
211
- this.state.circuitOpenedAt = null;
212
- }
213
- /**
214
- * Get current state for inspection
215
- */
216
- getState() {
217
- return { ...this.state };
218
- }
219
- /**
220
- * Get configuration
221
- */
222
- getConfig() {
223
- return { ...this.config };
224
- }
225
- /**
226
- * Reset all state (for testing)
227
- */
228
- reset() {
229
- this.state = {
230
- consecutiveFailures: 0,
231
- circuitOpen: false,
232
- circuitOpenedAt: null,
233
- lastCallTime: null,
234
- };
235
- }
236
- }
237
- // ============================================================================
238
- // Utility Functions
239
- // ============================================================================
240
- /**
241
- * Create a new rate limiter with default configuration
242
- */
243
- export function createRateLimiter(config) {
244
- return new RateLimiter(config);
245
- }
246
- /**
247
- * Determine if a status code is a 5xx error (server error)
248
- */
249
- export function is5xx(status) {
250
- return status >= 500 && status < 600;
251
- }
252
- /**
253
- * Determine if a status code should trigger retry
254
- */
255
- export function isRetryableStatus(status) {
256
- return status === 429 || is5xx(status);
257
- }
258
- //# sourceMappingURL=rate-limiting.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"rate-limiting.js","sourceRoot":"","sources":["../../src/enrich/rate-limiting.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAyBH,+EAA+E;AAC/E,2DAA2D;AAC3D,+EAA+E;AAE/E,MAAM,OAAO,WAAW;IACf,MAAM,CAAiB;IACvB,KAAK,CAAgB;IAE7B,YAAY,aAAwC;QACnD,8CAA8C;QAC9C,IAAI,CAAC,MAAM,GAAG;YACb,cAAc,EAAE,aAAa,EAAE,cAAc,IAAI,IAAI;YACrD,UAAU,EAAE,aAAa,EAAE,UAAU,IAAI,CAAC;YAC1C,uBAAuB,EAAE,aAAa,EAAE,uBAAuB,IAAI,CAAC;YACpE,qBAAqB,EAAE,aAAa,EAAE,qBAAqB,IAAI,KAAK;SACpE,CAAA;QAED,kBAAkB;QAClB,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QAEhC,mBAAmB;QACnB,IAAI,CAAC,KAAK,GAAG;YACZ,mBAAmB,EAAE,CAAC;YACtB,WAAW,EAAE,KAAK;YAClB,eAAe,EAAE,IAAI;YACrB,YAAY,EAAE,IAAI;SAClB,CAAA;IACF,CAAC;IAEO,cAAc,CAAC,MAAuB;QAC7C,IAAI,MAAM,CAAC,cAAc,GAAG,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAA;QACvD,IAAI,MAAM,CAAC,UAAU,GAAG,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAA;QACnD,IAAI,MAAM,CAAC,uBAAuB,GAAG,CAAC;YACrC,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAA;QACxD,IAAI,MAAM,CAAC,qBAAqB,GAAG,CAAC;YACnC,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAA;IAC/D,CAAC;IAED,+EAA+E;IAC/E,uCAAuC;IACvC,+EAA+E;IAE/E;;;OAGG;IACI,eAAe;QACrB,0BAA0B;QAC1B,IAAI,IAAI,CAAC,KAAK,CAAC,YAAY,KAAK,IAAI,EAAE,CAAC;YACtC,OAAO,CAAC,CAAA;QACT,CAAC;QAED,MAAM,iBAAiB,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAA;QAC9D,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,cAAc,CAAA;QAEhD,IAAI,iBAAiB,GAAG,aAAa,EAAE,CAAC;YACvC,OAAO,aAAa,GAAG,iBAAiB,CAAA;QACzC,CAAC;QAED,OAAO,CAAC,CAAA;IACT,CAAC;IAED;;OAEG;IACI,UAAU;QAChB,IAAI,CAAC,KAAK,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;IACrC,CAAC;IAED,+EAA+E;IAC/E,wCAAwC;IACxC,+EAA+E;IAE/E;;;;;OAKG;IACK,2BAA2B,CAAC,aAAqB;QACxD,gCAAgC;QAChC,MAAM,gBAAgB,GAAG,CAAC,IAAI,aAAa,CAAA;QAC3C,MAAM,WAAW,GAAG,gBAAgB,GAAG,IAAI,CAAA;QAE3C,oBAAoB;QACpB,MAAM,YAAY,GAAG,WAAW,GAAG,IAAI,CAAA;QACvC,MAAM,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,YAAY,CAAA;QAEvD,OAAO,WAAW,GAAG,MAAM,CAAA;IAC5B,CAAC;IAED,+EAA+E;IAC/E,mCAAmC;IACnC,+EAA+E;IAE/E;;;;OAIG;IACK,qBAAqB,CAC5B,eAA4C;QAE5C,IAAI,eAAe,KAAK,SAAS,IAAI,eAAe,KAAK,IAAI,EAAE,CAAC;YAC/D,OAAO,IAAI,CAAA;QACZ,CAAC;QAED,2BAA2B;QAC3B,IAAI,OAAO,eAAe,KAAK,QAAQ,EAAE,CAAC;YACzC,OAAO,eAAe,GAAG,IAAI,CAAA;QAC9B,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,CAAC,eAAe,CAAC,CAAC,IAAI,EAAE,CAAA;QAE/C,iCAAiC;QACjC,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAA;QAC7C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,OAAO,IAAI,CAAC,EAAE,CAAC;YAC5C,OAAO,OAAO,GAAG,IAAI,CAAA;QACtB,CAAC;QAED,2BAA2B;QAC3B,IAAI,CAAC;YACJ,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,QAAQ,CAAC,CAAA;YAC/B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;gBACnC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;gBAC3C,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAA;YAC5B,CAAC;QACF,CAAC;QAAC,MAAM,CAAC;YACR,sBAAsB;QACvB,CAAC;QAED,yDAAyD;QACzD,OAAO,IAAI,CAAA;IACZ,CAAC;IAED,+EAA+E;IAC/E,yDAAyD;IACzD,+EAA+E;IAE/E;;;;;OAKG;IACI,gBAAgB,CACtB,QAAqB,EACrB,aAAqB;QAErB,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,QAAQ,CAAA;QAEpC,+BAA+B;QAC/B,IAAI,MAAM,IAAI,GAAG,IAAI,MAAM,GAAG,GAAG,EAAE,CAAC;YACnC,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,EAAE,CAAA;QAC1C,CAAC;QAED,sBAAsB;QACtB,MAAM,iBAAiB,GAAG,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,IAAI,GAAG,IAAI,MAAM,GAAG,GAAG,CAAC,CAAA;QAC3E,IAAI,CAAC,iBAAiB,EAAE,CAAC;YACxB,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,EAAE,CAAA;QAC1C,CAAC;QAED,wDAAwD;QACxD,MAAM,YAAY,GAAG,IAAI,CAAC,qBAAqB,CAAC,OAAO,EAAE,CAAC,aAAa,CAAC,CAAC,CAAA;QACzE,IAAI,YAAY,KAAK,IAAI,EAAE,CAAC;YAC3B,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,OAAO,EAAE,YAAY,EAAE,CAAA;QACpD,CAAC;QAED,4CAA4C;QAC5C,MAAM,SAAS,GAAG,IAAI,CAAC,2BAA2B,CAAC,aAAa,CAAC,CAAA;QACjE,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,CAAA;IACjD,CAAC;IAED;;;;OAIG;IACI,kBAAkB,CAAC,aAAqB;QAC9C,OAAO,aAAa,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,CAAA;IAC/C,CAAC;IAED,+EAA+E;IAC/E,wBAAwB;IACxB,+EAA+E;IAE/E;;;OAGG;IACI,aAAa;QACnB,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;YAC7B,OAAO,KAAK,CAAA;QACb,CAAC;QAED,+CAA+C;QAC/C,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,eAAe,IAAI,CAAC,CAAC,CAAA;QACtE,IAAI,eAAe,IAAI,IAAI,CAAC,MAAM,CAAC,qBAAqB,EAAE,CAAC;YAC1D,wBAAwB;YACxB,IAAI,CAAC,mBAAmB,EAAE,CAAA;YAC1B,OAAO,KAAK,CAAA;QACb,CAAC;QAED,OAAO,IAAI,CAAA;IACZ,CAAC;IAED;;OAEG;IACI,aAAa;QACnB,IAAI,CAAC,KAAK,CAAC,mBAAmB,IAAI,CAAC,CAAA;QAEnC,IAAI,IAAI,CAAC,KAAK,CAAC,mBAAmB,IAAI,IAAI,CAAC,MAAM,CAAC,uBAAuB,EAAE,CAAC;YAC3E,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,IAAI,CAAA;YAC7B,IAAI,CAAC,KAAK,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QACxC,CAAC;IACF,CAAC;IAED;;OAEG;IACI,aAAa;QACnB,IAAI,CAAC,KAAK,CAAC,mBAAmB,GAAG,CAAC,CAAA;QAClC,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,KAAK,CAAA;QAC9B,IAAI,CAAC,KAAK,CAAC,eAAe,GAAG,IAAI,CAAA;IAClC,CAAC;IAED;;OAEG;IACI,mBAAmB;QACzB,IAAI,CAAC,KAAK,CAAC,mBAAmB,GAAG,CAAC,CAAA;QAClC,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,KAAK,CAAA;QAC9B,IAAI,CAAC,KAAK,CAAC,eAAe,GAAG,IAAI,CAAA;IAClC,CAAC;IAED;;OAEG;IACI,QAAQ;QACd,OAAO,EAAE,GAAG,IAAI,CAAC,KAAK,EAAE,CAAA;IACzB,CAAC;IAED;;OAEG;IACI,SAAS;QACf,OAAO,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,CAAA;IAC1B,CAAC;IAED;;OAEG;IACI,KAAK;QACX,IAAI,CAAC,KAAK,GAAG;YACZ,mBAAmB,EAAE,CAAC;YACtB,WAAW,EAAE,KAAK;YAClB,eAAe,EAAE,IAAI;YACrB,YAAY,EAAE,IAAI;SAClB,CAAA;IACF,CAAC;CACD;AAED,+EAA+E;AAC/E,oBAAoB;AACpB,+EAA+E;AAE/E;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAChC,MAAiC;IAEjC,OAAO,IAAI,WAAW,CAAC,MAAM,CAAC,CAAA;AAC/B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,KAAK,CAAC,MAAc;IACnC,OAAO,MAAM,IAAI,GAAG,IAAI,MAAM,GAAG,GAAG,CAAA;AACrC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,MAAc;IAC/C,OAAO,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,MAAM,CAAC,CAAA;AACvC,CAAC"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EACN,qBAAqB,EACrB,oBAAoB,EACpB,uBAAuB,GACvB,MAAM,uBAAuB,CAAA;AAE9B,OAAO,EACN,gBAAgB,EAChB,kBAAkB,EAClB,cAAc,EACd,UAAU,EACV,cAAc,EACd,WAAW,EACX,iBAAiB,GACjB,MAAM,oBAAoB,CAAA;AAC3B,YAAY,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAA;AAC9D,OAAO,EACN,oBAAoB,EACpB,cAAc,EACd,kBAAkB,EAClB,cAAc,EACd,kBAAkB,GAClB,MAAM,oBAAoB,CAAA;AAC3B,YAAY,EACX,WAAW,EACX,eAAe,EACf,cAAc,GACd,MAAM,2BAA2B,CAAA;AAElC,OAAO,EACN,iBAAiB,EACjB,KAAK,EACL,iBAAiB,EACjB,WAAW,GACX,MAAM,2BAA2B,CAAA;AAClC,YAAY,EACX,WAAW,IAAI,iBAAiB,EAChC,UAAU,GACV,MAAM,yBAAyB,CAAA;AAChC,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAA;AACvD,YAAY,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAA;AAEnE,OAAO,EAAE,oBAAoB,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAA;AAExE,YAAY,EACX,MAAM,EACN,cAAc,EACd,eAAe,EACf,SAAS,EACT,SAAS,EACT,eAAe,EACf,OAAO,EACP,WAAW,EACX,WAAW,EACX,SAAS,EACT,WAAW,GACX,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EACN,qBAAqB,EACrB,eAAe,EACf,qBAAqB,EACrB,iBAAiB,EACjB,eAAe,EACf,iBAAiB,GACjB,MAAM,qBAAqB,CAAA;AAC5B,YAAY,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAA;AAE7D,OAAO,EACN,WAAW,EACX,wBAAwB,EACxB,aAAa,EACb,eAAe,GACf,MAAM,4BAA4B,CAAA;AACnC,YAAY,EACX,YAAY,EACZ,WAAW,IAAI,qBAAqB,EACpC,eAAe,GACf,MAAM,6BAA6B,CAAA;AACpC,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAA"}
package/dist/index.js.map DELETED
@@ -1 +0,0 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EACN,qBAAqB,EACrB,oBAAoB,EACpB,uBAAuB,GACvB,MAAM,uBAAuB,CAAA;AAC9B,gCAAgC;AAChC,OAAO,EACN,gBAAgB,EAChB,kBAAkB,EAClB,cAAc,EACd,UAAU,EACV,cAAc,EACd,WAAW,EACX,iBAAiB,GACjB,MAAM,oBAAoB,CAAA;AAE3B,OAAO,EACN,oBAAoB,EACpB,cAAc,EACd,kBAAkB,EAClB,cAAc,EACd,kBAAkB,GAClB,MAAM,oBAAoB,CAAA;AAM3B,4BAA4B;AAC5B,OAAO,EACN,iBAAiB,EACjB,KAAK,EACL,iBAAiB,EACjB,WAAW,GACX,MAAM,2BAA2B,CAAA;AAKlC,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAA;AAEvD,+BAA+B;AAC/B,OAAO,EAAE,oBAAoB,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAA;AAexE,OAAO,EACN,qBAAqB,EACrB,eAAe,EACf,qBAAqB,EACrB,iBAAiB,EACjB,eAAe,EACf,iBAAiB,GACjB,MAAM,qBAAqB,CAAA;AAE5B,wBAAwB;AACxB,OAAO,EACN,WAAW,EACX,wBAAwB,EACxB,aAAa,EACb,eAAe,GACf,MAAM,4BAA4B,CAAA;AAMnC,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAA"}
@@ -1,82 +0,0 @@
1
- import type { Message } from '#schema/message';
2
- /**
3
- * Deduplication and merge logic for NORMALIZE--T04
4
- *
5
- * Merges CSV and DB message sources with:
6
- * AC01: Exact GUID matching (primary)
7
- * AC02: DB authoritiveness for conflicts
8
- * AC03: Content equivalence detection
9
- * AC04: Data loss verification
10
- * AC05: Deterministic GUID assignment
11
- */
12
- export type MergeStats = {
13
- csvCount: number;
14
- dbCount: number;
15
- outputCount: number;
16
- exactMatches: number;
17
- contentMatches: number;
18
- conflicts: number;
19
- noMatches: number;
20
- };
21
- export type ContentMatch = {
22
- message: Message;
23
- confidence: number;
24
- reasons: string[];
25
- };
26
- export type MergeResult = {
27
- messages: Message[];
28
- stats: MergeStats;
29
- conflicts?: Array<{
30
- csvMsg: Message;
31
- dbMsg: Message;
32
- confidence: number;
33
- }>;
34
- warnings?: string[];
35
- };
36
- /**
37
- * AC01 + AC02 + AC03 + AC04 + AC05: Main dedup and merge function
38
- *
39
- * Strategy:
40
- * 1. Build GUID index for fast lookup
41
- * 2. For each CSV message:
42
- * a. Try exact GUID match (AC01)
43
- * b. Try content equivalence (AC03)
44
- * c. Apply DB authoritiveness if merging (AC02)
45
- * d. Keep separate if no match
46
- * 3. Add unmatched DB messages
47
- * 4. Verify no data loss (AC04)
48
- * 5. Ensure determinism (AC05)
49
- */
50
- export declare function dedupAndMerge(csvMessages: Message[], dbMessages: Message[]): MergeResult;
51
- /**
52
- * AC01: Find exact GUID match in DB messages
53
- */
54
- export declare function findExactMatch(message: Message, dbMessages: Message[]): Message | null;
55
- /**
56
- * AC03: Detect content equivalence
57
- *
58
- * Normalizes text and compares:
59
- * - Normalized text content (lowercase, trimmed, punctuation removed)
60
- * - messageKind must match
61
- * - sender (handle) must match
62
- *
63
- * Returns match with confidence score (1.0 = exact match)
64
- */
65
- export declare function detectContentEquivalence(csvMsg: Message, candidates: Message[], threshold?: number): ContentMatch | null;
66
- /**
67
- * AC02: Merge messages with DB authoritiveness
68
- *
69
- * DB is authoritative for:
70
- * - All timestamps (date, dateRead, dateDelivered, dateEdited)
71
- * - Associations (replyingTo.targetMessageGuid)
72
- * - handle
73
- *
74
- * CSV fields are preserved when DB doesn't have them
75
- */
76
- export declare function applyDbAuthoritiveness(csvMsg: Message, dbMsg: Message): Message;
77
- /**
78
- * AC04: Verify count invariants to prevent data loss
79
- */
80
- export declare function verifyNoDataLoss(csvCount: number, dbCount: number, outputCount: number): boolean;
81
- export type { Message };
82
- //# sourceMappingURL=dedup-merge.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"dedup-merge.d.ts","sourceRoot":"","sources":["../../src/ingest/dedup-merge.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAA;AAE9C;;;;;;;;;GASG;AAEH,MAAM,MAAM,UAAU,GAAG;IACxB,QAAQ,EAAE,MAAM,CAAA;IAChB,OAAO,EAAE,MAAM,CAAA;IACf,WAAW,EAAE,MAAM,CAAA;IACnB,YAAY,EAAE,MAAM,CAAA;IACpB,cAAc,EAAE,MAAM,CAAA;IACtB,SAAS,EAAE,MAAM,CAAA;IACjB,SAAS,EAAE,MAAM,CAAA;CACjB,CAAA;AAED,MAAM,MAAM,YAAY,GAAG;IAC1B,OAAO,EAAE,OAAO,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;IAClB,OAAO,EAAE,MAAM,EAAE,CAAA;CACjB,CAAA;AAED,MAAM,MAAM,WAAW,GAAG;IACzB,QAAQ,EAAE,OAAO,EAAE,CAAA;IACnB,KAAK,EAAE,UAAU,CAAA;IACjB,SAAS,CAAC,EAAE,KAAK,CAAC;QAAE,MAAM,EAAE,OAAO,CAAC;QAAC,KAAK,EAAE,OAAO,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;IAC1E,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAA;CACnB,CAAA;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,aAAa,CAC5B,WAAW,EAAE,OAAO,EAAE,EACtB,UAAU,EAAE,OAAO,EAAE,GACnB,WAAW,CA8Fb;AAED;;GAEG;AACH,wBAAgB,cAAc,CAC7B,OAAO,EAAE,OAAO,EAChB,UAAU,EAAE,OAAO,EAAE,GACnB,OAAO,GAAG,IAAI,CAEhB;AAED;;;;;;;;;GASG;AACH,wBAAgB,wBAAwB,CACvC,MAAM,EAAE,OAAO,EACf,UAAU,EAAE,OAAO,EAAE,EACrB,SAAS,SAAM,GACb,YAAY,GAAG,IAAI,CA2DrB;AAiED;;;;;;;;;GASG;AACH,wBAAgB,sBAAsB,CACrC,MAAM,EAAE,OAAO,EACf,KAAK,EAAE,OAAO,GACZ,OAAO,CA6BT;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAC/B,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,MAAM,EACf,WAAW,EAAE,MAAM,GACjB,OAAO,CAIT;AAED,YAAY,EAAE,OAAO,EAAE,CAAA"}