@nathanvale/chatline 0.0.1 → 0.0.2-next.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/dist/bin/index.js +1 -1
- package/dist/index.js +1 -1
- package/package.json +1 -1
- package/dist/cli/commands/clean.d.ts +0 -17
- package/dist/cli/commands/clean.d.ts.map +0 -1
- package/dist/cli/commands/clean.js +0 -142
- package/dist/cli/commands/clean.js.map +0 -1
- package/dist/cli/commands/doctor.d.ts +0 -17
- package/dist/cli/commands/doctor.d.ts.map +0 -1
- package/dist/cli/commands/doctor.js +0 -202
- package/dist/cli/commands/doctor.js.map +0 -1
- package/dist/cli/commands/enrich-ai.d.ts +0 -17
- package/dist/cli/commands/enrich-ai.d.ts.map +0 -1
- package/dist/cli/commands/enrich-ai.js +0 -371
- package/dist/cli/commands/enrich-ai.js.map +0 -1
- package/dist/cli/commands/index.d.ts +0 -16
- package/dist/cli/commands/index.d.ts.map +0 -1
- package/dist/cli/commands/index.js +0 -16
- package/dist/cli/commands/index.js.map +0 -1
- package/dist/cli/commands/ingest-csv.d.ts +0 -17
- package/dist/cli/commands/ingest-csv.d.ts.map +0 -1
- package/dist/cli/commands/ingest-csv.js +0 -138
- package/dist/cli/commands/ingest-csv.js.map +0 -1
- package/dist/cli/commands/ingest-db.d.ts +0 -17
- package/dist/cli/commands/ingest-db.d.ts.map +0 -1
- package/dist/cli/commands/ingest-db.js +0 -159
- package/dist/cli/commands/ingest-db.js.map +0 -1
- package/dist/cli/commands/init.d.ts +0 -17
- package/dist/cli/commands/init.d.ts.map +0 -1
- package/dist/cli/commands/init.js +0 -110
- package/dist/cli/commands/init.js.map +0 -1
- package/dist/cli/commands/normalize-link.d.ts +0 -16
- package/dist/cli/commands/normalize-link.d.ts.map +0 -1
- package/dist/cli/commands/normalize-link.js +0 -144
- package/dist/cli/commands/normalize-link.js.map +0 -1
- package/dist/cli/commands/render-markdown.d.ts +0 -17
- package/dist/cli/commands/render-markdown.d.ts.map +0 -1
- package/dist/cli/commands/render-markdown.js +0 -218
- package/dist/cli/commands/render-markdown.js.map +0 -1
- package/dist/cli/commands/stats.d.ts +0 -17
- package/dist/cli/commands/stats.d.ts.map +0 -1
- package/dist/cli/commands/stats.js +0 -175
- package/dist/cli/commands/stats.js.map +0 -1
- package/dist/cli/commands/validate.d.ts +0 -17
- package/dist/cli/commands/validate.d.ts.map +0 -1
- package/dist/cli/commands/validate.js +0 -152
- package/dist/cli/commands/validate.js.map +0 -1
- package/dist/cli/index.d.ts +0 -13
- package/dist/cli/index.d.ts.map +0 -1
- package/dist/cli/index.js +0 -121
- package/dist/cli/index.js.map +0 -1
- package/dist/cli/types.d.ts +0 -93
- package/dist/cli/types.d.ts.map +0 -1
- package/dist/cli/types.js +0 -7
- package/dist/cli/types.js.map +0 -1
- package/dist/cli/utils.d.ts +0 -29
- package/dist/cli/utils.d.ts.map +0 -1
- package/dist/cli/utils.js +0 -53
- package/dist/cli/utils.js.map +0 -1
- package/dist/cli.d.ts +0 -9
- package/dist/cli.d.ts.map +0 -1
- package/dist/cli.js +0 -1805
- package/dist/config/generator.d.ts +0 -90
- package/dist/config/generator.d.ts.map +0 -1
- package/dist/config/generator.js +0 -320
- package/dist/config/generator.js.map +0 -1
- package/dist/config/loader.d.ts +0 -107
- package/dist/config/loader.d.ts.map +0 -1
- package/dist/config/loader.js +0 -251
- package/dist/config/loader.js.map +0 -1
- package/dist/config/schema.d.ts +0 -107
- package/dist/config/schema.d.ts.map +0 -1
- package/dist/config/schema.js +0 -169
- package/dist/config/schema.js.map +0 -1
- package/dist/enrich/audio-transcription.d.ts +0 -77
- package/dist/enrich/audio-transcription.d.ts.map +0 -1
- package/dist/enrich/audio-transcription.js +0 -370
- package/dist/enrich/audio-transcription.js.map +0 -1
- package/dist/enrich/checkpoint.d.ts +0 -137
- package/dist/enrich/checkpoint.d.ts.map +0 -1
- package/dist/enrich/checkpoint.js +0 -205
- package/dist/enrich/checkpoint.js.map +0 -1
- package/dist/enrich/idempotency.d.ts +0 -90
- package/dist/enrich/idempotency.d.ts.map +0 -1
- package/dist/enrich/idempotency.js +0 -188
- package/dist/enrich/idempotency.js.map +0 -1
- package/dist/enrich/image-analysis.d.ts +0 -62
- package/dist/enrich/image-analysis.d.ts.map +0 -1
- package/dist/enrich/image-analysis.js +0 -264
- package/dist/enrich/image-analysis.js.map +0 -1
- package/dist/enrich/index.d.ts +0 -60
- package/dist/enrich/index.d.ts.map +0 -1
- package/dist/enrich/index.js +0 -74
- package/dist/enrich/index.js.map +0 -1
- package/dist/enrich/link-enrichment.d.ts +0 -37
- package/dist/enrich/link-enrichment.d.ts.map +0 -1
- package/dist/enrich/link-enrichment.js +0 -202
- package/dist/enrich/link-enrichment.js.map +0 -1
- package/dist/enrich/pdf-video-handling.d.ts +0 -49
- package/dist/enrich/pdf-video-handling.d.ts.map +0 -1
- package/dist/enrich/pdf-video-handling.js +0 -325
- package/dist/enrich/pdf-video-handling.js.map +0 -1
- package/dist/enrich/progress-tracker.d.ts +0 -120
- package/dist/enrich/progress-tracker.d.ts.map +0 -1
- package/dist/enrich/progress-tracker.js +0 -220
- package/dist/enrich/progress-tracker.js.map +0 -1
- package/dist/enrich/providers/firecrawl.d.ts +0 -18
- package/dist/enrich/providers/firecrawl.d.ts.map +0 -1
- package/dist/enrich/providers/firecrawl.js +0 -48
- package/dist/enrich/providers/firecrawl.js.map +0 -1
- package/dist/enrich/providers/generic.d.ts +0 -16
- package/dist/enrich/providers/generic.d.ts.map +0 -1
- package/dist/enrich/providers/generic.js +0 -36
- package/dist/enrich/providers/generic.js.map +0 -1
- package/dist/enrich/providers/index.d.ts +0 -14
- package/dist/enrich/providers/index.d.ts.map +0 -1
- package/dist/enrich/providers/index.js +0 -13
- package/dist/enrich/providers/index.js.map +0 -1
- package/dist/enrich/providers/instagram.d.ts +0 -16
- package/dist/enrich/providers/instagram.d.ts.map +0 -1
- package/dist/enrich/providers/instagram.js +0 -43
- package/dist/enrich/providers/instagram.js.map +0 -1
- package/dist/enrich/providers/spotify.d.ts +0 -16
- package/dist/enrich/providers/spotify.d.ts.map +0 -1
- package/dist/enrich/providers/spotify.js +0 -45
- package/dist/enrich/providers/spotify.js.map +0 -1
- package/dist/enrich/providers/twitter.d.ts +0 -16
- package/dist/enrich/providers/twitter.d.ts.map +0 -1
- package/dist/enrich/providers/twitter.js +0 -43
- package/dist/enrich/providers/twitter.js.map +0 -1
- package/dist/enrich/providers/types.d.ts +0 -47
- package/dist/enrich/providers/types.d.ts.map +0 -1
- package/dist/enrich/providers/types.js +0 -15
- package/dist/enrich/providers/types.js.map +0 -1
- package/dist/enrich/providers/youtube.d.ts +0 -16
- package/dist/enrich/providers/youtube.d.ts.map +0 -1
- package/dist/enrich/providers/youtube.js +0 -43
- package/dist/enrich/providers/youtube.js.map +0 -1
- package/dist/enrich/rate-limiting.d.ts +0 -118
- package/dist/enrich/rate-limiting.d.ts.map +0 -1
- package/dist/enrich/rate-limiting.js +0 -258
- package/dist/enrich/rate-limiting.js.map +0 -1
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/ingest/dedup-merge.d.ts +0 -82
- package/dist/ingest/dedup-merge.d.ts.map +0 -1
- package/dist/ingest/dedup-merge.js +0 -262
- package/dist/ingest/dedup-merge.js.map +0 -1
- package/dist/ingest/ingest-csv.d.ts +0 -62
- package/dist/ingest/ingest-csv.d.ts.map +0 -1
- package/dist/ingest/ingest-csv.js +0 -300
- package/dist/ingest/ingest-csv.js.map +0 -1
- package/dist/ingest/ingest-db.d.ts +0 -64
- package/dist/ingest/ingest-db.d.ts.map +0 -1
- package/dist/ingest/ingest-db.js +0 -172
- package/dist/ingest/ingest-db.js.map +0 -1
- package/dist/ingest/link-replies-and-tapbacks.d.ts +0 -53
- package/dist/ingest/link-replies-and-tapbacks.d.ts.map +0 -1
- package/dist/ingest/link-replies-and-tapbacks.js +0 -381
- package/dist/ingest/link-replies-and-tapbacks.js.map +0 -1
- package/dist/normalize/date-converters.d.ts +0 -45
- package/dist/normalize/date-converters.d.ts.map +0 -1
- package/dist/normalize/date-converters.js +0 -166
- package/dist/normalize/date-converters.js.map +0 -1
- package/dist/normalize/path-validator.d.ts +0 -65
- package/dist/normalize/path-validator.d.ts.map +0 -1
- package/dist/normalize/path-validator.js +0 -221
- package/dist/normalize/path-validator.js.map +0 -1
- package/dist/normalize/validate-normalized.d.ts +0 -45
- package/dist/normalize/validate-normalized.d.ts.map +0 -1
- package/dist/normalize/validate-normalized.js +0 -144
- package/dist/normalize/validate-normalized.js.map +0 -1
- package/dist/render/embeds-blockquotes.d.ts +0 -84
- package/dist/render/embeds-blockquotes.d.ts.map +0 -1
- package/dist/render/embeds-blockquotes.js +0 -204
- package/dist/render/embeds-blockquotes.js.map +0 -1
- package/dist/render/grouping.d.ts +0 -78
- package/dist/render/grouping.d.ts.map +0 -1
- package/dist/render/grouping.js +0 -134
- package/dist/render/grouping.js.map +0 -1
- package/dist/render/index.d.ts +0 -47
- package/dist/render/index.d.ts.map +0 -1
- package/dist/render/index.js +0 -245
- package/dist/render/index.js.map +0 -1
- package/dist/render/reply-rendering.d.ts +0 -88
- package/dist/render/reply-rendering.d.ts.map +0 -1
- package/dist/render/reply-rendering.js +0 -196
- package/dist/render/reply-rendering.js.map +0 -1
- package/dist/schema/message.d.ts +0 -125
- package/dist/schema/message.d.ts.map +0 -1
- package/dist/schema/message.js +0 -331
- package/dist/schema/message.js.map +0 -1
- package/dist/utils/delta-detection.d.ts +0 -107
- package/dist/utils/delta-detection.d.ts.map +0 -1
- package/dist/utils/delta-detection.js +0 -199
- package/dist/utils/delta-detection.js.map +0 -1
- package/dist/utils/enrichment-merge.d.ts +0 -135
- package/dist/utils/enrichment-merge.d.ts.map +0 -1
- package/dist/utils/enrichment-merge.js +0 -280
- package/dist/utils/enrichment-merge.js.map +0 -1
- package/dist/utils/human.d.ts +0 -15
- package/dist/utils/human.d.ts.map +0 -1
- package/dist/utils/human.js +0 -27
- package/dist/utils/human.js.map +0 -1
- package/dist/utils/incremental-state.d.ts +0 -133
- package/dist/utils/incremental-state.d.ts.map +0 -1
- package/dist/utils/incremental-state.js +0 -237
- package/dist/utils/incremental-state.js.map +0 -1
- package/dist/utils/logger.d.ts +0 -40
- package/dist/utils/logger.d.ts.map +0 -1
- package/dist/utils/logger.js +0 -176
- package/dist/utils/logger.js.map +0 -1
|
@@ -1,47 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Link Enrichment Provider Types
|
|
3
|
-
*
|
|
4
|
-
* Shared types for link enrichment providers.
|
|
5
|
-
*/
|
|
6
|
-
/**
|
|
7
|
-
* Link context extracted from a URL
|
|
8
|
-
*/
|
|
9
|
-
export type LinkContext = {
|
|
10
|
-
url: string;
|
|
11
|
-
title?: string;
|
|
12
|
-
description?: string;
|
|
13
|
-
provider: 'gemini' | 'firecrawl' | 'local' | 'youtube' | 'spotify' | 'twitter' | 'instagram' | 'generic';
|
|
14
|
-
usedFallback?: boolean;
|
|
15
|
-
failedProviders?: string[];
|
|
16
|
-
};
|
|
17
|
-
/**
|
|
18
|
-
* Configuration for link enrichment
|
|
19
|
-
*/
|
|
20
|
-
export type LinkEnrichmentConfig = {
|
|
21
|
-
enableLinkAnalysis: boolean;
|
|
22
|
-
firecrawlApiKey?: string;
|
|
23
|
-
youtubeApiKey?: string;
|
|
24
|
-
spotifyClientId?: string;
|
|
25
|
-
spotifyClientSecret?: string;
|
|
26
|
-
rateLimitDelay?: number;
|
|
27
|
-
maxRetries?: number;
|
|
28
|
-
};
|
|
29
|
-
/**
|
|
30
|
-
* Provider interface for cascading fallbacks
|
|
31
|
-
*/
|
|
32
|
-
export interface Provider {
|
|
33
|
-
name: string;
|
|
34
|
-
priority: number;
|
|
35
|
-
detect: (url: string) => boolean;
|
|
36
|
-
extract: (url: string) => Promise<LinkContext>;
|
|
37
|
-
}
|
|
38
|
-
/**
|
|
39
|
-
* URL detection patterns for different provider types
|
|
40
|
-
*/
|
|
41
|
-
export declare const URL_PATTERNS: {
|
|
42
|
-
youtube: RegExp;
|
|
43
|
-
spotify: RegExp;
|
|
44
|
-
twitter: RegExp;
|
|
45
|
-
instagram: RegExp;
|
|
46
|
-
};
|
|
47
|
-
//# sourceMappingURL=types.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/enrich/providers/types.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH;;GAEG;AACH,MAAM,MAAM,WAAW,GAAG;IACzB,GAAG,EAAE,MAAM,CAAA;IACX,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,QAAQ,EACL,QAAQ,GACR,WAAW,GACX,OAAO,GACP,SAAS,GACT,SAAS,GACT,SAAS,GACT,WAAW,GACX,SAAS,CAAA;IACZ,YAAY,CAAC,EAAE,OAAO,CAAA;IACtB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAA;CAC1B,CAAA;AAED;;GAEG;AACH,MAAM,MAAM,oBAAoB,GAAG;IAClC,kBAAkB,EAAE,OAAO,CAAA;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,UAAU,CAAC,EAAE,MAAM,CAAA;CACnB,CAAA;AAED;;GAEG;AACH,MAAM,WAAW,QAAQ;IACxB,IAAI,EAAE,MAAM,CAAA;IACZ,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAA;IAChC,OAAO,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,WAAW,CAAC,CAAA;CAC9C;AAED;;GAEG;AACH,eAAO,MAAM,YAAY;;;;;CAMxB,CAAA"}
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Link Enrichment Provider Types
|
|
3
|
-
*
|
|
4
|
-
* Shared types for link enrichment providers.
|
|
5
|
-
*/
|
|
6
|
-
/**
|
|
7
|
-
* URL detection patterns for different provider types
|
|
8
|
-
*/
|
|
9
|
-
export const URL_PATTERNS = {
|
|
10
|
-
youtube: /^https?:\/\/(www\.)?(youtube\.com|youtu\.be|youtube-nocookie\.com)/i,
|
|
11
|
-
spotify: /^https?:\/\/open\.spotify\.com\/(track|album|playlist|artist)/i,
|
|
12
|
-
twitter: /^https?:\/\/(www\.)?(twitter\.com|x\.com)/i,
|
|
13
|
-
instagram: /^https?:\/\/(www\.)?instagram\.com/i,
|
|
14
|
-
};
|
|
15
|
-
//# sourceMappingURL=types.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/enrich/providers/types.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AA6CH;;GAEG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG;IAC3B,OAAO,EACN,qEAAqE;IACtE,OAAO,EAAE,gEAAgE;IACzE,OAAO,EAAE,4CAA4C;IACrD,SAAS,EAAE,qCAAqC;CAChD,CAAA"}
|
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* YouTube Provider (ENRICH--T04-AC02)
|
|
3
|
-
*
|
|
4
|
-
* Fallback provider for YouTube video metadata extraction.
|
|
5
|
-
*/
|
|
6
|
-
import type { LinkContext, Provider } from './types.js';
|
|
7
|
-
/**
|
|
8
|
-
* AC02: YouTube Provider (fallback)
|
|
9
|
-
*/
|
|
10
|
-
export declare class YouTubeProvider implements Provider {
|
|
11
|
-
name: string;
|
|
12
|
-
priority: number;
|
|
13
|
-
detect(url: string): boolean;
|
|
14
|
-
extract(url: string): Promise<LinkContext>;
|
|
15
|
-
}
|
|
16
|
-
//# sourceMappingURL=youtube.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"youtube.d.ts","sourceRoot":"","sources":["../../../src/enrich/providers/youtube.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAA;AAKvD;;GAEG;AACH,qBAAa,eAAgB,YAAW,QAAQ;IAC/C,IAAI,SAAY;IAChB,QAAQ,SAAI;IAEZ,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO;IAItB,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;CA4BhD"}
|
|
@@ -1,43 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* YouTube Provider (ENRICH--T04-AC02)
|
|
3
|
-
*
|
|
4
|
-
* Fallback provider for YouTube video metadata extraction.
|
|
5
|
-
*/
|
|
6
|
-
import { createLogger } from '#utils/logger';
|
|
7
|
-
import { URL_PATTERNS } from './types.js';
|
|
8
|
-
const logger = createLogger('enrich:youtube-provider');
|
|
9
|
-
/**
|
|
10
|
-
* AC02: YouTube Provider (fallback)
|
|
11
|
-
*/
|
|
12
|
-
export class YouTubeProvider {
|
|
13
|
-
name = 'youtube';
|
|
14
|
-
priority = 1;
|
|
15
|
-
detect(url) {
|
|
16
|
-
return URL_PATTERNS.youtube.test(url);
|
|
17
|
-
}
|
|
18
|
-
async extract(url) {
|
|
19
|
-
try {
|
|
20
|
-
// Extract video ID from URL
|
|
21
|
-
const videoIdMatch = url.match(/(?:youtube\.com\/watch\?v=|youtu\.be\/)([a-zA-Z0-9_-]{11})/);
|
|
22
|
-
const videoId = videoIdMatch?.[1];
|
|
23
|
-
if (!videoId) {
|
|
24
|
-
throw new Error('Could not extract YouTube video ID');
|
|
25
|
-
}
|
|
26
|
-
// In production, would fetch from YouTube API or parse HTML
|
|
27
|
-
const title = `YouTube Video: ${videoId}`;
|
|
28
|
-
logger.debug(`YouTube extracted metadata for ${url}`);
|
|
29
|
-
return {
|
|
30
|
-
url,
|
|
31
|
-
title,
|
|
32
|
-
description: 'YouTube video',
|
|
33
|
-
provider: 'youtube',
|
|
34
|
-
usedFallback: true,
|
|
35
|
-
};
|
|
36
|
-
}
|
|
37
|
-
catch (error) {
|
|
38
|
-
logger.error(`YouTube extraction failed for ${url}`, { error });
|
|
39
|
-
throw error;
|
|
40
|
-
}
|
|
41
|
-
}
|
|
42
|
-
}
|
|
43
|
-
//# sourceMappingURL=youtube.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"youtube.js","sourceRoot":"","sources":["../../../src/enrich/providers/youtube.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAE5C,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AAEzC,MAAM,MAAM,GAAG,YAAY,CAAC,yBAAyB,CAAC,CAAA;AAEtD;;GAEG;AACH,MAAM,OAAO,eAAe;IAC3B,IAAI,GAAG,SAAS,CAAA;IAChB,QAAQ,GAAG,CAAC,CAAA;IAEZ,MAAM,CAAC,GAAW;QACjB,OAAO,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IACtC,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,GAAW;QACxB,IAAI,CAAC;YACJ,4BAA4B;YAC5B,MAAM,YAAY,GAAG,GAAG,CAAC,KAAK,CAC7B,4DAA4D,CAC5D,CAAA;YACD,MAAM,OAAO,GAAG,YAAY,EAAE,CAAC,CAAC,CAAC,CAAA;YAEjC,IAAI,CAAC,OAAO,EAAE,CAAC;gBACd,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAA;YACtD,CAAC;YAED,4DAA4D;YAC5D,MAAM,KAAK,GAAG,kBAAkB,OAAO,EAAE,CAAA;YAEzC,MAAM,CAAC,KAAK,CAAC,kCAAkC,GAAG,EAAE,CAAC,CAAA;YACrD,OAAO;gBACN,GAAG;gBACH,KAAK;gBACL,WAAW,EAAE,eAAe;gBAC5B,QAAQ,EAAE,SAAS;gBACnB,YAAY,EAAE,IAAI;aAClB,CAAA;QACF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,CAAC,KAAK,CAAC,iCAAiC,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CAAA;YAC/D,MAAM,KAAK,CAAA;QACZ,CAAC;IACF,CAAC;CACD"}
|
|
@@ -1,118 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Rate Limiting and Retry Logic Module (ENRICH--T07)
|
|
3
|
-
*
|
|
4
|
-
* Implements comprehensive rate limiting with:
|
|
5
|
-
* - AC01: Configurable delays between API calls (default 1000ms)
|
|
6
|
-
* - AC02: Exponential backoff for 429 responses with ±25% jitter
|
|
7
|
-
* - AC03: Retry 5xx errors with maxRetries limit (default 3)
|
|
8
|
-
* - AC04: Respect Retry-After header for 429/503 responses
|
|
9
|
-
* - AC05: Circuit breaker after N consecutive failures (default 5)
|
|
10
|
-
*
|
|
11
|
-
* Architecture:
|
|
12
|
-
* - RateLimiter: Main class managing all rate limiting logic
|
|
13
|
-
* - Configuration with sensible defaults
|
|
14
|
-
* - Per-provider state isolation
|
|
15
|
-
* - Deterministic jitter calculation
|
|
16
|
-
*/
|
|
17
|
-
export type RateLimitConfig = {
|
|
18
|
-
rateLimitDelay: number;
|
|
19
|
-
maxRetries: number;
|
|
20
|
-
circuitBreakerThreshold: number;
|
|
21
|
-
circuitBreakerResetMs: number;
|
|
22
|
-
};
|
|
23
|
-
export type RateLimitState = {
|
|
24
|
-
consecutiveFailures: number;
|
|
25
|
-
circuitOpen: boolean;
|
|
26
|
-
circuitOpenedAt: number | null;
|
|
27
|
-
lastCallTime: number | null;
|
|
28
|
-
};
|
|
29
|
-
export type ApiResponse = {
|
|
30
|
-
status: number;
|
|
31
|
-
headers?: Record<string, string | number | undefined>;
|
|
32
|
-
};
|
|
33
|
-
export declare class RateLimiter {
|
|
34
|
-
private config;
|
|
35
|
-
private state;
|
|
36
|
-
constructor(partialConfig?: Partial<RateLimitConfig>);
|
|
37
|
-
private validateConfig;
|
|
38
|
-
/**
|
|
39
|
-
* Check if rate limiting should delay the next call
|
|
40
|
-
* @returns delay in ms, or 0 if no delay needed
|
|
41
|
-
*/
|
|
42
|
-
shouldRateLimit(): number;
|
|
43
|
-
/**
|
|
44
|
-
* Record a successful API call for rate limiting tracking
|
|
45
|
-
*/
|
|
46
|
-
recordCall(): void;
|
|
47
|
-
/**
|
|
48
|
-
* Calculate exponential backoff with ±25% jitter
|
|
49
|
-
* Formula: 2^n seconds ± 25%
|
|
50
|
-
* @param attemptNumber - retry attempt number (1-based)
|
|
51
|
-
* @returns delay in ms
|
|
52
|
-
*/
|
|
53
|
-
private calculateExponentialBackoff;
|
|
54
|
-
/**
|
|
55
|
-
* Parse Retry-After header (can be integer seconds or HTTP date)
|
|
56
|
-
* @param retryAfterValue - header value
|
|
57
|
-
* @returns delay in ms, or null if invalid
|
|
58
|
-
*/
|
|
59
|
-
private parseRetryAfterHeader;
|
|
60
|
-
/**
|
|
61
|
-
* Determine if response should be retried and calculate delay
|
|
62
|
-
* @param response - API response with status and optional headers
|
|
63
|
-
* @param attemptNumber - current attempt (1-based)
|
|
64
|
-
* @returns { shouldRetry: boolean, delayMs: number }
|
|
65
|
-
*/
|
|
66
|
-
getRetryStrategy(response: ApiResponse, attemptNumber: number): {
|
|
67
|
-
shouldRetry: boolean;
|
|
68
|
-
delayMs: number;
|
|
69
|
-
};
|
|
70
|
-
/**
|
|
71
|
-
* Check if we should retry based on attempt count
|
|
72
|
-
* @param attemptNumber - current attempt (1-based)
|
|
73
|
-
* @returns true if we should retry
|
|
74
|
-
*/
|
|
75
|
-
shouldRetryAttempt(attemptNumber: number): boolean;
|
|
76
|
-
/**
|
|
77
|
-
* Check if circuit breaker is open
|
|
78
|
-
* @returns true if circuit is open (should fail fast)
|
|
79
|
-
*/
|
|
80
|
-
isCircuitOpen(): boolean;
|
|
81
|
-
/**
|
|
82
|
-
* Record a failure for circuit breaker tracking
|
|
83
|
-
*/
|
|
84
|
-
recordFailure(): void;
|
|
85
|
-
/**
|
|
86
|
-
* Record a success to reset failure counter
|
|
87
|
-
*/
|
|
88
|
-
recordSuccess(): void;
|
|
89
|
-
/**
|
|
90
|
-
* Manually reset circuit breaker
|
|
91
|
-
*/
|
|
92
|
-
resetCircuitBreaker(): void;
|
|
93
|
-
/**
|
|
94
|
-
* Get current state for inspection
|
|
95
|
-
*/
|
|
96
|
-
getState(): RateLimitState;
|
|
97
|
-
/**
|
|
98
|
-
* Get configuration
|
|
99
|
-
*/
|
|
100
|
-
getConfig(): RateLimitConfig;
|
|
101
|
-
/**
|
|
102
|
-
* Reset all state (for testing)
|
|
103
|
-
*/
|
|
104
|
-
reset(): void;
|
|
105
|
-
}
|
|
106
|
-
/**
|
|
107
|
-
* Create a new rate limiter with default configuration
|
|
108
|
-
*/
|
|
109
|
-
export declare function createRateLimiter(config?: Partial<RateLimitConfig>): RateLimiter;
|
|
110
|
-
/**
|
|
111
|
-
* Determine if a status code is a 5xx error (server error)
|
|
112
|
-
*/
|
|
113
|
-
export declare function is5xx(status: number): boolean;
|
|
114
|
-
/**
|
|
115
|
-
* Determine if a status code should trigger retry
|
|
116
|
-
*/
|
|
117
|
-
export declare function isRetryableStatus(status: number): boolean;
|
|
118
|
-
//# sourceMappingURL=rate-limiting.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"rate-limiting.d.ts","sourceRoot":"","sources":["../../src/enrich/rate-limiting.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAMH,MAAM,MAAM,eAAe,GAAG;IAC7B,cAAc,EAAE,MAAM,CAAA;IACtB,UAAU,EAAE,MAAM,CAAA;IAClB,uBAAuB,EAAE,MAAM,CAAA;IAC/B,qBAAqB,EAAE,MAAM,CAAA;CAC7B,CAAA;AAED,MAAM,MAAM,cAAc,GAAG;IAC5B,mBAAmB,EAAE,MAAM,CAAA;IAC3B,WAAW,EAAE,OAAO,CAAA;IACpB,eAAe,EAAE,MAAM,GAAG,IAAI,CAAA;IAC9B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAA;CAC3B,CAAA;AAED,MAAM,MAAM,WAAW,GAAG;IACzB,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC,CAAA;CACrD,CAAA;AAMD,qBAAa,WAAW;IACvB,OAAO,CAAC,MAAM,CAAiB;IAC/B,OAAO,CAAC,KAAK,CAAgB;gBAEjB,aAAa,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC;IAqBpD,OAAO,CAAC,cAAc;IAetB;;;OAGG;IACI,eAAe,IAAI,MAAM;IAgBhC;;OAEG;IACI,UAAU,IAAI,IAAI;IAQzB;;;;;OAKG;IACH,OAAO,CAAC,2BAA2B;IAgBnC;;;;OAIG;IACH,OAAO,CAAC,qBAAqB;IAuC7B;;;;;OAKG;IACI,gBAAgB,CACtB,QAAQ,EAAE,WAAW,EACrB,aAAa,EAAE,MAAM,GACnB;QAAE,WAAW,EAAE,OAAO,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE;IAyB5C;;;;OAIG;IACI,kBAAkB,CAAC,aAAa,EAAE,MAAM,GAAG,OAAO;IAQzD;;;OAGG;IACI,aAAa,IAAI,OAAO;IAgB/B;;OAEG;IACI,aAAa,IAAI,IAAI;IAS5B;;OAEG;IACI,aAAa,IAAI,IAAI;IAM5B;;OAEG;IACI,mBAAmB,IAAI,IAAI;IAMlC;;OAEG;IACI,QAAQ,IAAI,cAAc;IAIjC;;OAEG;IACI,SAAS,IAAI,eAAe;IAInC;;OAEG;IACI,KAAK,IAAI,IAAI;CAQpB;AAMD;;GAEG;AACH,wBAAgB,iBAAiB,CAChC,MAAM,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,GAC/B,WAAW,CAEb;AAED;;GAEG;AACH,wBAAgB,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAE7C;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAEzD"}
|
|
@@ -1,258 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Rate Limiting and Retry Logic Module (ENRICH--T07)
|
|
3
|
-
*
|
|
4
|
-
* Implements comprehensive rate limiting with:
|
|
5
|
-
* - AC01: Configurable delays between API calls (default 1000ms)
|
|
6
|
-
* - AC02: Exponential backoff for 429 responses with ±25% jitter
|
|
7
|
-
* - AC03: Retry 5xx errors with maxRetries limit (default 3)
|
|
8
|
-
* - AC04: Respect Retry-After header for 429/503 responses
|
|
9
|
-
* - AC05: Circuit breaker after N consecutive failures (default 5)
|
|
10
|
-
*
|
|
11
|
-
* Architecture:
|
|
12
|
-
* - RateLimiter: Main class managing all rate limiting logic
|
|
13
|
-
* - Configuration with sensible defaults
|
|
14
|
-
* - Per-provider state isolation
|
|
15
|
-
* - Deterministic jitter calculation
|
|
16
|
-
*/
|
|
17
|
-
// ============================================================================
|
|
18
|
-
// AC01 + AC02 + AC03 + AC04 + AC05: Main RateLimiter Class
|
|
19
|
-
// ============================================================================
|
|
20
|
-
export class RateLimiter {
|
|
21
|
-
config;
|
|
22
|
-
state;
|
|
23
|
-
constructor(partialConfig) {
|
|
24
|
-
// Set defaults and merge with provided config
|
|
25
|
-
this.config = {
|
|
26
|
-
rateLimitDelay: partialConfig?.rateLimitDelay ?? 1000,
|
|
27
|
-
maxRetries: partialConfig?.maxRetries ?? 3,
|
|
28
|
-
circuitBreakerThreshold: partialConfig?.circuitBreakerThreshold ?? 5,
|
|
29
|
-
circuitBreakerResetMs: partialConfig?.circuitBreakerResetMs ?? 60000,
|
|
30
|
-
};
|
|
31
|
-
// Validate config
|
|
32
|
-
this.validateConfig(this.config);
|
|
33
|
-
// Initialize state
|
|
34
|
-
this.state = {
|
|
35
|
-
consecutiveFailures: 0,
|
|
36
|
-
circuitOpen: false,
|
|
37
|
-
circuitOpenedAt: null,
|
|
38
|
-
lastCallTime: null,
|
|
39
|
-
};
|
|
40
|
-
}
|
|
41
|
-
validateConfig(config) {
|
|
42
|
-
if (config.rateLimitDelay < 0)
|
|
43
|
-
throw new Error('rateLimitDelay must be non-negative');
|
|
44
|
-
if (config.maxRetries < 0)
|
|
45
|
-
throw new Error('maxRetries must be non-negative');
|
|
46
|
-
if (config.circuitBreakerThreshold < 1)
|
|
47
|
-
throw new Error('circuitBreakerThreshold must be >= 1');
|
|
48
|
-
if (config.circuitBreakerResetMs < 0)
|
|
49
|
-
throw new Error('circuitBreakerResetMs must be non-negative');
|
|
50
|
-
}
|
|
51
|
-
// ============================================================================
|
|
52
|
-
// AC01: Configurable Rate Limit Delays
|
|
53
|
-
// ============================================================================
|
|
54
|
-
/**
|
|
55
|
-
* Check if rate limiting should delay the next call
|
|
56
|
-
* @returns delay in ms, or 0 if no delay needed
|
|
57
|
-
*/
|
|
58
|
-
shouldRateLimit() {
|
|
59
|
-
// No delay for first call
|
|
60
|
-
if (this.state.lastCallTime === null) {
|
|
61
|
-
return 0;
|
|
62
|
-
}
|
|
63
|
-
const timeSinceLastCall = Date.now() - this.state.lastCallTime;
|
|
64
|
-
const requiredDelay = this.config.rateLimitDelay;
|
|
65
|
-
if (timeSinceLastCall < requiredDelay) {
|
|
66
|
-
return requiredDelay - timeSinceLastCall;
|
|
67
|
-
}
|
|
68
|
-
return 0;
|
|
69
|
-
}
|
|
70
|
-
/**
|
|
71
|
-
* Record a successful API call for rate limiting tracking
|
|
72
|
-
*/
|
|
73
|
-
recordCall() {
|
|
74
|
-
this.state.lastCallTime = Date.now();
|
|
75
|
-
}
|
|
76
|
-
// ============================================================================
|
|
77
|
-
// AC02: Exponential Backoff with Jitter
|
|
78
|
-
// ============================================================================
|
|
79
|
-
/**
|
|
80
|
-
* Calculate exponential backoff with ±25% jitter
|
|
81
|
-
* Formula: 2^n seconds ± 25%
|
|
82
|
-
* @param attemptNumber - retry attempt number (1-based)
|
|
83
|
-
* @returns delay in ms
|
|
84
|
-
*/
|
|
85
|
-
calculateExponentialBackoff(attemptNumber) {
|
|
86
|
-
// Base calculation: 2^n seconds
|
|
87
|
-
const baseDelaySeconds = 2 ** attemptNumber;
|
|
88
|
-
const baseDelayMs = baseDelaySeconds * 1000;
|
|
89
|
-
// Apply ±25% jitter
|
|
90
|
-
const jitterAmount = baseDelayMs * 0.25;
|
|
91
|
-
const jitter = (Math.random() - 0.5) * 2 * jitterAmount;
|
|
92
|
-
return baseDelayMs + jitter;
|
|
93
|
-
}
|
|
94
|
-
// ============================================================================
|
|
95
|
-
// AC04: Retry-After Header Parsing
|
|
96
|
-
// ============================================================================
|
|
97
|
-
/**
|
|
98
|
-
* Parse Retry-After header (can be integer seconds or HTTP date)
|
|
99
|
-
* @param retryAfterValue - header value
|
|
100
|
-
* @returns delay in ms, or null if invalid
|
|
101
|
-
*/
|
|
102
|
-
parseRetryAfterHeader(retryAfterValue) {
|
|
103
|
-
if (retryAfterValue === undefined || retryAfterValue === null) {
|
|
104
|
-
return null;
|
|
105
|
-
}
|
|
106
|
-
// Handle numeric (seconds)
|
|
107
|
-
if (typeof retryAfterValue === 'number') {
|
|
108
|
-
return retryAfterValue * 1000;
|
|
109
|
-
}
|
|
110
|
-
const strValue = String(retryAfterValue).trim();
|
|
111
|
-
// Try parsing as integer seconds
|
|
112
|
-
const seconds = Number.parseInt(strValue, 10);
|
|
113
|
-
if (!Number.isNaN(seconds) && seconds >= 0) {
|
|
114
|
-
return seconds * 1000;
|
|
115
|
-
}
|
|
116
|
-
// Try parsing as HTTP date
|
|
117
|
-
try {
|
|
118
|
-
const date = new Date(strValue);
|
|
119
|
-
if (!Number.isNaN(date.getTime())) {
|
|
120
|
-
const delayMs = date.getTime() - Date.now();
|
|
121
|
-
return Math.max(0, delayMs);
|
|
122
|
-
}
|
|
123
|
-
}
|
|
124
|
-
catch {
|
|
125
|
-
// Invalid date format
|
|
126
|
-
}
|
|
127
|
-
// Invalid format - return null to use calculated backoff
|
|
128
|
-
return null;
|
|
129
|
-
}
|
|
130
|
-
// ============================================================================
|
|
131
|
-
// AC03: Retry 5xx Errors + AC04: Retry-After Integration
|
|
132
|
-
// ============================================================================
|
|
133
|
-
/**
|
|
134
|
-
* Determine if response should be retried and calculate delay
|
|
135
|
-
* @param response - API response with status and optional headers
|
|
136
|
-
* @param attemptNumber - current attempt (1-based)
|
|
137
|
-
* @returns { shouldRetry: boolean, delayMs: number }
|
|
138
|
-
*/
|
|
139
|
-
getRetryStrategy(response, attemptNumber) {
|
|
140
|
-
const { status, headers } = response;
|
|
141
|
-
// Don't retry on success (2xx)
|
|
142
|
-
if (status >= 200 && status < 300) {
|
|
143
|
-
return { shouldRetry: false, delayMs: 0 };
|
|
144
|
-
}
|
|
145
|
-
// Retry on 429 or 5xx
|
|
146
|
-
const isRetryableStatus = status === 429 || (status >= 500 && status < 600);
|
|
147
|
-
if (!isRetryableStatus) {
|
|
148
|
-
return { shouldRetry: false, delayMs: 0 };
|
|
149
|
-
}
|
|
150
|
-
// AC04: Check for Retry-After header (takes precedence)
|
|
151
|
-
const retryAfterMs = this.parseRetryAfterHeader(headers?.['Retry-After']);
|
|
152
|
-
if (retryAfterMs !== null) {
|
|
153
|
-
return { shouldRetry: true, delayMs: retryAfterMs };
|
|
154
|
-
}
|
|
155
|
-
// AC02: Use exponential backoff with jitter
|
|
156
|
-
const backoffMs = this.calculateExponentialBackoff(attemptNumber);
|
|
157
|
-
return { shouldRetry: true, delayMs: backoffMs };
|
|
158
|
-
}
|
|
159
|
-
/**
|
|
160
|
-
* Check if we should retry based on attempt count
|
|
161
|
-
* @param attemptNumber - current attempt (1-based)
|
|
162
|
-
* @returns true if we should retry
|
|
163
|
-
*/
|
|
164
|
-
shouldRetryAttempt(attemptNumber) {
|
|
165
|
-
return attemptNumber <= this.config.maxRetries;
|
|
166
|
-
}
|
|
167
|
-
// ============================================================================
|
|
168
|
-
// AC05: Circuit Breaker
|
|
169
|
-
// ============================================================================
|
|
170
|
-
/**
|
|
171
|
-
* Check if circuit breaker is open
|
|
172
|
-
* @returns true if circuit is open (should fail fast)
|
|
173
|
-
*/
|
|
174
|
-
isCircuitOpen() {
|
|
175
|
-
if (!this.state.circuitOpen) {
|
|
176
|
-
return false;
|
|
177
|
-
}
|
|
178
|
-
// Check if circuit breaker timeout has elapsed
|
|
179
|
-
const timeSinceOpened = Date.now() - (this.state.circuitOpenedAt ?? 0);
|
|
180
|
-
if (timeSinceOpened >= this.config.circuitBreakerResetMs) {
|
|
181
|
-
// Reset circuit breaker
|
|
182
|
-
this.resetCircuitBreaker();
|
|
183
|
-
return false;
|
|
184
|
-
}
|
|
185
|
-
return true;
|
|
186
|
-
}
|
|
187
|
-
/**
|
|
188
|
-
* Record a failure for circuit breaker tracking
|
|
189
|
-
*/
|
|
190
|
-
recordFailure() {
|
|
191
|
-
this.state.consecutiveFailures += 1;
|
|
192
|
-
if (this.state.consecutiveFailures >= this.config.circuitBreakerThreshold) {
|
|
193
|
-
this.state.circuitOpen = true;
|
|
194
|
-
this.state.circuitOpenedAt = Date.now();
|
|
195
|
-
}
|
|
196
|
-
}
|
|
197
|
-
/**
|
|
198
|
-
* Record a success to reset failure counter
|
|
199
|
-
*/
|
|
200
|
-
recordSuccess() {
|
|
201
|
-
this.state.consecutiveFailures = 0;
|
|
202
|
-
this.state.circuitOpen = false;
|
|
203
|
-
this.state.circuitOpenedAt = null;
|
|
204
|
-
}
|
|
205
|
-
/**
|
|
206
|
-
* Manually reset circuit breaker
|
|
207
|
-
*/
|
|
208
|
-
resetCircuitBreaker() {
|
|
209
|
-
this.state.consecutiveFailures = 0;
|
|
210
|
-
this.state.circuitOpen = false;
|
|
211
|
-
this.state.circuitOpenedAt = null;
|
|
212
|
-
}
|
|
213
|
-
/**
|
|
214
|
-
* Get current state for inspection
|
|
215
|
-
*/
|
|
216
|
-
getState() {
|
|
217
|
-
return { ...this.state };
|
|
218
|
-
}
|
|
219
|
-
/**
|
|
220
|
-
* Get configuration
|
|
221
|
-
*/
|
|
222
|
-
getConfig() {
|
|
223
|
-
return { ...this.config };
|
|
224
|
-
}
|
|
225
|
-
/**
|
|
226
|
-
* Reset all state (for testing)
|
|
227
|
-
*/
|
|
228
|
-
reset() {
|
|
229
|
-
this.state = {
|
|
230
|
-
consecutiveFailures: 0,
|
|
231
|
-
circuitOpen: false,
|
|
232
|
-
circuitOpenedAt: null,
|
|
233
|
-
lastCallTime: null,
|
|
234
|
-
};
|
|
235
|
-
}
|
|
236
|
-
}
|
|
237
|
-
// ============================================================================
|
|
238
|
-
// Utility Functions
|
|
239
|
-
// ============================================================================
|
|
240
|
-
/**
|
|
241
|
-
* Create a new rate limiter with default configuration
|
|
242
|
-
*/
|
|
243
|
-
export function createRateLimiter(config) {
|
|
244
|
-
return new RateLimiter(config);
|
|
245
|
-
}
|
|
246
|
-
/**
|
|
247
|
-
* Determine if a status code is a 5xx error (server error)
|
|
248
|
-
*/
|
|
249
|
-
export function is5xx(status) {
|
|
250
|
-
return status >= 500 && status < 600;
|
|
251
|
-
}
|
|
252
|
-
/**
|
|
253
|
-
* Determine if a status code should trigger retry
|
|
254
|
-
*/
|
|
255
|
-
export function isRetryableStatus(status) {
|
|
256
|
-
return status === 429 || is5xx(status);
|
|
257
|
-
}
|
|
258
|
-
//# sourceMappingURL=rate-limiting.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"rate-limiting.js","sourceRoot":"","sources":["../../src/enrich/rate-limiting.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAyBH,+EAA+E;AAC/E,2DAA2D;AAC3D,+EAA+E;AAE/E,MAAM,OAAO,WAAW;IACf,MAAM,CAAiB;IACvB,KAAK,CAAgB;IAE7B,YAAY,aAAwC;QACnD,8CAA8C;QAC9C,IAAI,CAAC,MAAM,GAAG;YACb,cAAc,EAAE,aAAa,EAAE,cAAc,IAAI,IAAI;YACrD,UAAU,EAAE,aAAa,EAAE,UAAU,IAAI,CAAC;YAC1C,uBAAuB,EAAE,aAAa,EAAE,uBAAuB,IAAI,CAAC;YACpE,qBAAqB,EAAE,aAAa,EAAE,qBAAqB,IAAI,KAAK;SACpE,CAAA;QAED,kBAAkB;QAClB,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QAEhC,mBAAmB;QACnB,IAAI,CAAC,KAAK,GAAG;YACZ,mBAAmB,EAAE,CAAC;YACtB,WAAW,EAAE,KAAK;YAClB,eAAe,EAAE,IAAI;YACrB,YAAY,EAAE,IAAI;SAClB,CAAA;IACF,CAAC;IAEO,cAAc,CAAC,MAAuB;QAC7C,IAAI,MAAM,CAAC,cAAc,GAAG,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAA;QACvD,IAAI,MAAM,CAAC,UAAU,GAAG,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAA;QACnD,IAAI,MAAM,CAAC,uBAAuB,GAAG,CAAC;YACrC,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAA;QACxD,IAAI,MAAM,CAAC,qBAAqB,GAAG,CAAC;YACnC,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAA;IAC/D,CAAC;IAED,+EAA+E;IAC/E,uCAAuC;IACvC,+EAA+E;IAE/E;;;OAGG;IACI,eAAe;QACrB,0BAA0B;QAC1B,IAAI,IAAI,CAAC,KAAK,CAAC,YAAY,KAAK,IAAI,EAAE,CAAC;YACtC,OAAO,CAAC,CAAA;QACT,CAAC;QAED,MAAM,iBAAiB,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAA;QAC9D,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,cAAc,CAAA;QAEhD,IAAI,iBAAiB,GAAG,aAAa,EAAE,CAAC;YACvC,OAAO,aAAa,GAAG,iBAAiB,CAAA;QACzC,CAAC;QAED,OAAO,CAAC,CAAA;IACT,CAAC;IAED;;OAEG;IACI,UAAU;QAChB,IAAI,CAAC,KAAK,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;IACrC,CAAC;IAED,+EAA+E;IAC/E,wCAAwC;IACxC,+EAA+E;IAE/E;;;;;OAKG;IACK,2BAA2B,CAAC,aAAqB;QACxD,gCAAgC;QAChC,MAAM,gBAAgB,GAAG,CAAC,IAAI,aAAa,CAAA;QAC3C,MAAM,WAAW,GAAG,gBAAgB,GAAG,IAAI,CAAA;QAE3C,oBAAoB;QACpB,MAAM,YAAY,GAAG,WAAW,GAAG,IAAI,CAAA;QACvC,MAAM,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,YAAY,CAAA;QAEvD,OAAO,WAAW,GAAG,MAAM,CAAA;IAC5B,CAAC;IAED,+EAA+E;IAC/E,mCAAmC;IACnC,+EAA+E;IAE/E;;;;OAIG;IACK,qBAAqB,CAC5B,eAA4C;QAE5C,IAAI,eAAe,KAAK,SAAS,IAAI,eAAe,KAAK,IAAI,EAAE,CAAC;YAC/D,OAAO,IAAI,CAAA;QACZ,CAAC;QAED,2BAA2B;QAC3B,IAAI,OAAO,eAAe,KAAK,QAAQ,EAAE,CAAC;YACzC,OAAO,eAAe,GAAG,IAAI,CAAA;QAC9B,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,CAAC,eAAe,CAAC,CAAC,IAAI,EAAE,CAAA;QAE/C,iCAAiC;QACjC,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAA;QAC7C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,OAAO,IAAI,CAAC,EAAE,CAAC;YAC5C,OAAO,OAAO,GAAG,IAAI,CAAA;QACtB,CAAC;QAED,2BAA2B;QAC3B,IAAI,CAAC;YACJ,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,QAAQ,CAAC,CAAA;YAC/B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;gBACnC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;gBAC3C,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAA;YAC5B,CAAC;QACF,CAAC;QAAC,MAAM,CAAC;YACR,sBAAsB;QACvB,CAAC;QAED,yDAAyD;QACzD,OAAO,IAAI,CAAA;IACZ,CAAC;IAED,+EAA+E;IAC/E,yDAAyD;IACzD,+EAA+E;IAE/E;;;;;OAKG;IACI,gBAAgB,CACtB,QAAqB,EACrB,aAAqB;QAErB,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,QAAQ,CAAA;QAEpC,+BAA+B;QAC/B,IAAI,MAAM,IAAI,GAAG,IAAI,MAAM,GAAG,GAAG,EAAE,CAAC;YACnC,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,EAAE,CAAA;QAC1C,CAAC;QAED,sBAAsB;QACtB,MAAM,iBAAiB,GAAG,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,IAAI,GAAG,IAAI,MAAM,GAAG,GAAG,CAAC,CAAA;QAC3E,IAAI,CAAC,iBAAiB,EAAE,CAAC;YACxB,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,EAAE,CAAA;QAC1C,CAAC;QAED,wDAAwD;QACxD,MAAM,YAAY,GAAG,IAAI,CAAC,qBAAqB,CAAC,OAAO,EAAE,CAAC,aAAa,CAAC,CAAC,CAAA;QACzE,IAAI,YAAY,KAAK,IAAI,EAAE,CAAC;YAC3B,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,OAAO,EAAE,YAAY,EAAE,CAAA;QACpD,CAAC;QAED,4CAA4C;QAC5C,MAAM,SAAS,GAAG,IAAI,CAAC,2BAA2B,CAAC,aAAa,CAAC,CAAA;QACjE,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,CAAA;IACjD,CAAC;IAED;;;;OAIG;IACI,kBAAkB,CAAC,aAAqB;QAC9C,OAAO,aAAa,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,CAAA;IAC/C,CAAC;IAED,+EAA+E;IAC/E,wBAAwB;IACxB,+EAA+E;IAE/E;;;OAGG;IACI,aAAa;QACnB,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;YAC7B,OAAO,KAAK,CAAA;QACb,CAAC;QAED,+CAA+C;QAC/C,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,eAAe,IAAI,CAAC,CAAC,CAAA;QACtE,IAAI,eAAe,IAAI,IAAI,CAAC,MAAM,CAAC,qBAAqB,EAAE,CAAC;YAC1D,wBAAwB;YACxB,IAAI,CAAC,mBAAmB,EAAE,CAAA;YAC1B,OAAO,KAAK,CAAA;QACb,CAAC;QAED,OAAO,IAAI,CAAA;IACZ,CAAC;IAED;;OAEG;IACI,aAAa;QACnB,IAAI,CAAC,KAAK,CAAC,mBAAmB,IAAI,CAAC,CAAA;QAEnC,IAAI,IAAI,CAAC,KAAK,CAAC,mBAAmB,IAAI,IAAI,CAAC,MAAM,CAAC,uBAAuB,EAAE,CAAC;YAC3E,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,IAAI,CAAA;YAC7B,IAAI,CAAC,KAAK,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QACxC,CAAC;IACF,CAAC;IAED;;OAEG;IACI,aAAa;QACnB,IAAI,CAAC,KAAK,CAAC,mBAAmB,GAAG,CAAC,CAAA;QAClC,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,KAAK,CAAA;QAC9B,IAAI,CAAC,KAAK,CAAC,eAAe,GAAG,IAAI,CAAA;IAClC,CAAC;IAED;;OAEG;IACI,mBAAmB;QACzB,IAAI,CAAC,KAAK,CAAC,mBAAmB,GAAG,CAAC,CAAA;QAClC,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,KAAK,CAAA;QAC9B,IAAI,CAAC,KAAK,CAAC,eAAe,GAAG,IAAI,CAAA;IAClC,CAAC;IAED;;OAEG;IACI,QAAQ;QACd,OAAO,EAAE,GAAG,IAAI,CAAC,KAAK,EAAE,CAAA;IACzB,CAAC;IAED;;OAEG;IACI,SAAS;QACf,OAAO,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,CAAA;IAC1B,CAAC;IAED;;OAEG;IACI,KAAK;QACX,IAAI,CAAC,KAAK,GAAG;YACZ,mBAAmB,EAAE,CAAC;YACtB,WAAW,EAAE,KAAK;YAClB,eAAe,EAAE,IAAI;YACrB,YAAY,EAAE,IAAI;SAClB,CAAA;IACF,CAAC;CACD;AAED,+EAA+E;AAC/E,oBAAoB;AACpB,+EAA+E;AAE/E;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAChC,MAAiC;IAEjC,OAAO,IAAI,WAAW,CAAC,MAAM,CAAC,CAAA;AAC/B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,KAAK,CAAC,MAAc;IACnC,OAAO,MAAM,IAAI,GAAG,IAAI,MAAM,GAAG,GAAG,CAAA;AACrC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,MAAc;IAC/C,OAAO,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,MAAM,CAAC,CAAA;AACvC,CAAC"}
|
package/dist/index.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EACN,qBAAqB,EACrB,oBAAoB,EACpB,uBAAuB,GACvB,MAAM,uBAAuB,CAAA;AAE9B,OAAO,EACN,gBAAgB,EAChB,kBAAkB,EAClB,cAAc,EACd,UAAU,EACV,cAAc,EACd,WAAW,EACX,iBAAiB,GACjB,MAAM,oBAAoB,CAAA;AAC3B,YAAY,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAA;AAC9D,OAAO,EACN,oBAAoB,EACpB,cAAc,EACd,kBAAkB,EAClB,cAAc,EACd,kBAAkB,GAClB,MAAM,oBAAoB,CAAA;AAC3B,YAAY,EACX,WAAW,EACX,eAAe,EACf,cAAc,GACd,MAAM,2BAA2B,CAAA;AAElC,OAAO,EACN,iBAAiB,EACjB,KAAK,EACL,iBAAiB,EACjB,WAAW,GACX,MAAM,2BAA2B,CAAA;AAClC,YAAY,EACX,WAAW,IAAI,iBAAiB,EAChC,UAAU,GACV,MAAM,yBAAyB,CAAA;AAChC,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAA;AACvD,YAAY,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAA;AAEnE,OAAO,EAAE,oBAAoB,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAA;AAExE,YAAY,EACX,MAAM,EACN,cAAc,EACd,eAAe,EACf,SAAS,EACT,SAAS,EACT,eAAe,EACf,OAAO,EACP,WAAW,EACX,WAAW,EACX,SAAS,EACT,WAAW,GACX,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EACN,qBAAqB,EACrB,eAAe,EACf,qBAAqB,EACrB,iBAAiB,EACjB,eAAe,EACf,iBAAiB,GACjB,MAAM,qBAAqB,CAAA;AAC5B,YAAY,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAA;AAE7D,OAAO,EACN,WAAW,EACX,wBAAwB,EACxB,aAAa,EACb,eAAe,GACf,MAAM,4BAA4B,CAAA;AACnC,YAAY,EACX,YAAY,EACZ,WAAW,IAAI,qBAAqB,EACpC,eAAe,GACf,MAAM,6BAA6B,CAAA;AACpC,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAA"}
|
package/dist/index.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EACN,qBAAqB,EACrB,oBAAoB,EACpB,uBAAuB,GACvB,MAAM,uBAAuB,CAAA;AAC9B,gCAAgC;AAChC,OAAO,EACN,gBAAgB,EAChB,kBAAkB,EAClB,cAAc,EACd,UAAU,EACV,cAAc,EACd,WAAW,EACX,iBAAiB,GACjB,MAAM,oBAAoB,CAAA;AAE3B,OAAO,EACN,oBAAoB,EACpB,cAAc,EACd,kBAAkB,EAClB,cAAc,EACd,kBAAkB,GAClB,MAAM,oBAAoB,CAAA;AAM3B,4BAA4B;AAC5B,OAAO,EACN,iBAAiB,EACjB,KAAK,EACL,iBAAiB,EACjB,WAAW,GACX,MAAM,2BAA2B,CAAA;AAKlC,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAA;AAEvD,+BAA+B;AAC/B,OAAO,EAAE,oBAAoB,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAA;AAexE,OAAO,EACN,qBAAqB,EACrB,eAAe,EACf,qBAAqB,EACrB,iBAAiB,EACjB,eAAe,EACf,iBAAiB,GACjB,MAAM,qBAAqB,CAAA;AAE5B,wBAAwB;AACxB,OAAO,EACN,WAAW,EACX,wBAAwB,EACxB,aAAa,EACb,eAAe,GACf,MAAM,4BAA4B,CAAA;AAMnC,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAA"}
|
|
@@ -1,82 +0,0 @@
|
|
|
1
|
-
import type { Message } from '#schema/message';
|
|
2
|
-
/**
|
|
3
|
-
* Deduplication and merge logic for NORMALIZE--T04
|
|
4
|
-
*
|
|
5
|
-
* Merges CSV and DB message sources with:
|
|
6
|
-
* AC01: Exact GUID matching (primary)
|
|
7
|
-
* AC02: DB authoritiveness for conflicts
|
|
8
|
-
* AC03: Content equivalence detection
|
|
9
|
-
* AC04: Data loss verification
|
|
10
|
-
* AC05: Deterministic GUID assignment
|
|
11
|
-
*/
|
|
12
|
-
export type MergeStats = {
|
|
13
|
-
csvCount: number;
|
|
14
|
-
dbCount: number;
|
|
15
|
-
outputCount: number;
|
|
16
|
-
exactMatches: number;
|
|
17
|
-
contentMatches: number;
|
|
18
|
-
conflicts: number;
|
|
19
|
-
noMatches: number;
|
|
20
|
-
};
|
|
21
|
-
export type ContentMatch = {
|
|
22
|
-
message: Message;
|
|
23
|
-
confidence: number;
|
|
24
|
-
reasons: string[];
|
|
25
|
-
};
|
|
26
|
-
export type MergeResult = {
|
|
27
|
-
messages: Message[];
|
|
28
|
-
stats: MergeStats;
|
|
29
|
-
conflicts?: Array<{
|
|
30
|
-
csvMsg: Message;
|
|
31
|
-
dbMsg: Message;
|
|
32
|
-
confidence: number;
|
|
33
|
-
}>;
|
|
34
|
-
warnings?: string[];
|
|
35
|
-
};
|
|
36
|
-
/**
|
|
37
|
-
* AC01 + AC02 + AC03 + AC04 + AC05: Main dedup and merge function
|
|
38
|
-
*
|
|
39
|
-
* Strategy:
|
|
40
|
-
* 1. Build GUID index for fast lookup
|
|
41
|
-
* 2. For each CSV message:
|
|
42
|
-
* a. Try exact GUID match (AC01)
|
|
43
|
-
* b. Try content equivalence (AC03)
|
|
44
|
-
* c. Apply DB authoritiveness if merging (AC02)
|
|
45
|
-
* d. Keep separate if no match
|
|
46
|
-
* 3. Add unmatched DB messages
|
|
47
|
-
* 4. Verify no data loss (AC04)
|
|
48
|
-
* 5. Ensure determinism (AC05)
|
|
49
|
-
*/
|
|
50
|
-
export declare function dedupAndMerge(csvMessages: Message[], dbMessages: Message[]): MergeResult;
|
|
51
|
-
/**
|
|
52
|
-
* AC01: Find exact GUID match in DB messages
|
|
53
|
-
*/
|
|
54
|
-
export declare function findExactMatch(message: Message, dbMessages: Message[]): Message | null;
|
|
55
|
-
/**
|
|
56
|
-
* AC03: Detect content equivalence
|
|
57
|
-
*
|
|
58
|
-
* Normalizes text and compares:
|
|
59
|
-
* - Normalized text content (lowercase, trimmed, punctuation removed)
|
|
60
|
-
* - messageKind must match
|
|
61
|
-
* - sender (handle) must match
|
|
62
|
-
*
|
|
63
|
-
* Returns match with confidence score (1.0 = exact match)
|
|
64
|
-
*/
|
|
65
|
-
export declare function detectContentEquivalence(csvMsg: Message, candidates: Message[], threshold?: number): ContentMatch | null;
|
|
66
|
-
/**
|
|
67
|
-
* AC02: Merge messages with DB authoritiveness
|
|
68
|
-
*
|
|
69
|
-
* DB is authoritative for:
|
|
70
|
-
* - All timestamps (date, dateRead, dateDelivered, dateEdited)
|
|
71
|
-
* - Associations (replyingTo.targetMessageGuid)
|
|
72
|
-
* - handle
|
|
73
|
-
*
|
|
74
|
-
* CSV fields are preserved when DB doesn't have them
|
|
75
|
-
*/
|
|
76
|
-
export declare function applyDbAuthoritiveness(csvMsg: Message, dbMsg: Message): Message;
|
|
77
|
-
/**
|
|
78
|
-
* AC04: Verify count invariants to prevent data loss
|
|
79
|
-
*/
|
|
80
|
-
export declare function verifyNoDataLoss(csvCount: number, dbCount: number, outputCount: number): boolean;
|
|
81
|
-
export type { Message };
|
|
82
|
-
//# sourceMappingURL=dedup-merge.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"dedup-merge.d.ts","sourceRoot":"","sources":["../../src/ingest/dedup-merge.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAA;AAE9C;;;;;;;;;GASG;AAEH,MAAM,MAAM,UAAU,GAAG;IACxB,QAAQ,EAAE,MAAM,CAAA;IAChB,OAAO,EAAE,MAAM,CAAA;IACf,WAAW,EAAE,MAAM,CAAA;IACnB,YAAY,EAAE,MAAM,CAAA;IACpB,cAAc,EAAE,MAAM,CAAA;IACtB,SAAS,EAAE,MAAM,CAAA;IACjB,SAAS,EAAE,MAAM,CAAA;CACjB,CAAA;AAED,MAAM,MAAM,YAAY,GAAG;IAC1B,OAAO,EAAE,OAAO,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;IAClB,OAAO,EAAE,MAAM,EAAE,CAAA;CACjB,CAAA;AAED,MAAM,MAAM,WAAW,GAAG;IACzB,QAAQ,EAAE,OAAO,EAAE,CAAA;IACnB,KAAK,EAAE,UAAU,CAAA;IACjB,SAAS,CAAC,EAAE,KAAK,CAAC;QAAE,MAAM,EAAE,OAAO,CAAC;QAAC,KAAK,EAAE,OAAO,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;IAC1E,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAA;CACnB,CAAA;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,aAAa,CAC5B,WAAW,EAAE,OAAO,EAAE,EACtB,UAAU,EAAE,OAAO,EAAE,GACnB,WAAW,CA8Fb;AAED;;GAEG;AACH,wBAAgB,cAAc,CAC7B,OAAO,EAAE,OAAO,EAChB,UAAU,EAAE,OAAO,EAAE,GACnB,OAAO,GAAG,IAAI,CAEhB;AAED;;;;;;;;;GASG;AACH,wBAAgB,wBAAwB,CACvC,MAAM,EAAE,OAAO,EACf,UAAU,EAAE,OAAO,EAAE,EACrB,SAAS,SAAM,GACb,YAAY,GAAG,IAAI,CA2DrB;AAiED;;;;;;;;;GASG;AACH,wBAAgB,sBAAsB,CACrC,MAAM,EAAE,OAAO,EACf,KAAK,EAAE,OAAO,GACZ,OAAO,CA6BT;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAC/B,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,MAAM,EACf,WAAW,EAAE,MAAM,GACjB,OAAO,CAIT;AAED,YAAY,EAAE,OAAO,EAAE,CAAA"}
|