@orgloop/transform-dedup 0.1.8 → 0.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/dedup.d.ts +9 -0
- package/dist/dedup.d.ts.map +1 -1
- package/dist/dedup.js +16 -1
- package/dist/dedup.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -0
- package/dist/index.js.map +1 -1
- package/package.json +2 -2
package/dist/dedup.d.ts
CHANGED
|
@@ -8,14 +8,23 @@
|
|
|
8
8
|
* Using `fields` is equivalent to `key` — if both are provided, `key` wins.
|
|
9
9
|
*/
|
|
10
10
|
import type { OrgLoopEvent, Transform, TransformContext } from '@orgloop/sdk';
|
|
11
|
+
export interface DedupStats {
|
|
12
|
+
seen: number;
|
|
13
|
+
dropped: number;
|
|
14
|
+
delivered: number;
|
|
15
|
+
windowMs: number;
|
|
16
|
+
}
|
|
11
17
|
export declare class DedupTransform implements Transform {
|
|
12
18
|
readonly id = "dedup";
|
|
13
19
|
private config;
|
|
14
20
|
private windowMs;
|
|
15
21
|
private seen;
|
|
16
22
|
private cleanupTimer;
|
|
23
|
+
private droppedCount;
|
|
24
|
+
private deliveredCount;
|
|
17
25
|
init(config: Record<string, unknown>): Promise<void>;
|
|
18
26
|
execute(event: OrgLoopEvent, _context: TransformContext): Promise<OrgLoopEvent | null>;
|
|
27
|
+
getStats(): DedupStats;
|
|
19
28
|
shutdown(): Promise<void>;
|
|
20
29
|
private buildHash;
|
|
21
30
|
private cleanup;
|
package/dist/dedup.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dedup.d.ts","sourceRoot":"","sources":["../src/dedup.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"dedup.d.ts","sourceRoot":"","sources":["../src/dedup.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAU9E,MAAM,WAAW,UAAU;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;CACjB;AA6BD,qBAAa,cAAe,YAAW,SAAS;IAC/C,QAAQ,CAAC,EAAE,WAAW;IACtB,OAAO,CAAC,MAAM,CAA0C;IACxD,OAAO,CAAC,QAAQ,CAAiB;IACjC,OAAO,CAAC,IAAI,CAA6B;IACzC,OAAO,CAAC,YAAY,CAA+C;IACnE,OAAO,CAAC,YAAY,CAAK;IACzB,OAAO,CAAC,cAAc,CAAK;IAErB,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IA+BpD,OAAO,CAAC,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,gBAAgB,GAAG,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;IAiB5F,QAAQ,IAAI,UAAU;IAShB,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAU/B,OAAO,CAAC,SAAS;IAajB,OAAO,CAAC,OAAO;CAQf"}
|
package/dist/dedup.js
CHANGED
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
import { createHash } from 'node:crypto';
|
|
11
11
|
import { parseDuration } from '@orgloop/sdk';
|
|
12
12
|
/** Known config properties for validation */
|
|
13
|
-
const KNOWN_CONFIG_KEYS = new Set(['key', 'fields', 'window', 'store']);
|
|
13
|
+
const KNOWN_CONFIG_KEYS = new Set(['key', 'fields', 'window', 'store', 'track_delivery']);
|
|
14
14
|
/**
|
|
15
15
|
* Get a value from a nested object using a dot-separated path.
|
|
16
16
|
*/
|
|
@@ -31,6 +31,8 @@ export class DedupTransform {
|
|
|
31
31
|
windowMs = 5 * 60 * 1000;
|
|
32
32
|
seen = new Map();
|
|
33
33
|
cleanupTimer = null;
|
|
34
|
+
droppedCount = 0;
|
|
35
|
+
deliveredCount = 0;
|
|
34
36
|
async init(config) {
|
|
35
37
|
// Validate config keys — warn on unknown properties to catch typos early
|
|
36
38
|
const unknownKeys = Object.keys(config).filter((k) => !KNOWN_CONFIG_KEYS.has(k));
|
|
@@ -44,6 +46,7 @@ export class DedupTransform {
|
|
|
44
46
|
this.config = {
|
|
45
47
|
key: keyPaths,
|
|
46
48
|
window: c.window ?? '5m',
|
|
49
|
+
track_delivery: c.track_delivery ?? false,
|
|
47
50
|
};
|
|
48
51
|
this.windowMs = parseDuration(this.config.window);
|
|
49
52
|
// Clean up expired entries every window period (minimum 10s)
|
|
@@ -60,18 +63,30 @@ export class DedupTransform {
|
|
|
60
63
|
const lastSeen = this.seen.get(hash);
|
|
61
64
|
if (lastSeen !== undefined && now - lastSeen < this.windowMs) {
|
|
62
65
|
// Duplicate within window — drop
|
|
66
|
+
this.droppedCount++;
|
|
63
67
|
return null;
|
|
64
68
|
}
|
|
65
69
|
// New or expired — pass and record
|
|
66
70
|
this.seen.set(hash, now);
|
|
71
|
+
this.deliveredCount++;
|
|
67
72
|
return event;
|
|
68
73
|
}
|
|
74
|
+
getStats() {
|
|
75
|
+
return {
|
|
76
|
+
seen: this.seen.size,
|
|
77
|
+
dropped: this.droppedCount,
|
|
78
|
+
delivered: this.deliveredCount,
|
|
79
|
+
windowMs: this.windowMs,
|
|
80
|
+
};
|
|
81
|
+
}
|
|
69
82
|
async shutdown() {
|
|
70
83
|
if (this.cleanupTimer) {
|
|
71
84
|
clearInterval(this.cleanupTimer);
|
|
72
85
|
this.cleanupTimer = null;
|
|
73
86
|
}
|
|
74
87
|
this.seen.clear();
|
|
88
|
+
this.droppedCount = 0;
|
|
89
|
+
this.deliveredCount = 0;
|
|
75
90
|
}
|
|
76
91
|
buildHash(event) {
|
|
77
92
|
const eventObj = event;
|
package/dist/dedup.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dedup.js","sourceRoot":"","sources":["../src/dedup.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"dedup.js","sourceRoot":"","sources":["../src/dedup.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAyB7C,6CAA6C;AAC7C,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,gBAAgB,CAAC,CAAC,CAAC;AAE1F;;GAEG;AACH,SAAS,SAAS,CAAC,GAAY,EAAE,IAAY;IAC5C,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACjC,IAAI,OAAO,GAAY,GAAG,CAAC;IAC3B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAChC,IAAI,OAAO,KAAK,IAAI,IAAI,OAAO,KAAK,SAAS,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;YAC9E,OAAO,SAAS,CAAC;QAClB,CAAC;QACD,OAAO,GAAI,OAAmC,CAAC,OAAO,CAAC,CAAC;IACzD,CAAC;IACD,OAAO,OAAO,CAAC;AAChB,CAAC;AAED,MAAM,OAAO,cAAc;IACjB,EAAE,GAAG,OAAO,CAAC;IACd,MAAM,GAAgB,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IAChD,QAAQ,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;IACzB,IAAI,GAAG,IAAI,GAAG,EAAkB,CAAC;IACjC,YAAY,GAA0C,IAAI,CAAC;IAC3D,YAAY,GAAG,CAAC,CAAC;IACjB,cAAc,GAAG,CAAC,CAAC;IAE3B,KAAK,CAAC,IAAI,CAAC,MAA+B;QACzC,yEAAyE;QACzE,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACjF,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CACd,yCAAyC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI;gBAClE,mBAAmB,CAAC,GAAG,iBAAiB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACvD,CAAC;QACH,CAAC;QAED,MAAM,CAAC,GAAG,MAAmC,CAAC;QAE9C,8EAA8E;QAC9E,MAAM,QAAQ,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;QAE/D,IAAI,CAAC,MAAM,GAAG;YACb,GAAG,EAAE,QAAQ;YACb,MAAM,EAAE,CAAC,CAAC,MAAM,IAAI,IAAI;YACxB,cAAc,EAAE,CAAC,CAAC,cAAc,IAAI,KAAK;SACzC,CAAC;QACF,IAAI,CAAC,QAAQ,GAAG,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAElD,6DAA6D;QAC7D,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QACxD,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,eAAe,CAAC,CAAC;QACvE,6DAA6D;QAC7D,IAAI,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;YAC7B,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;QAC3B,CAAC;IACF,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,KAAmB,EAAE,QAA0B;QAC5D,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QACnC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAErC,IAAI,QAAQ,KAAK,SAAS,IAAI,GAAG,GAAG,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC9D,iCAAiC;YACjC,IAAI,CAAC,YAAY,EAAE,CAAC;YACpB,OAAO,IAAI,CAAC;QACb,CAAC;QAED,mCAAmC;QACnC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QACzB,IAAI,CAAC,cAAc,EAAE,CAAC;QACtB,OAAO,KAAK,CAAC;IACd,CAAC;IAED,QAAQ;QACP,OAAO;YACN,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI;YACpB,OAAO,EAAE,IAAI,CAAC,YAAY;YAC1B,SAAS,EAAE,IAAI,CAAC,cAAc;YAC9B,QAAQ,EAAE,IAAI,CAAC,QAAQ;SACvB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,QAAQ;QACb,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACvB,aAAa,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YACjC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QAC1B,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QAClB,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;QACtB,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC;IACzB,CAAC;IAEO,SAAS,CAAC,KAAmB;QACpC,MAAM,QAAQ,GAAG,KAA2C,CAAC;QAC7D,MAAM,KAAK,GAAa,EAAE,CAAC;QAE3B,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;YACvC,MAAM,KAAK,GAAG,SAAS,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAC3C,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC;QACjC,CAAC;QAED,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChC,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC1D,CAAC;IAEO,OAAO;QACd,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,KAAK,MAAM,CAAC,IAAI,EAAE,SAAS,CAAC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YAC3C,IAAI,GAAG,GAAG,SAAS,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACtC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YACxB,CAAC;QACF,CAAC;IACF,CAAC;CACD"}
|
package/dist/index.d.ts
CHANGED
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC;AAG1D,wBAAgB,QAAQ,IAAI,qBAAqB,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC;AAG1D,wBAAgB,QAAQ,IAAI,qBAAqB,CAsChD;AAED,YAAY,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -30,6 +30,11 @@ export function register() {
|
|
|
30
30
|
description: 'Storage backend (only "memory" for MVP).',
|
|
31
31
|
default: 'memory',
|
|
32
32
|
},
|
|
33
|
+
track_delivery: {
|
|
34
|
+
type: 'boolean',
|
|
35
|
+
description: 'Track delivery stats (seen vs delivered vs dropped).',
|
|
36
|
+
default: false,
|
|
37
|
+
},
|
|
33
38
|
},
|
|
34
39
|
required: ['window'],
|
|
35
40
|
additionalProperties: false,
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAE5C,MAAM,UAAU,QAAQ;IACvB,OAAO;QACN,EAAE,EAAE,OAAO;QACX,SAAS,EAAE,cAAc;QACzB,YAAY,EAAE;YACb,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACX,GAAG,EAAE;oBACJ,IAAI,EAAE,OAAO;oBACb,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;oBACzB,WAAW,EAAE,wCAAwC;iBACrD;gBACD,MAAM,EAAE;oBACP,IAAI,EAAE,OAAO;oBACb,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;oBACzB,WAAW,EAAE,0DAA0D;iBACvE;gBACD,MAAM,EAAE;oBACP,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,iDAAiD;oBAC9D,OAAO,EAAE,IAAI;iBACb;gBACD,KAAK,EAAE;oBACN,IAAI,EAAE,QAAQ;oBACd,IAAI,EAAE,CAAC,QAAQ,CAAC;oBAChB,WAAW,EAAE,0CAA0C;oBACvD,OAAO,EAAE,QAAQ;iBACjB;aACD;YACD,QAAQ,EAAE,CAAC,QAAQ,CAAC;YACpB,oBAAoB,EAAE,KAAK;SAC3B;KACD,CAAC;AACH,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAE5C,MAAM,UAAU,QAAQ;IACvB,OAAO;QACN,EAAE,EAAE,OAAO;QACX,SAAS,EAAE,cAAc;QACzB,YAAY,EAAE;YACb,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACX,GAAG,EAAE;oBACJ,IAAI,EAAE,OAAO;oBACb,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;oBACzB,WAAW,EAAE,wCAAwC;iBACrD;gBACD,MAAM,EAAE;oBACP,IAAI,EAAE,OAAO;oBACb,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;oBACzB,WAAW,EAAE,0DAA0D;iBACvE;gBACD,MAAM,EAAE;oBACP,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,iDAAiD;oBAC9D,OAAO,EAAE,IAAI;iBACb;gBACD,KAAK,EAAE;oBACN,IAAI,EAAE,QAAQ;oBACd,IAAI,EAAE,CAAC,QAAQ,CAAC;oBAChB,WAAW,EAAE,0CAA0C;oBACvD,OAAO,EAAE,QAAQ;iBACjB;gBACD,cAAc,EAAE;oBACf,IAAI,EAAE,SAAS;oBACf,WAAW,EAAE,sDAAsD;oBACnE,OAAO,EAAE,KAAK;iBACd;aACD;YACD,QAAQ,EAAE,CAAC,QAAQ,CAAC;YACpB,oBAAoB,EAAE,KAAK;SAC3B;KACD,CAAC;AACH,CAAC;AAGD,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@orgloop/transform-dedup",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.9",
|
|
4
4
|
"description": "OrgLoop dedup transform — deduplicate events within a time window",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
7
7
|
"types": "dist/index.d.ts",
|
|
8
8
|
"dependencies": {
|
|
9
|
-
"@orgloop/sdk": "0.1.
|
|
9
|
+
"@orgloop/sdk": "0.1.9"
|
|
10
10
|
},
|
|
11
11
|
"orgloop": {
|
|
12
12
|
"type": "transform",
|