@orgloop/transform-dedup 0.1.0

package/LICENSE.md

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 OrgLoop contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,71 @@
+# @orgloop/transform-dedup
+
+Deduplicates events within a configurable time window. Events with the same key hash seen within the window are dropped; the first occurrence passes through.
+
+## Install
+
+```bash
+npm install @orgloop/transform-dedup
+```
+
+## Configuration
+
+```yaml
+transforms:
+  - name: dedup-5m
+    type: package
+    package: "@orgloop/transform-dedup"
+    config:
+      key:                                # fields used to build the dedup hash
+        - source
+        - type
+        - provenance.platform_event
+        - payload.pr_number
+      window: "5m"                        # time window for dedup (default: 5m)
+      store: "memory"                     # storage backend (only "memory" for now)
+```
+
+### Config options
+
+| Field | Type | Required | Default | Description |
+|-------|------|----------|---------|-------------|
+| `key` | `string[]` | yes | `["source", "type", "id"]` | Dot-path fields used to build the dedup hash. Values are concatenated and SHA-256 hashed |
+| `window` | `string` | yes | `"5m"` | Duration window. Supported units: `ms`, `s`, `m`, `h`, `d` |
+| `store` | `string` | no | `"memory"` | Storage backend. Only `"memory"` is supported in the current version |
+
+### How it works
+
+1. For each incoming event, the transform extracts values at the configured `key` field paths.
+2. The values are concatenated (null-separated) and hashed with SHA-256.
+3. If the hash has been seen within the `window` duration, the event is dropped (returns `null`).
+4. If the hash is new or expired, the event passes through and the hash is recorded with the current timestamp.
+
+A periodic cleanup timer evicts expired entries from the in-memory store.
+
+## Example route
+
+```yaml
+routes:
+  - name: deduped-pr-reviews
+    when:
+      source: github-eng
+      events:
+        - resource.changed
+    transforms:
+      - ref: dedup-5m
+      - ref: humans-only
+    then:
+      actor: openclaw-agent
+```
+
+## Auth / prerequisites
+
+None.
+
+## Limitations / known issues
+
+- **Memory-only store** -- Dedup state is held entirely in memory and is lost on engine restart. After a restart, previously seen events may be processed again until the window catches up.
+- **No distributed dedup** -- The in-memory store does not support multiple engine instances. Running multiple instances results in each instance maintaining its own independent dedup state.
+- **Hash collisions** -- SHA-256 collisions are theoretically possible but practically negligible.
+- **Cleanup interval** -- Expired entries are cleaned up on a timer interval equal to the dedup window (minimum 10 seconds). Between cleanups, the memory footprint grows proportionally to event throughput.
+- **Key field ordering matters** -- The hash is built from key fields in the order specified. Changing the `key` array order produces different hashes.

package/dist/dedup.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Dedup transform — deduplicates events within a configurable time window.
+ *
+ * Builds a hash from specified key fields and drops events that have
+ * been seen within the configured window.
+ */
+import type { OrgLoopEvent, Transform, TransformContext } from '@orgloop/sdk';
+export declare class DedupTransform implements Transform {
+    readonly id = "dedup";
+    private config;
+    private windowMs;
+    private seen;
+    private cleanupTimer;
+    init(config: Record<string, unknown>): Promise<void>;
+    execute(event: OrgLoopEvent, _context: TransformContext): Promise<OrgLoopEvent | null>;
+    shutdown(): Promise<void>;
+    private buildHash;
+    private cleanup;
+}
+//# sourceMappingURL=dedup.d.ts.map

package/dist/dedup.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dedup.d.ts","sourceRoot":"","sources":["../src/dedup.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAwB9E,qBAAa,cAAe,YAAW,SAAS;IAC/C,QAAQ,CAAC,EAAE,WAAW;IACtB,OAAO,CAAC,MAAM,CAA0C;IACxD,OAAO,CAAC,QAAQ,CAAiB;IACjC,OAAO,CAAC,IAAI,CAA6B;IACzC,OAAO,CAAC,YAAY,CAA+C;IAE7D,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAiBpD,OAAO,CAAC,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,gBAAgB,GAAG,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;IAetF,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAQ/B,OAAO,CAAC,SAAS;IAajB,OAAO,CAAC,OAAO;CAQf"}

package/dist/dedup.js

@@ -0,0 +1,82 @@
+/**
+ * Dedup transform — deduplicates events within a configurable time window.
+ *
+ * Builds a hash from specified key fields and drops events that have
+ * been seen within the configured window.
+ */
+import { createHash } from 'node:crypto';
+import { parseDuration } from '@orgloop/sdk';
+/**
+ * Get a value from a nested object using a dot-separated path.
+ */
+function getByPath(obj, path) {
+    const segments = path.split('.');
+    let current = obj;
+    for (const segment of segments) {
+        if (current === null || current === undefined || typeof current !== 'object') {
+            return undefined;
+        }
+        current = current[segment];
+    }
+    return current;
+}
+export class DedupTransform {
+    id = 'dedup';
+    config = { key: [], window: '5m' };
+    windowMs = 5 * 60 * 1000;
+    seen = new Map();
+    cleanupTimer = null;
+    async init(config) {
+        const c = config;
+        this.config = {
+            key: c.key ?? ['source', 'type', 'id'],
+            window: c.window ?? '5m',
+        };
+        this.windowMs = parseDuration(this.config.window);
+        // Clean up expired entries every window period (minimum 10s)
+        const cleanupInterval = Math.max(this.windowMs, 10_000);
+        this.cleanupTimer = setInterval(() => this.cleanup(), cleanupInterval);
+        // Allow the process to exit even if cleanup timer is pending
+        if (this.cleanupTimer.unref) {
+            this.cleanupTimer.unref();
+        }
+    }
+    async execute(event, _context) {
+        const hash = this.buildHash(event);
+        const now = Date.now();
+        const lastSeen = this.seen.get(hash);
+        if (lastSeen !== undefined && now - lastSeen < this.windowMs) {
+            // Duplicate within window — drop
+            return null;
+        }
+        // New or expired — pass and record
+        this.seen.set(hash, now);
+        return event;
+    }
+    async shutdown() {
+        if (this.cleanupTimer) {
+            clearInterval(this.cleanupTimer);
+            this.cleanupTimer = null;
+        }
+        this.seen.clear();
+    }
+    buildHash(event) {
+        const eventObj = event;
+        const parts = [];
+        for (const keyPath of this.config.key) {
+            const value = getByPath(eventObj, keyPath);
+            parts.push(String(value ?? ''));
+        }
+        const joined = parts.join('\0');
+        return createHash('sha256').update(joined).digest('hex');
+    }
+    cleanup() {
+        const now = Date.now();
+        for (const [hash, timestamp] of this.seen) {
+            if (now - timestamp >= this.windowMs) {
+                this.seen.delete(hash);
+            }
+        }
+    }
+}
+//# sourceMappingURL=dedup.js.map

package/dist/dedup.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dedup.js","sourceRoot":"","sources":["../src/dedup.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAQ7C;;GAEG;AACH,SAAS,SAAS,CAAC,GAAY,EAAE,IAAY;IAC5C,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACjC,IAAI,OAAO,GAAY,GAAG,CAAC;IAC3B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAChC,IAAI,OAAO,KAAK,IAAI,IAAI,OAAO,KAAK,SAAS,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;YAC9E,OAAO,SAAS,CAAC;QAClB,CAAC;QACD,OAAO,GAAI,OAAmC,CAAC,OAAO,CAAC,CAAC;IACzD,CAAC;IACD,OAAO,OAAO,CAAC;AAChB,CAAC;AAED,MAAM,OAAO,cAAc;IACjB,EAAE,GAAG,OAAO,CAAC;IACd,MAAM,GAAgB,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IAChD,QAAQ,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;IACzB,IAAI,GAAG,IAAI,GAAG,EAAkB,CAAC;IACjC,YAAY,GAA0C,IAAI,CAAC;IAEnE,KAAK,CAAC,IAAI,CAAC,MAA+B;QACzC,MAAM,CAAC,GAAG,MAAyC,CAAC;QACpD,IAAI,CAAC,MAAM,GAAG;YACb,GAAG,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC;YACtC,MAAM,EAAE,CAAC,CAAC,MAAM,IAAI,IAAI;SACxB,CAAC;QACF,IAAI,CAAC,QAAQ,GAAG,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAElD,6DAA6D;QAC7D,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QACxD,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,eAAe,CAAC,CAAC;QACvE,6DAA6D;QAC7D,IAAI,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;YAC7B,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;QAC3B,CAAC;IACF,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,KAAmB,EAAE,QAA0B;QAC5D,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QACnC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAErC,IAAI,QAAQ,KAAK,SAAS,IAAI,GAAG,GAAG,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC9D,iCAAiC;YACjC,OAAO,IAAI,CAAC;QACb,CAAC;QAED,mCAAmC;QACnC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QACzB,OAAO,KAAK,CAAC;IACd,CAAC;IAED,KAAK,CAAC,QAAQ;QACb,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACvB,aAAa,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YACjC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QAC1B,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;IACnB,CAAC;IAEO,SAAS,CAAC,KAAmB;QACpC,MAAM,QAAQ,GAAG,KAA2C,CAAC;QAC7D,MAAM,KAAK,GAAa,EAAE,CAAC;QAE3B,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;YACvC,MAAM,KAAK,GAAG,SAAS,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAC3C,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC;QACjC,CAAC;QAED,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChC,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC1D,CAAC;IAEO,OAAO;QACd,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,KAAK,MAAM,CAAC,IAAI,EAAE,SAAS,CAAC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YAC3C,IAAI,GAAG,GAAG,SAAS,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACtC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YACxB,CAAC;QACF,CAAC;IACF,CAAC;CACD"}

package/dist/index.d.ts

@@ -0,0 +1,7 @@
+/**
+ * @orgloop/transform-dedup — registration entry point.
+ */
+import type { TransformRegistration } from '@orgloop/sdk';
+export declare function register(): TransformRegistration;
+export { DedupTransform } from './dedup.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC;AAG1D,wBAAgB,QAAQ,IAAI,qBAAqB,CA4BhD;AAED,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC"}

package/dist/index.js

@@ -0,0 +1,35 @@
+/**
+ * @orgloop/transform-dedup — registration entry point.
+ */
+import { DedupTransform } from './dedup.js';
+export function register() {
+    return {
+        id: 'dedup',
+        transform: DedupTransform,
+        configSchema: {
+            type: 'object',
+            properties: {
+                key: {
+                    type: 'array',
+                    items: { type: 'string' },
+                    description: 'Event field paths to use as dedup key.',
+                },
+                window: {
+                    type: 'string',
+                    description: 'Duration window for deduplication (e.g., "5m").',
+                    default: '5m',
+                },
+                store: {
+                    type: 'string',
+                    enum: ['memory'],
+                    description: 'Storage backend (only "memory" for MVP).',
+                    default: 'memory',
+                },
+            },
+            required: ['key', 'window'],
+            additionalProperties: false,
+        },
+    };
+}
+export { DedupTransform } from './dedup.js';
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAE5C,MAAM,UAAU,QAAQ;IACvB,OAAO;QACN,EAAE,EAAE,OAAO;QACX,SAAS,EAAE,cAAc;QACzB,YAAY,EAAE;YACb,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACX,GAAG,EAAE;oBACJ,IAAI,EAAE,OAAO;oBACb,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;oBACzB,WAAW,EAAE,wCAAwC;iBACrD;gBACD,MAAM,EAAE;oBACP,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,iDAAiD;oBAC9D,OAAO,EAAE,IAAI;iBACb;gBACD,KAAK,EAAE;oBACN,IAAI,EAAE,QAAQ;oBACd,IAAI,EAAE,CAAC,QAAQ,CAAC;oBAChB,WAAW,EAAE,0CAA0C;oBACvD,OAAO,EAAE,QAAQ;iBACjB;aACD;YACD,QAAQ,EAAE,CAAC,KAAK,EAAE,QAAQ,CAAC;YAC3B,oBAAoB,EAAE,KAAK;SAC3B;KACD,CAAC;AACH,CAAC;AAED,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC"}

package/package.json

@@ -0,0 +1,28 @@
+{
+  "name": "@orgloop/transform-dedup",
+  "version": "0.1.0",
+  "description": "OrgLoop dedup transform — deduplicate events within a time window",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "dependencies": {
+    "@orgloop/sdk": "0.1.0"
+  },
+  "orgloop": {
+    "type": "transform",
+    "id": "dedup"
+  },
+  "files": [
+    "dist"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "license": "MIT",
+  "scripts": {
+    "build": "tsc",
+    "clean": "rm -rf dist",
+    "typecheck": "tsc --noEmit",
+    "test": "vitest run"
+  }
+}