@rubytech/create-maxy 1.0.808 → 1.0.809

package/payload/platform/plugins/whatsapp-import/bin/whatsapp-ingest.sh

@@ -1,18 +1,28 @@
 #!/usr/bin/env bash
 # =============================================================================
 # whatsapp-ingest.sh — single deterministic Bash entry for WhatsApp archive
-# ingestion (Task 855). Thin wrapper: arg-validate, resolve env, invoke
-# ingest.mjs in-process. The script is the database-operator subagent's only
-# operator-facing handle on the parse → archive-write → insight pipeline.
+# ingestion (Task 855 / Task 871). Thin wrapper: arg-validate, resolve env,
+# invoke ingest.mjs in-process. The script is the database-operator
+# subagent's only operator-facing handle on the parse → filter → archive-write
+# pipeline. Phase 1 has NO LLM. The Haiku insight pass is Phase 2 — invoked
+# consciously via `mcp__memory__whatsapp-export-insight-pass`.
 #
 # Usage:
 #   bash whatsapp-ingest.sh <archive.zip|dir|_chat.txt>
 #         --owner-element-id <id>
 #         --scope <admin|public>
+#         --filter <all|senders=<csv>|date-range=<isoFrom>..<isoTo>>
 #         [--account-id <accountId>]
 #         [--timezone <iana-zone>]
 #         [--date-format <DD/MM/YY|MM/DD/YY|DD/MM/YYYY|MM/DD/YYYY>]
-#         [--no-insight]
+#
+# `--filter` is mandatory (Task 871). Forms:
+#   all                              — write every parsed row
+#   senders=Alice,Bob Carter         — keep rows whose senderName ∈ csv
+#   date-range=2024-01-01..2024-06-30
+#                                    — keep rows whose dateSent falls inside
+#                                      the inclusive range (date-only or full
+#                                      ISO 8601 endpoints both accepted)
 #
 # Exit 0  + JSON summary on stdout on success.
 # Exit !0 + one [whatsapp-ingest] FAIL line on stderr on failure.
@@ -45,18 +55,20 @@ fi
 ARCHIVE=""
 OWNER_VAL=""
 SCOPE_VAL=""
+FILTER_VAL=""
 HAS_OWNER=0
 HAS_SCOPE=0
+HAS_FILTER=0
 
 ARGS=("$@")
 i=0
 while [ $i -lt ${#ARGS[@]} ]; do
   a="${ARGS[$i]}"
   case "$a" in
-    --owner-element-id) HAS_OWNER=1; OWNER_VAL="${ARGS[$((i + 1))]:-}"; i=$((i + 2)); continue ;;
-    --scope)            HAS_SCOPE=1; SCOPE_VAL="${ARGS[$((i + 1))]:-}"; i=$((i + 2)); continue ;;
+    --owner-element-id) HAS_OWNER=1;  OWNER_VAL="${ARGS[$((i + 1))]:-}";  i=$((i + 2)); continue ;;
+    --scope)            HAS_SCOPE=1;  SCOPE_VAL="${ARGS[$((i + 1))]:-}";  i=$((i + 2)); continue ;;
+    --filter)           HAS_FILTER=1; FILTER_VAL="${ARGS[$((i + 1))]:-}"; i=$((i + 2)); continue ;;
     --account-id|--timezone|--date-format) i=$((i + 2)); continue ;;
-    --no-insight)       i=$((i + 1)); continue ;;
     --*)                i=$((i + 2)); continue ;;
     *)
       if [ -z "$ARCHIVE" ]; then ARCHIVE="$a"; fi
@@ -66,13 +78,19 @@ while [ $i -lt ${#ARGS[@]} ]; do
   esac
 done
 
-[ -n "$ARCHIVE" ]                          || arg_fail "missing positional <archive>"
-[ "$HAS_OWNER" -eq 1 ] && [ -n "$OWNER_VAL" ] || arg_fail "missing --owner-element-id (or empty value)"
-[ "$HAS_SCOPE" -eq 1 ] && [ -n "$SCOPE_VAL" ] || arg_fail "missing --scope (or empty value)"
+[ -n "$ARCHIVE" ]                              || arg_fail "missing positional <archive>"
+[ "$HAS_OWNER"  -eq 1 ] && [ -n "$OWNER_VAL"  ] || arg_fail "missing --owner-element-id (or empty value)"
+[ "$HAS_SCOPE"  -eq 1 ] && [ -n "$SCOPE_VAL"  ] || arg_fail "missing --scope (or empty value)"
 case "$SCOPE_VAL" in
   admin|public) : ;;
   *) arg_fail "invalid --scope \"$SCOPE_VAL\" (admin|public)" ;;
 esac
+if [ "$HAS_FILTER" -ne 1 ] || [ -z "$FILTER_VAL" ]; then
+  # Mirror ingest.mjs's pinned LOUD-FAIL line so a single grep covers both
+  # layers — the operator's runbook recipe is `grep '\[whatsapp-ingest\] FAIL filter-required'`.
+  echo "[whatsapp-ingest] FAIL filter-required reason=\"bulk-archive-gate (Task 871) — operator must specify --filter (one of all, senders=<csv>, date-range=<isoFrom>..<isoTo>)\"" >&2
+  arg_fail "missing --filter (one of all, senders=<csv>, date-range=<isoFrom>..<isoTo>)"
+fi
 
 # Lift NEO4J_PASSWORD from the install's config file when env doesn't carry it
 # (e.g. operator running directly from a shell rather than via the platform

package/payload/platform/plugins/whatsapp-import/lib/dist/derive-keys.d.ts

@@ -0,0 +1,21 @@
+export declare function normaliseSenderName(name: string): string;
+export declare function sha256Hex(input: string): string;
+export interface DeriveMessageIdInput {
+    /** SHA-256 of the source `_chat.txt` bytes — stable across re-imports. */
+    conversationSha256: string;
+    /** ISO 8601 with timezone offset, as emitted by parseExport. */
+    dateSent: string;
+    /** Raw senderName from the export line. Normalised internally. */
+    senderName: string;
+    /** Raw message body. Hashed internally. */
+    body: string;
+}
+export declare function deriveMessageId(input: DeriveMessageIdInput): string;
+export interface ObservationContentFields {
+    summary?: string | null;
+    from?: string | null;
+    to?: string | null;
+    subject?: string | null;
+}
+export declare function observationContentHash(fields: ObservationContentFields): string;
+//# sourceMappingURL=derive-keys.d.ts.map

package/payload/platform/plugins/whatsapp-import/lib/dist/derive-keys.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"derive-keys.d.ts","sourceRoot":"","sources":["../src/derive-keys.ts"],"names":[],"mappings":"AAqBA,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAExD;AAED,wBAAgB,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAE/C;AAED,MAAM,WAAW,oBAAoB;IACnC,0EAA0E;IAC1E,kBAAkB,EAAE,MAAM,CAAC;IAC3B,gEAAgE;IAChE,QAAQ,EAAE,MAAM,CAAC;IACjB,kEAAkE;IAClE,UAAU,EAAE,MAAM,CAAC;IACnB,2CAA2C;IAC3C,IAAI,EAAE,MAAM,CAAC;CACd;AAED,wBAAgB,eAAe,CAAC,KAAK,EAAE,oBAAoB,GAAG,MAAM,CAInE;AAED,MAAM,WAAW,wBAAwB;IACvC,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,EAAE,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB;AAED,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,wBAAwB,GAAG,MAAM,CAK/E"}

package/payload/platform/plugins/whatsapp-import/lib/dist/derive-keys.js

@@ -0,0 +1,41 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.normaliseSenderName = normaliseSenderName;
+exports.sha256Hex = sha256Hex;
+exports.deriveMessageId = deriveMessageId;
+exports.observationContentHash = observationContentHash;
+const node_crypto_1 = require("node:crypto");
+// ---------------------------------------------------------------------------
+// derive-keys — natural-key derivation for whatsapp-import (Task 870).
+//
+// Pure functions. No I/O. The whole point is that re-imports of the same
+// archive collapse to the same Message identity regardless of release-level
+// drift in array indices, hash widths, or arbitrary tiebreakers.
+//
+// Key shape (Task 870 brief):
+//
+//   messageId = whatsapp-export:msg:<conversationSha256>:<dateSentISO>
+//                                  :<NFKC-trim-lower(senderName)>
+//                                  :<sha256-hex(body)>
+//
+// Operator constraint: the same archive must be re-imported with the same
+// `--timezone` flag. Different timezones reinterpret wall-clock instants and
+// will produce drifted messageIds — that is correct semantics, not a bug.
+// Documented in .docs/whatsapp.md natural-key contract section.
+// ---------------------------------------------------------------------------
+function normaliseSenderName(name) {
+    return name.normalize("NFKC").trim().toLowerCase();
+}
+function sha256Hex(input) {
+    return (0, node_crypto_1.createHash)("sha256").update(input).digest("hex");
+}
+function deriveMessageId(input) {
+    const norm = normaliseSenderName(input.senderName);
+    const bodyHash = sha256Hex(input.body);
+    return `whatsapp-export:msg:${input.conversationSha256}:${input.dateSent}:${norm}:${bodyHash}`;
+}
+function observationContentHash(fields) {
+    const parts = [fields.summary, fields.from, fields.to, fields.subject].map((p) => (p ?? "").normalize("NFKC").trim().toLowerCase());
+    return sha256Hex(parts.join("|"));
+}
+//# sourceMappingURL=derive-keys.js.map

package/payload/platform/plugins/whatsapp-import/lib/dist/derive-keys.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"derive-keys.js","sourceRoot":"","sources":["../src/derive-keys.ts"],"names":[],"mappings":";;AAqBA,kDAEC;AAED,8BAEC;AAaD,0CAIC;AASD,wDAKC;AA1DD,6CAAyC;AAEzC,8EAA8E;AAC9E,uEAAuE;AACvE,EAAE;AACF,yEAAyE;AACzE,4EAA4E;AAC5E,iEAAiE;AACjE,EAAE;AACF,8BAA8B;AAC9B,EAAE;AACF,uEAAuE;AACvE,kEAAkE;AAClE,uDAAuD;AACvD,EAAE;AACF,0EAA0E;AAC1E,6EAA6E;AAC7E,0EAA0E;AAC1E,gEAAgE;AAChE,8EAA8E;AAE9E,SAAgB,mBAAmB,CAAC,IAAY;IAC9C,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;AACrD,CAAC;AAED,SAAgB,SAAS,CAAC,KAAa;IACrC,OAAO,IAAA,wBAAU,EAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC1D,CAAC;AAaD,SAAgB,eAAe,CAAC,KAA2B;IACzD,MAAM,IAAI,GAAG,mBAAmB,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IACnD,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACvC,OAAO,uBAAuB,KAAK,CAAC,kBAAkB,IAAI,KAAK,CAAC,QAAQ,IAAI,IAAI,IAAI,QAAQ,EAAE,CAAC;AACjG,CAAC;AASD,SAAgB,sBAAsB,CAAC,MAAgC;IACrE,MAAM,KAAK,GAAG,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CACxE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CACxD,CAAC;IACF,OAAO,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AACpC,CAAC"}

package/payload/platform/plugins/whatsapp-import/lib/dist/filter.d.ts

@@ -0,0 +1,29 @@
+import type { ParsedLine } from "./parse-export.js";
+export type Filter = {
+    kind: "all";
+} | {
+    kind: "senders";
+    senders: string[];
+} | {
+    kind: "date-range";
+    fromIso: string;
+    toIso: string;
+};
+/**
+ * Parse a CLI `--filter` argument into a structured Filter.
+ *
+ * Throws Error with message starting "filter: …" on malformed input. The
+ * caller (ingest.mjs / vitest) surfaces the reason verbatim — the brief
+ * mandates `[whatsapp-ingest] FAIL filter-required reason="…"` so the
+ * operator can grep one line.
+ */
+export declare function parseFilterArg(raw: string | undefined | null): Filter;
+/**
+ * Apply a parsed Filter to ParsedLine[]. Returns a new array of kept lines
+ * with the parser's original `sequenceIndex` preserved (the filter never
+ * reorders). ingest.mjs re-stamps `sequenceIndex` to its post-filter position
+ * during row construction for archive-write — re-stamping here too would be
+ * redundant.
+ */
+export declare function applyFilter(parsedLines: readonly ParsedLine[], filter: Filter): ParsedLine[];
+//# sourceMappingURL=filter.d.ts.map

package/payload/platform/plugins/whatsapp-import/lib/dist/filter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"filter.d.ts","sourceRoot":"","sources":["../src/filter.ts"],"names":[],"mappings":"AA0BA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAEpD,MAAM,MAAM,MAAM,GACd;IAAE,IAAI,EAAE,KAAK,CAAA;CAAE,GACf;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,OAAO,EAAE,MAAM,EAAE,CAAA;CAAE,GACtC;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC;AAE3D;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,GAAG,MAAM,CA+CrE;AAED;;;;;;GAMG;AACH,wBAAgB,WAAW,CACzB,WAAW,EAAE,SAAS,UAAU,EAAE,EAClC,MAAM,EAAE,MAAM,GACb,UAAU,EAAE,CAQd"}

package/payload/platform/plugins/whatsapp-import/lib/dist/filter.js

@@ -0,0 +1,123 @@
+"use strict";
+// ---------------------------------------------------------------------------
+// filter — operator-supplied gate over ParsedLine[] (Task 871).
+//
+// Phase 1 ingest is now mandatory-filter: the deterministic Bash entry refuses
+// to write a bulk archive without `--filter`. Three forms cover the operator
+// patterns named in the brief:
+//
+//   --filter all                                     → no row drop
+//   --filter senders=Alice,Bob Carter                → keep rows whose
+//                                                       senderName matches
+//                                                       any csv entry exactly
+//   --filter date-range=2024-01-01..2024-06-30       → keep rows whose
+//                                                       dateSent ISO falls
+//                                                       inside the inclusive
+//                                                       range (date or full
+//                                                       ISO 8601)
+//
+// Doctrine alignment:
+//   - feedback_compress_at_ingest_for_bulk_archives.md — the gate is
+//     mandatory at write-time, not after.
+//   - feedback_deterministic_means_remove_llm.md — the filter parser is a
+//     pure function, no LLM in the per-row decision path.
+//   - feedback_loud_failures.md — malformed `--filter` raises a structured
+//     error with a named reason rather than silently coercing to `all`.
+// ---------------------------------------------------------------------------
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseFilterArg = parseFilterArg;
+exports.applyFilter = applyFilter;
+/**
+ * Parse a CLI `--filter` argument into a structured Filter.
+ *
+ * Throws Error with message starting "filter: …" on malformed input. The
+ * caller (ingest.mjs / vitest) surfaces the reason verbatim — the brief
+ * mandates `[whatsapp-ingest] FAIL filter-required reason="…"` so the
+ * operator can grep one line.
+ */
+function parseFilterArg(raw) {
+    if (raw == null || raw.trim() === "") {
+        throw new Error('filter: --filter is required (one of "all", "senders=<csv>", "date-range=<isoFrom>..<isoTo>")');
+    }
+    const value = raw.trim();
+    if (value === "all")
+        return { kind: "all" };
+    if (value.startsWith("senders=")) {
+        const csv = value.slice("senders=".length);
+        const senders = csv
+            .split(",")
+            .map((s) => s.trim())
+            .filter((s) => s.length > 0);
+        if (senders.length === 0) {
+            throw new Error('filter: senders= requires at least one comma-separated name');
+        }
+        return { kind: "senders", senders };
+    }
+    if (value.startsWith("date-range=")) {
+        const range = value.slice("date-range=".length);
+        const parts = range.split("..");
+        if (parts.length !== 2) {
+            throw new Error(`filter: date-range must be "<isoFrom>..<isoTo>" — got "${range}"`);
+        }
+        const [fromIso, toIso] = parts.map((p) => p.trim());
+        if (!fromIso || !toIso) {
+            throw new Error(`filter: date-range requires both endpoints — got "${range}"`);
+        }
+        if (Number.isNaN(Date.parse(fromIso))) {
+            throw new Error(`filter: date-range fromIso="${fromIso}" is not parseable as ISO 8601`);
+        }
+        if (Number.isNaN(Date.parse(toIso))) {
+            throw new Error(`filter: date-range toIso="${toIso}" is not parseable as ISO 8601`);
+        }
+        if (Date.parse(fromIso) > Date.parse(toIso)) {
+            throw new Error(`filter: date-range fromIso="${fromIso}" is later than toIso="${toIso}"`);
+        }
+        return { kind: "date-range", fromIso, toIso };
+    }
+    throw new Error(`filter: unrecognised form "${value}" — must be "all", "senders=<csv>", or "date-range=<isoFrom>..<isoTo>"`);
+}
+/**
+ * Apply a parsed Filter to ParsedLine[]. Returns a new array of kept lines
+ * with the parser's original `sequenceIndex` preserved (the filter never
+ * reorders). ingest.mjs re-stamps `sequenceIndex` to its post-filter position
+ * during row construction for archive-write — re-stamping here too would be
+ * redundant.
+ */
+function applyFilter(parsedLines, filter) {
+    const predicate = makePredicate(filter);
+    const kept = [];
+    for (const line of parsedLines) {
+        if (!predicate(line))
+            continue;
+        kept.push(line);
+    }
+    return kept;
+}
+function makePredicate(filter) {
+    if (filter.kind === "all")
+        return () => true;
+    if (filter.kind === "senders") {
+        const allow = new Set(filter.senders);
+        return (line) => allow.has(line.senderName);
+    }
+    // date-range: inclusive on both ends. Date-only endpoints widen to whole-
+    // day semantics: `from=YYYY-MM-DD` → `T00:00:00Z`, `to=YYYY-MM-DD` →
+    // `T23:59:59.999Z`. Full ISO 8601 endpoints with `T` are passed through.
+    // Without this widening, `--filter date-range=2024-01-01..2024-06-30`
+    // would silently drop every message later than 2024-06-30T00:00:00Z on the
+    // last day — a UX trap that contradicts the operator's reading.
+    const fromMs = parseRangeEndpoint(filter.fromIso, "start");
+    const toMs = parseRangeEndpoint(filter.toIso, "end");
+    return (line) => {
+        const ms = Date.parse(line.dateSent);
+        return ms >= fromMs && ms <= toMs;
+    };
+}
+function parseRangeEndpoint(iso, edge) {
+    if (/T/.test(iso))
+        return Date.parse(iso);
+    // Date-only — widen to whole-day inclusive on the requested edge.
+    const suffix = edge === "start" ? "T00:00:00.000Z" : "T23:59:59.999Z";
+    return Date.parse(iso + suffix);
+}
+//# sourceMappingURL=filter.js.map

package/payload/platform/plugins/whatsapp-import/lib/dist/filter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"filter.js","sourceRoot":"","sources":["../src/filter.ts"],"names":[],"mappings":";AAAA,8EAA8E;AAC9E,gEAAgE;AAChE,EAAE;AACF,+EAA+E;AAC/E,6EAA6E;AAC7E,+BAA+B;AAC/B,EAAE;AACF,mEAAmE;AACnE,uEAAuE;AACvE,2EAA2E;AAC3E,8EAA8E;AAC9E,uEAAuE;AACvE,2EAA2E;AAC3E,6EAA6E;AAC7E,4EAA4E;AAC5E,kEAAkE;AAClE,EAAE;AACF,sBAAsB;AACtB,qEAAqE;AACrE,0CAA0C;AAC1C,0EAA0E;AAC1E,0DAA0D;AAC1D,2EAA2E;AAC3E,wEAAwE;AACxE,8EAA8E;;AAiB9E,wCA+CC;AASD,kCAWC;AA3ED;;;;;;;GAOG;AACH,SAAgB,cAAc,CAAC,GAA8B;IAC3D,IAAI,GAAG,IAAI,IAAI,IAAI,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;QACrC,MAAM,IAAI,KAAK,CACb,+FAA+F,CAChG,CAAC;IACJ,CAAC;IACD,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;IACzB,IAAI,KAAK,KAAK,KAAK;QAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;IAC5C,IAAI,KAAK,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QACjC,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QAC3C,MAAM,OAAO,GAAG,GAAG;aAChB,KAAK,CAAC,GAAG,CAAC;aACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC/B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CAAC,6DAA6D,CAAC,CAAC;QACjF,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC;IACtC,CAAC;IACD,IAAI,KAAK,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;QAChD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAChC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CACb,0DAA0D,KAAK,GAAG,CACnE,CAAC;QACJ,CAAC;QACD,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACpD,IAAI,CAAC,OAAO,IAAI,CAAC,KAAK,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CACb,qDAAqD,KAAK,GAAG,CAC9D,CAAC;QACJ,CAAC;QACD,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;YACtC,MAAM,IAAI,KAAK,CAAC,+BAA+B,OAAO,gCAAgC,CAAC,CAAC;QAC1F,CAAC;QACD,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,6BAA6B,KAAK,gCAAgC,CAAC,CAAC;QACtF,CAAC;QACD,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;YAC5C,MAAM,IAAI,KAAK,CAAC,+BAA+B,OAAO,0BAA0B,KAAK,GAAG,CAAC,CAAC;QAC5F,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IAChD,CAAC;IACD,MAAM,IAAI,KAAK,CACb,8BAA8B,KAAK,wEAAwE,CAC5G,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,SAAgB,WAAW,CACzB,WAAkC,EAClC,MAAc;IAEd,MAAM,SAAS,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IACxC,MAAM,IAAI,GAAiB,EAAE,CAAC;IAC9B,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAC/B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;YAAE,SAAS;QAC/B,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAClB,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,aAAa,CAAC,MAAc;IACnC,IAAI,MAAM,CAAC,IAAI,KAAK,KAAK;QAAE,OAAO,GAAG,EAAE,CAAC,IAAI,CAAC;IAC7C,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACtC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAC9C,CAAC;IACD,0EAA0E;IAC1E,qEAAqE;IACrE,yEAAyE;IACzE,sEAAsE;IACtE,2EAA2E;IAC3E,gEAAgE;IAChE,MAAM,MAAM,GAAG,kBAAkB,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC3D,MAAM,IAAI,GAAG,kBAAkB,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;IACrD,OAAO,CAAC,IAAI,EAAE,EAAE;QACd,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACrC,OAAO,EAAE,IAAI,MAAM,IAAI,EAAE,IAAI,IAAI,CAAC;IACpC,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,GAAW,EAAE,IAAqB;IAC5D,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC1C,kEAAkE;IAClE,MAAM,MAAM,GAAG,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,gBAAgB,CAAC;IACtE,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,MAAM,CAAC,CAAC;AAClC,CAAC"}

package/payload/platform/plugins/whatsapp-import/lib/dist/index.d.ts

@@ -1,3 +1,7 @@
 export { parseExport } from "./parse-export.js";
 export type { ParseExportInput, ParseExportResult, ParseExportCounters, ParsedLine, } from "./parse-export.js";
+export { parseFilterArg, applyFilter } from "./filter.js";
+export type { Filter } from "./filter.js";
+export { normaliseSenderName, sha256Hex, deriveMessageId, observationContentHash, } from "./derive-keys.js";
+export type { DeriveMessageIdInput, ObservationContentFields, } from "./derive-keys.js";
 //# sourceMappingURL=index.d.ts.map

package/payload/platform/plugins/whatsapp-import/lib/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,YAAY,EACV,gBAAgB,EAChB,iBAAiB,EACjB,mBAAmB,EACnB,UAAU,GACX,MAAM,mBAAmB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,YAAY,EACV,gBAAgB,EAChB,iBAAiB,EACjB,mBAAmB,EACnB,UAAU,GACX,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1D,YAAY,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EACL,mBAAmB,EACnB,SAAS,EACT,eAAe,EACf,sBAAsB,GACvB,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EACV,oBAAoB,EACpB,wBAAwB,GACzB,MAAM,kBAAkB,CAAC"}

package/payload/platform/plugins/whatsapp-import/lib/dist/index.js

@@ -1,6 +1,14 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.parseExport = void 0;
+exports.observationContentHash = exports.deriveMessageId = exports.sha256Hex = exports.normaliseSenderName = exports.applyFilter = exports.parseFilterArg = exports.parseExport = void 0;
 var parse_export_js_1 = require("./parse-export.js");
 Object.defineProperty(exports, "parseExport", { enumerable: true, get: function () { return parse_export_js_1.parseExport; } });
+var filter_js_1 = require("./filter.js");
+Object.defineProperty(exports, "parseFilterArg", { enumerable: true, get: function () { return filter_js_1.parseFilterArg; } });
+Object.defineProperty(exports, "applyFilter", { enumerable: true, get: function () { return filter_js_1.applyFilter; } });
+var derive_keys_js_1 = require("./derive-keys.js");
+Object.defineProperty(exports, "normaliseSenderName", { enumerable: true, get: function () { return derive_keys_js_1.normaliseSenderName; } });
+Object.defineProperty(exports, "sha256Hex", { enumerable: true, get: function () { return derive_keys_js_1.sha256Hex; } });
+Object.defineProperty(exports, "deriveMessageId", { enumerable: true, get: function () { return derive_keys_js_1.deriveMessageId; } });
+Object.defineProperty(exports, "observationContentHash", { enumerable: true, get: function () { return derive_keys_js_1.observationContentHash; } });
 //# sourceMappingURL=index.js.map

package/payload/platform/plugins/whatsapp-import/lib/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;AAAA,qDAAgD;AAAvC,8GAAA,WAAW,OAAA"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;AAAA,qDAAgD;AAAvC,8GAAA,WAAW,OAAA;AAOpB,yCAA0D;AAAjD,2GAAA,cAAc,OAAA;AAAE,wGAAA,WAAW,OAAA;AAEpC,mDAK0B;AAJxB,qHAAA,mBAAmB,OAAA;AACnB,2GAAA,SAAS,OAAA;AACT,iHAAA,eAAe,OAAA;AACf,wHAAA,sBAAsB,OAAA"}

package/payload/platform/plugins/whatsapp-import/lib/src/__tests__/filter-gate.test.ts

@@ -0,0 +1,170 @@
+import { describe, it, expect } from "vitest";
+import { spawnSync } from "node:child_process";
+import { mkdtempSync, writeFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join, resolve, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+
+import { parseFilterArg, applyFilter } from "../filter.js";
+import type { ParsedLine } from "../parse-export.js";
+
+// ---------------------------------------------------------------------------
+// filter-gate — Task 871 contract: --filter is mandatory, three forms.
+//
+// The unit tests cover the helper module's pure-function surface (parse +
+// apply). The integration test spawns ingest.mjs as a subprocess and
+// asserts the no-flag invocation exits non-zero with the LOUD-FAIL line
+// (`[whatsapp-ingest] FAIL phase=argv reason="--filter is required …"`)
+// before any Neo4j connection — argv parsing fires before imports/sessions.
+// ---------------------------------------------------------------------------
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+// src/__tests__/ → ../../bin/ingest.mjs (up to src/, up to lib/, up to plugin
+// root, then bin/).
+const INGEST_MJS = resolve(__dirname, "..", "..", "..", "bin", "ingest.mjs");
+
+function lines(parsed: ParsedLine[]): string[] {
+  return parsed.map((p) => `${p.senderName}@${p.dateSent}:${p.body}`);
+}
+
+const SAMPLE: ParsedLine[] = [
+  { senderName: "Alice", dateSent: "2024-01-15T09:00:00+00:00", body: "morning", sequenceIndex: 0 },
+  { senderName: "Bob",   dateSent: "2024-01-15T09:05:00+00:00", body: "hi",      sequenceIndex: 1 },
+  { senderName: "Alice", dateSent: "2024-03-20T11:00:00+00:00", body: "lunch?",  sequenceIndex: 2 },
+  { senderName: "Carol", dateSent: "2024-06-30T23:59:59+00:00: ", body: "bye",   sequenceIndex: 3 },
+  { senderName: "Bob",   dateSent: "2024-07-01T00:00:01+00:00", body: "next",    sequenceIndex: 4 },
+];
+
+// Repair the malformed dateSent fixture (extra ':' suffix would Date.parse to
+// NaN). Keep this in code, not in the const above, so the fixture reads
+// natural while we still hold a strict tuple.
+SAMPLE[3] = { ...SAMPLE[3], dateSent: "2024-06-30T23:59:59+00:00" };
+
+describe("parseFilterArg — happy path", () => {
+  it('parses --filter all', () => {
+    expect(parseFilterArg("all")).toEqual({ kind: "all" });
+  });
+
+  it('parses senders=<csv> with whitespace tolerance', () => {
+    expect(parseFilterArg("senders=Alice, Bob Carter ,  Carol")).toEqual({
+      kind: "senders",
+      senders: ["Alice", "Bob Carter", "Carol"],
+    });
+  });
+
+  it('parses date-range=<from>..<to>', () => {
+    expect(parseFilterArg("date-range=2024-01-01..2024-06-30")).toEqual({
+      kind: "date-range",
+      fromIso: "2024-01-01",
+      toIso: "2024-06-30",
+    });
+  });
+});
+
+describe("parseFilterArg — LOUD-FAIL on malformed input", () => {
+  it("rejects undefined / empty", () => {
+    expect(() => parseFilterArg(undefined)).toThrow(/--filter is required/);
+    expect(() => parseFilterArg("")).toThrow(/--filter is required/);
+    expect(() => parseFilterArg("   ")).toThrow(/--filter is required/);
+  });
+
+  it("rejects unknown form", () => {
+    expect(() => parseFilterArg("everyone")).toThrow(/unrecognised form/);
+    expect(() => parseFilterArg("nope=yes")).toThrow(/unrecognised form/);
+  });
+
+  it("rejects senders= with no names", () => {
+    expect(() => parseFilterArg("senders=")).toThrow(/at least one/);
+    expect(() => parseFilterArg("senders=,,,")).toThrow(/at least one/);
+  });
+
+  it("rejects date-range with malformed shape or unparseable ISO", () => {
+    expect(() => parseFilterArg("date-range=2024-01-01")).toThrow(/<isoFrom>\.\.<isoTo>/);
+    expect(() => parseFilterArg("date-range=..2024-06-30")).toThrow(/both endpoints/);
+    expect(() => parseFilterArg("date-range=2024-01-01..")).toThrow(/both endpoints/);
+    expect(() => parseFilterArg("date-range=not-a-date..2024-06-30")).toThrow(/not parseable/);
+    expect(() => parseFilterArg("date-range=2024-06-30..2024-01-01")).toThrow(/later than/);
+  });
+});
+
+describe("applyFilter — semantics", () => {
+  it("--filter all keeps every row, preserves the parser's original sequenceIndex", () => {
+    const out = applyFilter(SAMPLE, parseFilterArg("all"));
+    expect(out.length).toBe(SAMPLE.length);
+    expect(out.map((l) => l.sequenceIndex)).toEqual([0, 1, 2, 3, 4]);
+  });
+
+  it("--filter senders=Alice,Bob keeps only those senders, sequenceIndex stays as the parser-stamped value (Carol's index 3 drops out)", () => {
+    const out = applyFilter(SAMPLE, parseFilterArg("senders=Alice,Bob"));
+    expect(lines(out)).toEqual([
+      "Alice@2024-01-15T09:00:00+00:00:morning",
+      "Bob@2024-01-15T09:05:00+00:00:hi",
+      "Alice@2024-03-20T11:00:00+00:00:lunch?",
+      "Bob@2024-07-01T00:00:01+00:00:next",
+    ]);
+    expect(out.map((l) => l.sequenceIndex)).toEqual([0, 1, 2, 4]);
+  });
+
+  it("--filter date-range clips inclusively on both ends", () => {
+    const out = applyFilter(SAMPLE, parseFilterArg("date-range=2024-01-01..2024-06-30"));
+    // 2024-06-30T23:59:59 IS inclusive; 2024-07-01 is NOT.
+    expect(out.map((l) => `${l.senderName}/${l.dateSent}`)).toEqual([
+      "Alice/2024-01-15T09:00:00+00:00",
+      "Bob/2024-01-15T09:05:00+00:00",
+      "Alice/2024-03-20T11:00:00+00:00",
+      "Carol/2024-06-30T23:59:59+00:00",
+    ]);
+  });
+
+  it("--filter date-range with full ISO 8601 endpoints", () => {
+    const out = applyFilter(
+      SAMPLE,
+      parseFilterArg("date-range=2024-03-01T00:00:00Z..2024-06-15T00:00:00Z"),
+    );
+    expect(out).toHaveLength(1);
+    expect(out[0].senderName).toBe("Alice");
+    expect(out[0].body).toBe("lunch?");
+  });
+
+  it("an unmatched senders filter returns []", () => {
+    const out = applyFilter(SAMPLE, parseFilterArg("senders=Nobody"));
+    expect(out).toEqual([]);
+  });
+});
+
+describe("ingest.mjs — missing --filter exits non-zero with LOUD-FAIL line", () => {
+  it("emits [whatsapp-ingest] FAIL phase=argv reason=\"--filter required ...\" before touching Neo4j", () => {
+    // Write a stub _chat.txt; argv parse fires first and rejects the call
+    // before resolveChatTxt/Neo4j connection. We do NOT need a real archive
+    // to test the gate.
+    const work = mkdtempSync(join(tmpdir(), "filter-gate-"));
+    try {
+      const stub = join(work, "_chat.txt");
+      writeFileSync(stub, "[14/03/26, 10:15:23] Joel: stub\n");
+
+      const res = spawnSync(
+        "node",
+        [
+          INGEST_MJS,
+          stub,
+          "--owner-element-id",
+          "stub-owner",
+          "--scope",
+          "admin",
+        ],
+        { encoding: "utf8" },
+      );
+
+      expect(res.status).not.toBe(0);
+      const stderr = res.stderr ?? "";
+      expect(stderr).toMatch(/\[whatsapp-ingest\] FAIL phase=argv/);
+      expect(stderr).toMatch(/reason="--filter is required/);
+      // Confirm the gate fired BEFORE any Neo4j or import work — no
+      // [whatsapp-ingest] start line, no archive-write log line.
+      expect(stderr).not.toMatch(/\[whatsapp-ingest\] start /);
+      expect(stderr).not.toMatch(/\[memory-archive-write\]/);
+    } finally {
+      rmSync(work, { recursive: true, force: true });
+    }
+  });
+});