@rubytech/create-realagent 1.0.828 → 1.0.829

package/payload/platform/plugins/whatsapp-import/bin/whatsapp-ingest.sh

@@ -1,98 +0,0 @@
-#!/usr/bin/env bash
-# =============================================================================
-# whatsapp-ingest.sh — single deterministic Bash entry for WhatsApp archive
-# ingestion (Task 891 — chunked :ConversationArchive shape).
-#
-# Pipeline: parse → bind canonical sender set → derive conversationIdentity →
-# look up prior :ConversationArchive (delta cursor) → sessionize delta at
-# gap-hours boundary → classify each session via Haiku (mode='chat') →
-# memory-ingest with parentLabel='ConversationArchive'.
-#
-# Usage:
-#   bash whatsapp-ingest.sh <archive.zip|dir|_chat.txt>
-#         --owner-element-id <id>
-#         --participant-person-ids <csv>
-#         --scope <admin|public>
-#         [--session-gap-hours <N>]    (default 12)
-#         [--account-id <accountId>]
-#         [--timezone <iana-zone>]
-#         [--date-format <DD/MM/YY|MM/DD/YY|DD/MM/YYYY|MM/DD/YYYY>]
-#
-# `--owner-element-id` + `--participant-person-ids` form the closed sender
-# set; any parsed senderName outside that set LOUD-FAILs with `parser-miss`
-# and exits non-zero. `--filter` and `--subject-person-id` are gone — chunked
-# Section:Conversation rows bound the operator surface naturally.
-#
-# Exit 0  + JSON summary on stdout on success.
-# Exit !0 + one [whatsapp-import] FAIL line on stderr on failure.
-# =============================================================================
-
-set -euo pipefail
-
-arg_fail() {
-  local reason="$1"
-  echo "[whatsapp-import] FAIL phase=argv reason=\"${reason}\"" >&2
-  exit 1
-}
-
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-PLATFORM_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
-INGEST_MJS="$SCRIPT_DIR/ingest.mjs"
-
-if [ ! -f "$INGEST_MJS" ]; then
-  arg_fail "ingest.mjs not found at $INGEST_MJS — run from a built install"
-fi
-
-ARCHIVE=""
-HAS_OWNER=0
-OWNER_VAL=""
-HAS_PARTICIPANTS=0
-PARTICIPANTS_VAL=""
-HAS_SCOPE=0
-SCOPE_VAL=""
-
-ARGS=("$@")
-i=0
-while [ $i -lt ${#ARGS[@]} ]; do
-  a="${ARGS[$i]}"
-  case "$a" in
-    --owner-element-id)        HAS_OWNER=1;        OWNER_VAL="${ARGS[$((i + 1))]:-}";        i=$((i + 2)); continue ;;
-    --participant-person-ids)  HAS_PARTICIPANTS=1; PARTICIPANTS_VAL="${ARGS[$((i + 1))]:-}"; i=$((i + 2)); continue ;;
-    --scope)                   HAS_SCOPE=1;        SCOPE_VAL="${ARGS[$((i + 1))]:-}";        i=$((i + 2)); continue ;;
-    --session-gap-hours|--account-id|--timezone|--date-format) i=$((i + 2)); continue ;;
-    --*)                       i=$((i + 2)); continue ;;
-    *)
-      if [ -z "$ARCHIVE" ]; then ARCHIVE="$a"; fi
-      i=$((i + 1))
-      continue
-      ;;
-  esac
-done
-
-[ -n "$ARCHIVE" ]                                              || arg_fail "missing positional <archive>"
-[ "$HAS_OWNER"        -eq 1 ] && [ -n "$OWNER_VAL"        ]    || arg_fail "missing --owner-element-id (or empty value)"
-[ "$HAS_PARTICIPANTS" -eq 1 ] && [ -n "$PARTICIPANTS_VAL" ]    || arg_fail "missing --participant-person-ids (csv of operator-confirmed :Person/:AdminUser elementIds, owner excluded)"
-[ "$HAS_SCOPE"        -eq 1 ] && [ -n "$SCOPE_VAL"        ]    || arg_fail "missing --scope (or empty value)"
-case "$SCOPE_VAL" in
-  admin|public) : ;;
-  *) arg_fail "invalid --scope \"$SCOPE_VAL\" (admin|public)" ;;
-esac
-
-if [ -z "${NEO4J_PASSWORD:-}" ]; then
-  NEO4J_PASSWORD_FILE="$PLATFORM_ROOT/config/.neo4j-password"
-  if [ -f "$NEO4J_PASSWORD_FILE" ]; then
-    NEO4J_PASSWORD="$(cat "$NEO4J_PASSWORD_FILE")"
-    export NEO4J_PASSWORD
-  else
-    arg_fail "NEO4J_PASSWORD not in env and $NEO4J_PASSWORD_FILE not found"
-  fi
-fi
-
-if [ -z "${NEO4J_URI:-}" ]; then
-  arg_fail "NEO4J_URI not set (no default — set in env)"
-fi
-
-export NEO4J_USER="${NEO4J_USER:-neo4j}"
-export MAXY_PLATFORM_ROOT="$PLATFORM_ROOT"
-
-exec node "$INGEST_MJS" "$@"

package/payload/platform/plugins/whatsapp-import/lib/src/__tests__/delta-append.test.ts

@@ -1,163 +0,0 @@
-import { describe, it, expect } from "vitest";
-import { findDeltaCursor } from "../delta-cursor.js";
-import {
-  deriveConversationIdentity,
-  deriveMessageContentHash,
-  normaliseSenderName,
-} from "../derive-keys.js";
-import type { ParsedLine } from "../parse-export.js";
-
-function mk(dateSent: string, senderName: string, body: string): ParsedLine {
-  return { dateSent, senderName, body, sequenceIndex: 0 };
-}
-
-describe("deriveConversationIdentity", () => {
-  it("is stable across participant order (sorted internally)", () => {
-    const a = deriveConversationIdentity({
-      accountId: "acct-1",
-      participantElementIds: ["1:abc:1", "1:abc:2"],
-    });
-    const b = deriveConversationIdentity({
-      accountId: "acct-1",
-      participantElementIds: ["1:abc:2", "1:abc:1"],
-    });
-    expect(a).toBe(b);
-  });
-
-  it("changes when accountId differs", () => {
-    const a = deriveConversationIdentity({
-      accountId: "acct-1",
-      participantElementIds: ["1:abc:1", "1:abc:2"],
-    });
-    const b = deriveConversationIdentity({
-      accountId: "acct-2",
-      participantElementIds: ["1:abc:1", "1:abc:2"],
-    });
-    expect(a).not.toBe(b);
-  });
-
-  it("rejects empty participant array", () => {
-    expect(() =>
-      deriveConversationIdentity({
-        accountId: "acct-1",
-        participantElementIds: [],
-      }),
-    ).toThrow(/non-empty/);
-  });
-
-  it("rejects empty accountId", () => {
-    expect(() =>
-      deriveConversationIdentity({
-        accountId: "",
-        participantElementIds: ["1:abc:1"],
-      }),
-    ).toThrow(/accountId/);
-  });
-
-  it("treats DM and group identically (same formula, different array length)", () => {
-    const dm = deriveConversationIdentity({
-      accountId: "acct-1",
-      participantElementIds: ["1:a:1", "1:a:2"],
-    });
-    const group = deriveConversationIdentity({
-      accountId: "acct-1",
-      participantElementIds: ["1:a:1", "1:a:2", "1:a:3"],
-    });
-    expect(dm).not.toBe(group);
-    // Both must hex-decode to a sha256 (64 hex chars).
-    expect(dm).toMatch(/^[a-f0-9]{64}$/);
-    expect(group).toMatch(/^[a-f0-9]{64}$/);
-  });
-});
-
-describe("deriveMessageContentHash", () => {
-  it("is content-only — no archive bytes contribute", () => {
-    const a = deriveMessageContentHash({
-      dateSent: "2026-03-14T10:00:00+00:00",
-      senderName: "Joel",
-      body: "hi",
-    });
-    const b = deriveMessageContentHash({
-      dateSent: "2026-03-14T10:00:00+00:00",
-      senderName: "Joel",
-      body: "hi",
-    });
-    expect(a).toBe(b);
-  });
-
-  it("changes when any field changes", () => {
-    const base = {
-      dateSent: "2026-03-14T10:00:00+00:00",
-      senderName: "Joel",
-      body: "hi",
-    };
-    expect(deriveMessageContentHash(base)).not.toBe(
-      deriveMessageContentHash({ ...base, body: "hello" }),
-    );
-    expect(deriveMessageContentHash(base)).not.toBe(
-      deriveMessageContentHash({ ...base, senderName: "Adam" }),
-    );
-    expect(deriveMessageContentHash(base)).not.toBe(
-      deriveMessageContentHash({ ...base, dateSent: "2026-03-14T10:00:01+00:00" }),
-    );
-  });
-
-  it("collapses NFKC-equivalent sender names to one hash", () => {
-    const a = deriveMessageContentHash({
-      dateSent: "2026-03-14T10:00:00+00:00",
-      senderName: "  ADAM Mackay  ",
-      body: "hi",
-    });
-    const b = deriveMessageContentHash({
-      dateSent: "2026-03-14T10:00:00+00:00",
-      senderName: "adam mackay",
-      body: "hi",
-    });
-    expect(a).toBe(b);
-  });
-});
-
-describe("normaliseSenderName", () => {
-  it("returns NFKC-trim-lower form", () => {
-    expect(normaliseSenderName("  Adam Mackay  ")).toBe("adam mackay");
-  });
-});
-
-describe("findDeltaCursor", () => {
-  const lines: ParsedLine[] = [
-    mk("2026-03-14T10:00:00+00:00", "Joel", "first"),
-    mk("2026-03-14T10:01:00+00:00", "Adam", "second"),
-    mk("2026-03-14T10:02:00+00:00", "Joel", "third"),
-  ];
-  const hashFor = (l: ParsedLine) =>
-    deriveMessageContentHash({
-      dateSent: l.dateSent,
-      senderName: l.senderName,
-      body: l.body,
-    });
-
-  it("returns 'found' with deltaStart = cursor + 1 when the hash matches a prior line", () => {
-    const result = findDeltaCursor(lines, hashFor(lines[0]));
-    expect(result).toEqual({ kind: "found", deltaStart: 1 });
-  });
-
-  it("returns 'empty' when the cursor is the last line (no new messages)", () => {
-    const result = findDeltaCursor(lines, hashFor(lines[2]));
-    expect(result).toEqual({ kind: "empty" });
-  });
-
-  it("returns 'missing' when no parsed line matches (LOUD-FAIL upstream)", () => {
-    const result = findDeltaCursor(lines, "deadbeef".repeat(8));
-    expect(result).toEqual({ kind: "missing" });
-  });
-
-  it("returns 'missing' for an empty parsed sequence (cursor cannot exist)", () => {
-    const result = findDeltaCursor([], hashFor(lines[0]));
-    expect(result).toEqual({ kind: "missing" });
-  });
-
-  it("rejects empty cursor hash", () => {
-    expect(() => findDeltaCursor(lines, "")).toThrow(/non-empty/);
-    expect(() => findDeltaCursor(lines, "   ")).toThrow(/non-empty/);
-  });
-});

package/payload/platform/plugins/whatsapp-import/lib/src/__tests__/parse-export-lrm.test.ts

@@ -1,83 +0,0 @@
-import { describe, it, expect, beforeEach, afterEach } from "vitest";
-import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { parseExport } from "../parse-export.js";
-
-// Task 887 — bidi-strip regression. Some WhatsApp exports prefix every
-// timestamp header with U+200E (LEFT-TO-RIGHT MARK) or U+200F (RTL MARK).
-// Pre-fix `decodeAndNormalise` left those bytes in place; the timestamp
-// regex (`^\[(\d{2})\/...`) failed; the LRM-prefixed line was glued onto
-// the previous body as a continuation; the next clean header parsed its
-// senderName off the polluted body — leaking 23 :Person nodes per import
-// in the Adam Mackay archive. The fix strips U+200E/U+200F before
-// tokenisation; this test reproduces the failure shape.
-
-let workDir: string;
-
-beforeEach(() => {
-  workDir = mkdtempSync(join(tmpdir(), "whatsapp-export-lrm-"));
-});
-
-afterEach(() => {
-  rmSync(workDir, { recursive: true, force: true });
-});
-
-function writeChat(name: string, content: string): string {
-  const filePath = join(workDir, name);
-  writeFileSync(filePath, content);
-  return filePath;
-}
-
-describe("parseExport — bidi-strip (Task 887)", () => {
-  it("strips U+200E from timestamp headers and parses each row independently", () => {
-    const LRM = "‎";
-    const filePath = writeChat(
-      "_chat.txt",
-      [
-        `${LRM}[04/02/26, 11:52:16] Adam Mackay: hi`,
-        `${LRM}[04/02/26, 11:52:30] Joel Smalley: hey`,
-        "",
-      ].join("\n"),
-    );
-
-    const result = parseExport({
-      filePath,
-      accountId: "acct-887",
-      timezone: "Europe/London",
-    });
-
-    expect(result.parsedLines).toHaveLength(2);
-    expect(result.parsedLines.map((l) => l.senderName).sort()).toEqual([
-      "Adam Mackay",
-      "Joel Smalley",
-    ]);
-    for (const line of result.parsedLines) {
-      expect(line.senderName).not.toContain("\n");
-      expect(line.senderName).not.toContain("[");
-      expect(line.senderName).not.toContain(LRM);
-    }
-  });
-
-  it("strips U+200F (RLM) on the timestamp line", () => {
-    const RLM = "‏";
-    const filePath = writeChat(
-      "_chat.txt",
-      [
-        `${RLM}[14/03/26, 10:15:23] Joel: Hello`,
-        `${RLM}[14/03/26, 10:16:01] Sarah: Hi back`,
-        "",
-      ].join("\n"),
-    );
-
-    const result = parseExport({
-      filePath,
-      accountId: "acct-887",
-      timezone: "Europe/London",
-    });
-
-    expect(result.parsedLines).toHaveLength(2);
-    expect(result.parsedLines[0].senderName).toBe("Joel");
-    expect(result.parsedLines[1].senderName).toBe("Sarah");
-  });
-});