@rubytech/create-realagent 1.0.826 → 1.0.828

package/payload/platform/plugins/whatsapp-import/lib/src/__tests__/filter-gate.test.ts

@@ -1,172 +0,0 @@
-import { describe, it, expect } from "vitest";
-import { spawnSync } from "node:child_process";
-import { mkdtempSync, writeFileSync, rmSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { join, resolve, dirname } from "node:path";
-import { fileURLToPath } from "node:url";
-
-import { parseFilterArg, applyFilter } from "../filter.js";
-import type { ParsedLine } from "../parse-export.js";
-
-// ---------------------------------------------------------------------------
-// filter-gate — Task 871 contract: --filter is mandatory, three forms.
-//
-// The unit tests cover the helper module's pure-function surface (parse +
-// apply). The integration test spawns ingest.mjs as a subprocess and
-// asserts the no-flag invocation exits non-zero with the LOUD-FAIL line
-// (`[whatsapp-ingest] FAIL phase=argv reason="--filter is required …"`)
-// before any Neo4j connection — argv parsing fires before imports/sessions.
-// ---------------------------------------------------------------------------
-
-const __dirname = dirname(fileURLToPath(import.meta.url));
-// src/__tests__/ → ../../bin/ingest.mjs (up to src/, up to lib/, up to plugin
-// root, then bin/).
-const INGEST_MJS = resolve(__dirname, "..", "..", "..", "bin", "ingest.mjs");
-
-function lines(parsed: ParsedLine[]): string[] {
-  return parsed.map((p) => `${p.senderName}@${p.dateSent}:${p.body}`);
-}
-
-const SAMPLE: ParsedLine[] = [
-  { senderName: "Alice", dateSent: "2024-01-15T09:00:00+00:00", body: "morning", sequenceIndex: 0 },
-  { senderName: "Bob",   dateSent: "2024-01-15T09:05:00+00:00", body: "hi",      sequenceIndex: 1 },
-  { senderName: "Alice", dateSent: "2024-03-20T11:00:00+00:00", body: "lunch?",  sequenceIndex: 2 },
-  { senderName: "Carol", dateSent: "2024-06-30T23:59:59+00:00: ", body: "bye",   sequenceIndex: 3 },
-  { senderName: "Bob",   dateSent: "2024-07-01T00:00:01+00:00", body: "next",    sequenceIndex: 4 },
-];
-
-// Repair the malformed dateSent fixture (extra ':' suffix would Date.parse to
-// NaN). Keep this in code, not in the const above, so the fixture reads
-// natural while we still hold a strict tuple.
-SAMPLE[3] = { ...SAMPLE[3], dateSent: "2024-06-30T23:59:59+00:00" };
-
-describe("parseFilterArg — happy path", () => {
-  it('parses --filter all', () => {
-    expect(parseFilterArg("all")).toEqual({ kind: "all" });
-  });
-
-  it('parses senders=<csv> with whitespace tolerance', () => {
-    expect(parseFilterArg("senders=Alice, Bob Carter ,  Carol")).toEqual({
-      kind: "senders",
-      senders: ["Alice", "Bob Carter", "Carol"],
-    });
-  });
-
-  it('parses date-range=<from>..<to>', () => {
-    expect(parseFilterArg("date-range=2024-01-01..2024-06-30")).toEqual({
-      kind: "date-range",
-      fromIso: "2024-01-01",
-      toIso: "2024-06-30",
-    });
-  });
-});
-
-describe("parseFilterArg — LOUD-FAIL on malformed input", () => {
-  it("rejects undefined / empty", () => {
-    expect(() => parseFilterArg(undefined)).toThrow(/--filter is required/);
-    expect(() => parseFilterArg("")).toThrow(/--filter is required/);
-    expect(() => parseFilterArg("   ")).toThrow(/--filter is required/);
-  });
-
-  it("rejects unknown form", () => {
-    expect(() => parseFilterArg("everyone")).toThrow(/unrecognised form/);
-    expect(() => parseFilterArg("nope=yes")).toThrow(/unrecognised form/);
-  });
-
-  it("rejects senders= with no names", () => {
-    expect(() => parseFilterArg("senders=")).toThrow(/at least one/);
-    expect(() => parseFilterArg("senders=,,,")).toThrow(/at least one/);
-  });
-
-  it("rejects date-range with malformed shape or unparseable ISO", () => {
-    expect(() => parseFilterArg("date-range=2024-01-01")).toThrow(/<isoFrom>\.\.<isoTo>/);
-    expect(() => parseFilterArg("date-range=..2024-06-30")).toThrow(/both endpoints/);
-    expect(() => parseFilterArg("date-range=2024-01-01..")).toThrow(/both endpoints/);
-    expect(() => parseFilterArg("date-range=not-a-date..2024-06-30")).toThrow(/not parseable/);
-    expect(() => parseFilterArg("date-range=2024-06-30..2024-01-01")).toThrow(/later than/);
-  });
-});
-
-describe("applyFilter — semantics", () => {
-  it("--filter all keeps every row, preserves the parser's original sequenceIndex", () => {
-    const out = applyFilter(SAMPLE, parseFilterArg("all"));
-    expect(out.length).toBe(SAMPLE.length);
-    expect(out.map((l) => l.sequenceIndex)).toEqual([0, 1, 2, 3, 4]);
-  });
-
-  it("--filter senders=Alice,Bob keeps only those senders, sequenceIndex stays as the parser-stamped value (Carol's index 3 drops out)", () => {
-    const out = applyFilter(SAMPLE, parseFilterArg("senders=Alice,Bob"));
-    expect(lines(out)).toEqual([
-      "Alice@2024-01-15T09:00:00+00:00:morning",
-      "Bob@2024-01-15T09:05:00+00:00:hi",
-      "Alice@2024-03-20T11:00:00+00:00:lunch?",
-      "Bob@2024-07-01T00:00:01+00:00:next",
-    ]);
-    expect(out.map((l) => l.sequenceIndex)).toEqual([0, 1, 2, 4]);
-  });
-
-  it("--filter date-range clips inclusively on both ends", () => {
-    const out = applyFilter(SAMPLE, parseFilterArg("date-range=2024-01-01..2024-06-30"));
-    // 2024-06-30T23:59:59 IS inclusive; 2024-07-01 is NOT.
-    expect(out.map((l) => `${l.senderName}/${l.dateSent}`)).toEqual([
-      "Alice/2024-01-15T09:00:00+00:00",
-      "Bob/2024-01-15T09:05:00+00:00",
-      "Alice/2024-03-20T11:00:00+00:00",
-      "Carol/2024-06-30T23:59:59+00:00",
-    ]);
-  });
-
-  it("--filter date-range with full ISO 8601 endpoints", () => {
-    const out = applyFilter(
-      SAMPLE,
-      parseFilterArg("date-range=2024-03-01T00:00:00Z..2024-06-15T00:00:00Z"),
-    );
-    expect(out).toHaveLength(1);
-    expect(out[0].senderName).toBe("Alice");
-    expect(out[0].body).toBe("lunch?");
-  });
-
-  it("an unmatched senders filter returns []", () => {
-    const out = applyFilter(SAMPLE, parseFilterArg("senders=Nobody"));
-    expect(out).toEqual([]);
-  });
-});
-
-describe("ingest.mjs — missing --filter exits non-zero with LOUD-FAIL line", () => {
-  it("emits [whatsapp-ingest] FAIL phase=argv reason=\"--filter required ...\" before touching Neo4j", () => {
-    // Write a stub _chat.txt; argv parse fires first and rejects the call
-    // before resolveChatTxt/Neo4j connection. We do NOT need a real archive
-    // to test the gate.
-    const work = mkdtempSync(join(tmpdir(), "filter-gate-"));
-    try {
-      const stub = join(work, "_chat.txt");
-      writeFileSync(stub, "[14/03/26, 10:15:23] Joel: stub\n");
-
-      const res = spawnSync(
-        "node",
-        [
-          INGEST_MJS,
-          stub,
-          "--owner-element-id",
-          "stub-owner",
-          "--subject-person-id",
-          "stub-subject",
-          "--scope",
-          "admin",
-        ],
-        { encoding: "utf8" },
-      );
-
-      expect(res.status).not.toBe(0);
-      const stderr = res.stderr ?? "";
-      expect(stderr).toMatch(/\[whatsapp-ingest\] FAIL phase=argv/);
-      expect(stderr).toMatch(/reason="--filter is required/);
-      // Confirm the gate fired BEFORE any Neo4j or import work — no
-      // [whatsapp-ingest] start line, no archive-write log line.
-      expect(stderr).not.toMatch(/\[whatsapp-ingest\] start /);
-      expect(stderr).not.toMatch(/\[memory-archive-write\]/);
-    } finally {
-      rmSync(work, { recursive: true, force: true });
-    }
-  });
-});

package/payload/platform/plugins/whatsapp-import/lib/src/__tests__/ingest-idempotence.test.ts

@@ -1,141 +0,0 @@
-import { describe, it, expect } from "vitest";
-import {
-  normaliseSenderName,
-  sha256Hex,
-  deriveMessageId,
-  observationContentHash,
-} from "../derive-keys.js";
-
-describe("normaliseSenderName", () => {
-  it("returns NFKC-trim-lower form", () => {
-    expect(normaliseSenderName("  Adam Mackay  ")).toBe("adam mackay");
-  });
-
-  it("collapses NFKC equivalent forms (composed vs decomposed accents)", () => {
-    const composed = "Adám";
-    const decomposed = "Adám";
-    expect(normaliseSenderName(composed)).toBe(normaliseSenderName(decomposed));
-  });
-
-  it("collapses full-width characters to ASCII via NFKC", () => {
-    expect(normaliseSenderName("Ａｄａｍ")).toBe("adam");
-  });
-
-  it("returns empty string for empty input without throwing", () => {
-    expect(normaliseSenderName("")).toBe("");
-  });
-});
-
-describe("sha256Hex", () => {
-  it("matches the canonical sha256 of an empty string", () => {
-    expect(sha256Hex("")).toBe(
-      "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
-    );
-  });
-
-  it("produces a deterministic hex digest", () => {
-    expect(sha256Hex("hello")).toBe(sha256Hex("hello"));
-    expect(sha256Hex("hello")).not.toBe(sha256Hex("world"));
-  });
-});
-
-describe("deriveMessageId", () => {
-  const baseInputs = {
-    conversationSha256: "abc123",
-    dateSent: "2026-03-14T10:15:23+00:00",
-    senderName: "Adam Mackay",
-    body: "Hello there",
-  };
-
-  it("produces a stable id for identical inputs", () => {
-    const id1 = deriveMessageId(baseInputs);
-    const id2 = deriveMessageId({ ...baseInputs });
-    expect(id1).toBe(id2);
-  });
-
-  it("collapses identical (sender, dateSent, body) tuples to one id under NFKC-trim-lower (correct for export duplicates)", () => {
-    const id1 = deriveMessageId(baseInputs);
-    const id2 = deriveMessageId({ ...baseInputs, senderName: "  ADAM Mackay  " });
-    expect(id1).toBe(id2);
-  });
-
-  it("changes when the body differs", () => {
-    const id1 = deriveMessageId(baseInputs);
-    const id2 = deriveMessageId({ ...baseInputs, body: "Hello there!" });
-    expect(id1).not.toBe(id2);
-  });
-
-  it("changes when the sender differs (after normalisation)", () => {
-    const id1 = deriveMessageId(baseInputs);
-    const id2 = deriveMessageId({ ...baseInputs, senderName: "Joel" });
-    expect(id1).not.toBe(id2);
-  });
-
-  it("changes when the dateSent differs", () => {
-    const id1 = deriveMessageId(baseInputs);
-    const id2 = deriveMessageId({
-      ...baseInputs,
-      dateSent: "2026-03-14T10:15:24+00:00",
-    });
-    expect(id1).not.toBe(id2);
-  });
-
-  it("changes when the conversation changes", () => {
-    const id1 = deriveMessageId(baseInputs);
-    const id2 = deriveMessageId({ ...baseInputs, conversationSha256: "def456" });
-    expect(id1).not.toBe(id2);
-  });
-
-  it("starts with the whatsapp-export:msg prefix and embeds normalised sender", () => {
-    const id = deriveMessageId(baseInputs);
-    expect(id.startsWith("whatsapp-export:msg:")).toBe(true);
-    expect(id).toContain(":adam mackay:");
-  });
-
-  it("does not embed array-position or FNV32 collapse (Task 870 contract)", () => {
-    const id = deriveMessageId(baseInputs);
-    expect(id).toContain(":msg:");
-    expect(id).toMatch(/:[a-f0-9]{64}$/);
-    expect(id).not.toMatch(/:\d+:[a-f0-9]{8}$/);
-  });
-
-  it("produces a stable id for empty body", () => {
-    const id = deriveMessageId({ ...baseInputs, body: "" });
-    expect(id).toBe(deriveMessageId({ ...baseInputs, body: "" }));
-  });
-});
-
-describe("observationContentHash", () => {
-  it("is deterministic for identical inputs", () => {
-    const fields = { summary: "Adam said hi", from: "Adam", to: "Joel", subject: null };
-    expect(observationContentHash(fields)).toBe(observationContentHash(fields));
-  });
-
-  it("normalises NFKC + trim + lowercase across all fields", () => {
-    const a = { summary: "  Hello  ", from: "ADAM", to: null, subject: null };
-    const b = { summary: "hello", from: "adam", to: null, subject: null };
-    expect(observationContentHash(a)).toBe(observationContentHash(b));
-  });
-
-  it("treats null and empty string equivalently", () => {
-    const withNull = { summary: "x", from: null, to: null, subject: null };
-    const withEmpty = { summary: "x", from: "", to: "", subject: "" };
-    expect(observationContentHash(withNull)).toBe(observationContentHash(withEmpty));
-  });
-
-  it("changes when any field changes", () => {
-    const base = { summary: "x", from: null, to: null, subject: null };
-    expect(observationContentHash(base)).not.toBe(
-      observationContentHash({ ...base, summary: "y" }),
-    );
-    expect(observationContentHash(base)).not.toBe(
-      observationContentHash({ ...base, from: "z" }),
-    );
-  });
-
-  it("collapses NFKC equivalent forms in summary", () => {
-    const composed = { summary: "Adám", from: null, to: null, subject: null };
-    const decomposed = { summary: "Adám", from: null, to: null, subject: null };
-    expect(observationContentHash(composed)).toBe(observationContentHash(decomposed));
-  });
-});

package/payload/platform/plugins/whatsapp-import/lib/src/filter.ts

@@ -1,136 +0,0 @@
-// ---------------------------------------------------------------------------
-// filter — operator-supplied gate over ParsedLine[] (Task 871).
-//
-// Phase 1 ingest is now mandatory-filter: the deterministic Bash entry refuses
-// to write a bulk archive without `--filter`. Three forms cover the operator
-// patterns named in the brief:
-//
-//   --filter all                                     → no row drop
-//   --filter senders=Alice,Bob Carter                → keep rows whose
-//                                                       senderName matches
-//                                                       any csv entry exactly
-//   --filter date-range=2024-01-01..2024-06-30       → keep rows whose
-//                                                       dateSent ISO falls
-//                                                       inside the inclusive
-//                                                       range (date or full
-//                                                       ISO 8601)
-//
-// Doctrine alignment:
-//   - feedback_compress_at_ingest_for_bulk_archives.md — the gate is
-//     mandatory at write-time, not after.
-//   - feedback_deterministic_means_remove_llm.md — the filter parser is a
-//     pure function, no LLM in the per-row decision path.
-//   - feedback_loud_failures.md — malformed `--filter` raises a structured
-//     error with a named reason rather than silently coercing to `all`.
-// ---------------------------------------------------------------------------
-
-import type { ParsedLine } from "./parse-export.js";
-
-export type Filter =
-  | { kind: "all" }
-  | { kind: "senders"; senders: string[] }
-  | { kind: "date-range"; fromIso: string; toIso: string };
-
-/**
- * Parse a CLI `--filter` argument into a structured Filter.
- *
- * Throws Error with message starting "filter: …" on malformed input. The
- * caller (ingest.mjs / vitest) surfaces the reason verbatim — the brief
- * mandates `[whatsapp-ingest] FAIL filter-required reason="…"` so the
- * operator can grep one line.
- */
-export function parseFilterArg(raw: string | undefined | null): Filter {
-  if (raw == null || raw.trim() === "") {
-    throw new Error(
-      'filter: --filter is required (one of "all", "senders=<csv>", "date-range=<isoFrom>..<isoTo>")',
-    );
-  }
-  const value = raw.trim();
-  if (value === "all") return { kind: "all" };
-  if (value.startsWith("senders=")) {
-    const csv = value.slice("senders=".length);
-    const senders = csv
-      .split(",")
-      .map((s) => s.trim())
-      .filter((s) => s.length > 0);
-    if (senders.length === 0) {
-      throw new Error('filter: senders= requires at least one comma-separated name');
-    }
-    return { kind: "senders", senders };
-  }
-  if (value.startsWith("date-range=")) {
-    const range = value.slice("date-range=".length);
-    const parts = range.split("..");
-    if (parts.length !== 2) {
-      throw new Error(
-        `filter: date-range must be "<isoFrom>..<isoTo>" — got "${range}"`,
-      );
-    }
-    const [fromIso, toIso] = parts.map((p) => p.trim());
-    if (!fromIso || !toIso) {
-      throw new Error(
-        `filter: date-range requires both endpoints — got "${range}"`,
-      );
-    }
-    if (Number.isNaN(Date.parse(fromIso))) {
-      throw new Error(`filter: date-range fromIso="${fromIso}" is not parseable as ISO 8601`);
-    }
-    if (Number.isNaN(Date.parse(toIso))) {
-      throw new Error(`filter: date-range toIso="${toIso}" is not parseable as ISO 8601`);
-    }
-    if (Date.parse(fromIso) > Date.parse(toIso)) {
-      throw new Error(`filter: date-range fromIso="${fromIso}" is later than toIso="${toIso}"`);
-    }
-    return { kind: "date-range", fromIso, toIso };
-  }
-  throw new Error(
-    `filter: unrecognised form "${value}" — must be "all", "senders=<csv>", or "date-range=<isoFrom>..<isoTo>"`,
-  );
-}
-
-/**
- * Apply a parsed Filter to ParsedLine[]. Returns a new array of kept lines
- * with the parser's original `sequenceIndex` preserved (the filter never
- * reorders). ingest.mjs re-stamps `sequenceIndex` to its post-filter position
- * during row construction for archive-write — re-stamping here too would be
- * redundant.
- */
-export function applyFilter(
-  parsedLines: readonly ParsedLine[],
-  filter: Filter,
-): ParsedLine[] {
-  const predicate = makePredicate(filter);
-  const kept: ParsedLine[] = [];
-  for (const line of parsedLines) {
-    if (!predicate(line)) continue;
-    kept.push(line);
-  }
-  return kept;
-}
-
-function makePredicate(filter: Filter): (line: ParsedLine) => boolean {
-  if (filter.kind === "all") return () => true;
-  if (filter.kind === "senders") {
-    const allow = new Set(filter.senders);
-    return (line) => allow.has(line.senderName);
-  }
-  // date-range: inclusive on both ends. Date-only endpoints widen to whole-
-  // day semantics: `from=YYYY-MM-DD` → `T00:00:00Z`, `to=YYYY-MM-DD` →
-  // `T23:59:59.999Z`. Full ISO 8601 endpoints with `T` are passed through.
-  // Without this widening, `--filter date-range=2024-01-01..2024-06-30`
-  // would silently drop every message later than 2024-06-30T00:00:00Z on the
-  // last day — a UX trap that contradicts the operator's reading.
-  const fromMs = parseRangeEndpoint(filter.fromIso, "start");
-  const toMs = parseRangeEndpoint(filter.toIso, "end");
-  return (line) => {
-    const ms = Date.parse(line.dateSent);
-    return ms >= fromMs && ms <= toMs;
-  };
-}
-
-function parseRangeEndpoint(iso: string, edge: "start" | "end"): number {
-  if (/T/.test(iso)) return Date.parse(iso);
-  // Date-only — widen to whole-day inclusive on the requested edge.
-  const suffix = edge === "start" ? "T00:00:00.000Z" : "T23:59:59.999Z";
-  return Date.parse(iso + suffix);
-}