@rubytech/create-realagent 1.0.829 → 1.0.831
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/payload/platform/config/brand.json +1 -1
- package/payload/platform/lib/oauth-llm/dist/index.d.ts +9 -2
- package/payload/platform/lib/oauth-llm/dist/index.d.ts.map +1 -1
- package/payload/platform/lib/oauth-llm/dist/index.js +26 -1
- package/payload/platform/lib/oauth-llm/dist/index.js.map +1 -1
- package/payload/platform/lib/oauth-llm/src/index.ts +43 -4
- package/payload/platform/neo4j/migrations/007-conversation-archive-source.ts +116 -0
- package/payload/platform/neo4j/migrations/008-adminuser-accountid-backfill.ts +85 -0
- package/payload/platform/neo4j/schema.cypher +12 -3
- package/payload/platform/plugins/admin/hooks/__tests__/archive-ingest-surface-gate.test.sh +54 -39
- package/payload/platform/plugins/admin/hooks/archive-ingest-surface-gate.sh +64 -26
- package/payload/platform/plugins/admin/mcp/dist/index.js +25 -3
- package/payload/platform/plugins/admin/mcp/dist/index.js.map +1 -1
- package/payload/platform/plugins/contacts/mcp/dist/index.js +5 -5
- package/payload/platform/plugins/contacts/mcp/dist/index.js.map +1 -1
- package/payload/platform/plugins/contacts/mcp/dist/tools/contact-create.d.ts +1 -1
- package/payload/platform/plugins/contacts/mcp/dist/tools/contact-create.d.ts.map +1 -1
- package/payload/platform/plugins/contacts/mcp/dist/tools/contact-create.js +29 -23
- package/payload/platform/plugins/contacts/mcp/dist/tools/contact-create.js.map +1 -1
- package/payload/platform/plugins/docs/references/internals.md +1 -1
- package/payload/platform/plugins/docs/references/plugins-guide.md +1 -1
- package/payload/platform/plugins/memory/PLUGIN.md +2 -1
- package/payload/platform/plugins/memory/bin/conversation-archive-ingest.mjs +564 -0
- package/payload/platform/plugins/memory/bin/conversation-archive-ingest.sh +106 -0
- package/payload/platform/plugins/memory/mcp/dist/index.js +30 -16
- package/payload/platform/plugins/memory/mcp/dist/index.js.map +1 -1
- package/payload/platform/plugins/memory/mcp/dist/lib/__tests__/llm-classifier.test.js +4 -3
- package/payload/platform/plugins/memory/mcp/dist/lib/__tests__/llm-classifier.test.js.map +1 -1
- package/payload/platform/plugins/memory/mcp/dist/lib/__tests__/schema-loader.test.js +11 -6
- package/payload/platform/plugins/memory/mcp/dist/lib/__tests__/schema-loader.test.js.map +1 -1
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-normalisers/index.d.ts +5 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-normalisers/index.d.ts.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-normalisers/index.js +30 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-normalisers/index.js.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-normalisers/timestamp-scanner.d.ts +49 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-normalisers/timestamp-scanner.d.ts.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-normalisers/timestamp-scanner.js +35 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-normalisers/timestamp-scanner.js.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-normalisers/types.d.ts +47 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-normalisers/types.d.ts.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-normalisers/types.js +31 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-normalisers/types.js.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-normalisers/whatsapp-text.d.ts +3 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-normalisers/whatsapp-text.d.ts.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-normalisers/whatsapp-text.js +155 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-normalisers/whatsapp-text.js.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-pipeline/delta-cursor.d.ts +11 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-pipeline/delta-cursor.d.ts.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-pipeline/delta-cursor.js +20 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-pipeline/delta-cursor.js.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-pipeline/derive-keys.d.ts +14 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-pipeline/derive-keys.d.ts.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-pipeline/derive-keys.js +38 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-pipeline/derive-keys.js.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-pipeline/sender-bind.d.ts +16 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-pipeline/sender-bind.d.ts.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-pipeline/sender-bind.js +59 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-pipeline/sender-bind.js.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-pipeline/sessionize.d.ts +9 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-pipeline/sessionize.d.ts.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-pipeline/sessionize.js +32 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-pipeline/sessionize.js.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-pipeline/to-turn-text.d.ts +3 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-pipeline/to-turn-text.d.ts.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-pipeline/to-turn-text.js +29 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-pipeline/to-turn-text.js.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/document-chunker.d.ts +45 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/document-chunker.d.ts.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/document-chunker.js +125 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/document-chunker.js.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/llm-classifier.d.ts +34 -9
- package/payload/platform/plugins/memory/mcp/dist/lib/llm-classifier.d.ts.map +1 -1
- package/payload/platform/plugins/memory/mcp/dist/lib/llm-classifier.js +360 -35
- package/payload/platform/plugins/memory/mcp/dist/lib/llm-classifier.js.map +1 -1
- package/payload/platform/plugins/memory/mcp/dist/lib/llm-ranker.d.ts +3 -2
- package/payload/platform/plugins/memory/mcp/dist/lib/llm-ranker.d.ts.map +1 -1
- package/payload/platform/plugins/memory/mcp/dist/lib/llm-ranker.js +46 -17
- package/payload/platform/plugins/memory/mcp/dist/lib/llm-ranker.js.map +1 -1
- package/payload/platform/plugins/memory/mcp/dist/tools/__tests__/conversation-normalisers-source-agnosticism.test.d.ts +2 -0
- package/payload/platform/plugins/memory/mcp/dist/tools/__tests__/conversation-normalisers-source-agnosticism.test.d.ts.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/tools/__tests__/conversation-normalisers-source-agnosticism.test.js +73 -0
- package/payload/platform/plugins/memory/mcp/dist/tools/__tests__/conversation-normalisers-source-agnosticism.test.js.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/tools/__tests__/conversation-normalisers-whatsapp-text.test.d.ts +2 -0
- package/payload/platform/plugins/memory/mcp/dist/tools/__tests__/conversation-normalisers-whatsapp-text.test.d.ts.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/tools/__tests__/conversation-normalisers-whatsapp-text.test.js +109 -0
- package/payload/platform/plugins/memory/mcp/dist/tools/__tests__/conversation-normalisers-whatsapp-text.test.js.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/tools/__tests__/memory-ingest.test.js +34 -3
- package/payload/platform/plugins/memory/mcp/dist/tools/__tests__/memory-ingest.test.js.map +1 -1
- package/payload/platform/plugins/memory/mcp/dist/tools/memory-archive-write.d.ts +17 -0
- package/payload/platform/plugins/memory/mcp/dist/tools/memory-archive-write.d.ts.map +1 -1
- package/payload/platform/plugins/memory/mcp/dist/tools/memory-archive-write.js +34 -13
- package/payload/platform/plugins/memory/mcp/dist/tools/memory-archive-write.js.map +1 -1
- package/payload/platform/plugins/memory/mcp/dist/tools/memory-ingest.d.ts +18 -7
- package/payload/platform/plugins/memory/mcp/dist/tools/memory-ingest.d.ts.map +1 -1
- package/payload/platform/plugins/memory/mcp/dist/tools/memory-ingest.js +24 -8
- package/payload/platform/plugins/memory/mcp/dist/tools/memory-ingest.js.map +1 -1
- package/payload/platform/plugins/memory/mcp/dist/tools/memory-rank.js +2 -2
- package/payload/platform/plugins/memory/mcp/dist/tools/memory-rank.js.map +1 -1
- package/payload/platform/plugins/memory/references/schema-base.md +2 -2
- package/payload/platform/plugins/memory/skills/conversation-archive/SKILL.md +133 -0
- package/payload/platform/plugins/memory/skills/document-ingest/SKILL.md +5 -2
- package/payload/platform/plugins/whatsapp/PLUGIN.md +1 -1
- package/payload/platform/scripts/seed-neo4j.sh +15 -15
- package/payload/platform/templates/specialists/agents/database-operator.md +8 -9
- package/payload/server/chunk-7BO5HDJC.js +10093 -0
- package/payload/server/chunk-BCFM2UPH.js +2305 -0
- package/payload/server/chunk-CV3HPX46.js +10097 -0
- package/payload/server/chunk-EL4DZ56X.js +1116 -0
- package/payload/server/chunk-J6YWEJBN.js +1116 -0
- package/payload/server/chunk-OCPJGZ6S.js +654 -0
- package/payload/server/chunk-QOJ2D26Z.js +654 -0
- package/payload/server/chunk-RC46ZYGT.js +2305 -0
- package/payload/server/client-pool-7NTEFNVQ.js +32 -0
- package/payload/server/client-pool-ZNGN66GN.js +32 -0
- package/payload/server/cloudflare-task-tracker-MHALDN54.js +19 -0
- package/payload/server/cloudflare-task-tracker-WE77WXSI.js +19 -0
- package/payload/server/maxy-edge.js +3 -3
- package/payload/server/neo4j-migrations-4XPNJNM6.js +490 -0
- package/payload/server/neo4j-migrations-6RW423E2.js +530 -0
- package/payload/server/server.js +30 -19
package/payload/platform/plugins/memory/mcp/dist/lib/conversation-normalisers/timestamp-scanner.js
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Walk `text` line by line. Lines that match `matchPrefix` start a new entry;
|
|
3
|
+
* lines that don't match glue onto the preceding entry's body with `\n`.
|
|
4
|
+
*
|
|
5
|
+
* Returns an empty `parsedLines` array when no line matches the prefix —
|
|
6
|
+
* the caller decides whether to route the input to a non-conversation path
|
|
7
|
+
* (document chunker) or loud-fail.
|
|
8
|
+
*/
|
|
9
|
+
export function scanTimestampedLines(input) {
|
|
10
|
+
const { text, matchPrefix, formatIso } = input;
|
|
11
|
+
const lines = text.split("\n");
|
|
12
|
+
const raw = [];
|
|
13
|
+
for (let i = 0; i < lines.length; i++) {
|
|
14
|
+
const line = lines[i];
|
|
15
|
+
if (line.length === 0 && i === lines.length - 1)
|
|
16
|
+
continue;
|
|
17
|
+
const m = matchPrefix(line);
|
|
18
|
+
if (m) {
|
|
19
|
+
const { remainder, ...parts } = m;
|
|
20
|
+
raw.push({ parts, remainder });
|
|
21
|
+
}
|
|
22
|
+
else {
|
|
23
|
+
const last = raw[raw.length - 1];
|
|
24
|
+
if (last)
|
|
25
|
+
last.remainder += "\n" + line;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
const parsedLines = raw.map((r, idx) => ({
|
|
29
|
+
dateSent: formatIso(r.parts),
|
|
30
|
+
body: r.remainder.replace(/\s+$/, ""),
|
|
31
|
+
sequenceIndex: idx,
|
|
32
|
+
}));
|
|
33
|
+
return { parsedLines, matchedCount: parsedLines.length };
|
|
34
|
+
}
|
|
35
|
+
//# sourceMappingURL=timestamp-scanner.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"timestamp-scanner.js","sourceRoot":"","sources":["../../../src/lib/conversation-normalisers/timestamp-scanner.ts"],"names":[],"mappings":"AAyDA;;;;;;;GAOG;AACH,MAAM,UAAU,oBAAoB,CAAC,KAAgB;IACnD,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,GAAG,KAAK,CAAC;IAC/C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAM/B,MAAM,GAAG,GAAiB,EAAE,CAAC;IAE7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACtB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,SAAS;QAC1D,MAAM,CAAC,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QAC5B,IAAI,CAAC,EAAE,CAAC;YACN,MAAM,EAAE,SAAS,EAAE,GAAG,KAAK,EAAE,GAAG,CAAC,CAAC;YAClC,GAAG,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;QACjC,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,GAAG,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YACjC,IAAI,IAAI;gBAAE,IAAI,CAAC,SAAS,IAAI,IAAI,GAAG,IAAI,CAAC;QAC1C,CAAC;IACH,CAAC;IAED,MAAM,WAAW,GAAiB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;QACrD,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC;QAC5B,IAAI,EAAE,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;QACrC,aAAa,EAAE,GAAG;KACnB,CAAC,CAAC,CAAC;IAEJ,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,WAAW,CAAC,MAAM,EAAE,CAAC;AAC3D,CAAC"}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
export declare const CONVERSATION_SOURCES: readonly ["whatsapp", "telegram", "signal", "linkedin-messages", "zoom-transcript", "meeting-minutes", "imessage", "slack", "other"];
|
|
2
|
+
export type ConversationSource = (typeof CONVERSATION_SOURCES)[number];
|
|
3
|
+
export declare function isConversationSource(s: unknown): s is ConversationSource;
|
|
4
|
+
/**
|
|
5
|
+
* One message after source-specific timestamp scanning. Identical shape across
|
|
6
|
+
* every source. Bodies are the raw post-prefix remainder — for WhatsApp this
|
|
7
|
+
* still carries the `Sender: ...` segment; the chat-mode classifier prompt
|
|
8
|
+
* preserves these prefixes verbatim in chunk bodies.
|
|
9
|
+
*/
|
|
10
|
+
export interface ParsedLine {
|
|
11
|
+
/** ISO 8601 instant with timezone offset (e.g. "2026-04-30T18:42:00+01:00"). */
|
|
12
|
+
dateSent: string;
|
|
13
|
+
/** Everything after the timestamp prefix; multi-line continuations glued on `\n`. */
|
|
14
|
+
body: string;
|
|
15
|
+
/** Position within the parsed sequence, 0-based. */
|
|
16
|
+
sequenceIndex: number;
|
|
17
|
+
}
|
|
18
|
+
export interface NormaliserCounters {
|
|
19
|
+
parsed: number;
|
|
20
|
+
systemSkipped: number;
|
|
21
|
+
mediaSkipped: number;
|
|
22
|
+
parseErrors: number;
|
|
23
|
+
}
|
|
24
|
+
export interface NormaliserInput {
|
|
25
|
+
/** Absolute path to the source file or directory the normaliser knows. */
|
|
26
|
+
filePath: string;
|
|
27
|
+
/** Account scope for any provenance composition. */
|
|
28
|
+
accountId: string;
|
|
29
|
+
/** IANA timezone the operator confirmed (e.g. "Europe/London"). */
|
|
30
|
+
timezone: string;
|
|
31
|
+
/**
|
|
32
|
+
* Source-specific options. WhatsApp: `dateFormat`. Other sources may add
|
|
33
|
+
* their own keys; unknown keys are ignored by every normaliser.
|
|
34
|
+
*/
|
|
35
|
+
opts?: Record<string, unknown>;
|
|
36
|
+
}
|
|
37
|
+
export interface NormaliserResult {
|
|
38
|
+
parsedLines: ParsedLine[];
|
|
39
|
+
/** SHA-256 hex of the source file bytes (or the canonical digest the
|
|
40
|
+
* normaliser computes for non-file sources). */
|
|
41
|
+
archiveSha256: string;
|
|
42
|
+
/** Basename suitable for operator audit (e.g. "_chat.txt"). */
|
|
43
|
+
archiveSourceFile: string;
|
|
44
|
+
counters: NormaliserCounters;
|
|
45
|
+
}
|
|
46
|
+
export type ConversationNormaliser = (input: NormaliserInput) => Promise<NormaliserResult> | NormaliserResult;
|
|
47
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/lib/conversation-normalisers/types.ts"],"names":[],"mappings":"AAiBA,eAAO,MAAM,oBAAoB,sIAUvB,CAAC;AAEX,MAAM,MAAM,kBAAkB,GAAG,CAAC,OAAO,oBAAoB,CAAC,CAAC,MAAM,CAAC,CAAC;AAEvE,wBAAgB,oBAAoB,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,IAAI,kBAAkB,CAExE;AAED;;;;;GAKG;AACH,MAAM,WAAW,UAAU;IACzB,gFAAgF;IAChF,QAAQ,EAAE,MAAM,CAAC;IACjB,qFAAqF;IACrF,IAAI,EAAE,MAAM,CAAC;IACb,oDAAoD;IACpD,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,eAAe;IAC9B,0EAA0E;IAC1E,QAAQ,EAAE,MAAM,CAAC;IACjB,oDAAoD;IACpD,SAAS,EAAE,MAAM,CAAC;IAClB,mEAAmE;IACnE,QAAQ,EAAE,MAAM,CAAC;IACjB;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAChC;AAED,MAAM,WAAW,gBAAgB;IAC/B,WAAW,EAAE,UAAU,EAAE,CAAC;IAC1B;qDACiD;IACjD,aAAa,EAAE,MAAM,CAAC;IACtB,+DAA+D;IAC/D,iBAAiB,EAAE,MAAM,CAAC;IAC1B,QAAQ,EAAE,kBAAkB,CAAC;CAC9B;AAED,MAAM,MAAM,sBAAsB,GAAG,CACnC,KAAK,EAAE,eAAe,KACnB,OAAO,CAAC,gBAAgB,CAAC,GAAG,gBAAgB,CAAC"}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
// Conversation-archive ingest contracts (Task 894 / 897).
|
|
2
|
+
//
|
|
3
|
+
// One skill, one writer, one bash entry — source becomes a property on
|
|
4
|
+
// :ConversationArchive and a pluggable normaliser, never a separate skill.
|
|
5
|
+
//
|
|
6
|
+
// Task 897 dropped per-line `senderName`. The normaliser is now a
|
|
7
|
+
// timestamp-prefix scanner — it walks lines, matches a source-specific
|
|
8
|
+
// regex, glues continuations onto the preceding match, and returns body
|
|
9
|
+
// strings verbatim. Bodies are opaque: vCards, system messages, forwarded
|
|
10
|
+
// blocks, and the bare `Sender:` segment all flow through unchanged.
|
|
11
|
+
// Operator-confirmed participants attach to the parent `:ConversationArchive`
|
|
12
|
+
// via `:PARTICIPANT_IN`, never per-message.
|
|
13
|
+
//
|
|
14
|
+
// `ConversationSource` is the closed enum stamped on every :ConversationArchive
|
|
15
|
+
// and :Section:Conversation. Adding a new source = adding a normaliser
|
|
16
|
+
// implementation here + extending this enum, never a new skill or plugin.
|
|
17
|
+
export const CONVERSATION_SOURCES = [
|
|
18
|
+
"whatsapp",
|
|
19
|
+
"telegram",
|
|
20
|
+
"signal",
|
|
21
|
+
"linkedin-messages",
|
|
22
|
+
"zoom-transcript",
|
|
23
|
+
"meeting-minutes",
|
|
24
|
+
"imessage",
|
|
25
|
+
"slack",
|
|
26
|
+
"other",
|
|
27
|
+
];
|
|
28
|
+
export function isConversationSource(s) {
|
|
29
|
+
return typeof s === "string" && CONVERSATION_SOURCES.includes(s);
|
|
30
|
+
}
|
|
31
|
+
//# sourceMappingURL=types.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/lib/conversation-normalisers/types.ts"],"names":[],"mappings":"AAAA,0DAA0D;AAC1D,EAAE;AACF,uEAAuE;AACvE,2EAA2E;AAC3E,EAAE;AACF,kEAAkE;AAClE,uEAAuE;AACvE,wEAAwE;AACxE,0EAA0E;AAC1E,qEAAqE;AACrE,8EAA8E;AAC9E,4CAA4C;AAC5C,EAAE;AACF,gFAAgF;AAChF,uEAAuE;AACvE,0EAA0E;AAE1E,MAAM,CAAC,MAAM,oBAAoB,GAAG;IAClC,UAAU;IACV,UAAU;IACV,QAAQ;IACR,mBAAmB;IACnB,iBAAiB;IACjB,iBAAiB;IACjB,UAAU;IACV,OAAO;IACP,OAAO;CACC,CAAC;AAIX,MAAM,UAAU,oBAAoB,CAAC,CAAU;IAC7C,OAAO,OAAO,CAAC,KAAK,QAAQ,IAAK,oBAA0C,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;AAC1F,CAAC"}
|
package/payload/platform/plugins/memory/mcp/dist/lib/conversation-normalisers/whatsapp-text.d.ts.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"whatsapp-text.d.ts","sourceRoot":"","sources":["../../../src/lib/conversation-normalisers/whatsapp-text.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAiCpE,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,eAAe,GAAG,gBAAgB,CA+C/E"}
|
package/payload/platform/plugins/memory/mcp/dist/lib/conversation-normalisers/whatsapp-text.js
ADDED
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { readFileSync } from "node:fs";
|
|
3
|
+
import { basename } from "node:path";
|
|
4
|
+
import { scanTimestampedLines } from "./timestamp-scanner.js";
|
|
5
|
+
const TIMESTAMP_PREFIX = /^\[(\d{2})\/(\d{2})\/(\d{4}|\d{2}),\s+(\d{1,2}):(\d{2})(?::(\d{2}))?\]\s*(.*)$/;
|
|
6
|
+
export function whatsappTextNormaliser(input) {
|
|
7
|
+
const { filePath, accountId, timezone } = input;
|
|
8
|
+
const opts = (input.opts ?? {});
|
|
9
|
+
if (!accountId || !accountId.trim()) {
|
|
10
|
+
throw new Error("whatsapp-text: accountId is required.");
|
|
11
|
+
}
|
|
12
|
+
if (!timezone || !timezone.trim()) {
|
|
13
|
+
throw new Error("whatsapp-text: timezone is required (e.g. 'Europe/London').");
|
|
14
|
+
}
|
|
15
|
+
const rawBytes = readFileSync(filePath);
|
|
16
|
+
const archiveSha256 = createHash("sha256").update(rawBytes).digest("hex");
|
|
17
|
+
const text = decodeAndNormalise(rawBytes);
|
|
18
|
+
if (text.length === 0) {
|
|
19
|
+
throw new Error(`whatsapp-text: file is empty — not a _chat.txt. file=${filePath}`);
|
|
20
|
+
}
|
|
21
|
+
const ordering = resolveOrdering(opts.dateFormat, text);
|
|
22
|
+
const matchPrefix = makeWhatsappPrefixMatcher(ordering);
|
|
23
|
+
const { parsedLines } = scanTimestampedLines({
|
|
24
|
+
text,
|
|
25
|
+
matchPrefix,
|
|
26
|
+
formatIso: (parts) => isoWithOffset(parts.year, parts.month, parts.day, parts.hour, parts.minute, parts.second, timezone),
|
|
27
|
+
});
|
|
28
|
+
// `parsed` is the matched-prefix count (= parsedLines.length).
|
|
29
|
+
// `parseErrors` is structurally zero — the scanner cannot fail on body
|
|
30
|
+
// shape post-Task-897. `systemSkipped` / `mediaSkipped` are zero too:
|
|
31
|
+
// skip detection lived in the pre-897 colon-parser path that's gone.
|
|
32
|
+
// The bin emits `[conversation-archive] parsed lines=N` and routes a
|
|
33
|
+
// zero-parsed file to the document-ingest chunker rather than failing.
|
|
34
|
+
return {
|
|
35
|
+
parsedLines,
|
|
36
|
+
archiveSha256,
|
|
37
|
+
archiveSourceFile: basename(filePath),
|
|
38
|
+
counters: {
|
|
39
|
+
parsed: parsedLines.length,
|
|
40
|
+
systemSkipped: 0,
|
|
41
|
+
mediaSkipped: 0,
|
|
42
|
+
parseErrors: 0,
|
|
43
|
+
},
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
function decodeAndNormalise(bytes) {
|
|
47
|
+
let text;
|
|
48
|
+
try {
|
|
49
|
+
text = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
|
|
50
|
+
}
|
|
51
|
+
catch (err) {
|
|
52
|
+
throw new Error(`whatsapp-text: UTF-8 decode failed — ${err instanceof Error ? err.message : String(err)}. The file is not valid UTF-8; re-export from WhatsApp.`);
|
|
53
|
+
}
|
|
54
|
+
if (text.charCodeAt(0) === 0xfeff)
|
|
55
|
+
text = text.slice(1);
|
|
56
|
+
text = text.replace(/\r\n/g, "\n").replace(/\r/g, "\n");
|
|
57
|
+
// Strip leading bidi marks (U+200E LRM, U+200F RLM) only at line-start —
|
|
58
|
+
// some WhatsApp builds prefix the timestamp header. Body-internal bidi
|
|
59
|
+
// marks are preserved (semantic). See Task 887.
|
|
60
|
+
const leadingBidiMatches = text.match(/(?:^|\n)[]+/g) || [];
|
|
61
|
+
let lrmStripped = 0;
|
|
62
|
+
let rlmStripped = 0;
|
|
63
|
+
for (const m of leadingBidiMatches) {
|
|
64
|
+
for (const ch of m) {
|
|
65
|
+
if (ch === "")
|
|
66
|
+
lrmStripped++;
|
|
67
|
+
else if (ch === "")
|
|
68
|
+
rlmStripped++;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
if (leadingBidiMatches.length > 0) {
|
|
72
|
+
text = text.replace(/(^|\n)[]+/g, "$1");
|
|
73
|
+
process.stderr.write(`[conversation-archive] source=whatsapp decoded normalised lrm-stripped=${lrmStripped} rlm-stripped=${rlmStripped}\n`);
|
|
74
|
+
}
|
|
75
|
+
return text;
|
|
76
|
+
}
|
|
77
|
+
function makeWhatsappPrefixMatcher(ordering) {
|
|
78
|
+
return (line) => {
|
|
79
|
+
const m = line.match(TIMESTAMP_PREFIX);
|
|
80
|
+
if (!m)
|
|
81
|
+
return null;
|
|
82
|
+
const a = parseInt(m[1], 10);
|
|
83
|
+
const b = parseInt(m[2], 10);
|
|
84
|
+
const yearRaw = m[3];
|
|
85
|
+
const hour = parseInt(m[4], 10);
|
|
86
|
+
const minute = parseInt(m[5], 10);
|
|
87
|
+
const second = m[6] !== undefined ? parseInt(m[6], 10) : 0;
|
|
88
|
+
const remainder = m[7] ?? "";
|
|
89
|
+
const day = ordering === "MMDD" ? b : a;
|
|
90
|
+
const month = ordering === "MMDD" ? a : b;
|
|
91
|
+
if (month < 1 || month > 12 || day < 1 || day > 31)
|
|
92
|
+
return null;
|
|
93
|
+
if (hour > 23 || minute > 59 || second > 59)
|
|
94
|
+
return null;
|
|
95
|
+
const year = yearRaw.length === 2 ? 2000 + parseInt(yearRaw, 10) : parseInt(yearRaw, 10);
|
|
96
|
+
return { year, month, day, hour, minute, second, remainder };
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
function resolveOrdering(explicit, text) {
|
|
100
|
+
if (explicit === "MM/DD/YY" || explicit === "MM/DD/YYYY")
|
|
101
|
+
return "MMDD";
|
|
102
|
+
if (explicit === "DD/MM/YY" || explicit === "DD/MM/YYYY")
|
|
103
|
+
return "DDMM";
|
|
104
|
+
// Probe the file: first valid match wins. DD/MM precedence is consistent
|
|
105
|
+
// with WhatsApp's UK/EU default; explicit MM/DD is an opt-in.
|
|
106
|
+
const ddmm = makeWhatsappPrefixMatcher("DDMM");
|
|
107
|
+
const mmdd = makeWhatsappPrefixMatcher("MMDD");
|
|
108
|
+
for (const line of text.split("\n")) {
|
|
109
|
+
if (ddmm(line))
|
|
110
|
+
return "DDMM";
|
|
111
|
+
if (mmdd(line))
|
|
112
|
+
return "MMDD";
|
|
113
|
+
}
|
|
114
|
+
return "DDMM";
|
|
115
|
+
}
|
|
116
|
+
function isoWithOffset(year, month, day, hour, minute, second, timezone) {
|
|
117
|
+
const guessUtcMs = Date.UTC(year, month - 1, day, hour, minute, second);
|
|
118
|
+
let offMin = offsetMinutesAt(new Date(guessUtcMs), timezone);
|
|
119
|
+
const refinedUtcMs = guessUtcMs - offMin * 60_000;
|
|
120
|
+
offMin = offsetMinutesAt(new Date(refinedUtcMs), timezone);
|
|
121
|
+
const sign = offMin >= 0 ? "+" : "-";
|
|
122
|
+
const absOff = Math.abs(offMin);
|
|
123
|
+
const offHH = String(Math.floor(absOff / 60)).padStart(2, "0");
|
|
124
|
+
const offMM = String(absOff % 60).padStart(2, "0");
|
|
125
|
+
const Y = String(year).padStart(4, "0");
|
|
126
|
+
const M = String(month).padStart(2, "0");
|
|
127
|
+
const D = String(day).padStart(2, "0");
|
|
128
|
+
const H = String(hour).padStart(2, "0");
|
|
129
|
+
const Mi = String(minute).padStart(2, "0");
|
|
130
|
+
const S = String(second).padStart(2, "0");
|
|
131
|
+
return `${Y}-${M}-${D}T${H}:${Mi}:${S}${sign}${offHH}:${offMM}`;
|
|
132
|
+
}
|
|
133
|
+
function offsetMinutesAt(date, timezone) {
|
|
134
|
+
const formatter = new Intl.DateTimeFormat("en-US", {
|
|
135
|
+
timeZone: timezone,
|
|
136
|
+
timeZoneName: "longOffset",
|
|
137
|
+
});
|
|
138
|
+
const parts = formatter.formatToParts(date);
|
|
139
|
+
const tzPart = parts.find((p) => p.type === "timeZoneName");
|
|
140
|
+
if (!tzPart) {
|
|
141
|
+
throw new Error(`whatsapp-text: unable to read offset for timezone "${timezone}".`);
|
|
142
|
+
}
|
|
143
|
+
const value = tzPart.value;
|
|
144
|
+
if (value === "GMT" || value === "UTC")
|
|
145
|
+
return 0;
|
|
146
|
+
const m = value.match(/^(?:GMT|UTC)([+-])(\d{1,2}):?(\d{2})?$/);
|
|
147
|
+
if (!m) {
|
|
148
|
+
throw new Error(`whatsapp-text: cannot parse timezone offset "${value}" for IANA zone "${timezone}".`);
|
|
149
|
+
}
|
|
150
|
+
const sign = m[1] === "+" ? 1 : -1;
|
|
151
|
+
const hh = parseInt(m[2], 10);
|
|
152
|
+
const mm = m[3] ? parseInt(m[3], 10) : 0;
|
|
153
|
+
return sign * (hh * 60 + mm);
|
|
154
|
+
}
|
|
155
|
+
//# sourceMappingURL=whatsapp-text.js.map
|
package/payload/platform/plugins/memory/mcp/dist/lib/conversation-normalisers/whatsapp-text.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"whatsapp-text.js","sourceRoot":"","sources":["../../../src/lib/conversation-normalisers/whatsapp-text.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,oBAAoB,EAA+B,MAAM,wBAAwB,CAAC;AA+B3F,MAAM,gBAAgB,GACpB,gFAAgF,CAAC;AAEnF,MAAM,UAAU,sBAAsB,CAAC,KAAsB;IAC3D,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,KAAK,CAAC;IAChD,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,CAAqB,CAAC;IACpD,IAAI,CAAC,SAAS,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,CAAC;QACpC,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAC3D,CAAC;IACD,IAAI,CAAC,QAAQ,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,6DAA6D,CAAC,CAAC;IACjF,CAAC;IAED,MAAM,QAAQ,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;IACxC,MAAM,aAAa,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAE1E,MAAM,IAAI,GAAG,kBAAkB,CAAC,QAAQ,CAAC,CAAC;IAC1C,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CACb,wDAAwD,QAAQ,EAAE,CACnE,CAAC;IACJ,CAAC;IAED,MAAM,QAAQ,GAAG,eAAe,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;IACxD,MAAM,WAAW,GAAG,yBAAyB,CAAC,QAAQ,CAAC,CAAC;IAExD,MAAM,EAAE,WAAW,EAAE,GAAG,oBAAoB,CAAC;QAC3C,IAAI;QACJ,WAAW;QACX,SAAS,EAAE,CAAC,KAAK,EAAE,EAAE,CACnB,aAAa,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,QAAQ,CAAC;KACtG,CAAC,CAAC;IAEH,+DAA+D;IAC/D,uEAAuE;IACvE,sEAAsE;IACtE,qEAAqE;IACrE,qEAAqE;IACrE,uEAAuE;IACvE,OAAO;QACL,WAAW;QACX,aAAa;QACb,iBAAiB,EAAE,QAAQ,CAAC,QAAQ,CAAC;QACrC,QAAQ,EAAE;YACR,MAAM,EAAE,WAAW,CAAC,MAAM;YAC1B,aAAa,EAAE,CAAC;YAChB,YAAY,EAAE,CAAC;YACf,WAAW,EAAE,CAAC;SACf;KACF,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAa;IACvC,IAAI,IAAY,CAAC;IACjB,IAAI,CAAC;QACH,IAAI,GAAG,IAAI,WAAW,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACjE,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CACb,wCAAwC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,yDAAyD,CAClJ,CAAC;IACJ,CAAC;IACD,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,MAAM;QAAE,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACxD,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAExD,yEAAyE;IACzE,uEAAuE;IACvE,gDAAgD;IAChD,MAAM,kBAAkB,GAAG,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC;IAC9D,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,KAAK,MAAM,CAAC,IAAI,kBAAkB,EAAE,CAAC;QACnC,KAAK,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC;YACnB,IAAI,EAAE,KAAK,GAAG;gBAAE,WAAW,EAAE,CAAC;iBACzB,IAAI,EAAE,KAAK,GAAG;gBAAE,WAAW,EAAE,CAAC;QACrC,CAAC;IACH,CAAC;IACD,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC;QAC1C,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,0EAA0E,WAAW,iBAAiB,WAAW,IAAI,CACtH,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAID,SAAS,yBAAyB,CAAC,QAAkB;IACnD,OAAO,CAAC,IAAY,EAAE,EAAE;QACtB,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;QACvC,IAAI,CAAC,CAAC;YAAE,OAAO,IAAI,CAAC;QACpB,MAAM,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC7B,MAAM,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC7B,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAChC,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAClC,MAAM,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3D,MAAM,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC7B,MAAM,GAAG,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACxC,MAAM,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1C,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,EAAE,IAAI,GAAG,GAAG,CAAC,IAAI,GAAG,GAAG,EAAE;YAAE,OAAO,IAAI,CAAC;QAChE,IAAI,IAAI,GAAG,EAAE,IAAI,MAAM,GAAG,EAAE,IAAI,MAAM,GAAG,EAAE;YAAE,OAAO,IAAI,CAAC;QACzD,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QACzF,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;IAC/D,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CAAC,QAAwC,EAAE,IAAY;IAC7E,IAAI,QAAQ,KAAK,UAAU,IAAI,QAAQ,KAAK,YAAY;QAAE,OAAO,MAAM,CAAC;IACxE,IAAI,QAAQ,KAAK,UAAU,IAAI,QAAQ,KAAK,YAAY;QAAE,OAAO,MAAM,CAAC;IACxE,yEAAyE;IACzE,8DAA8D;IAC9D,MAAM,IAAI,GAAG,yBAAyB,CAAC,MAAM,CAAC,CAAC;IAC/C,MAAM,IAAI,GAAG,yBAAyB,CAAC,MAAM,CAAC,CAAC;IAC/C,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACpC,IAAI,IAAI,CAAC,IAAI,CAAC;YAAE,OAAO,MAAM,CAAC;QAC9B,IAAI,IAAI,CAAC,IAAI,CAAC;YAAE,OAAO,MAAM,CAAC;IAChC,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,aAAa,CACpB,IAAY,EACZ,KAAa,EACb,GAAW,EACX,IAAY,EACZ,MAAc,EACd,MAAc,EACd,QAAgB;IAEhB,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,GAAG,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IACxE,IAAI,MAAM,GAAG,eAAe,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,EAAE,QAAQ,CAAC,CAAC;IAC7D,MAAM,YAAY,GAAG,UAAU,GAAG,MAAM,GAAG,MAAM,CAAC;IAClD,MAAM,GAAG,eAAe,CAAC,IAAI,IAAI,CAAC,YAAY,CAAC,EAAE,QAAQ,CAAC,CAAC;IAC3D,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IACrC,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAChC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IAC/D,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACnD,MAAM,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACxC,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACzC,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACvC,MAAM,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACxC,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IAC3C,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IAC1C,OAAO,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,IAAI,GAAG,KAAK,IAAI,KAAK,EAAE,CAAC;AAClE,CAAC;AAED,SAAS,eAAe,CAAC,IAAU,EAAE,QAAgB;IACnD,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE;QACjD,QAAQ,EAAE,QAAQ;QAClB,YAAY,EAAE,YAAY;KAC3B,CAAC,CAAC;IACH,MAAM,KAAK,GAAG,SAAS,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;IAC5C,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,cAAc,CAAC,CAAC;IAC5D,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CAAC,sDAAsD,QAAQ,IAAI,CAAC,CAAC;IACtF,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;IAC3B,IAAI,KAAK,KAAK,KAAK,IAAI,KAAK,KAAK,KAAK;QAAE,OAAO,CAAC,CAAC;IACjD,MAAM,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,wCAAwC,CAAC,CAAC;IAChE,IAAI,CAAC,CAAC,EAAE,CAAC;QACP,MAAM,IAAI,KAAK,CACb,gDAAgD,KAAK,oBAAoB,QAAQ,IAAI,CACtF,CAAC;IACJ,CAAC;IACD,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACnC,MAAM,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC9B,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzC,OAAO,IAAI,GAAG,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;AAC/B,CAAC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { ParsedLine } from "../conversation-normalisers/types.js";
|
|
2
|
+
export type CursorResult = {
|
|
3
|
+
kind: "found";
|
|
4
|
+
deltaStart: number;
|
|
5
|
+
} | {
|
|
6
|
+
kind: "empty";
|
|
7
|
+
} | {
|
|
8
|
+
kind: "missing";
|
|
9
|
+
};
|
|
10
|
+
export declare function findDeltaCursor(parsedLines: readonly ParsedLine[], lastIngestedMessageHash: string): CursorResult;
|
|
11
|
+
//# sourceMappingURL=delta-cursor.d.ts.map
|
package/payload/platform/plugins/memory/mcp/dist/lib/conversation-pipeline/delta-cursor.d.ts.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"delta-cursor.d.ts","sourceRoot":"","sources":["../../../src/lib/conversation-pipeline/delta-cursor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,sCAAsC,CAAC;AAqBvE,MAAM,MAAM,YAAY,GACpB;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,GACrC;IAAE,IAAI,EAAE,OAAO,CAAA;CAAE,GACjB;IAAE,IAAI,EAAE,SAAS,CAAA;CAAE,CAAC;AAExB,wBAAgB,eAAe,CAC7B,WAAW,EAAE,SAAS,UAAU,EAAE,EAClC,uBAAuB,EAAE,MAAM,GAC9B,YAAY,CAgBd"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { deriveMessageContentHash } from "./derive-keys.js";
|
|
2
|
+
export function findDeltaCursor(parsedLines, lastIngestedMessageHash) {
|
|
3
|
+
if (!lastIngestedMessageHash || !lastIngestedMessageHash.trim()) {
|
|
4
|
+
throw new Error("findDeltaCursor: lastIngestedMessageHash must be non-empty");
|
|
5
|
+
}
|
|
6
|
+
for (let i = 0; i < parsedLines.length; i++) {
|
|
7
|
+
const line = parsedLines[i];
|
|
8
|
+
const hash = deriveMessageContentHash({
|
|
9
|
+
dateSent: line.dateSent,
|
|
10
|
+
body: line.body,
|
|
11
|
+
});
|
|
12
|
+
if (hash === lastIngestedMessageHash) {
|
|
13
|
+
if (i === parsedLines.length - 1)
|
|
14
|
+
return { kind: "empty" };
|
|
15
|
+
return { kind: "found", deltaStart: i + 1 };
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
return { kind: "missing" };
|
|
19
|
+
}
|
|
20
|
+
//# sourceMappingURL=delta-cursor.js.map
|
package/payload/platform/plugins/memory/mcp/dist/lib/conversation-pipeline/delta-cursor.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"delta-cursor.js","sourceRoot":"","sources":["../../../src/lib/conversation-pipeline/delta-cursor.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,wBAAwB,EAAE,MAAM,kBAAkB,CAAC;AAyB5D,MAAM,UAAU,eAAe,CAC7B,WAAkC,EAClC,uBAA+B;IAE/B,IAAI,CAAC,uBAAuB,IAAI,CAAC,uBAAuB,CAAC,IAAI,EAAE,EAAE,CAAC;QAChE,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAC;IAChF,CAAC;IACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5C,MAAM,IAAI,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,IAAI,GAAG,wBAAwB,CAAC;YACpC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,IAAI,EAAE,IAAI,CAAC,IAAI;SAChB,CAAC,CAAC;QACH,IAAI,IAAI,KAAK,uBAAuB,EAAE,CAAC;YACrC,IAAI,CAAC,KAAK,WAAW,CAAC,MAAM,GAAG,CAAC;gBAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;YAC3D,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC;QAC9C,CAAC;IACH,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;AAC7B,CAAC"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export declare function sha256Hex(input: string): string;
|
|
2
|
+
export interface DeriveConversationIdentityInput {
|
|
3
|
+
accountId: string;
|
|
4
|
+
/** Element IDs of every confirmed participant (owner + others). Order is
|
|
5
|
+
* not significant; the function sorts internally. */
|
|
6
|
+
participantElementIds: readonly string[];
|
|
7
|
+
}
|
|
8
|
+
export declare function deriveConversationIdentity(input: DeriveConversationIdentityInput): string;
|
|
9
|
+
export interface DeriveMessageContentHashInput {
|
|
10
|
+
dateSent: string;
|
|
11
|
+
body: string;
|
|
12
|
+
}
|
|
13
|
+
export declare function deriveMessageContentHash(input: DeriveMessageContentHashInput): string;
|
|
14
|
+
//# sourceMappingURL=derive-keys.d.ts.map
|
package/payload/platform/plugins/memory/mcp/dist/lib/conversation-pipeline/derive-keys.d.ts.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"derive-keys.d.ts","sourceRoot":"","sources":["../../../src/lib/conversation-pipeline/derive-keys.ts"],"names":[],"mappings":"AAuBA,wBAAgB,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAE/C;AAED,MAAM,WAAW,+BAA+B;IAC9C,SAAS,EAAE,MAAM,CAAC;IAClB;0DACsD;IACtD,qBAAqB,EAAE,SAAS,MAAM,EAAE,CAAC;CAC1C;AAED,wBAAgB,0BAA0B,CACxC,KAAK,EAAE,+BAA+B,GACrC,MAAM,CASR;AAED,MAAM,WAAW,6BAA6B;IAC5C,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,wBAAgB,wBAAwB,CACtC,KAAK,EAAE,6BAA6B,GACnC,MAAM,CAER"}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
// ---------------------------------------------------------------------------
|
|
3
|
+
// derive-keys — natural-key derivation for conversation-archive ingestion
|
|
4
|
+
// (Task 894 / 897).
|
|
5
|
+
//
|
|
6
|
+
// Pure functions. No I/O. The whole point is that re-imports of the same
|
|
7
|
+
// archive collapse to the same identity regardless of release-level drift in
|
|
8
|
+
// chunk indices, hash widths, or arbitrary tiebreakers.
|
|
9
|
+
//
|
|
10
|
+
// conversationIdentity = sha256(accountId + ":" + sortedParticipantElementIds.join(","))
|
|
11
|
+
// messageContentHash = sha256(dateSent + "|" + body)
|
|
12
|
+
//
|
|
13
|
+
// `conversationIdentity` is stable across re-exports — same operator + same
|
|
14
|
+
// participant set → same identity, regardless of file bytes.
|
|
15
|
+
//
|
|
16
|
+
// `messageContentHash` (Task 897): hashes `dateSent + body`. Per-line
|
|
17
|
+
// senderName is gone (the scanner is body-opaque); the body string still
|
|
18
|
+
// carries any `Sender:` prefix verbatim, so two senders posting different
|
|
19
|
+
// content at the same instant stay distinguishable. The cursor survives a
|
|
20
|
+
// fresh re-export as long as the source preserves the prefix in the body.
|
|
21
|
+
// ---------------------------------------------------------------------------
|
|
22
|
+
export function sha256Hex(input) {
|
|
23
|
+
return createHash("sha256").update(input).digest("hex");
|
|
24
|
+
}
|
|
25
|
+
export function deriveConversationIdentity(input) {
|
|
26
|
+
if (!input.accountId || !input.accountId.trim()) {
|
|
27
|
+
throw new Error("deriveConversationIdentity: accountId is required");
|
|
28
|
+
}
|
|
29
|
+
if (input.participantElementIds.length === 0) {
|
|
30
|
+
throw new Error("deriveConversationIdentity: participantElementIds must be non-empty");
|
|
31
|
+
}
|
|
32
|
+
const sorted = [...input.participantElementIds].sort();
|
|
33
|
+
return sha256Hex(`${input.accountId}:${sorted.join(",")}`);
|
|
34
|
+
}
|
|
35
|
+
export function deriveMessageContentHash(input) {
|
|
36
|
+
return sha256Hex(`${input.dateSent}|${input.body}`);
|
|
37
|
+
}
|
|
38
|
+
//# sourceMappingURL=derive-keys.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"derive-keys.js","sourceRoot":"","sources":["../../../src/lib/conversation-pipeline/derive-keys.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,8EAA8E;AAC9E,0EAA0E;AAC1E,oBAAoB;AACpB,EAAE;AACF,yEAAyE;AACzE,6EAA6E;AAC7E,wDAAwD;AACxD,EAAE;AACF,2FAA2F;AAC3F,yDAAyD;AACzD,EAAE;AACF,4EAA4E;AAC5E,6DAA6D;AAC7D,EAAE;AACF,sEAAsE;AACtE,yEAAyE;AACzE,0EAA0E;AAC1E,0EAA0E;AAC1E,0EAA0E;AAC1E,8EAA8E;AAE9E,MAAM,UAAU,SAAS,CAAC,KAAa;IACrC,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC1D,CAAC;AASD,MAAM,UAAU,0BAA0B,CACxC,KAAsC;IAEtC,IAAI,CAAC,KAAK,CAAC,SAAS,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,CAAC;QAChD,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;IACvE,CAAC;IACD,IAAI,KAAK,CAAC,qBAAqB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7C,MAAM,IAAI,KAAK,CAAC,qEAAqE,CAAC,CAAC;IACzF,CAAC;IACD,MAAM,MAAM,GAAG,CAAC,GAAG,KAAK,CAAC,qBAAqB,CAAC,CAAC,IAAI,EAAE,CAAC;IACvD,OAAO,SAAS,CAAC,GAAG,KAAK,CAAC,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;AAC7D,CAAC;AAOD,MAAM,UAAU,wBAAwB,CACtC,KAAoC;IAEpC,OAAO,SAAS,CAAC,GAAG,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;AACtD,CAAC"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { Session as Neo4jSession } from "neo4j-driver";
|
|
2
|
+
export declare class IngestUserFacingError extends Error {
|
|
3
|
+
readonly userFacing = true;
|
|
4
|
+
constructor(message: string);
|
|
5
|
+
}
|
|
6
|
+
export interface VerifyParticipantElementIdsInput {
|
|
7
|
+
session: Neo4jSession;
|
|
8
|
+
accountId: string;
|
|
9
|
+
ownerElementId: string;
|
|
10
|
+
participantElementIds: readonly string[];
|
|
11
|
+
}
|
|
12
|
+
export interface VerifyParticipantElementIdsResult {
|
|
13
|
+
participantsResolved: number;
|
|
14
|
+
}
|
|
15
|
+
export declare function verifyParticipantElementIds(input: VerifyParticipantElementIdsInput): Promise<VerifyParticipantElementIdsResult>;
|
|
16
|
+
//# sourceMappingURL=sender-bind.d.ts.map
|
package/payload/platform/plugins/memory/mcp/dist/lib/conversation-pipeline/sender-bind.d.ts.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sender-bind.d.ts","sourceRoot":"","sources":["../../../src/lib/conversation-pipeline/sender-bind.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,IAAI,YAAY,EAAE,MAAM,cAAc,CAAC;AAgB5D,qBAAa,qBAAsB,SAAQ,KAAK;IAC9C,QAAQ,CAAC,UAAU,QAAQ;gBACf,OAAO,EAAE,MAAM;CAI5B;AAED,MAAM,WAAW,gCAAgC;IAC/C,OAAO,EAAE,YAAY,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,qBAAqB,EAAE,SAAS,MAAM,EAAE,CAAC;CAC1C;AAED,MAAM,WAAW,iCAAiC;IAChD,oBAAoB,EAAE,MAAM,CAAC;CAC9B;AAUD,wBAAsB,2BAA2B,CAC/C,KAAK,EAAE,gCAAgC,GACtC,OAAO,CAAC,iCAAiC,CAAC,CAkC5C"}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
// ---------------------------------------------------------------------------
|
|
2
|
+
// sender-bind — defensive participant-elementId verification (Task 894 / 897).
|
|
3
|
+
//
|
|
4
|
+
// Task 897 dropped per-line senderName, so the closed-set sender-resolution
|
|
5
|
+
// step that lived here is gone. What remains is a defensive read pass: every
|
|
6
|
+
// supplied participant elementId must exist in the graph, must carry the
|
|
7
|
+
// supplied accountId, and must be labelled :Person or :AdminUser. If any
|
|
8
|
+
// fails, the bin LOUD-FAILs before the writer attaches PARTICIPANT_IN edges
|
|
9
|
+
// to a stale or cross-account node.
|
|
10
|
+
//
|
|
11
|
+
// The operator confirms participants up front; participants attach to the
|
|
12
|
+
// :ConversationArchive parent via :PARTICIPANT_IN, never per-message.
|
|
13
|
+
// ---------------------------------------------------------------------------
|
|
14
|
+
export class IngestUserFacingError extends Error {
|
|
15
|
+
userFacing = true;
|
|
16
|
+
constructor(message) {
|
|
17
|
+
super(message);
|
|
18
|
+
this.name = "IngestUserFacingError";
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
const CANONICAL_FETCH_CYPHER = `
|
|
22
|
+
UNWIND $ids AS id
|
|
23
|
+
MATCH (n) WHERE elementId(n) = id
|
|
24
|
+
RETURN elementId(n) AS elemId,
|
|
25
|
+
labels(n) AS labels,
|
|
26
|
+
coalesce(n.accountId, '') AS accountId
|
|
27
|
+
`;
|
|
28
|
+
export async function verifyParticipantElementIds(input) {
|
|
29
|
+
const { session, accountId, ownerElementId, participantElementIds } = input;
|
|
30
|
+
const allIds = [ownerElementId, ...participantElementIds];
|
|
31
|
+
const distinctIds = Array.from(new Set(allIds));
|
|
32
|
+
if (distinctIds.length !== allIds.length) {
|
|
33
|
+
throw new IngestUserFacingError(`participant id list contains duplicates (owner appears in --participant-person-ids?)`);
|
|
34
|
+
}
|
|
35
|
+
const res = await session.executeRead(async (tx) => tx.run(CANONICAL_FETCH_CYPHER, { ids: distinctIds }));
|
|
36
|
+
const seenIds = new Set();
|
|
37
|
+
for (const r of res.records) {
|
|
38
|
+
const elemId = r.get("elemId");
|
|
39
|
+
const labels = r.get("labels") || [];
|
|
40
|
+
const acct = r.get("accountId") || "";
|
|
41
|
+
if (!acct) {
|
|
42
|
+
throw new IngestUserFacingError(`node ${elemId} has no accountId — corrupt canonical Person/AdminUser`);
|
|
43
|
+
}
|
|
44
|
+
if (acct !== accountId) {
|
|
45
|
+
throw new IngestUserFacingError(`node ${elemId} belongs to account ${acct}, not ${accountId}`);
|
|
46
|
+
}
|
|
47
|
+
if (!labels.includes("Person") && !labels.includes("AdminUser")) {
|
|
48
|
+
throw new IngestUserFacingError(`node ${elemId} has labels [${labels.join(",")}]; expected :Person or :AdminUser`);
|
|
49
|
+
}
|
|
50
|
+
seenIds.add(elemId);
|
|
51
|
+
}
|
|
52
|
+
for (const id of distinctIds) {
|
|
53
|
+
if (!seenIds.has(id)) {
|
|
54
|
+
throw new IngestUserFacingError(`elementId ${id} not found in graph`);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
return { participantsResolved: seenIds.size };
|
|
58
|
+
}
|
|
59
|
+
//# sourceMappingURL=sender-bind.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sender-bind.js","sourceRoot":"","sources":["../../../src/lib/conversation-pipeline/sender-bind.ts"],"names":[],"mappings":"AAEA,8EAA8E;AAC9E,+EAA+E;AAC/E,EAAE;AACF,4EAA4E;AAC5E,6EAA6E;AAC7E,yEAAyE;AACzE,yEAAyE;AACzE,4EAA4E;AAC5E,oCAAoC;AACpC,EAAE;AACF,0EAA0E;AAC1E,sEAAsE;AACtE,8EAA8E;AAE9E,MAAM,OAAO,qBAAsB,SAAQ,KAAK;IACrC,UAAU,GAAG,IAAI,CAAC;IAC3B,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,uBAAuB,CAAC;IACtC,CAAC;CACF;AAaD,MAAM,sBAAsB,GAAG;;;;;;CAM9B,CAAC;AAEF,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAC/C,KAAuC;IAEvC,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,qBAAqB,EAAE,GAAG,KAAK,CAAC;IAC5E,MAAM,MAAM,GAAG,CAAC,cAAc,EAAE,GAAG,qBAAqB,CAAC,CAAC;IAC1D,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;IAChD,IAAI,WAAW,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC;QACzC,MAAM,IAAI,qBAAqB,CAC7B,sFAAsF,CACvF,CAAC;IACJ,CAAC;IACD,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,WAAW,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE,CACjD,EAAE,CAAC,GAAG,CAAC,sBAAsB,EAAE,EAAE,GAAG,EAAE,WAAW,EAAE,CAAC,CACrD,CAAC;IACF,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAW,CAAC;QACzC,MAAM,MAAM,GAAI,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAc,IAAI,EAAE,CAAC;QACnD,MAAM,IAAI,GAAI,CAAC,CAAC,GAAG,CAAC,WAAW,CAAY,IAAI,EAAE,CAAC;QAClD,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,MAAM,IAAI,qBAAqB,CAAC,QAAQ,MAAM,wDAAwD,CAAC,CAAC;QAC1G,CAAC;QACD,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvB,MAAM,IAAI,qBAAqB,CAAC,QAAQ,MAAM,uBAAuB,IAAI,SAAS,SAAS,EAAE,CAAC,CAAC;QACjG,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;YAChE,MAAM,IAAI,qBAAqB,CAAC,QAAQ,MAAM,gBAAgB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAC;QACrH,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACtB,CAAC;IACD,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE,CAAC;QAC7B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;YACrB,MAAM,IAAI,qBAAqB,CAAC,aAAa,EAAE,qBAAqB,CAAC,CAAC;QACxE,CAAC;IACH,CAAC;IACD,OAAO,EAAE,oBAAoB,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC;AAChD,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { ParsedLine } from "../conversation-normalisers/types.js";
|
|
2
|
+
export interface ConversationSession {
|
|
3
|
+
index: number;
|
|
4
|
+
firstMessageAt: string;
|
|
5
|
+
lastMessageAt: string;
|
|
6
|
+
messages: ParsedLine[];
|
|
7
|
+
}
|
|
8
|
+
export declare function sessionize(messages: readonly ParsedLine[], gapHours: number): ConversationSession[];
|
|
9
|
+
//# sourceMappingURL=sessionize.d.ts.map
|
package/payload/platform/plugins/memory/mcp/dist/lib/conversation-pipeline/sessionize.d.ts.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sessionize.d.ts","sourceRoot":"","sources":["../../../src/lib/conversation-pipeline/sessionize.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,sCAAsC,CAAC;AAcvE,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,UAAU,EAAE,CAAC;CACxB;AAED,wBAAgB,UAAU,CACxB,QAAQ,EAAE,SAAS,UAAU,EAAE,EAC/B,QAAQ,EAAE,MAAM,GACf,mBAAmB,EAAE,CA+BvB"}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
export function sessionize(messages, gapHours) {
|
|
2
|
+
if (gapHours <= 0) {
|
|
3
|
+
throw new Error(`sessionize: gapHours must be positive, got ${gapHours}`);
|
|
4
|
+
}
|
|
5
|
+
if (messages.length === 0)
|
|
6
|
+
return [];
|
|
7
|
+
const gapMs = gapHours * 60 * 60 * 1000;
|
|
8
|
+
const sessions = [];
|
|
9
|
+
let currentMessages = [messages[0]];
|
|
10
|
+
const flush = () => {
|
|
11
|
+
sessions.push({
|
|
12
|
+
index: sessions.length,
|
|
13
|
+
firstMessageAt: currentMessages[0].dateSent,
|
|
14
|
+
lastMessageAt: currentMessages[currentMessages.length - 1].dateSent,
|
|
15
|
+
messages: currentMessages,
|
|
16
|
+
});
|
|
17
|
+
};
|
|
18
|
+
for (let i = 1; i < messages.length; i++) {
|
|
19
|
+
const prevMs = Date.parse(messages[i - 1].dateSent);
|
|
20
|
+
const currMs = Date.parse(messages[i].dateSent);
|
|
21
|
+
if (currMs - prevMs >= gapMs) {
|
|
22
|
+
flush();
|
|
23
|
+
currentMessages = [messages[i]];
|
|
24
|
+
}
|
|
25
|
+
else {
|
|
26
|
+
currentMessages.push(messages[i]);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
flush();
|
|
30
|
+
return sessions;
|
|
31
|
+
}
|
|
32
|
+
//# sourceMappingURL=sessionize.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sessionize.js","sourceRoot":"","sources":["../../../src/lib/conversation-pipeline/sessionize.ts"],"names":[],"mappings":"AAqBA,MAAM,UAAU,UAAU,CACxB,QAA+B,EAC/B,QAAgB;IAEhB,IAAI,QAAQ,IAAI,CAAC,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CAAC,8CAA8C,QAAQ,EAAE,CAAC,CAAC;IAC5E,CAAC;IACD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAErC,MAAM,KAAK,GAAG,QAAQ,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;IACxC,MAAM,QAAQ,GAA0B,EAAE,CAAC;IAC3C,IAAI,eAAe,GAAiB,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;IAElD,MAAM,KAAK,GAAG,GAAG,EAAE;QACjB,QAAQ,CAAC,IAAI,CAAC;YACZ,KAAK,EAAE,QAAQ,CAAC,MAAM;YACtB,cAAc,EAAE,eAAe,CAAC,CAAC,CAAC,CAAC,QAAQ;YAC3C,aAAa,EAAE,eAAe,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,QAAQ;YACnE,QAAQ,EAAE,eAAe;SAC1B,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QACpD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QAChD,IAAI,MAAM,GAAG,MAAM,IAAI,KAAK,EAAE,CAAC;YAC7B,KAAK,EAAE,CAAC;YACR,eAAe,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC;aAAM,CAAC;YACN,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;QACpC,CAAC;IACH,CAAC;IACD,KAAK,EAAE,CAAC;IACR,OAAO,QAAQ,CAAC;AAClB,CAAC"}
|
package/payload/platform/plugins/memory/mcp/dist/lib/conversation-pipeline/to-turn-text.d.ts.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"to-turn-text.d.ts","sourceRoot":"","sources":["../../../src/lib/conversation-pipeline/to-turn-text.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAgB3D,wBAAgB,UAAU,CAAC,OAAO,EAAE,mBAAmB,GAAG,MAAM,CAM/D"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
// ---------------------------------------------------------------------------
|
|
2
|
+
// to-turn-text — render one ConversationSession as a timestamped text block
|
|
3
|
+
// for memory-classify (mode='chat'). Task 897: bodies are opaque; the
|
|
4
|
+
// scanner already preserved the source's `Sender:` segment inside `body`,
|
|
5
|
+
// so a WhatsApp session renders as:
|
|
6
|
+
//
|
|
7
|
+
// [YYYY-MM-DD HH:MM:SS ±HH:MM] Sender: body text
|
|
8
|
+
// [YYYY-MM-DD HH:MM:SS ±HH:MM] Sender: body text
|
|
9
|
+
//
|
|
10
|
+
// Other sources whose body shape is different (system message, vCard,
|
|
11
|
+
// forwarded block) flow through unchanged — the chat-mode prompt is
|
|
12
|
+
// instructed to keep these prefixes verbatim in chunk bodies.
|
|
13
|
+
// ---------------------------------------------------------------------------
|
|
14
|
+
export function toTurnText(session) {
|
|
15
|
+
const lines = [];
|
|
16
|
+
for (const m of session.messages) {
|
|
17
|
+
lines.push(`[${formatWallClock(m.dateSent)}] ${m.body}`);
|
|
18
|
+
}
|
|
19
|
+
return lines.join("\n");
|
|
20
|
+
}
|
|
21
|
+
function formatWallClock(iso) {
|
|
22
|
+
const m = iso.match(/^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.\d+)?(Z|[+-]\d{2}:?\d{2})$/);
|
|
23
|
+
if (!m)
|
|
24
|
+
return iso;
|
|
25
|
+
const [, y, mo, d, h, mi, s, off] = m;
|
|
26
|
+
const offsetLabel = off === "Z" ? "+00:00" : off;
|
|
27
|
+
return `${y}-${mo}-${d} ${h}:${mi}:${s} ${offsetLabel}`;
|
|
28
|
+
}
|
|
29
|
+
//# sourceMappingURL=to-turn-text.js.map
|
package/payload/platform/plugins/memory/mcp/dist/lib/conversation-pipeline/to-turn-text.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"to-turn-text.js","sourceRoot":"","sources":["../../../src/lib/conversation-pipeline/to-turn-text.ts"],"names":[],"mappings":"AAEA,8EAA8E;AAC9E,4EAA4E;AAC5E,sEAAsE;AACtE,0EAA0E;AAC1E,oCAAoC;AACpC,EAAE;AACF,mDAAmD;AACnD,mDAAmD;AACnD,EAAE;AACF,sEAAsE;AACtE,oEAAoE;AACpE,8DAA8D;AAC9D,8EAA8E;AAE9E,MAAM,UAAU,UAAU,CAAC,OAA4B;IACrD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACjC,KAAK,CAAC,IAAI,CAAC,IAAI,eAAe,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAC3D,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,eAAe,CAAC,GAAW;IAClC,MAAM,CAAC,GAAG,GAAG,CAAC,KAAK,CACjB,iFAAiF,CAClF,CAAC;IACF,IAAI,CAAC,CAAC;QAAE,OAAO,GAAG,CAAC;IACnB,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;IACtC,MAAM,WAAW,GAAG,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC;IACjD,OAAO,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,WAAW,EAAE,CAAC;AAC1D,CAAC"}
|