@rubytech/create-realagent 1.0.829 → 1.0.831
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/payload/platform/config/brand.json +1 -1
- package/payload/platform/lib/oauth-llm/dist/index.d.ts +9 -2
- package/payload/platform/lib/oauth-llm/dist/index.d.ts.map +1 -1
- package/payload/platform/lib/oauth-llm/dist/index.js +26 -1
- package/payload/platform/lib/oauth-llm/dist/index.js.map +1 -1
- package/payload/platform/lib/oauth-llm/src/index.ts +43 -4
- package/payload/platform/neo4j/migrations/007-conversation-archive-source.ts +116 -0
- package/payload/platform/neo4j/migrations/008-adminuser-accountid-backfill.ts +85 -0
- package/payload/platform/neo4j/schema.cypher +12 -3
- package/payload/platform/plugins/admin/hooks/__tests__/archive-ingest-surface-gate.test.sh +54 -39
- package/payload/platform/plugins/admin/hooks/archive-ingest-surface-gate.sh +64 -26
- package/payload/platform/plugins/admin/mcp/dist/index.js +25 -3
- package/payload/platform/plugins/admin/mcp/dist/index.js.map +1 -1
- package/payload/platform/plugins/contacts/mcp/dist/index.js +5 -5
- package/payload/platform/plugins/contacts/mcp/dist/index.js.map +1 -1
- package/payload/platform/plugins/contacts/mcp/dist/tools/contact-create.d.ts +1 -1
- package/payload/platform/plugins/contacts/mcp/dist/tools/contact-create.d.ts.map +1 -1
- package/payload/platform/plugins/contacts/mcp/dist/tools/contact-create.js +29 -23
- package/payload/platform/plugins/contacts/mcp/dist/tools/contact-create.js.map +1 -1
- package/payload/platform/plugins/docs/references/internals.md +1 -1
- package/payload/platform/plugins/docs/references/plugins-guide.md +1 -1
- package/payload/platform/plugins/memory/PLUGIN.md +2 -1
- package/payload/platform/plugins/memory/bin/conversation-archive-ingest.mjs +564 -0
- package/payload/platform/plugins/memory/bin/conversation-archive-ingest.sh +106 -0
- package/payload/platform/plugins/memory/mcp/dist/index.js +30 -16
- package/payload/platform/plugins/memory/mcp/dist/index.js.map +1 -1
- package/payload/platform/plugins/memory/mcp/dist/lib/__tests__/llm-classifier.test.js +4 -3
- package/payload/platform/plugins/memory/mcp/dist/lib/__tests__/llm-classifier.test.js.map +1 -1
- package/payload/platform/plugins/memory/mcp/dist/lib/__tests__/schema-loader.test.js +11 -6
- package/payload/platform/plugins/memory/mcp/dist/lib/__tests__/schema-loader.test.js.map +1 -1
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-normalisers/index.d.ts +5 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-normalisers/index.d.ts.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-normalisers/index.js +30 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-normalisers/index.js.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-normalisers/timestamp-scanner.d.ts +49 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-normalisers/timestamp-scanner.d.ts.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-normalisers/timestamp-scanner.js +35 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-normalisers/timestamp-scanner.js.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-normalisers/types.d.ts +47 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-normalisers/types.d.ts.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-normalisers/types.js +31 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-normalisers/types.js.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-normalisers/whatsapp-text.d.ts +3 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-normalisers/whatsapp-text.d.ts.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-normalisers/whatsapp-text.js +155 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-normalisers/whatsapp-text.js.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-pipeline/delta-cursor.d.ts +11 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-pipeline/delta-cursor.d.ts.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-pipeline/delta-cursor.js +20 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-pipeline/delta-cursor.js.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-pipeline/derive-keys.d.ts +14 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-pipeline/derive-keys.d.ts.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-pipeline/derive-keys.js +38 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-pipeline/derive-keys.js.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-pipeline/sender-bind.d.ts +16 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-pipeline/sender-bind.d.ts.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-pipeline/sender-bind.js +59 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-pipeline/sender-bind.js.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-pipeline/sessionize.d.ts +9 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-pipeline/sessionize.d.ts.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-pipeline/sessionize.js +32 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-pipeline/sessionize.js.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-pipeline/to-turn-text.d.ts +3 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-pipeline/to-turn-text.d.ts.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-pipeline/to-turn-text.js +29 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/conversation-pipeline/to-turn-text.js.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/document-chunker.d.ts +45 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/document-chunker.d.ts.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/document-chunker.js +125 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/document-chunker.js.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/lib/llm-classifier.d.ts +34 -9
- package/payload/platform/plugins/memory/mcp/dist/lib/llm-classifier.d.ts.map +1 -1
- package/payload/platform/plugins/memory/mcp/dist/lib/llm-classifier.js +360 -35
- package/payload/platform/plugins/memory/mcp/dist/lib/llm-classifier.js.map +1 -1
- package/payload/platform/plugins/memory/mcp/dist/lib/llm-ranker.d.ts +3 -2
- package/payload/platform/plugins/memory/mcp/dist/lib/llm-ranker.d.ts.map +1 -1
- package/payload/platform/plugins/memory/mcp/dist/lib/llm-ranker.js +46 -17
- package/payload/platform/plugins/memory/mcp/dist/lib/llm-ranker.js.map +1 -1
- package/payload/platform/plugins/memory/mcp/dist/tools/__tests__/conversation-normalisers-source-agnosticism.test.d.ts +2 -0
- package/payload/platform/plugins/memory/mcp/dist/tools/__tests__/conversation-normalisers-source-agnosticism.test.d.ts.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/tools/__tests__/conversation-normalisers-source-agnosticism.test.js +73 -0
- package/payload/platform/plugins/memory/mcp/dist/tools/__tests__/conversation-normalisers-source-agnosticism.test.js.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/tools/__tests__/conversation-normalisers-whatsapp-text.test.d.ts +2 -0
- package/payload/platform/plugins/memory/mcp/dist/tools/__tests__/conversation-normalisers-whatsapp-text.test.d.ts.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/tools/__tests__/conversation-normalisers-whatsapp-text.test.js +109 -0
- package/payload/platform/plugins/memory/mcp/dist/tools/__tests__/conversation-normalisers-whatsapp-text.test.js.map +1 -0
- package/payload/platform/plugins/memory/mcp/dist/tools/__tests__/memory-ingest.test.js +34 -3
- package/payload/platform/plugins/memory/mcp/dist/tools/__tests__/memory-ingest.test.js.map +1 -1
- package/payload/platform/plugins/memory/mcp/dist/tools/memory-archive-write.d.ts +17 -0
- package/payload/platform/plugins/memory/mcp/dist/tools/memory-archive-write.d.ts.map +1 -1
- package/payload/platform/plugins/memory/mcp/dist/tools/memory-archive-write.js +34 -13
- package/payload/platform/plugins/memory/mcp/dist/tools/memory-archive-write.js.map +1 -1
- package/payload/platform/plugins/memory/mcp/dist/tools/memory-ingest.d.ts +18 -7
- package/payload/platform/plugins/memory/mcp/dist/tools/memory-ingest.d.ts.map +1 -1
- package/payload/platform/plugins/memory/mcp/dist/tools/memory-ingest.js +24 -8
- package/payload/platform/plugins/memory/mcp/dist/tools/memory-ingest.js.map +1 -1
- package/payload/platform/plugins/memory/mcp/dist/tools/memory-rank.js +2 -2
- package/payload/platform/plugins/memory/mcp/dist/tools/memory-rank.js.map +1 -1
- package/payload/platform/plugins/memory/references/schema-base.md +2 -2
- package/payload/platform/plugins/memory/skills/conversation-archive/SKILL.md +133 -0
- package/payload/platform/plugins/memory/skills/document-ingest/SKILL.md +5 -2
- package/payload/platform/plugins/whatsapp/PLUGIN.md +1 -1
- package/payload/platform/scripts/seed-neo4j.sh +15 -15
- package/payload/platform/templates/specialists/agents/database-operator.md +8 -9
- package/payload/server/chunk-7BO5HDJC.js +10093 -0
- package/payload/server/chunk-BCFM2UPH.js +2305 -0
- package/payload/server/chunk-CV3HPX46.js +10097 -0
- package/payload/server/chunk-EL4DZ56X.js +1116 -0
- package/payload/server/chunk-J6YWEJBN.js +1116 -0
- package/payload/server/chunk-OCPJGZ6S.js +654 -0
- package/payload/server/chunk-QOJ2D26Z.js +654 -0
- package/payload/server/chunk-RC46ZYGT.js +2305 -0
- package/payload/server/client-pool-7NTEFNVQ.js +32 -0
- package/payload/server/client-pool-ZNGN66GN.js +32 -0
- package/payload/server/cloudflare-task-tracker-MHALDN54.js +19 -0
- package/payload/server/cloudflare-task-tracker-WE77WXSI.js +19 -0
- package/payload/server/maxy-edge.js +3 -3
- package/payload/server/neo4j-migrations-4XPNJNM6.js +490 -0
- package/payload/server/neo4j-migrations-6RW423E2.js +530 -0
- package/payload/server/server.js +30 -19
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"llm-ranker.d.ts","sourceRoot":"","sources":["../../src/lib/llm-ranker.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AASH,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACpC,OAAO,EAAE,KAAK,CAAC;QACb,YAAY,EAAE,MAAM,CAAC;QACrB,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,EAAE,CAAC;QACjB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACrC,CAAC,CAAC;CACJ;AAED,MAAM,WAAW,OAAO;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,MAAM,UAAU,GAClB;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,QAAQ,EAAE,OAAO,EAAE,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,GAC7D;IAAE,IAAI,EAAE,
|
|
1
|
+
{"version":3,"file":"llm-ranker.d.ts","sourceRoot":"","sources":["../../src/lib/llm-ranker.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AASH,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACpC,OAAO,EAAE,KAAK,CAAC;QACb,YAAY,EAAE,MAAM,CAAC;QACrB,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,EAAE,CAAC;QACjB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACrC,CAAC,CAAC;CACJ;AAED,MAAM,WAAW,OAAO;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,MAAM,UAAU,GAClB;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,QAAQ,EAAE,OAAO,EAAE,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,GAC7D;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAgGtC;;;;;;;;;;GAUG;AACH,wBAAsB,cAAc,CAClC,UAAU,EAAE,aAAa,EAAE,EAC3B,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,UAAU,CAAC,CAiHrB"}
|
|
@@ -74,11 +74,24 @@ function formatCandidates(candidates) {
|
|
|
74
74
|
});
|
|
75
75
|
return JSON.stringify(items, null, 2);
|
|
76
76
|
}
|
|
77
|
-
/**
|
|
77
|
+
/**
|
|
78
|
+
* Strip a balanced markdown fence (Task 897 sub-scope G):
|
|
79
|
+
* { kind: "ok", json } when the text was bare JSON or fully fenced.
|
|
80
|
+
* { kind: "truncated", reason } when the text opens a fence but never closes
|
|
81
|
+
* it — the structural fingerprint of `max_tokens` mid-response.
|
|
82
|
+
*/
|
|
78
83
|
function extractJson(raw) {
|
|
79
84
|
const trimmed = raw.trim();
|
|
80
|
-
const
|
|
81
|
-
|
|
85
|
+
const balanced = trimmed.match(/^```(?:json)?\s*([\s\S]*?)```$/);
|
|
86
|
+
if (balanced)
|
|
87
|
+
return { kind: "ok", json: balanced[1].trim() };
|
|
88
|
+
if (/^```(?:json)?/.test(trimmed) && !/```\s*$/.test(trimmed)) {
|
|
89
|
+
return {
|
|
90
|
+
kind: "truncated",
|
|
91
|
+
reason: "response-truncated: opening ``` fence with no closing — model likely hit max_tokens mid-response",
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
return { kind: "ok", json: trimmed };
|
|
82
95
|
}
|
|
83
96
|
/** Cap a reasoning string at REASONING_MAX_CHARS, preserving a trailing ellipsis. */
|
|
84
97
|
function capReasoning(reasoning) {
|
|
@@ -87,9 +100,9 @@ function capReasoning(reasoning) {
|
|
|
87
100
|
return cleaned;
|
|
88
101
|
return cleaned.slice(0, REASONING_MAX_CHARS - 1) + "…";
|
|
89
102
|
}
|
|
90
|
-
/** Log
|
|
91
|
-
function
|
|
92
|
-
process.stderr.write(`[memory-rank] [${accountId}]
|
|
103
|
+
/** Log an error with a consistent prefix for logs-read.sh grep (Task 897). */
|
|
104
|
+
function logError(accountId, reason) {
|
|
105
|
+
process.stderr.write(`[memory-rank] [${accountId}] ERROR reason="${reason}"\n`);
|
|
93
106
|
}
|
|
94
107
|
// ---------------------------------------------------------------------------
|
|
95
108
|
// Main entry point
|
|
@@ -100,9 +113,10 @@ function logFallback(accountId, reason) {
|
|
|
100
113
|
* Returns:
|
|
101
114
|
* { kind: "ranked", rankings, hallucinated } on success (hallucinated counts
|
|
102
115
|
* nodeIds Haiku invented that were filtered out)
|
|
103
|
-
* { kind: "
|
|
116
|
+
* { kind: "error", reason } when the LLM is unavailable or its output
|
|
104
117
|
* cannot be used — callers should return the unranked memory-search
|
|
105
118
|
* results prefixed with "[Ranking unavailable: <reason>]".
|
|
119
|
+
* (Task 897 vocabulary — was `kind: "fallback"`.)
|
|
106
120
|
*/
|
|
107
121
|
export async function rankCandidates(candidates, criteria, accountId) {
|
|
108
122
|
if (candidates.length === 0) {
|
|
@@ -131,26 +145,41 @@ export async function rankCandidates(candidates, criteria, accountId) {
|
|
|
131
145
|
maxTokens: MAX_OUTPUT_TOKENS,
|
|
132
146
|
});
|
|
133
147
|
if (llmResult.kind === "fallback") {
|
|
134
|
-
|
|
135
|
-
return { kind: "
|
|
148
|
+
logError(accountId, `${llmResult.cause}: ${llmResult.reason}`);
|
|
149
|
+
return { kind: "error", reason: llmResult.reason };
|
|
136
150
|
}
|
|
137
151
|
const responseText = llmResult.text;
|
|
152
|
+
const stopReason = llmResult.stopReason ?? "(unknown)";
|
|
153
|
+
const outputTokens = llmResult.outputTokens ?? -1;
|
|
138
154
|
// --- Parse response ---
|
|
139
|
-
|
|
155
|
+
// Task 897 sub-scope G — one-sided fence = truncation fingerprint.
|
|
156
|
+
const fence = extractJson(responseText);
|
|
157
|
+
if (fence.kind === "truncated") {
|
|
158
|
+
logError(accountId, `${fence.reason} stop-reason=${stopReason} output-tokens=${outputTokens} len=${responseText.length}`);
|
|
159
|
+
return { kind: "error", reason: "Haiku response truncated mid-fence (likely max_tokens hit)" };
|
|
160
|
+
}
|
|
161
|
+
const jsonText = fence.json;
|
|
140
162
|
let parsed;
|
|
141
163
|
try {
|
|
142
164
|
parsed = JSON.parse(jsonText);
|
|
143
165
|
}
|
|
144
|
-
catch {
|
|
145
|
-
|
|
146
|
-
|
|
166
|
+
catch (err) {
|
|
167
|
+
// Task 896 clause 5: mirrors llm-classifier — diagnostic payload on parse
|
|
168
|
+
// failure (parser message, raw length, fence-stripped flag, head + tail
|
|
169
|
+
// of the pre-strip text) so malformed JSON names its actual cause.
|
|
170
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
171
|
+
const fenceStripped = jsonText !== responseText;
|
|
172
|
+
logError(accountId, `malformed JSON: parse-error=${JSON.stringify(message)} len=${responseText.length} fence-stripped=${fenceStripped} ` +
|
|
173
|
+
`pre-strip-head=${JSON.stringify(responseText.slice(0, 200))} ` +
|
|
174
|
+
`pre-strip-tail=${JSON.stringify(responseText.slice(-200))}`);
|
|
175
|
+
return { kind: "error", reason: "Haiku returned malformed JSON" };
|
|
147
176
|
}
|
|
148
177
|
if (typeof parsed !== "object" ||
|
|
149
178
|
parsed === null ||
|
|
150
179
|
!("rankings" in parsed) ||
|
|
151
180
|
!Array.isArray(parsed.rankings)) {
|
|
152
|
-
|
|
153
|
-
return { kind: "
|
|
181
|
+
logError(accountId, "schema mismatch (missing rankings array)");
|
|
182
|
+
return { kind: "error", reason: "invalid response schema" };
|
|
154
183
|
}
|
|
155
184
|
const rawRankings = parsed.rankings;
|
|
156
185
|
const validNodeIds = new Set(candidates.map((c) => c.nodeId));
|
|
@@ -176,8 +205,8 @@ export async function rankCandidates(candidates, criteria, accountId) {
|
|
|
176
205
|
rankings.push({ nodeId, rank, reasoning: capReasoning(reasoning) });
|
|
177
206
|
}
|
|
178
207
|
if (rankings.length === 0 && rawRankings.length > 0) {
|
|
179
|
-
|
|
180
|
-
return { kind: "
|
|
208
|
+
logError(accountId, `all ${rawRankings.length} rankings hallucinated or invalid`);
|
|
209
|
+
return { kind: "error", reason: "Haiku returned no valid rankings" };
|
|
181
210
|
}
|
|
182
211
|
// Sort ascending by rank so rank=1 comes first
|
|
183
212
|
rankings.sort((a, b) => a.rank - b.rank);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"llm-ranker.js","sourceRoot":"","sources":["../../src/lib/llm-ranker.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,4CAA4C,CAAC;AAC1E,OAAO,EAAE,WAAW,EAAE,MAAM,yCAAyC,CAAC;AA4BtE,8EAA8E;AAC9E,YAAY;AACZ,8EAA8E;AAE9E,uCAAuC;AACvC,MAAM,iBAAiB,GAAG,IAAI,CAAC;AAC/B,MAAM,mBAAmB,GAAG,GAAG,CAAC;AAEhC,gFAAgF;AAChF,4EAA4E;AAC5E,MAAM,aAAa,GAAG;;;;;;;;;;;+EAWyD,CAAC;AAEhF,8EAA8E;AAC9E,UAAU;AACV,8EAA8E;AAE9E,+DAA+D;AAC/D,SAAS,gBAAgB,CAAC,UAA2B;IACnD,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACjC,8EAA8E;QAC9E,MAAM,KAAK,GAA4B,EAAE,CAAC;QAC1C,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE,CAAC;YAClD,IAAI,CAAC,KAAK,WAAW,IAAI,CAAC,KAAK,WAAW,IAAI,CAAC,KAAK,QAAQ;gBAAE,SAAS;YACvE,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACf,CAAC;QACD,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YAC/C,MAAM,QAAQ,GAA4B,EAAE,CAAC;YAC7C,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE,CAAC;gBAClD,IAAI,CAAC,KAAK,WAAW,IAAI,CAAC,KAAK,WAAW;oBAAE,SAAS;gBACrD,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YAClB,CAAC;YACD,OAAO;gBACL,YAAY,EAAE,CAAC,CAAC,YAAY;gBAC5B,SAAS,EAAE,CAAC,CAAC,SAAS;gBACtB,MAAM,EAAE,CAAC,CAAC,MAAM;gBAChB,UAAU,EAAE,QAAQ;aACrB,CAAC;QACJ,CAAC,CAAC,CAAC;QACH,OAAO;YACL,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,UAAU,EAAE,KAAK;YACjB,OAAO;SACR,CAAC;IACJ,CAAC,CAAC,CAAC;IACH,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;AACxC,CAAC;AAED
|
|
1
|
+
{"version":3,"file":"llm-ranker.js","sourceRoot":"","sources":["../../src/lib/llm-ranker.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,4CAA4C,CAAC;AAC1E,OAAO,EAAE,WAAW,EAAE,MAAM,yCAAyC,CAAC;AA4BtE,8EAA8E;AAC9E,YAAY;AACZ,8EAA8E;AAE9E,uCAAuC;AACvC,MAAM,iBAAiB,GAAG,IAAI,CAAC;AAC/B,MAAM,mBAAmB,GAAG,GAAG,CAAC;AAEhC,gFAAgF;AAChF,4EAA4E;AAC5E,MAAM,aAAa,GAAG;;;;;;;;;;;+EAWyD,CAAC;AAEhF,8EAA8E;AAC9E,UAAU;AACV,8EAA8E;AAE9E,+DAA+D;AAC/D,SAAS,gBAAgB,CAAC,UAA2B;IACnD,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACjC,8EAA8E;QAC9E,MAAM,KAAK,GAA4B,EAAE,CAAC;QAC1C,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE,CAAC;YAClD,IAAI,CAAC,KAAK,WAAW,IAAI,CAAC,KAAK,WAAW,IAAI,CAAC,KAAK,QAAQ;gBAAE,SAAS;YACvE,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACf,CAAC;QACD,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YAC/C,MAAM,QAAQ,GAA4B,EAAE,CAAC;YAC7C,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE,CAAC;gBAClD,IAAI,CAAC,KAAK,WAAW,IAAI,CAAC,KAAK,WAAW;oBAAE,SAAS;gBACrD,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YAClB,CAAC;YACD,OAAO;gBACL,YAAY,EAAE,CAAC,CAAC,YAAY;gBAC5B,SAAS,EAAE,CAAC,CAAC,SAAS;gBACtB,MAAM,EAAE,CAAC,CAAC,MAAM;gBAChB,UAAU,EAAE,QAAQ;aACrB,CAAC;QACJ,CAAC,CAAC,CAAC;QACH,OAAO;YACL,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,UAAU,EAAE,KAAK;YACjB,OAAO;SACR,CAAC;IACJ,CAAC,CAAC,CAAC;IACH,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;AACxC,CAAC;AAED;;;;;GAKG;AACH,SAAS,WAAW,CAAC,GAAW;IAC9B,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;IAC3B,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;IACjE,IAAI,QAAQ;QAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;IAC9D,IAAI,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC9D,OAAO;YACL,IAAI,EAAE,WAAW;YACjB,MAAM,EAAE,kGAAkG;SAC3G,CAAC;IACJ,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;AACvC,CAAC;AAED,qFAAqF;AACrF,SAAS,YAAY,CAAC,SAAiB;IACrC,MAAM,OAAO,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC;IACjC,IAAI,OAAO,CAAC,MAAM,IAAI,mBAAmB;QAAE,OAAO,OAAO,CAAC;IAC1D,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,mBAAmB,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC;AACzD,CAAC;AAED,8EAA8E;AAC9E,SAAS,QAAQ,CAAC,SAAiB,EAAE,MAAc;IACjD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,kBAAkB,SAAS,mBAAmB,MAAM,KAAK,CAAC,CAAC;AAClF,CAAC;AAED,8EAA8E;AAC9E,mBAAmB;AACnB,8EAA8E;AAE9E;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,UAA2B,EAC3B,QAAgB,EAChB,SAAiB;IAEjB,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC;IAC3D,CAAC;IAED,sDAAsD;IACtD,MAAM,gBAAgB,GAAG,gBAAgB,CAAC,UAAU,CAAC,CAAC;IACtD,MAAM,WAAW,GAAG;QAClB,sDAAsD;QACtD,cAAc;QACd,QAAQ;QACR,WAAW;QACX,EAAE;QACF,aAAa;QACb,eAAe;QACf,gBAAgB;QAChB,YAAY;QACZ,EAAE;QACF,6BAA6B;KAC9B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEb,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,kBAAkB,SAAS,+BAA+B,UAAU,CAAC,MAAM,KAAK,CACjF,CAAC;IAEF,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC;QACnC,KAAK,EAAE,WAAW;QAClB,MAAM,EAAE,aAAa;QACrB,WAAW;QACX,SAAS,EAAE,iBAAiB;KAC7B,CAAC,CAAC;IACH,IAAI,SAAS,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;QAClC,QAAQ,CAAC,SAAS,EAAE,GAAG,SAAS,CAAC,KAAK,KAAK,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;QAC/D,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,CAAC,MAAM,EAAE,CAAC;IACrD,CAAC;IACD,MAAM,YAAY,GAAG,SAAS,CAAC,IAAI,CAAC;IACpC,MAAM,UAAU,GAAG,SAAS,CAAC,UAAU,IAAI,WAAW,CAAC;IACvD,MAAM,YAAY,GAAG,SAAS,CAAC,YAAY,IAAI,CAAC,CAAC,CAAC;IAElD,yBAAyB;IACzB,mEAAmE;IACnE,MAAM,KAAK,GAAG,WAAW,CAAC,YAAY,CAAC,CAAC;IACxC,IAAI,KAAK,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;QAC/B,QAAQ,CACN,SAAS,EACT,GAAG,KAAK,CAAC,MAAM,gBAAgB,UAAU,kBAAkB,YAAY,QAAQ,YAAY,CAAC,MAAM,EAAE,CACrG,CAAC;QACF,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,4DAA4D,EAAE,CAAC;IACjG,CAAC;IACD,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC;IAC5B,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAChC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,0EAA0E;QAC1E,wEAAwE;QACxE,mEAAmE;QACnE,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjE,MAAM,aAAa,GAAG,QAAQ,KAAK,YAAY,CAAC;QAChD,QAAQ,CACN,SAAS,EACT,+BAA+B,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,YAAY,CAAC,MAAM,mBAAmB,aAAa,GAAG;YAClH,kBAAkB,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG;YAC/D,kBAAkB,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAC/D,CAAC;QACF,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,+BAA+B,EAAE,CAAC;IACpE,CAAC;IAED,IACE,OAAO,MAAM,KAAK,QAAQ;QAC1B,MAAM,KAAK,IAAI;QACf,CAAC,CAAC,UAAU,IAAI,MAAM,CAAC;QACvB,CAAC,KAAK,CAAC,OAAO,CAAE,MAAgC,CAAC,QAAQ,CAAC,EAC1D,CAAC;QACD,QAAQ,CAAC,SAAS,EAAE,0CAA0C,CAAC,CAAC;QAChE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,yBAAyB,EAAE,CAAC;IAC9D,CAAC;IAED,MAAM,WAAW,GAAI,MAAkC,CAAC,QAAQ,CAAC;IACjE,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;IAC9D,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;IACtC,MAAM,QAAQ,GAAc,EAAE,CAAC;IAC/B,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAC/B,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI;YAAE,SAAS;QACxD,MAAM,GAAG,GAAG,IAA+B,CAAC;QAC5C,MAAM,MAAM,GAAG,OAAO,GAAG,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;QAClE,MAAM,IAAI,GAAG,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;QACzF,MAAM,SAAS,GAAG,OAAO,GAAG,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;QACzE,IAAI,CAAC,MAAM,IAAI,IAAI,KAAK,IAAI;YAAE,SAAS;QACvC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YAC9B,YAAY,IAAI,CAAC,CAAC;YAClB,SAAS;QACX,CAAC;QACD,IAAI,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC;YAAE,SAAS,CAAC,yCAAyC;QAChF,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACxB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,YAAY,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;IACtE,CAAC;IAED,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpD,QAAQ,CAAC,SAAS,EAAE,OAAO,WAAW,CAAC,MAAM,mCAAmC,CAAC,CAAC;QAClF,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,kCAAkC,EAAE,CAAC;IACvE,CAAC;IAED,+CAA+C;IAC/C,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;IAEzC,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,kBAAkB,SAAS,sBAAsB,QAAQ,CAAC,MAAM,kBAAkB,YAAY,KAAK,CACpG,CAAC;IAEF,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC;AACpD,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"conversation-normalisers-source-agnosticism.test.d.ts","sourceRoot":"","sources":["../../../src/tools/__tests__/conversation-normalisers-source-agnosticism.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import { describe, it, expect } from "vitest";
|
|
2
|
+
import { CONVERSATION_SOURCES, isConversationSource, } from "../../lib/conversation-normalisers/types.js";
|
|
3
|
+
import { getNormaliser, whatsappTextNormaliser } from "../../lib/conversation-normalisers/index.js";
|
|
4
|
+
// ---------------------------------------------------------------------------
|
|
5
|
+
// Task 894 — source-agnosticism contract.
|
|
6
|
+
//
|
|
7
|
+
// The point of the refactor is that every normaliser produces the SAME
|
|
8
|
+
// `ParsedLine[]` shape and the SAME NormaliserResult contract regardless of
|
|
9
|
+
// source. Downstream pipeline code (sessionize, classify, memory-ingest)
|
|
10
|
+
// must not branch on `source` — only on the normaliser output.
|
|
11
|
+
//
|
|
12
|
+
// This file pins that contract: a stub normaliser and the live whatsapp-text
|
|
13
|
+
// normaliser are interchangeable from the writer's perspective; `source`
|
|
14
|
+
// flows through as a property only.
|
|
15
|
+
// ---------------------------------------------------------------------------
|
|
16
|
+
describe("ConversationSource enum", () => {
|
|
17
|
+
it("includes the closed Phase-0 set", () => {
|
|
18
|
+
const expected = [
|
|
19
|
+
"whatsapp",
|
|
20
|
+
"telegram",
|
|
21
|
+
"signal",
|
|
22
|
+
"linkedin-messages",
|
|
23
|
+
"zoom-transcript",
|
|
24
|
+
"meeting-minutes",
|
|
25
|
+
"imessage",
|
|
26
|
+
"slack",
|
|
27
|
+
"other",
|
|
28
|
+
];
|
|
29
|
+
expect([...CONVERSATION_SOURCES].sort()).toEqual(expected.sort());
|
|
30
|
+
});
|
|
31
|
+
it("isConversationSource accepts members and rejects everything else", () => {
|
|
32
|
+
expect(isConversationSource("whatsapp")).toBe(true);
|
|
33
|
+
expect(isConversationSource("slack")).toBe(true);
|
|
34
|
+
expect(isConversationSource("Whatsapp")).toBe(false); // case-sensitive enum
|
|
35
|
+
expect(isConversationSource("twitter")).toBe(false);
|
|
36
|
+
expect(isConversationSource(undefined)).toBe(false);
|
|
37
|
+
expect(isConversationSource(123)).toBe(false);
|
|
38
|
+
});
|
|
39
|
+
});
|
|
40
|
+
describe("getNormaliser registry", () => {
|
|
41
|
+
it("returns the whatsapp-text normaliser for source='whatsapp'", () => {
|
|
42
|
+
expect(getNormaliser("whatsapp")).toBe(whatsappTextNormaliser);
|
|
43
|
+
});
|
|
44
|
+
it("loud-fails on an enum member that has no normaliser yet", () => {
|
|
45
|
+
expect(() => getNormaliser("telegram")).toThrow(/no normaliser implemented for source="telegram"/);
|
|
46
|
+
});
|
|
47
|
+
});
|
|
48
|
+
describe("ConversationNormaliser shape parity", () => {
|
|
49
|
+
// A stub normaliser — proves the pipeline path is uniform across sources.
|
|
50
|
+
const stubNormaliser = () => ({
|
|
51
|
+
parsedLines: [
|
|
52
|
+
{ dateSent: "2026-04-30T10:00:00+00:00", body: "Alice: hi", sequenceIndex: 0 },
|
|
53
|
+
{ dateSent: "2026-04-30T10:01:00+00:00", body: "Bob: yo", sequenceIndex: 1 },
|
|
54
|
+
],
|
|
55
|
+
archiveSha256: "0".repeat(64),
|
|
56
|
+
archiveSourceFile: "stub.txt",
|
|
57
|
+
counters: { parsed: 2, systemSkipped: 0, mediaSkipped: 0, parseErrors: 0 },
|
|
58
|
+
});
|
|
59
|
+
it("a stub normaliser and the live whatsapp-text normaliser produce structurally identical NormaliserResult shapes", () => {
|
|
60
|
+
const stubResult = stubNormaliser({ filePath: "/dev/null", accountId: "acct", timezone: "UTC" });
|
|
61
|
+
expect("parsedLines" in stubResult).toBe(true);
|
|
62
|
+
expect("archiveSha256" in stubResult).toBe(true);
|
|
63
|
+
expect("archiveSourceFile" in stubResult).toBe(true);
|
|
64
|
+
expect("counters" in stubResult).toBe(true);
|
|
65
|
+
// Task 897: ParsedLine has dateSent, body, sequenceIndex — no senderName.
|
|
66
|
+
// Bodies carry the source's `Sender:` prefix verbatim.
|
|
67
|
+
const line = stubResult.parsedLines[0];
|
|
68
|
+
expect(typeof line.dateSent).toBe("string");
|
|
69
|
+
expect(typeof line.body).toBe("string");
|
|
70
|
+
expect(typeof line.sequenceIndex).toBe("number");
|
|
71
|
+
});
|
|
72
|
+
});
|
|
73
|
+
//# sourceMappingURL=conversation-normalisers-source-agnosticism.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"conversation-normalisers-source-agnosticism.test.js","sourceRoot":"","sources":["../../../src/tools/__tests__/conversation-normalisers-source-agnosticism.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EACL,oBAAoB,EACpB,oBAAoB,GAIrB,MAAM,6CAA6C,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,sBAAsB,EAAE,MAAM,6CAA6C,CAAC;AAEpG,8EAA8E;AAC9E,0CAA0C;AAC1C,EAAE;AACF,uEAAuE;AACvE,4EAA4E;AAC5E,yEAAyE;AACzE,+DAA+D;AAC/D,EAAE;AACF,6EAA6E;AAC7E,yEAAyE;AACzE,oCAAoC;AACpC,8EAA8E;AAE9E,QAAQ,CAAC,yBAAyB,EAAE,GAAG,EAAE;IACvC,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;QACzC,MAAM,QAAQ,GAAG;YACf,UAAU;YACV,UAAU;YACV,QAAQ;YACR,mBAAmB;YACnB,iBAAiB;YACjB,iBAAiB;YACjB,UAAU;YACV,OAAO;YACP,OAAO;SACR,CAAC;QACF,MAAM,CAAC,CAAC,GAAG,oBAAoB,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IACpE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kEAAkE,EAAE,GAAG,EAAE;QAC1E,MAAM,CAAC,oBAAoB,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpD,MAAM,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjD,MAAM,CAAC,oBAAoB,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,sBAAsB;QAC5E,MAAM,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACpD,MAAM,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACpD,MAAM,CAAC,oBAAoB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,wBAAwB,EAAE,GAAG,EAAE;IACtC,EAAE,CAAC,4DAA4D,EAAE,GAAG,EAAE;QACpE,MAAM,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;IACjE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yDAAyD,EAAE,GAAG,EAAE;QACjE,MAAM,CAAC,GAAG,EAAE,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,iDAAiD,CAAC,CAAC;IACrG,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,qCAAqC,EAAE,GAAG,EAAE;IACnD,0EAA0E;IAC1E,MAAM,cAAc,GAA2B,GAAqB,EAAE,CAAC,CAAC;QACtE,WAAW,EAAE;YACX,EAAE,QAAQ,EAAE,2BAA2B,EAAE,IAAI,EAAE,WAAW,EAAE,aAAa,EAAE,CAAC,EAAE;YAC9E,EAAE,QAAQ,EAAE,2BAA2B,EAAE,IAAI,EAAE,SAAS,EAAE,aAAa,EAAE,CAAC,EAAE;SAC7E;QACD,aAAa,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;QAC7B,iBAAiB,EAAE,UAAU;QAC7B,QAAQ,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE;KAC3E,CAAC,CAAC;IAEH,EAAE,CAAC,gHAAgH,EAAE,GAAG,EAAE;QACxH,MAAM,UAAU,GAAG,cAAc,CAAC,EAAE,QAAQ,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;QACjG,MAAM,CAAC,aAAa,IAAK,UAA+B,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrE,MAAM,CAAC,eAAe,IAAK,UAA+B,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvE,MAAM,CAAC,mBAAmB,IAAK,UAA+B,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC3E,MAAM,CAAC,UAAU,IAAK,UAA+B,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAElE,0EAA0E;QAC1E,uDAAuD;QACvD,MAAM,IAAI,GAAgB,UAA+B,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;QACzE,MAAM,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC5C,MAAM,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACxC,MAAM,CAAC,OAAO,IAAI,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"conversation-normalisers-whatsapp-text.test.d.ts","sourceRoot":"","sources":["../../../src/tools/__tests__/conversation-normalisers-whatsapp-text.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
|
2
|
+
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { tmpdir } from "node:os";
|
|
4
|
+
import { join } from "node:path";
|
|
5
|
+
import { whatsappTextNormaliser } from "../../lib/conversation-normalisers/whatsapp-text.js";
|
|
6
|
+
// ---------------------------------------------------------------------------
|
|
7
|
+
// Task 897 — production-shaped grammar grid for the WhatsApp `_chat.txt`
|
|
8
|
+
// normaliser. The pre-897 grid asserted a body-grammar throw on every line
|
|
9
|
+
// without `Sender: body` shape; that throw codified the Adam Mackay bug
|
|
10
|
+
// (vCards, system messages, forwarded blocks, bare-sender lines all triggered
|
|
11
|
+
// `parse-error reason=no-sender-body-separator`). The scanner is now
|
|
12
|
+
// body-opaque: every byte after the timestamp prefix is preserved verbatim
|
|
13
|
+
// and the chat-mode classifier handles all four shapes downstream.
|
|
14
|
+
// ---------------------------------------------------------------------------
|
|
15
|
+
describe("whatsapp-text normaliser", () => {
|
|
16
|
+
let tmp = "";
|
|
17
|
+
beforeEach(() => {
|
|
18
|
+
tmp = mkdtempSync(join(tmpdir(), "whatsapp-text-test-"));
|
|
19
|
+
});
|
|
20
|
+
afterEach(() => {
|
|
21
|
+
rmSync(tmp, { recursive: true, force: true });
|
|
22
|
+
});
|
|
23
|
+
function writeChat(content) {
|
|
24
|
+
const filePath = join(tmp, "_chat.txt");
|
|
25
|
+
writeFileSync(filePath, content, "utf-8");
|
|
26
|
+
return filePath;
|
|
27
|
+
}
|
|
28
|
+
it("parses a 4-digit-year DD/MM line with body carrying the Sender: prefix verbatim", () => {
|
|
29
|
+
const filePath = writeChat(`[15/04/2024, 09:30:00] Joel: hi adam\n`);
|
|
30
|
+
const result = whatsappTextNormaliser({ filePath, accountId: "acct-1", timezone: "Europe/London" });
|
|
31
|
+
expect(result.parsedLines).toHaveLength(1);
|
|
32
|
+
expect(result.parsedLines[0].body).toBe("Joel: hi adam");
|
|
33
|
+
expect(result.parsedLines[0].dateSent).toMatch(/^2024-04-15T09:30:00\+0[01]:00$/);
|
|
34
|
+
});
|
|
35
|
+
it("accepts both 2-digit and 4-digit years in the same file (Task 845)", () => {
|
|
36
|
+
const filePath = writeChat(`[15/04/24, 09:30:00] Joel: legacy line\n[15/04/2024, 09:31:00] Joel: modern line\n`);
|
|
37
|
+
const result = whatsappTextNormaliser({ filePath, accountId: "acct-1", timezone: "Europe/London" });
|
|
38
|
+
expect(result.parsedLines).toHaveLength(2);
|
|
39
|
+
expect(result.parsedLines[0].dateSent.startsWith("2024-")).toBe(true);
|
|
40
|
+
expect(result.parsedLines[1].dateSent.startsWith("2024-")).toBe(true);
|
|
41
|
+
});
|
|
42
|
+
it("strips leading bidi marks (LRM/RLM) without polluting the next message body (Task 887)", () => {
|
|
43
|
+
const filePath = writeChat(`[15/04/2024, 09:30:00] Joel: line one\n[15/04/2024, 09:31:00] Adam: line two\n`);
|
|
44
|
+
const result = whatsappTextNormaliser({ filePath, accountId: "acct-1", timezone: "Europe/London" });
|
|
45
|
+
expect(result.parsedLines).toHaveLength(2);
|
|
46
|
+
expect(result.parsedLines[1].body).toBe("Adam: line two");
|
|
47
|
+
});
|
|
48
|
+
it("bare-sender — line without `Sender: body` shape parses without throw (Task 897)", () => {
|
|
49
|
+
// Pre-897 this threw `parse-error reason=no-sender-body-separator`.
|
|
50
|
+
const filePath = writeChat(`[15/04/2024, 09:30:00] this is malformed\n`);
|
|
51
|
+
const result = whatsappTextNormaliser({ filePath, accountId: "acct-1", timezone: "Europe/London" });
|
|
52
|
+
expect(result.parsedLines).toHaveLength(1);
|
|
53
|
+
expect(result.parsedLines[0].body).toBe("this is malformed");
|
|
54
|
+
});
|
|
55
|
+
it("vCard — multi-line BEGIN:VCARD block glues onto the preceding match", () => {
|
|
56
|
+
// WhatsApp vCard exports surface as multi-line bodies after a header line.
|
|
57
|
+
const filePath = writeChat(`[15/04/2024, 09:30:00] Joel: contact card\nBEGIN:VCARD\nVERSION:3.0\nFN:Adam Mackay\nTEL:+447700900000\nEND:VCARD\n[15/04/2024, 09:31:00] Joel: next message\n`);
|
|
58
|
+
const result = whatsappTextNormaliser({ filePath, accountId: "acct-1", timezone: "Europe/London" });
|
|
59
|
+
expect(result.parsedLines).toHaveLength(2);
|
|
60
|
+
expect(result.parsedLines[0].body).toContain("BEGIN:VCARD");
|
|
61
|
+
expect(result.parsedLines[0].body).toContain("FN:Adam Mackay");
|
|
62
|
+
expect(result.parsedLines[1].body).toBe("Joel: next message");
|
|
63
|
+
});
|
|
64
|
+
it("system-message — `Joel created group ...` line parses without throw", () => {
|
|
65
|
+
// Pre-897 this branched into LINE_LEVEL_SYSTEM_PATTERNS and counted
|
|
66
|
+
// systemSkipped++; post-897 the scanner doesn't classify, so this is a
|
|
67
|
+
// regular ParsedLine with the system-message text in body.
|
|
68
|
+
const filePath = writeChat(`[15/04/2024, 09:30:00] Joel created group "Adam"\n[15/04/2024, 09:31:00] Joel: hi\n`);
|
|
69
|
+
const result = whatsappTextNormaliser({ filePath, accountId: "acct-1", timezone: "Europe/London" });
|
|
70
|
+
expect(result.parsedLines).toHaveLength(2);
|
|
71
|
+
expect(result.parsedLines[0].body).toBe(`Joel created group "Adam"`);
|
|
72
|
+
});
|
|
73
|
+
it("forwarded-block — leading `Forwarded\\n[ts] ...` block survives the scanner", () => {
|
|
74
|
+
const filePath = writeChat(`[15/04/2024, 09:30:00] Joel: Forwarded\nrelevant context body line one\nrelevant context body line two\n[15/04/2024, 09:31:00] Adam: thanks\n`);
|
|
75
|
+
const result = whatsappTextNormaliser({ filePath, accountId: "acct-1", timezone: "Europe/London" });
|
|
76
|
+
expect(result.parsedLines).toHaveLength(2);
|
|
77
|
+
expect(result.parsedLines[0].body).toContain("relevant context body line one");
|
|
78
|
+
expect(result.parsedLines[1].body).toBe("Adam: thanks");
|
|
79
|
+
});
|
|
80
|
+
it("no-timestamp-archive — file with zero matching prefixes returns empty parsedLines (no throw)", () => {
|
|
81
|
+
// This is the document-routing signal: bin sees parsed=0 and routes to
|
|
82
|
+
// document-ingest instead of failing.
|
|
83
|
+
const filePath = writeChat(`Just a paragraph of text.\nNo timestamps anywhere.\nLooks like a memo, not a chat.\n`);
|
|
84
|
+
const result = whatsappTextNormaliser({ filePath, accountId: "acct-1", timezone: "Europe/London" });
|
|
85
|
+
expect(result.parsedLines).toHaveLength(0);
|
|
86
|
+
expect(result.counters.parsed).toBe(0);
|
|
87
|
+
});
|
|
88
|
+
it("empty-archive — genuinely empty file throws", () => {
|
|
89
|
+
// Empty file is unambiguous: not a chat, not a document. Stay loud.
|
|
90
|
+
const filePath = writeChat("");
|
|
91
|
+
expect(() => whatsappTextNormaliser({ filePath, accountId: "acct-1", timezone: "Europe/London" })).toThrow(/file is empty/);
|
|
92
|
+
});
|
|
93
|
+
it("counts no skips post-897 (counters.systemSkipped/mediaSkipped are structural zeroes)", () => {
|
|
94
|
+
const filePath = writeChat(`[15/04/2024, 09:30:00] Joel: <Media omitted>\n[15/04/2024, 09:31:00] Joel: real msg\n`);
|
|
95
|
+
const result = whatsappTextNormaliser({ filePath, accountId: "acct-1", timezone: "Europe/London" });
|
|
96
|
+
// <Media omitted> is now a regular body — no skipping at the parser layer.
|
|
97
|
+
expect(result.parsedLines).toHaveLength(2);
|
|
98
|
+
expect(result.counters.mediaSkipped).toBe(0);
|
|
99
|
+
expect(result.counters.systemSkipped).toBe(0);
|
|
100
|
+
});
|
|
101
|
+
it("returns a stable archiveSha256 for identical bytes (cleanup discriminator)", () => {
|
|
102
|
+
const fp = writeChat(`[15/04/2024, 09:30:00] Joel: stable\n`);
|
|
103
|
+
const a = whatsappTextNormaliser({ filePath: fp, accountId: "acct-1", timezone: "Europe/London" });
|
|
104
|
+
const b = whatsappTextNormaliser({ filePath: fp, accountId: "acct-1", timezone: "Europe/London" });
|
|
105
|
+
expect(a.archiveSha256).toBe(b.archiveSha256);
|
|
106
|
+
expect(a.archiveSha256).toMatch(/^[a-f0-9]{64}$/);
|
|
107
|
+
});
|
|
108
|
+
});
|
|
109
|
+
//# sourceMappingURL=conversation-normalisers-whatsapp-text.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"conversation-normalisers-whatsapp-text.test.js","sourceRoot":"","sources":["../../../src/tools/__tests__/conversation-normalisers-whatsapp-text.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACrE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAC7D,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,sBAAsB,EAAE,MAAM,qDAAqD,CAAC;AAE7F,8EAA8E;AAC9E,yEAAyE;AACzE,2EAA2E;AAC3E,wEAAwE;AACxE,8EAA8E;AAC9E,qEAAqE;AACrE,2EAA2E;AAC3E,mEAAmE;AACnE,8EAA8E;AAE9E,QAAQ,CAAC,0BAA0B,EAAE,GAAG,EAAE;IACxC,IAAI,GAAG,GAAG,EAAE,CAAC;IACb,UAAU,CAAC,GAAG,EAAE;QACd,GAAG,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,qBAAqB,CAAC,CAAC,CAAC;IAC3D,CAAC,CAAC,CAAC;IACH,SAAS,CAAC,GAAG,EAAE;QACb,MAAM,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;IAEH,SAAS,SAAS,CAAC,OAAe;QAChC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;QACxC,aAAa,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QAC1C,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,EAAE,CAAC,iFAAiF,EAAE,GAAG,EAAE;QACzF,MAAM,QAAQ,GAAG,SAAS,CAAC,wCAAwC,CAAC,CAAC;QACrE,MAAM,MAAM,GAAG,sBAAsB,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,eAAe,EAAE,CAAC,CAAC;QACpG,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC3C,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QACzD,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,iCAAiC,CAAC,CAAC;IACpF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oEAAoE,EAAE,GAAG,EAAE;QAC5E,MAAM,QAAQ,GAAG,SAAS,CACxB,oFAAoF,CACrF,CAAC;QACF,MAAM,MAAM,GAAG,sBAAsB,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,eAAe,EAAE,CAAC,CAAC;QACpG,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC3C,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtE,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACxE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wFAAwF,EAAE,GAAG,EAAE;QAChG,MAAM,QAAQ,GAAG,SAAS,CACxB,iFAAiF,CAClF,CAAC;QACF,MAAM,MAAM,GAAG,sBAAsB,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,eAAe,EAAE,CAAC,CAAC;QACpG,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC3C,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC5D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iFAAiF,EAAE,GAAG,EAAE;QACzF,oEAAoE;QACpE,MAAM,QAAQ,GAAG,SAAS,CAAC,4CAA4C,CAAC,CAAC;QACzE,MAAM,MAAM,GAAG,sBAAsB,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,eAAe,EAAE,CAAC,CAAC;QACpG,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC3C,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;IAC/D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qEAAqE,EAAE,GAAG,EAAE;QAC7E,2EAA2E;QAC3E,MAAM,QAAQ,GAAG,SAAS,CACxB,iKAAiK,CAClK,CAAC;QACF,MAAM,MAAM,GAAG,sBAAsB,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,eAAe,EAAE,CAAC,CAAC;QACpG,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC3C,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;QAC5D,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC;QAC/D,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IAChE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qEAAqE,EAAE,GAAG,EAAE;QAC7E,oEAAoE;QACpE,uEAAuE;QACvE,2DAA2D;QAC3D,MAAM,QAAQ,GAAG,SAAS,CACxB,qFAAqF,CACtF,CAAC;QACF,MAAM,MAAM,GAAG,sBAAsB,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,eAAe,EAAE,CAAC,CAAC;QACpG,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC3C,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;IACvE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8EAA8E,EAAE,GAAG,EAAE;QACtF,MAAM,QAAQ,GAAG,SAAS,CACxB,gJAAgJ,CACjJ,CAAC;QACF,MAAM,MAAM,GAAG,sBAAsB,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,eAAe,EAAE,CAAC,CAAC;QACpG,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC3C,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,gCAAgC,CAAC,CAAC;QAC/E,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IAC1D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8FAA8F,EAAE,GAAG,EAAE;QACtG,uEAAuE;QACvE,sCAAsC;QACtC,MAAM,QAAQ,GAAG,SAAS,CACxB,sFAAsF,CACvF,CAAC;QACF,MAAM,MAAM,GAAG,sBAAsB,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,eAAe,EAAE,CAAC,CAAC;QACpG,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC3C,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6CAA6C,EAAE,GAAG,EAAE;QACrD,oEAAoE;QACpE,MAAM,QAAQ,GAAG,SAAS,CAAC,EAAE,CAAC,CAAC;QAC/B,MAAM,CAAC,GAAG,EAAE,CACV,sBAAsB,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,eAAe,EAAE,CAAC,CACrF,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;IAC7B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sFAAsF,EAAE,GAAG,EAAE;QAC9F,MAAM,QAAQ,GAAG,SAAS,CACxB,uFAAuF,CACxF,CAAC;QACF,MAAM,MAAM,GAAG,sBAAsB,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,eAAe,EAAE,CAAC,CAAC;QACpG,2EAA2E;QAC3E,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC3C,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC7C,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4EAA4E,EAAE,GAAG,EAAE;QACpF,MAAM,EAAE,GAAG,SAAS,CAAC,uCAAuC,CAAC,CAAC;QAC9D,MAAM,CAAC,GAAG,sBAAsB,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,eAAe,EAAE,CAAC,CAAC;QACnG,MAAM,CAAC,GAAG,sBAAsB,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,eAAe,EAAE,CAAC,CAAC;QACnG,MAAM,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC;QAC9C,MAAM,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;IACpD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -49,10 +49,10 @@ describe("memory-ingest — Task 891 ConversationArchive contract", () => {
|
|
|
49
49
|
it("MERGEs :PARTICIPANT_IN edges from operator-confirmed participants to the parent", () => {
|
|
50
50
|
expect(readIngest()).toMatch(/MERGE\s*\(p\)-\[r:PARTICIPANT_IN\]->\(a\)/);
|
|
51
51
|
});
|
|
52
|
-
it("
|
|
52
|
+
it("records lastIngestedMessageHash + lastIngestedMessageAt on the parent (cursor advance, set by the bash orchestrator)", () => {
|
|
53
53
|
const source = readIngest();
|
|
54
|
-
expect(source).
|
|
55
|
-
expect(source).
|
|
54
|
+
expect(source).toMatch(/a\.lastIngestedMessageHash\s*=\s*\$lastIngestedMessageHash/);
|
|
55
|
+
expect(source).toMatch(/a\.lastIngestedMessageAt\s*=\s*\$lastIngestedMessageAt/);
|
|
56
56
|
});
|
|
57
57
|
it("traps the Cypher silent-no-op pattern explicitly", () => {
|
|
58
58
|
expect(readIngest()).toMatch(/silent no-op/);
|
|
@@ -63,4 +63,35 @@ describe("memory-ingest — document mode preserved (Task 740 contract)", () =>
|
|
|
63
63
|
expect(readIngest()).toMatch(/MERGE\s*\(d:KnowledgeDocument\s*\{\s*attachmentId:\s*\$attachmentId\s*\}\s*\)/);
|
|
64
64
|
});
|
|
65
65
|
});
|
|
66
|
+
// ---------------------------------------------------------------------------
|
|
67
|
+
// Task 894 — `source` becomes a closed-enum property on :ConversationArchive
|
|
68
|
+
// and :Section:Conversation. Provenance agent renamed from 'whatsapp-import'
|
|
69
|
+
// (per-source) to 'conversation-archive' (source-agnostic). Source enum is
|
|
70
|
+
// imported from conversation-normalisers/types.ts; memory-ingest validates it
|
|
71
|
+
// at the boundary so silent typos cannot reach the index.
|
|
72
|
+
// ---------------------------------------------------------------------------
|
|
73
|
+
describe("memory-ingest — Task 894 source-agnostic provenance", () => {
|
|
74
|
+
it("declares source: ConversationSource on IngestParams", () => {
|
|
75
|
+
expect(readIngest()).toMatch(/source\??:\s*ConversationSource/);
|
|
76
|
+
});
|
|
77
|
+
it("imports the closed-enum guard from conversation-normalisers/types", () => {
|
|
78
|
+
expect(readIngest()).toMatch(/from\s+["']\.\.\/lib\/conversation-normalisers\/types\.js["']/);
|
|
79
|
+
expect(readIngest()).toMatch(/isConversationSource/);
|
|
80
|
+
});
|
|
81
|
+
it("validates source at the ConversationArchive entry — loud-fail on missing or invalid", () => {
|
|
82
|
+
expect(readIngest()).toMatch(/!isConversationSource\(source\)/);
|
|
83
|
+
expect(readIngest()).toMatch(/source is required for parentLabel='ConversationArchive'/);
|
|
84
|
+
});
|
|
85
|
+
it("renames the chat-mode createdByAgent to 'conversation-archive' (was 'whatsapp-import')", () => {
|
|
86
|
+
const source = readIngest();
|
|
87
|
+
expect(source).toMatch(/CHAT_PROVENANCE_AGENT\s*=\s*["']conversation-archive["']/);
|
|
88
|
+
expect(source).not.toMatch(/CHAT_PROVENANCE_AGENT\s*=\s*["']whatsapp-import["']/);
|
|
89
|
+
});
|
|
90
|
+
it("threads the dynamic `source` parameter into node + edge stamps (no hardcoded 'whatsapp')", () => {
|
|
91
|
+
const source = readIngest();
|
|
92
|
+
// The legacy `CHAT_PROVENANCE_SOURCE` constant is gone; every chat-branch
|
|
93
|
+
// SET clause that stamps `source` reads from the params now.
|
|
94
|
+
expect(source).not.toMatch(/CHAT_PROVENANCE_SOURCE/);
|
|
95
|
+
});
|
|
96
|
+
});
|
|
66
97
|
//# sourceMappingURL=memory-ingest.test.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"memory-ingest.test.js","sourceRoot":"","sources":["../../../src/tools/__tests__/memory-ingest.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAClD,MAAM,SAAS,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;AACtC,MAAM,YAAY,GAAG,OAAO,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;AAC1D,MAAM,WAAW,GAAG,OAAO,CAAC,YAAY,EAAE,4BAA4B,CAAC,CAAC;AAExE,SAAS,UAAU;IACjB,OAAO,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;AAC5C,CAAC;AAED,8EAA8E;AAC9E,+EAA+E;AAC/E,uEAAuE;AACvE,wEAAwE;AACxE,6EAA6E;AAC7E,kEAAkE;AAClE,EAAE;AACF,uEAAuE;AACvE,wEAAwE;AACxE,4EAA4E;AAC5E,8EAA8E;AAC9E,EAAE;AACF,4EAA4E;AAC5E,6DAA6D;AAC7D,EAAE;AACF,oEAAoE;AACpE,wEAAwE;AACxE,gEAAgE;AAChE,8EAA8E;AAE9E,QAAQ,CAAC,uDAAuD,EAAE,GAAG,EAAE;IACrE,EAAE,CAAC,mFAAmF,EAAE,GAAG,EAAE;QAC3F,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,OAAO,CAC1B,gFAAgF,CACjF,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mFAAmF,EAAE,GAAG,EAAE;QAC3F,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC;QAC5B,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,iDAAiD,CAAC,CAAC;QAC1E,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,gCAAgC,CAAC,CAAC;IAC3D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8DAA8D,EAAE,GAAG,EAAE;QACtE,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,OAAO,CAC1B,oGAAoG,CACrG,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oFAAoF,EAAE,GAAG,EAAE;QAC5F,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,OAAO,CAC1B,mEAAmE,CACpE,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wEAAwE,EAAE,GAAG,EAAE;QAChF,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,OAAO,CAC1B,0DAA0D,CAC3D,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iFAAiF,EAAE,GAAG,EAAE;QACzF,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,2CAA2C,CAAC,CAAC;IAC5E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,
|
|
1
|
+
{"version":3,"file":"memory-ingest.test.js","sourceRoot":"","sources":["../../../src/tools/__tests__/memory-ingest.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAClD,MAAM,SAAS,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;AACtC,MAAM,YAAY,GAAG,OAAO,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;AAC1D,MAAM,WAAW,GAAG,OAAO,CAAC,YAAY,EAAE,4BAA4B,CAAC,CAAC;AAExE,SAAS,UAAU;IACjB,OAAO,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;AAC5C,CAAC;AAED,8EAA8E;AAC9E,+EAA+E;AAC/E,uEAAuE;AACvE,wEAAwE;AACxE,6EAA6E;AAC7E,kEAAkE;AAClE,EAAE;AACF,uEAAuE;AACvE,wEAAwE;AACxE,4EAA4E;AAC5E,8EAA8E;AAC9E,EAAE;AACF,4EAA4E;AAC5E,6DAA6D;AAC7D,EAAE;AACF,oEAAoE;AACpE,wEAAwE;AACxE,gEAAgE;AAChE,8EAA8E;AAE9E,QAAQ,CAAC,uDAAuD,EAAE,GAAG,EAAE;IACrE,EAAE,CAAC,mFAAmF,EAAE,GAAG,EAAE;QAC3F,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,OAAO,CAC1B,gFAAgF,CACjF,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mFAAmF,EAAE,GAAG,EAAE;QAC3F,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC;QAC5B,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,iDAAiD,CAAC,CAAC;QAC1E,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,gCAAgC,CAAC,CAAC;IAC3D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8DAA8D,EAAE,GAAG,EAAE;QACtE,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,OAAO,CAC1B,oGAAoG,CACrG,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oFAAoF,EAAE,GAAG,EAAE;QAC5F,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,OAAO,CAC1B,mEAAmE,CACpE,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wEAAwE,EAAE,GAAG,EAAE;QAChF,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,OAAO,CAC1B,0DAA0D,CAC3D,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iFAAiF,EAAE,GAAG,EAAE;QACzF,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,2CAA2C,CAAC,CAAC;IAC5E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sHAAsH,EAAE,GAAG,EAAE;QAC9H,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC;QAC5B,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,4DAA4D,CAAC,CAAC;QACrF,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,wDAAwD,CAAC,CAAC;IACnF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;QAC1D,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,6DAA6D,EAAE,GAAG,EAAE;IAC3E,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;QAC9D,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,OAAO,CAC1B,+EAA+E,CAChF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,8EAA8E;AAC9E,6EAA6E;AAC7E,6EAA6E;AAC7E,2EAA2E;AAC3E,8EAA8E;AAC9E,0DAA0D;AAC1D,8EAA8E;AAE9E,QAAQ,CAAC,qDAAqD,EAAE,GAAG,EAAE;IACnE,EAAE,CAAC,qDAAqD,EAAE,GAAG,EAAE;QAC7D,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,iCAAiC,CAAC,CAAC;IAClE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mEAAmE,EAAE,GAAG,EAAE;QAC3E,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,OAAO,CAC1B,+DAA+D,CAChE,CAAC;QACF,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,sBAAsB,CAAC,CAAC;IACvD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qFAAqF,EAAE,GAAG,EAAE;QAC7F,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,iCAAiC,CAAC,CAAC;QAChE,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,0DAA0D,CAAC,CAAC;IAC3F,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wFAAwF,EAAE,GAAG,EAAE;QAChG,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC;QAC5B,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,0DAA0D,CAAC,CAAC;QACnF,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,qDAAqD,CAAC,CAAC;IACpF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0FAA0F,EAAE,GAAG,EAAE;QAClG,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC;QAC5B,0EAA0E;QAC1E,6DAA6D;QAC7D,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,wBAAwB,CAAC,CAAC;IACvD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -25,12 +25,29 @@ export interface ArchiveWriteResult {
|
|
|
25
25
|
rowIndex: number;
|
|
26
26
|
reason: string;
|
|
27
27
|
}>;
|
|
28
|
+
/** Present only for archiveTypes that run a `finalize` step. */
|
|
29
|
+
nextChainStatus?: "ok" | "failed" | "partial" | "skipped";
|
|
30
|
+
nextChainReason?: string;
|
|
28
31
|
}
|
|
29
32
|
export interface ArchiveHandlerWriteContext {
|
|
30
33
|
params: ArchiveWriteParams;
|
|
31
34
|
batchRows: ArchiveRow[];
|
|
32
35
|
sessionId: string | null;
|
|
33
36
|
}
|
|
37
|
+
export interface ArchiveHandlerFinalizeContext {
|
|
38
|
+
params: ArchiveWriteParams;
|
|
39
|
+
sessionId: string | null;
|
|
40
|
+
/** True when at least one writeBatch returned errors. The handler may
|
|
41
|
+
* still run finalize; it reports `status='partial'` on success. */
|
|
42
|
+
hadBatchErrors: boolean;
|
|
43
|
+
}
|
|
44
|
+
export interface ArchiveHandlerFinalizeResult {
|
|
45
|
+
counters: Record<string, number>;
|
|
46
|
+
/** Handlers report ok/partial/skipped. The outer entry-point sets `failed`
|
|
47
|
+
* on the public result when finalize itself throws. */
|
|
48
|
+
status: "ok" | "partial" | "skipped";
|
|
49
|
+
reason?: string;
|
|
50
|
+
}
|
|
34
51
|
export declare function memoryArchiveWrite(params: ArchiveWriteParams): Promise<ArchiveWriteResult>;
|
|
35
52
|
export declare const PERSON_AND_CONNECTED_EDGE_CYPHER = "\nMATCH (owner) WHERE elementId(owner) = $ownerNodeId\nWITH owner, $rows AS rows\nUNWIND rows AS row\nMERGE (p:Person {linkedinUrl: row.linkedinUrl})\n ON CREATE SET\n p.accountId = $accountId,\n p.source = 'linkedin',\n p.createdByAgent = 'linkedin-import',\n p.createdBySource = 'linkedin-import',\n p.createdBySession = $sessionId,\n p.createdAt = datetime(),\n p.scope = 'admin'\nSET\n p.givenName = row.givenName,\n p.familyName = row.familyName,\n p.currentTitle = row.title\nFOREACH (_ IN CASE WHEN row.email IS NOT NULL AND row.email <> '' THEN [1] ELSE [] END |\n SET p.email = row.email\n)\nMERGE (owner)-[c:CONNECTED_ON_LINKEDIN]->(p)\n ON CREATE SET\n c.connectedOn = date(row.connectedOn),\n c.source = 'linkedin',\n c.createdAt = datetime()\nRETURN count(*) AS processed\n";
|
|
36
53
|
export declare const ORG_AND_WORKS_FOR_CYPHER = "\nUNWIND $rows AS row\nWITH row WHERE row.company IS NOT NULL AND row.company <> ''\nMATCH (p:Person {linkedinUrl: row.linkedinUrl})\nMERGE (o:Organization {accountId: $accountId, name: row.company})\n ON CREATE SET\n o.source = 'linkedin',\n o.createdByAgent = 'linkedin-import',\n o.createdBySource = 'linkedin-import',\n o.createdBySession = $sessionId,\n o.createdAt = datetime(),\n o.scope = 'admin'\nMERGE (p)-[w:WORKS_FOR]->(o)\n ON CREATE SET\n w.title = row.title,\n w.source = 'linkedin',\n w.current = true,\n w.createdAt = datetime()\n ON MATCH SET\n w.title = coalesce(row.title, w.title)\nRETURN count(*) AS processed\n";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"memory-archive-write.d.ts","sourceRoot":"","sources":["../../src/tools/memory-archive-write.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"memory-archive-write.d.ts","sourceRoot":"","sources":["../../src/tools/memory-archive-write.ts"],"names":[],"mappings":"AA8BA,MAAM,WAAW,qBAAqB;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,2EAA2E;IAC3E,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,MAAM,UAAU,GAAG,qBAAqB,CAAC;AAE/C,MAAM,WAAW,kBAAkB;IACjC,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,UAAU,EAAE,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,kBAAkB;IACjC,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,oFAAoF;IACpF,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,MAAM,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACpD,gEAAgE;IAChE,eAAe,CAAC,EAAE,IAAI,GAAG,QAAQ,GAAG,SAAS,GAAG,SAAS,CAAC;IAC1D,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAMD,MAAM,WAAW,0BAA0B;IACzC,MAAM,EAAE,kBAAkB,CAAC;IAC3B,SAAS,EAAE,UAAU,EAAE,CAAC;IACxB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B;AAED,MAAM,WAAW,6BAA6B;IAC5C,MAAM,EAAE,kBAAkB,CAAC;IAC3B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB;wEACoE;IACpE,cAAc,EAAE,OAAO,CAAC;CACzB;AAED,MAAM,WAAW,4BAA4B;IAC3C,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC;4DACwD;IACxD,MAAM,EAAE,IAAI,GAAG,SAAS,GAAG,SAAS,CAAC;IACrC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAqBD,wBAAsB,kBAAkB,CACtC,MAAM,EAAE,kBAAkB,GACzB,OAAO,CAAC,kBAAkB,CAAC,CA2F7B;AAYD,eAAO,MAAM,gCAAgC,m3BA0B5C,CAAC;AAEF,eAAO,MAAM,wBAAwB,8sBAqBpC,CAAC"}
|
|
@@ -1,27 +1,23 @@
|
|
|
1
1
|
import { getSession } from "../lib/neo4j.js";
|
|
2
2
|
/**
|
|
3
|
-
* memory-archive-write — deterministic bulk-archive write surface
|
|
3
|
+
* memory-archive-write — deterministic bulk-archive write surface for flat
|
|
4
|
+
* datasets only (LinkedIn Connections.csv and future CRM-type seeds).
|
|
4
5
|
*
|
|
5
|
-
* Writes a flat dataset (LinkedIn Connections.csv, future CRM-type seeds)
|
|
6
|
-
* into the graph as first-class entity nodes plus their natural edges.
|
|
7
6
|
* Distinct from memory-write (per-node, adjacency-required) and memory-ingest
|
|
8
|
-
* (narrative document →
|
|
9
|
-
*
|
|
7
|
+
* (narrative document → Section / ConversationArchive). Conversation
|
|
8
|
+
* transcripts (WhatsApp, Telegram, Signal, …) flow through the
|
|
9
|
+
* `conversation-archive` skill instead, which writes :ConversationArchive
|
|
10
|
+
* + :Section:Conversation chunks via memory-ingest — see Task 894.
|
|
10
11
|
*
|
|
11
12
|
* Doctrine:
|
|
12
13
|
* - The Cypher body for each archiveType is fixed inside this tool.
|
|
13
14
|
* The agent supplies parsed rows + the archiveType discriminant,
|
|
14
15
|
* never raw Cypher.
|
|
15
|
-
* - 500-row UNWIND batches per executeWrite transaction.
|
|
16
|
-
* batches risk Neo4j Community heap pressure on a Pi 5.
|
|
16
|
+
* - 500-row UNWIND batches per executeWrite transaction.
|
|
17
17
|
* - Per-node provenance stamps (createdByAgent, createdBySession,
|
|
18
18
|
* source, createdAt) come from this server, not the agent.
|
|
19
|
-
* - Owner / participant nodes match by elementId so both :AdminUser
|
|
20
|
-
* and external :Person flow through one path.
|
|
21
19
|
* - Counters are per-handler — each handler picks the keys that
|
|
22
|
-
* describe its archiveType.
|
|
23
|
-
* JSON; nothing programmatic depends on the linkedin-specific
|
|
24
|
-
* legacy field names.
|
|
20
|
+
* describe its archiveType.
|
|
25
21
|
*/
|
|
26
22
|
const BATCH_SIZE = 500;
|
|
27
23
|
const HANDLERS = {
|
|
@@ -68,16 +64,41 @@ export async function memoryArchiveWrite(params) {
|
|
|
68
64
|
process.stderr.write(`[memory-archive-write] batch-failed archiveType=${archiveType} offset=${offset} reason=${reason}\n`);
|
|
69
65
|
}
|
|
70
66
|
}
|
|
67
|
+
let nextChainStatus;
|
|
68
|
+
let nextChainReason;
|
|
69
|
+
if (handler.finalize) {
|
|
70
|
+
try {
|
|
71
|
+
const finalizeResult = await handler.finalize(session, {
|
|
72
|
+
params,
|
|
73
|
+
sessionId,
|
|
74
|
+
hadBatchErrors: errors.length > 0,
|
|
75
|
+
});
|
|
76
|
+
mergeCounters(counters, finalizeResult.counters);
|
|
77
|
+
nextChainStatus = finalizeResult.status;
|
|
78
|
+
nextChainReason = finalizeResult.reason;
|
|
79
|
+
}
|
|
80
|
+
catch (err) {
|
|
81
|
+
const reason = err instanceof Error ? err.message : String(err);
|
|
82
|
+
nextChainStatus = "failed";
|
|
83
|
+
nextChainReason = reason;
|
|
84
|
+
process.stderr.write(`[memory-archive-write] finalize-failed archiveType=${archiveType} reason=${reason}\n`);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
71
87
|
const elapsedMs = Date.now() - startedMs;
|
|
72
88
|
const counterStr = Object.entries(counters)
|
|
73
89
|
.map(([k, v]) => `${k}=${v}`)
|
|
74
90
|
.join(" ");
|
|
75
|
-
|
|
91
|
+
const statusStr = nextChainStatus
|
|
92
|
+
? ` nextChainStatus=${nextChainStatus}${nextChainReason ? ` reason=${nextChainReason}` : ""}`
|
|
93
|
+
: "";
|
|
94
|
+
process.stderr.write(`[memory-archive-write] archiveType=${archiveType} rows=${rows.length} ${counterStr} errors=${errors.length}${statusStr} ms=${elapsedMs}\n`);
|
|
76
95
|
return {
|
|
77
96
|
archiveType,
|
|
78
97
|
processedRows: rows.length,
|
|
79
98
|
counters,
|
|
80
99
|
errors,
|
|
100
|
+
...(nextChainStatus ? { nextChainStatus } : {}),
|
|
101
|
+
...(nextChainReason ? { nextChainReason } : {}),
|
|
81
102
|
};
|
|
82
103
|
}
|
|
83
104
|
finally {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"memory-archive-write.js","sourceRoot":"","sources":["../../src/tools/memory-archive-write.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAE7C
|
|
1
|
+
{"version":3,"file":"memory-archive-write.js","sourceRoot":"","sources":["../../src/tools/memory-archive-write.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAE7C;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,MAAM,UAAU,GAAG,GAAG,CAAC;AA2EvB,MAAM,QAAQ,GAAmC;IAC/C,sBAAsB,EAAE,0BAA0B,EAAE;CACrD,CAAC;AAEF,8EAA8E;AAC9E,cAAc;AACd,8EAA8E;AAE9E,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,MAA0B;IAE1B,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,MAAM,CAAC;IAE7D,IAAI,CAAC,SAAS,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,CAAC;QACpC,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;IAClE,CAAC;IACD,IAAI,CAAC,WAAW,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;IACpE,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9C,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;IAC3E,CAAC;IAED,MAAM,OAAO,GAAG,QAAQ,CAAC,WAAW,CAAC,CAAC;IACtC,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CACb,8CAA8C,WAAW,aAAa,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACzG,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,UAAU,EAAE,CAAC;IAC7B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,MAAM,GAAiC,EAAE,CAAC;IAChD,MAAM,QAAQ,GAA2B,EAAE,CAAC;IAC5C,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,IAAI,CAAC;IAEnD,IAAI,CAAC;QACH,MAAM,OAAO,CAAC,aAAa,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAE7C,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,MAAM,IAAI,UAAU,EAAE,CAAC;YAChE,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,UAAU,CAAC,CAAC;YAC1D,IAAI,CAAC;gBACH,MAAM,aAAa,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,OAAO,EAAE;oBACtD,MAAM;oBACN,SAAS;oBACT,SAAS;iBACV,CAAC,CAAC;gBACH,aAAa,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;YACzC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,MAAM,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAChE,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,yBAAyB,MAAM,KAAK,MAAM,EAAE,EAAE,CAAC,CAAC;gBACxF,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,mDAAmD,WAAW,WAAW,MAAM,WAAW,MAAM,IAAI,CACrG,CAAC;YACJ,CAAC;QACH,CAAC;QAED,IAAI,eAAsD,CAAC;QAC3D,IAAI,eAAmC,CAAC;QACxC,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACrB,IAAI,CAAC;gBACH,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE;oBACrD,MAAM;oBACN,SAAS;oBACT,cAAc,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC;iBAClC,CAAC,CAAC;gBACH,aAAa,CAAC,QAAQ,EAAE,cAAc,CAAC,QAAQ,CAAC,CAAC;gBACjD,eAAe,GAAG,cAAc,CAAC,MAAM,CAAC;gBACxC,eAAe,GAAG,cAAc,CAAC,MAAM,CAAC;YAC1C,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,MAAM,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAChE,eAAe,GAAG,QAAQ,CAAC;gBAC3B,eAAe,GAAG,MAAM,CAAC;gBACzB,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,sDAAsD,WAAW,WAAW,MAAM,IAAI,CACvF,CAAC;YACJ,CAAC;QACH,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;QACzC,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC;aACxC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;aAC5B,IAAI,CAAC,GAAG,CAAC,CAAC;QACb,MAAM,SAAS,GAAG,eAAe;YAC/B,CAAC,CAAC,oBAAoB,eAAe,GAAG,eAAe,CAAC,CAAC,CAAC,WAAW,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE;YAC7F,CAAC,CAAC,EAAE,CAAC;QACP,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,sCAAsC,WAAW,SAAS,IAAI,CAAC,MAAM,IAAI,UAAU,WAAW,MAAM,CAAC,MAAM,GAAG,SAAS,OAAO,SAAS,IAAI,CAC5I,CAAC;QAEF,OAAO;YACL,WAAW;YACX,aAAa,EAAE,IAAI,CAAC,MAAM;YAC1B,QAAQ;YACR,MAAM;YACN,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/C,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAChD,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;AACH,CAAC;AAED,SAAS,aAAa,CAAC,MAA8B,EAAE,MAA8B;IACnF,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC5C,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IACnC,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,iFAAiF;AACjF,8EAA8E;AAE9E,MAAM,CAAC,MAAM,gCAAgC,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;CA0B/C,CAAC;AAEF,MAAM,CAAC,MAAM,wBAAwB,GAAG;;;;;;;;;;;;;;;;;;;;;CAqBvC,CAAC;AAEF,SAAS,0BAA0B;IACjC,OAAO;QACL,KAAK,CAAC,aAAa,CAAC,OAAO,EAAE,MAAM;YACjC,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,GAAG,MAAM,CAAC;YAC1C,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9B;sEAC8D,EAC9D,EAAE,WAAW,EAAE,CAChB,CAAC;YACF,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAChC,MAAM,IAAI,KAAK,CACb,eAAe,WAAW,8FAA8F,CACzH,CAAC;YACJ,CAAC;YACD,MAAM,MAAM,GAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAc,IAAI,EAAE,CAAC;YACnE,MAAM,cAAc,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,CAAkB,CAAC;YAC3E,MAAM,WAAW,GAAG,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;YACjD,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YAC3C,IAAI,CAAC,WAAW,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAC9B,MAAM,IAAI,KAAK,CACb,eAAe,WAAW,gBAAgB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,oCAAoC,CAChG,CAAC;YACJ,CAAC;YACD,IAAI,cAAc,IAAI,cAAc,KAAK,SAAS,EAAE,CAAC;gBACnD,MAAM,IAAI,KAAK,CACb,eAAe,WAAW,uBAAuB,cAAc,SAAS,SAAS,yCAAyC,CAC3H,CAAC;YACJ,CAAC;QACH,CAAC;QAED,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG;YAC3B,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,GAAG,CAAC;YAC7C,MAAM,WAAW,GAAG;gBAClB,WAAW,EAAE,MAAM,CAAC,WAAW;gBAC/B,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,SAAS;gBACT,IAAI,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,oBAAoB,CAAC,CAA0B,CAAC,CAAC;aAC7E,CAAC;YACF,OAAO,MAAM,OAAO,CAAC,YAAY,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;gBAC7C,MAAM,SAAS,GAAG,MAAM,EAAE,CAAC,GAAG,CAAC,gCAAgC,EAAE,WAAW,CAAC,CAAC;gBAC9E,MAAM,cAAc,GAAG,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;gBAE5D,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,GAAG,CAAC,wBAAwB,EAAE,WAAW,CAAC,CAAC;gBACnE,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;gBAEtD,MAAM,cAAc,GAAG,cAAc,CAAC,YAAY,CAAC;gBACnD,MAAM,WAAW,GAAG,WAAW,CAAC,YAAY,CAAC;gBAC7C,MAAM,cAAc,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC;gBAC/C,MAAM,aAAa,GAAG,cAAc,GAAG,cAAc,CAAC;gBACtD,MAAM,YAAY,GAAG,cAAc,CAAC,oBAAoB,GAAG,WAAW,CAAC,oBAAoB,CAAC;gBAE5F,OAAO;oBACL,cAAc;oBACd,aAAa;oBACb,oBAAoB,EAAE,WAAW;oBACjC,mBAAmB,EAAE,CAAC;oBACtB,YAAY;iBACb,CAAC;YACJ,CAAC,CAAC,CAAC;QACL,CAAC;KACF,CAAC;AACJ,CAAC;AAED,SAAS,oBAAoB,CAAC,GAA0B;IACtD,IAAI,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,EAAE,EAAE,CAAC;QAChD,MAAM,IAAI,KAAK,CAAC,6DAA6D,CAAC,CAAC;IACjF,CAAC;IACD,IAAI,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;QACrE,MAAM,IAAI,KAAK,CACb,yCAAyC,GAAG,CAAC,WAAW,mFAAmF,CAC5I,CAAC;IACJ,CAAC;IACD,OAAO;QACL,SAAS,EAAE,CAAC,GAAG,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE;QACvC,UAAU,EAAE,CAAC,GAAG,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE;QACzC,WAAW,EAAE,GAAG,CAAC,WAAW,CAAC,IAAI,EAAE;QACnC,KAAK,EAAE,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI;QAC9D,OAAO,EAAE,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI;QACtE,KAAK,EAAE,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI;QAC9D,WAAW,EAAE,GAAG,CAAC,WAAW;KAC7B,CAAC;AACJ,CAAC"}
|