deepadata-edm-sdk 0.8.8 → 0.8.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -0
- package/dist/assembler.d.ts +33 -4
- package/dist/assembler.d.ts.map +1 -1
- package/dist/assembler.js +146 -14
- package/dist/assembler.js.map +1 -1
- package/dist/conversation.d.ts +62 -0
- package/dist/conversation.d.ts.map +1 -0
- package/dist/conversation.js +96 -0
- package/dist/conversation.js.map +1 -0
- package/dist/extractors/generate-field-block.d.ts +44 -0
- package/dist/extractors/generate-field-block.d.ts.map +1 -0
- package/dist/extractors/generate-field-block.js +253 -0
- package/dist/extractors/generate-field-block.js.map +1 -0
- package/dist/extractors/kimi-extractor.d.ts +2 -2
- package/dist/extractors/kimi-extractor.d.ts.map +1 -1
- package/dist/extractors/kimi-extractor.js +21 -13
- package/dist/extractors/kimi-extractor.js.map +1 -1
- package/dist/extractors/llm-extractor.d.ts +24 -2
- package/dist/extractors/llm-extractor.d.ts.map +1 -1
- package/dist/extractors/llm-extractor.js +95 -56
- package/dist/extractors/llm-extractor.js.map +1 -1
- package/dist/extractors/openai-extractor.d.ts +2 -2
- package/dist/extractors/openai-extractor.d.ts.map +1 -1
- package/dist/extractors/openai-extractor.js +11 -6
- package/dist/extractors/openai-extractor.js.map +1 -1
- package/dist/extractors/output-sanitizer.d.ts +26 -0
- package/dist/extractors/output-sanitizer.d.ts.map +1 -0
- package/dist/extractors/output-sanitizer.js +129 -0
- package/dist/extractors/output-sanitizer.js.map +1 -0
- package/dist/extractors/profile-prompts.d.ts +2 -2
- package/dist/extractors/profile-prompts.d.ts.map +1 -1
- package/dist/extractors/profile-prompts.js +66 -41
- package/dist/extractors/profile-prompts.js.map +1 -1
- package/dist/extractors/stance-guard.d.ts +75 -0
- package/dist/extractors/stance-guard.d.ts.map +1 -0
- package/dist/extractors/stance-guard.js +171 -0
- package/dist/extractors/stance-guard.js.map +1 -0
- package/dist/index.d.ts +12 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +12 -3
- package/dist/index.js.map +1 -1
- package/dist/schema/edm-schema.d.ts +42 -26
- package/dist/schema/edm-schema.d.ts.map +1 -1
- package/dist/schema/edm-schema.js +18 -3
- package/dist/schema/edm-schema.js.map +1 -1
- package/dist/schema/types.d.ts +33 -1
- package/dist/schema/types.d.ts.map +1 -1
- package/dist/schema/types.js +12 -1
- package/dist/schema/types.js.map +1 -1
- package/package.json +4 -1
|
@@ -6,6 +6,27 @@
|
|
|
6
6
|
import Anthropic from "@anthropic-ai/sdk";
|
|
7
7
|
import { LlmExtractedFieldsSchema, LlmEssentialFieldsSchema, LlmExtendedFieldsSchema, } from "../schema/edm-schema.js";
|
|
8
8
|
import { getProfilePrompt, calculateProfileConfidence } from "./profile-prompts.js";
|
|
9
|
+
import { sanitizeLlmOutput } from "./output-sanitizer.js";
|
|
10
|
+
import { frameTranscript } from "../conversation.js";
|
|
11
|
+
/** Default output budget for non-thinking models */
|
|
12
|
+
export const DEFAULT_MAX_TOKENS = 4096;
|
|
13
|
+
/**
|
|
14
|
+
* Default output budget for thinking models, whose reasoning tokens count
|
|
15
|
+
* against max_tokens. 4096 silently truncated extraction JSON on exactly
|
|
16
|
+
* the most emotionally dense inputs (archive-sample run, 2026-06-10).
|
|
17
|
+
*/
|
|
18
|
+
export const THINKING_MODEL_MAX_TOKENS = 16_384;
|
|
19
|
+
/** Models that spend output tokens on reasoning before emitting JSON */
|
|
20
|
+
const THINKING_MODEL_RE = /k2\.[5-9]|k2\.\d{2,}|k3|thinking|reasoner|o[13](-|$)|gpt-5/i;
|
|
21
|
+
export function defaultMaxTokens(model) {
|
|
22
|
+
return THINKING_MODEL_RE.test(model) ? THINKING_MODEL_MAX_TOKENS : DEFAULT_MAX_TOKENS;
|
|
23
|
+
}
|
|
24
|
+
/** Apply conversation framing when the input declares transcript content */
|
|
25
|
+
export function prepareInputText(input) {
|
|
26
|
+
if (!input.text)
|
|
27
|
+
return input.text;
|
|
28
|
+
return input.inputType === "conversation" ? frameTranscript(input.text) : input.text;
|
|
29
|
+
}
|
|
9
30
|
/**
|
|
10
31
|
* System prompt for EDM extraction
|
|
11
32
|
* See CHANGELOG.md for version-specific schema changes
|
|
@@ -22,6 +43,21 @@ Rules
|
|
|
22
43
|
- Output JSON only — no commentary, markdown, or extra text.
|
|
23
44
|
- If motivation is ambiguous, choose the most conservative option (e.g., "curiosity" vs "fear") or return null.
|
|
24
45
|
|
|
46
|
+
SUBJECT ANCHORING (critical)
|
|
47
|
+
- The SUBJECT is the person this artifact will belong to. In a chat transcript the SUBJECT is the USER speaker; ASSISTANT text is context only, never a source of the subject's experience.
|
|
48
|
+
- Score every field relative to the SUBJECT, not the passage. emotional_weight measures what this content meant TO THE SUBJECT — not how vivid, dramatic, or emotionally rich the text itself is.
|
|
49
|
+
- Routine work content (debugging, drafting, planning, logistics) is 0.1–0.4 even when the subject expresses momentary relief or frustration. Reserve 0.7+ for events with personal stakes the subject states or plainly carries. Do not invent somatic or emotional detail the subject never expressed.
|
|
50
|
+
- transformational_pivot is true ONLY if the subject explicitly marks the experience as life-changing. Finishing a task, fixing a bug, or shipping a feature is not a transformational pivot.
|
|
51
|
+
|
|
52
|
+
EXPERIENTIAL STANCE (critical)
|
|
53
|
+
Classify whose experience the emotionally salient material is, in the top-level "experiential_stance" key:
|
|
54
|
+
- "lived" — the subject's own first-hand experience
|
|
55
|
+
- "witnessed" — events the subject personally witnessed or is directly affected by (a loved one's death, a family crisis)
|
|
56
|
+
- "quoted_third_party" — someone else's story the subject quoted, pasted, or retold without being a participant (an article, test data, a stranger's anecdote)
|
|
57
|
+
- "assistant_generated" — fiction, examples, or anecdotes produced by the assistant, not reported by the subject
|
|
58
|
+
- "hypothetical" — imagined scenarios, drafts about invented people, role-play
|
|
59
|
+
If the stance is quoted_third_party, assistant_generated, or hypothetical: do NOT encode that material into wound, identity_thread, expressed_insight, somatic_signature, transformational_pivot, the impulse domain, or high emotional_weight — those fields describe the SUBJECT. Extract only what the content reveals about the subject themselves (e.g. why they engaged with it), or return null fields with low weight.
|
|
60
|
+
|
|
25
61
|
CRITICAL: Enum Field Constraints
|
|
26
62
|
- Many fields below have CANONICAL values — preferred values for cross-artifact comparability.
|
|
27
63
|
- Use canonical values where they fit. If no canonical value accurately represents the content, use the most accurate free-text term. Accuracy takes precedence over canonical conformance.
|
|
@@ -41,14 +77,15 @@ Normalization (very important)
|
|
|
41
77
|
|
|
42
78
|
Schema
|
|
43
79
|
{
|
|
80
|
+
"experiential_stance": "", // STRICT ENUM: lived | witnessed | quoted_third_party | assistant_generated | hypothetical (pick ONE or null)
|
|
44
81
|
"core": {
|
|
45
|
-
"anchor": "",
|
|
46
|
-
"spark": "",
|
|
47
|
-
"wound": "",
|
|
48
|
-
"fuel": "",
|
|
49
|
-
"bridge": "",
|
|
50
|
-
"echo": "",
|
|
51
|
-
"narrative": ""
|
|
82
|
+
"anchor": "", // central theme (e.g., "dad's toolbox", "nana's traditions")
|
|
83
|
+
"spark": "", // what triggered the memory (e.g., "finding the cassette", "first snow")
|
|
84
|
+
"wound": "", // The specific vulnerability, loss, or pain present — NOT generic labels like 'loss' or 'grief' but what exactly was lost or why it hurts. Examples: 'unlived travel dream', 'war silence never spoken', 'father died before I knew him', 'shame of not fitting in'. If no wound is present, use null.
|
|
85
|
+
"fuel": "", // what energized the experience (e.g., "shared laughter", "curiosity")
|
|
86
|
+
"bridge": "", // connection between past and present (e.g., "replaying old tape", "returning to the porch")
|
|
87
|
+
"echo": "", // what still resonates (e.g., "her laugh", "smell of oil", "city lights on water")
|
|
88
|
+
"narrative": "" // 3–5 sentences. REQUIRED: include ALL of the following — ≥1 concrete sensory detail (sight, sound, smell, texture), ≥1 temporal cue that anchors the memory in time, ≥1 symbolic callback that connects past to present. Write from the subject's perspective. Do not compress or summarise — give the memory space to breathe. Faithful and specific. Never generic.
|
|
52
89
|
},
|
|
53
90
|
"constellation": {
|
|
54
91
|
"emotion_primary": "", // CANONICAL: joy | sadness | fear | anger | wonder | peace | tenderness | reverence | pride | anxiety | gratitude | longing | hope | shame | disappointment | relief | frustration (free text accepted if none fits)
|
|
@@ -60,8 +97,8 @@ Schema
|
|
|
60
97
|
"relational_dynamics": "", // CANONICAL: parent_child | grandparent_grandchild | romantic_partnership | couple | sibling_bond | family | friendship | friend | companionship | colleague | mentorship | reunion | community_ritual | grief | self_reflection | professional | therapeutic | service | adversarial (free text accepted if none fits)
|
|
61
98
|
"temporal_context": "", // STRICT ENUM: childhood | early_adulthood | midlife | late_life | recent | future | timeless (pick ONE or null)
|
|
62
99
|
"memory_type": "", // STRICT ENUM: legacy_artifact | fleeting_moment | milestone | reflection | formative_experience (pick ONE or null)
|
|
63
|
-
"media_format": "", // photo
|
|
64
|
-
"narrative_archetype": "", // STRICT ENUM: hero | caregiver | seeker | sage | lover | outlaw | innocent |
|
|
100
|
+
"media_format": "", // STRICT ENUM: photo | video | audio | text | photo_with_story (pick ONE or null)
|
|
101
|
+
"narrative_archetype": "", // STRICT ENUM: hero | caregiver | seeker | sage | lover | outlaw | innocent | magician | creator | everyman | jester | ruler (pick ONE or null; lowercase)
|
|
65
102
|
"symbolic_anchor": "", // concrete object/place/ritual (or null)
|
|
66
103
|
"relational_perspective": "", // STRICT ENUM: self | partner | family | friends | community | humanity (pick ONE or null)
|
|
67
104
|
"temporal_rhythm": "", // STRICT ENUM: still | sudden | rising | fading | recurring | spiraling | dragging | suspended | looping | cyclic (pick ONE or null)
|
|
@@ -72,42 +109,43 @@ Schema
|
|
|
72
109
|
"arc_type": "" // CANONICAL: betrayal | liberation | grief | discovery | resistance | bond | moral_awakening | transformation | reconciliation | reckoning | threshold | exile | gratitude | authenticity (free text accepted if none fits). gratitude = moments of thankfulness, appreciation, acknowledging blessing; authenticity = feeling fully oneself, self-alignment, identity congruence
|
|
73
110
|
},
|
|
74
111
|
"milky_way": {
|
|
75
|
-
"event_type": "",
|
|
76
|
-
"location_context": "",
|
|
77
|
-
"associated_people": [],
|
|
78
|
-
"visibility_context": "",
|
|
79
|
-
"tone_shift": ""
|
|
112
|
+
"event_type": "", // e.g., family gathering, farewell, birthday (or null)
|
|
113
|
+
"location_context": "", // place from text or image (or null)
|
|
114
|
+
"associated_people": [], // names or roles (proper case allowed)
|
|
115
|
+
"visibility_context": "", // STRICT ENUM: private | family_only | shared_publicly (pick ONE or null)
|
|
116
|
+
"tone_shift": "" // e.g., loss to gratitude (or null)
|
|
80
117
|
},
|
|
81
118
|
"gravity": {
|
|
82
|
-
"emotional_weight": 0.0,
|
|
83
|
-
"emotional_density": "",
|
|
84
|
-
"valence": "",
|
|
85
|
-
"viscosity": "",
|
|
86
|
-
"gravity_type": "",
|
|
87
|
-
"tether_type": "",
|
|
88
|
-
"recall_triggers": [],
|
|
89
|
-
"retrieval_keys": [],
|
|
90
|
-
"nearby_themes": [],
|
|
91
|
-
"recurrence_pattern": "",
|
|
92
|
-
"strength_score": 0.0,
|
|
93
|
-
"temporal_decay": "",
|
|
94
|
-
"resilience_markers": [],
|
|
95
|
-
"adaptation_trajectory": ""
|
|
119
|
+
"emotional_weight": 0.0, // 0.0–1.0 (felt intensity IN THE MOMENT). Calibration: 0.9+ life-altering irreversible moments; 0.7-0.9 significant personal events with strong emotional response; 0.4-0.7 meaningful but routine emotional experiences; 0.1-0.4 mild passing emotional content
|
|
120
|
+
"emotional_density": "", // STRICT ENUM: low | medium | high (pick ONE or null)
|
|
121
|
+
"valence": "", // STRICT ENUM: positive | negative | mixed (pick ONE or null)
|
|
122
|
+
"viscosity": "", // STRICT ENUM: low | medium | high | enduring | fluid (pick ONE or null)
|
|
123
|
+
"gravity_type": "", // short phrase (e.g., symbolic resonance)
|
|
124
|
+
"tether_type": "", // CANONICAL: person | symbol | event | place | ritual | object | tradition | identity | self (free text accepted if none fits)
|
|
125
|
+
"recall_triggers": [], // sensory or symbolic cues (lowercase tokens)
|
|
126
|
+
"retrieval_keys": [], // compact hooks (3–6 tokens recommended)
|
|
127
|
+
"nearby_themes": [], // adjacent concepts
|
|
128
|
+
"recurrence_pattern": "", // CANONICAL: cyclical | isolated | chronic | emerging (free text accepted if none fits)
|
|
129
|
+
"strength_score": 0.0, // 0.0–1.0 (how BOUND/STUCK this memory is)
|
|
130
|
+
"temporal_decay": "", // STRICT ENUM: fast | moderate | slow (pick ONE or null)
|
|
131
|
+
"resilience_markers": [], // 1–3 (e.g., acceptance, optimism, continuity)
|
|
132
|
+
"adaptation_trajectory": "" // STRICT ENUM: improving | stable | declining | integrative | emerging (pick ONE or null)
|
|
96
133
|
},
|
|
97
134
|
"impulse": {
|
|
98
|
-
"primary_energy": "",
|
|
99
|
-
"drive_state": "",
|
|
100
|
-
"motivational_orientation": "",
|
|
101
|
-
"temporal_focus": "",
|
|
102
|
-
"directionality": "",
|
|
103
|
-
"social_visibility": "",
|
|
104
|
-
"urgency": "",
|
|
105
|
-
"risk_posture": "",
|
|
106
|
-
"agency_level": "",
|
|
107
|
-
"regulation_state": "",
|
|
108
|
-
"attachment_style": "",
|
|
109
|
-
"coping_style": ""
|
|
135
|
+
"primary_energy": "", // free text: e.g., curiosity, fear, compassion (or null; lowercase)
|
|
136
|
+
"drive_state": "", // STRICT ENUM: explore | approach | avoid | repair | persevere | share | confront | protect | process (pick ONE or null)
|
|
137
|
+
"motivational_orientation": "", // STRICT ENUM: belonging | safety | mastery | meaning | autonomy | authenticity (pick ONE or null)
|
|
138
|
+
"temporal_focus": "", // STRICT ENUM: past | present | future (pick ONE or null)
|
|
139
|
+
"directionality": "", // STRICT ENUM: inward | outward | transcendent (pick ONE or null)
|
|
140
|
+
"social_visibility": "", // STRICT ENUM: private | relational | collective (pick ONE or null)
|
|
141
|
+
"urgency": "", // STRICT ENUM: calm | elevated | pressing | acute (pick ONE or null)
|
|
142
|
+
"risk_posture": "", // STRICT ENUM: cautious | balanced | bold (pick ONE or null)
|
|
143
|
+
"agency_level": "", // STRICT ENUM: low | medium | high (pick ONE or null)
|
|
144
|
+
"regulation_state": "", // STRICT ENUM: regulated | wavering | dysregulated (pick ONE or null)
|
|
145
|
+
"attachment_style": "", // STRICT ENUM: secure | anxious | avoidant | disorganized (pick ONE or null)
|
|
146
|
+
"coping_style": "" // CANONICAL: reframe_meaning | seek_support | distract | ritualize | confront | detach | process (free text accepted if none fits)
|
|
110
147
|
}
|
|
148
|
+
}
|
|
111
149
|
|
|
112
150
|
// Calibration — Impulse (helps apply the fields consistently)
|
|
113
151
|
// - temporal_focus: past (reminisce), present (here-and-now coping), future (plans/longing).
|
|
@@ -140,17 +178,15 @@ Schema
|
|
|
140
178
|
// - coping_style: the STRATEGY for managing emotions (reframe_meaning, seek_support, confront, etc.)
|
|
141
179
|
// - "confront" is valid in BOTH - use drive_state for action impulse, coping_style for emotion management
|
|
142
180
|
//
|
|
143
|
-
// emotion_primary (
|
|
144
|
-
// - emotion_primary
|
|
145
|
-
//
|
|
146
|
-
// - Use higher_order_emotion for complex emotions
|
|
181
|
+
// emotion_primary (two-tier free text) vs higher_order_emotion (free text):
|
|
182
|
+
// - emotion_primary: prefer a canonical value when one fits; otherwise use the most accurate
|
|
183
|
+
// free-text term (accuracy takes precedence over canonical conformance)
|
|
184
|
+
// - Use higher_order_emotion for complex/second-order emotions layered on the primary feeling
|
|
147
185
|
//
|
|
148
|
-
// narrative_arc (
|
|
149
|
-
// - Describes the STORY TRAJECTORY (overcoming, transformation, connection, reflection, closure)
|
|
150
|
-
// - "confrontation"
|
|
151
|
-
// -
|
|
152
|
-
// or "transformation" (fundamental change through conflict)
|
|
153
|
-
// - "confront" belongs in drive_state or coping_style, NOT in narrative_arc
|
|
186
|
+
// narrative_arc (story trajectory):
|
|
187
|
+
// - Describes the STORY TRAJECTORY (overcoming, transformation, connection, reflection, closure, loss, confrontation)
|
|
188
|
+
// - "confrontation" IS a canonical narrative_arc value (a confronting trajectory); free text accepted if none fits
|
|
189
|
+
// - Distinct from the impulse-domain "confront" (drive_state / coping_style), which describes action/coping, not story shape
|
|
154
190
|
//
|
|
155
191
|
// emotional_weight vs strength_score (CRITICAL - different concepts):
|
|
156
192
|
// - emotional_weight: The felt INTENSITY of the experience in the moment.
|
|
@@ -164,7 +200,6 @@ Schema
|
|
|
164
200
|
// SYNONYM CORRECTIONS (use the canonical form):
|
|
165
201
|
// - drive_state: Use "process" NOT "reflect". The enum value is "process" for internal processing/reflection.
|
|
166
202
|
// - narrative_archetype: Use "caregiver" NOT "caretaker". The Jungian archetype label is "caregiver".
|
|
167
|
-
}
|
|
168
203
|
`;
|
|
169
204
|
/**
|
|
170
205
|
* Get the appropriate schema for profile-specific validation
|
|
@@ -188,13 +223,14 @@ function getProfileSchema(profile) {
|
|
|
188
223
|
* @param model - Model to use (default: claude-sonnet-4-20250514)
|
|
189
224
|
* @param profile - EDM profile (default: 'full')
|
|
190
225
|
*/
|
|
191
|
-
export async function extractWithLlm(client, input, model = "claude-sonnet-4-20250514", profile = "full") {
|
|
226
|
+
export async function extractWithLlm(client, input, model = "claude-sonnet-4-20250514", profile = "full", options = {}) {
|
|
192
227
|
const userContent = [];
|
|
193
|
-
// Add text content
|
|
194
|
-
|
|
228
|
+
// Add text content (conversation inputs get source-material framing)
|
|
229
|
+
const text = prepareInputText(input);
|
|
230
|
+
if (text) {
|
|
195
231
|
userContent.push({
|
|
196
232
|
type: "text",
|
|
197
|
-
text
|
|
233
|
+
text,
|
|
198
234
|
});
|
|
199
235
|
}
|
|
200
236
|
// Add image if provided
|
|
@@ -213,7 +249,7 @@ export async function extractWithLlm(client, input, model = "claude-sonnet-4-202
|
|
|
213
249
|
const systemPrompt = profilePrompt || EXTRACTION_SYSTEM_PROMPT;
|
|
214
250
|
const response = await client.messages.create({
|
|
215
251
|
model,
|
|
216
|
-
max_tokens:
|
|
252
|
+
max_tokens: options.maxTokens ?? defaultMaxTokens(model),
|
|
217
253
|
system: systemPrompt,
|
|
218
254
|
messages: [
|
|
219
255
|
{
|
|
@@ -240,6 +276,9 @@ export async function extractWithLlm(client, input, model = "claude-sonnet-4-202
|
|
|
240
276
|
catch {
|
|
241
277
|
throw new Error(`Failed to parse LLM response as JSON: ${textBlock.text.slice(0, 200)}...`);
|
|
242
278
|
}
|
|
279
|
+
// Sanitize before validation: clamp array caps, coerce invalid
|
|
280
|
+
// strict-enum values to null (prefer a null field over a dropped artifact)
|
|
281
|
+
sanitizeLlmOutput(parsed);
|
|
243
282
|
// Validate against profile-specific schema
|
|
244
283
|
const schema = getProfileSchema(profile);
|
|
245
284
|
const result = schema.safeParse(parsed);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"llm-extractor.js","sourceRoot":"","sources":["../../src/extractors/llm-extractor.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,SAAS,MAAM,mBAAmB,CAAC;AAE1C,OAAO,EACL,wBAAwB,EACxB,wBAAwB,EACxB,uBAAuB,GACxB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,gBAAgB,EAAE,0BAA0B,EAAE,MAAM,sBAAsB,CAAC;
|
|
1
|
+
{"version":3,"file":"llm-extractor.js","sourceRoot":"","sources":["../../src/extractors/llm-extractor.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,SAAS,MAAM,mBAAmB,CAAC;AAE1C,OAAO,EACL,wBAAwB,EACxB,wBAAwB,EACxB,uBAAuB,GACxB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,gBAAgB,EAAE,0BAA0B,EAAE,MAAM,sBAAsB,CAAC;AACpF,OAAO,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAC1D,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAUrD,oDAAoD;AACpD,MAAM,CAAC,MAAM,kBAAkB,GAAG,IAAI,CAAC;AACvC;;;;GAIG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAG,MAAM,CAAC;AAEhD,wEAAwE;AACxE,MAAM,iBAAiB,GAAG,6DAA6D,CAAC;AAExF,MAAM,UAAU,gBAAgB,CAAC,KAAa;IAC5C,OAAO,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,yBAAyB,CAAC,CAAC,CAAC,kBAAkB,CAAC;AACxF,CAAC;AAED,4EAA4E;AAC5E,MAAM,UAAU,gBAAgB,CAAC,KAAsB;IACrD,IAAI,CAAC,KAAK,CAAC,IAAI;QAAE,OAAO,KAAK,CAAC,IAAI,CAAC;IACnC,OAAO,KAAK,CAAC,SAAS,KAAK,cAAc,CAAC,CAAC,CAAC,eAAe,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC;AACvF,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyKvC,CAAC;AAwCF;;GAEG;AACH,SAAS,gBAAgB,CAAC,OAAmB;IAC3C,QAAQ,OAAO,EAAE,CAAC;QAChB,KAAK,WAAW;YACd,OAAO,wBAAwB,CAAC;QAClC,KAAK,UAAU;YACb,OAAO,uBAAuB,CAAC;QACjC,KAAK,MAAM,CAAC;QACZ;YACE,OAAO,wBAAwB,CAAC;IACpC,CAAC;AACH,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,MAAiB,EACjB,KAAsB,EACtB,QAAgB,0BAA0B,EAC1C,UAAsB,MAAM,EAC5B,UAAgC,EAAE;IAElC,MAAM,WAAW,GAA4D,EAAE,CAAC;IAEhF,qEAAqE;IACrE,MAAM,IAAI,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;IACrC,IAAI,IAAI,EAAE,CAAC;QACT,WAAW,CAAC,IAAI,CAAC;YACf,IAAI,EAAE,MAAM;YACZ,IAAI;SACL,CAAC,CAAC;IACL,CAAC;IAED,wBAAwB;IACxB,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QAChB,WAAW,CAAC,IAAI,CAAC;YACf,IAAI,EAAE,OAAO;YACb,MAAM,EAAE;gBACN,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE,KAAK,CAAC,cAAc,IAAI,YAAY;gBAChD,IAAI,EAAE,KAAK,CAAC,KAAK;aAClB;SACF,CAAC,CAAC;IACL,CAAC;IAED,+DAA+D;IAC/D,MAAM,aAAa,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAChD,MAAM,YAAY,GAAG,aAAa,IAAI,wBAAwB,CAAC;IAE/D,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;QAC5C,KAAK;QACL,UAAU,EAAE,OAAO,CAAC,SAAS,IAAI,gBAAgB,CAAC,KAAK,CAAC;QACxD,MAAM,EAAE,YAAY;QACpB,QAAQ,EAAE;YACR;gBACE,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE,WAAW;aACrB;SACF;KACF,CAAC,CAAC;IAEH,wBAAwB;IACxB,MAAM,SAAS,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;IAC1E,IAAI,CAAC,SAAS,IAAI,SAAS,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAC/C,CAAC;IAED,8DAA8D;IAC9D,IAAI,QAAQ,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;IACrC,MAAM,UAAU,GAAG,QAAQ,CAAC,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAC7E,IAAI,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACpB,QAAQ,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAClC,CAAC;IACD,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAChC,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,yCAAyC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;IAC9F,CAAC;IAED,+DAA+D;IAC/D,2EAA2E;IAC3E,iBAAiB,CAAC,MAAM,CAAC,CAAC;IAE1B,2CAA2C;IAC3C,MAAM,MAAM,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IACxC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,YAAY,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM;aACrC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC;aAC/C,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,MAAM,IAAI,KAAK,CAAC,0CAA0C,YAAY,EAAE,CAAC,CAAC;IAC5E,CAAC;IAED,qCAAqC;IACrC,MAAM,UAAU,GAAG,0BAA0B,CAC3C,MAAM,CAAC,IAA0D,EACjE,OAAO,CACR,CAAC;IAEF,OAAO;QACL,SAAS,EAAE,MAAM,CAAC,IAAyE;QAC3F,UAAU;QACV,KAAK;QACL,OAAO;QACP,KAAK,EAAE,IAAI;KACZ,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,SAA6B;IAC/D,MAAM,OAAO,GAAG;QACd,IAAI,EAAE,IAAI;QACV,aAAa,EAAE,IAAI;QACnB,SAAS,EAAE,IAAI;QACf,OAAO,EAAE,GAAG;QACZ,OAAO,EAAE,IAAI;KACd,CAAC;IAEF,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,cAAc;IACd,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACjD,MAAM,aAAa,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC;IAC9E,UAAU,IAAI,OAAO,CAAC,IAAI,GAAG,CAAC,aAAa,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;IAEjE,qDAAqD;IACrD,MAAM,qBAAqB,GAAG,CAAC,iBAAiB,EAAE,wBAAwB,CAAC,CAAC;IAC5E,MAAM,qBAAqB,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,MAAM,CACvE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,qBAAqB,CAAC,QAAQ,CAAC,CAAC,CAAC,CAC1C,CAAC;IACF,MAAM,sBAAsB,GAC1B,qBAAqB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;QACjC,MAAM,GAAG,GAAG,SAAS,CAAC,aAAa,CAAC,CAAyC,CAAC,CAAC;QAC/E,OAAO,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,EAAE,CAAC;IACpC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,yBAAyB;IAC1C,UAAU,IAAI,OAAO,CAAC,aAAa,GAAG,CAAC,sBAAsB,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC,CAAC;IAE7G,kBAAkB;IAClB,MAAM,iBAAiB,GAAG,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,MAAM,CACjE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAC1E,CAAC,MAAM,CAAC;IACT,UAAU,IAAI,OAAO,CAAC,SAAS,GAAG,CAAC,iBAAiB,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC;IAEhG,iBAAiB;IACjB,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,MAAM,CAC9D,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAC1E,CAAC,MAAM,CAAC;IACT,UAAU,IAAI,OAAO,CAAC,OAAO,GAAG,CAAC,gBAAgB,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC;IAE3F,iBAAiB;IACjB,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC;IACvG,UAAU,IAAI,OAAO,CAAC,OAAO,GAAG,CAAC,gBAAgB,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC;IAE3F,OAAO,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC;AAC5C,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB,CAAC,MAAe;IACnD,OAAO,IAAI,SAAS,CAAC;QACnB,MAAM,EAAE,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC;KACnD,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -5,11 +5,11 @@
|
|
|
5
5
|
*/
|
|
6
6
|
import OpenAI from "openai";
|
|
7
7
|
import type { ExtractionInput, EdmProfile } from "../schema/types.js";
|
|
8
|
-
import { type LlmExtractionResult } from "./llm-extractor.js";
|
|
8
|
+
import { type ExtractorCallOptions, type LlmExtractionResult } from "./llm-extractor.js";
|
|
9
9
|
/**
|
|
10
10
|
* Extract EDM fields from content using OpenAI
|
|
11
11
|
*/
|
|
12
|
-
export declare function extractWithOpenAI(client: OpenAI, input: ExtractionInput, model?: string, temperature?: number, profile?: EdmProfile): Promise<LlmExtractionResult>;
|
|
12
|
+
export declare function extractWithOpenAI(client: OpenAI, input: ExtractionInput, model?: string, temperature?: number, profile?: EdmProfile, options?: ExtractorCallOptions): Promise<LlmExtractionResult>;
|
|
13
13
|
/**
|
|
14
14
|
* Create an OpenAI client
|
|
15
15
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"openai-extractor.d.ts","sourceRoot":"","sources":["../../src/extractors/openai-extractor.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,OAAO,KAAK,EAAsB,eAAe,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAE1F,OAAO,
|
|
1
|
+
{"version":3,"file":"openai-extractor.d.ts","sourceRoot":"","sources":["../../src/extractors/openai-extractor.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,OAAO,KAAK,EAAsB,eAAe,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAE1F,OAAO,EAIL,KAAK,oBAAoB,EACzB,KAAK,mBAAmB,EACzB,MAAM,oBAAoB,CAAC;AAI5B;;GAEG;AACH,wBAAsB,iBAAiB,CACrC,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,eAAe,EACtB,KAAK,GAAE,MAAsB,EAC7B,WAAW,GAAE,MAAU,EACvB,OAAO,GAAE,UAAmB,EAC5B,OAAO,GAAE,oBAAyB,GACjC,OAAO,CAAC,mBAAmB,CAAC,CAyF9B;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,CAQ1D"}
|
|
@@ -5,18 +5,20 @@
|
|
|
5
5
|
*/
|
|
6
6
|
import OpenAI from "openai";
|
|
7
7
|
import { LlmExtractedFieldsSchema } from "../schema/edm-schema.js";
|
|
8
|
-
import { EXTRACTION_SYSTEM_PROMPT, } from "./llm-extractor.js";
|
|
8
|
+
import { EXTRACTION_SYSTEM_PROMPT, defaultMaxTokens, prepareInputText, } from "./llm-extractor.js";
|
|
9
9
|
import { getProfilePrompt, calculateProfileConfidence } from "./profile-prompts.js";
|
|
10
|
+
import { sanitizeLlmOutput } from "./output-sanitizer.js";
|
|
10
11
|
/**
|
|
11
12
|
* Extract EDM fields from content using OpenAI
|
|
12
13
|
*/
|
|
13
|
-
export async function extractWithOpenAI(client, input, model = "gpt-4o-mini", temperature = 0, profile = "full") {
|
|
14
|
+
export async function extractWithOpenAI(client, input, model = "gpt-4o-mini", temperature = 0, profile = "full", options = {}) {
|
|
14
15
|
const userContent = [];
|
|
15
|
-
// Add text content
|
|
16
|
-
|
|
16
|
+
// Add text content (conversation inputs get source-material framing)
|
|
17
|
+
const inputText = prepareInputText(input);
|
|
18
|
+
if (inputText) {
|
|
17
19
|
userContent.push({
|
|
18
20
|
type: "text",
|
|
19
|
-
text:
|
|
21
|
+
text: inputText,
|
|
20
22
|
});
|
|
21
23
|
}
|
|
22
24
|
// Add image if provided (OpenAI uses image_url with data URI)
|
|
@@ -34,7 +36,7 @@ export async function extractWithOpenAI(client, input, model = "gpt-4o-mini", te
|
|
|
34
36
|
const systemPrompt = profilePrompt || EXTRACTION_SYSTEM_PROMPT;
|
|
35
37
|
const response = await client.chat.completions.create({
|
|
36
38
|
model,
|
|
37
|
-
max_tokens:
|
|
39
|
+
max_tokens: options.maxTokens ?? defaultMaxTokens(model),
|
|
38
40
|
response_format: { type: "json_object" },
|
|
39
41
|
temperature,
|
|
40
42
|
messages: [
|
|
@@ -66,6 +68,9 @@ export async function extractWithOpenAI(client, input, model = "gpt-4o-mini", te
|
|
|
66
68
|
catch {
|
|
67
69
|
throw new Error(`Failed to parse OpenAI response as JSON: ${text.slice(0, 200)}...`);
|
|
68
70
|
}
|
|
71
|
+
// Sanitize before validation: clamp array caps, coerce invalid
|
|
72
|
+
// strict-enum values to null (prefer a null field over a dropped artifact)
|
|
73
|
+
sanitizeLlmOutput(parsed);
|
|
69
74
|
// Validate against schema
|
|
70
75
|
const result = LlmExtractedFieldsSchema.safeParse(parsed);
|
|
71
76
|
if (!result.success) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"openai-extractor.js","sourceRoot":"","sources":["../../src/extractors/openai-extractor.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,MAAM,MAAM,QAAQ,CAAC;AAG5B,OAAO,EAAE,wBAAwB,EAAE,MAAM,yBAAyB,CAAC;AACnE,OAAO,EACL,wBAAwB,
|
|
1
|
+
{"version":3,"file":"openai-extractor.js","sourceRoot":"","sources":["../../src/extractors/openai-extractor.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,MAAM,MAAM,QAAQ,CAAC;AAG5B,OAAO,EAAE,wBAAwB,EAAE,MAAM,yBAAyB,CAAC;AACnE,OAAO,EACL,wBAAwB,EACxB,gBAAgB,EAChB,gBAAgB,GAGjB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,gBAAgB,EAAE,0BAA0B,EAAE,MAAM,sBAAsB,CAAC;AACpF,OAAO,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAE1D;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,MAAc,EACd,KAAsB,EACtB,QAAgB,aAAa,EAC7B,cAAsB,CAAC,EACvB,UAAsB,MAAM,EAC5B,UAAgC,EAAE;IAElC,MAAM,WAAW,GAAgC,EAAE,CAAC;IAEpD,qEAAqE;IACrE,MAAM,SAAS,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;IAC1C,IAAI,SAAS,EAAE,CAAC;QACd,WAAW,CAAC,IAAI,CAAC;YACf,IAAI,EAAE,MAAM;YACZ,IAAI,EAAE,SAAS;SAChB,CAAC,CAAC;IACL,CAAC;IAED,8DAA8D;IAC9D,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QAChB,MAAM,SAAS,GAAG,KAAK,CAAC,cAAc,IAAI,YAAY,CAAC;QACvD,WAAW,CAAC,IAAI,CAAC;YACf,IAAI,EAAE,WAAW;YACjB,SAAS,EAAE;gBACT,GAAG,EAAE,QAAQ,SAAS,WAAW,KAAK,CAAC,KAAK,EAAE;aAC/C;SACF,CAAC,CAAC;IACL,CAAC;IAED,+DAA+D;IAC/D,MAAM,aAAa,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAChD,MAAM,YAAY,GAAG,aAAa,IAAI,wBAAwB,CAAC;IAE/D,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC;QACpD,KAAK;QACL,UAAU,EAAE,OAAO,CAAC,SAAS,IAAI,gBAAgB,CAAC,KAAK,CAAC;QACxD,eAAe,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE;QACxC,WAAW;QACX,QAAQ,EAAE;YACR;gBACE,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,YAAY;aACtB;YACD;gBACE,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE,WAAW;aACrB;SACF;KACF,CAAC,CAAC;IAEH,wBAAwB;IACxB,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC;IACnD,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;IAClD,CAAC;IAED,8DAA8D;IAC9D,IAAI,QAAQ,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC3B,MAAM,UAAU,GAAG,QAAQ,CAAC,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAC7E,IAAI,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACpB,QAAQ,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAClC,CAAC;IACD,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAChC,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,4CAA4C,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;IACvF,CAAC;IAED,+DAA+D;IAC/D,2EAA2E;IAC3E,iBAAiB,CAAC,MAAM,CAAC,CAAC;IAE1B,0BAA0B;IAC1B,MAAM,MAAM,GAAG,wBAAwB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAC1D,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,YAAY,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM;aACrC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC;aAC/C,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,MAAM,IAAI,KAAK,CAAC,6CAA6C,YAAY,EAAE,CAAC,CAAC;IAC/E,CAAC;IAED,qCAAqC;IACrC,MAAM,UAAU,GAAG,0BAA0B,CAC3C,MAAM,CAAC,IAA0D,EACjE,OAAO,CACR,CAAC;IAEF,OAAO;QACL,SAAS,EAAE,MAAM,CAAC,IAAI;QACtB,UAAU;QACV,KAAK;QACL,OAAO;QACP,KAAK,EAAE,IAAI;KACZ,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,MAAe;IAChD,MAAM,GAAG,GAAG,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;IACpD,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,MAAM,IAAI,KAAK,CACb,8FAA8F,CAC/F,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,MAAM,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;AACrC,CAAC"}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LLM output sanitation
|
|
3
|
+
*
|
|
4
|
+
* Runs between JSON.parse and zod validation in every extractor. Two
|
|
5
|
+
* failure modes from the 2026-06-10 archive-sample run motivate it:
|
|
6
|
+
*
|
|
7
|
+
* - emotion_subtone with >4 items on emotionally rich threads (model
|
|
8
|
+
* enthusiasm exceeds the spec cap) — hard-failed validation/sealing.
|
|
9
|
+
* - Free-text values in STRICT ENUM fields (e.g. narrative_archetype:
|
|
10
|
+
* "observer") — hard-failed validation/sealing.
|
|
11
|
+
*
|
|
12
|
+
* Policy: prefer a null/clamped field over a dropped artifact. Strict-enum
|
|
13
|
+
* coercion to null matches the prompts' own instruction ("pick ONE or
|
|
14
|
+
* null"); array caps match the JSON schema. Free-text-tolerant fields
|
|
15
|
+
* (emotion_primary, narrative_arc, arc_type, relational_dynamics,
|
|
16
|
+
* tether_type, recurrence_pattern, coping_style) are NOT touched.
|
|
17
|
+
*/
|
|
18
|
+
export interface SanitationNote {
|
|
19
|
+
path: string;
|
|
20
|
+
action: "coerced_to_null" | "truncated" | "clamped";
|
|
21
|
+
original: unknown;
|
|
22
|
+
}
|
|
23
|
+
export declare function sanitizeLlmOutput(parsed: unknown): SanitationNote[];
|
|
24
|
+
/** Render sanitation notes for telemetry.extraction_notes */
|
|
25
|
+
export declare function formatSanitationNotes(notes: SanitationNote[]): string | null;
|
|
26
|
+
//# sourceMappingURL=output-sanitizer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"output-sanitizer.d.ts","sourceRoot":"","sources":["../../src/extractors/output-sanitizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AA+CH,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,iBAAiB,GAAG,WAAW,GAAG,SAAS,CAAC;IACpD,QAAQ,EAAE,OAAO,CAAC;CACnB;AAQD,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,OAAO,GAAG,cAAc,EAAE,CAwDnE;AAED,6DAA6D;AAC7D,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,cAAc,EAAE,GAAG,MAAM,GAAG,IAAI,CAM5E"}
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LLM output sanitation
|
|
3
|
+
*
|
|
4
|
+
* Runs between JSON.parse and zod validation in every extractor. Two
|
|
5
|
+
* failure modes from the 2026-06-10 archive-sample run motivate it:
|
|
6
|
+
*
|
|
7
|
+
* - emotion_subtone with >4 items on emotionally rich threads (model
|
|
8
|
+
* enthusiasm exceeds the spec cap) — hard-failed validation/sealing.
|
|
9
|
+
* - Free-text values in STRICT ENUM fields (e.g. narrative_archetype:
|
|
10
|
+
* "observer") — hard-failed validation/sealing.
|
|
11
|
+
*
|
|
12
|
+
* Policy: prefer a null/clamped field over a dropped artifact. Strict-enum
|
|
13
|
+
* coercion to null matches the prompts' own instruction ("pick ONE or
|
|
14
|
+
* null"); array caps match the JSON schema. Free-text-tolerant fields
|
|
15
|
+
* (emotion_primary, narrative_arc, arc_type, relational_dynamics,
|
|
16
|
+
* tether_type, recurrence_pattern, coping_style) are NOT touched.
|
|
17
|
+
*/
|
|
18
|
+
/** Fields documented as STRICT ENUM — invalid values coerce to null */
|
|
19
|
+
const STRICT_ENUMS = {
|
|
20
|
+
constellation: {
|
|
21
|
+
temporal_context: ["childhood", "early_adulthood", "midlife", "late_life", "recent", "future", "timeless"],
|
|
22
|
+
memory_type: ["legacy_artifact", "fleeting_moment", "milestone", "reflection", "formative_experience"],
|
|
23
|
+
media_format: ["photo", "video", "audio", "text", "photo_with_story"],
|
|
24
|
+
narrative_archetype: ["hero", "caregiver", "seeker", "sage", "lover", "outlaw", "innocent", "magician", "creator", "everyman", "jester", "ruler"],
|
|
25
|
+
relational_perspective: ["self", "partner", "family", "friends", "community", "humanity"],
|
|
26
|
+
temporal_rhythm: ["still", "sudden", "rising", "fading", "recurring", "spiraling", "dragging", "suspended", "looping", "cyclic"],
|
|
27
|
+
},
|
|
28
|
+
milky_way: {
|
|
29
|
+
visibility_context: ["private", "family_only", "shared_publicly"],
|
|
30
|
+
},
|
|
31
|
+
gravity: {
|
|
32
|
+
emotional_density: ["low", "medium", "high"],
|
|
33
|
+
valence: ["positive", "negative", "mixed"],
|
|
34
|
+
viscosity: ["low", "medium", "high", "enduring", "fluid"],
|
|
35
|
+
temporal_decay: ["fast", "moderate", "slow"],
|
|
36
|
+
adaptation_trajectory: ["improving", "stable", "declining", "integrative", "emerging"],
|
|
37
|
+
},
|
|
38
|
+
impulse: {
|
|
39
|
+
drive_state: ["explore", "approach", "avoid", "repair", "persevere", "share", "confront", "protect", "process"],
|
|
40
|
+
motivational_orientation: ["belonging", "safety", "mastery", "meaning", "autonomy", "authenticity"],
|
|
41
|
+
temporal_focus: ["past", "present", "future"],
|
|
42
|
+
directionality: ["inward", "outward", "transcendent"],
|
|
43
|
+
social_visibility: ["private", "relational", "collective"],
|
|
44
|
+
urgency: ["calm", "elevated", "pressing", "acute"],
|
|
45
|
+
risk_posture: ["cautious", "balanced", "bold"],
|
|
46
|
+
agency_level: ["low", "medium", "high"],
|
|
47
|
+
regulation_state: ["regulated", "wavering", "dysregulated"],
|
|
48
|
+
attachment_style: ["secure", "anxious", "avoidant", "disorganized"],
|
|
49
|
+
},
|
|
50
|
+
};
|
|
51
|
+
/** Array fields with spec caps — overflow is truncated, not rejected */
|
|
52
|
+
const ARRAY_CAPS = {
|
|
53
|
+
constellation: { emotion_subtone: 4 },
|
|
54
|
+
gravity: { resilience_markers: 3 },
|
|
55
|
+
};
|
|
56
|
+
/** Numeric fields clamped to [0, 1] */
|
|
57
|
+
const UNIT_INTERVAL_FIELDS = {
|
|
58
|
+
gravity: ["emotional_weight", "strength_score"],
|
|
59
|
+
};
|
|
60
|
+
/**
|
|
61
|
+
* Sanitize parsed LLM output in place. Returns notes describing every
|
|
62
|
+
* change so callers can surface them in telemetry.
|
|
63
|
+
*/
|
|
64
|
+
const STANCE_VALUES = ["lived", "witnessed", "quoted_third_party", "assistant_generated", "hypothetical"];
|
|
65
|
+
export function sanitizeLlmOutput(parsed) {
|
|
66
|
+
const notes = [];
|
|
67
|
+
if (typeof parsed !== "object" || parsed === null)
|
|
68
|
+
return notes;
|
|
69
|
+
const root = parsed;
|
|
70
|
+
// Normalize the top-level experiential_stance before validation:
|
|
71
|
+
// "quoted third-party" -> "quoted_third_party"; anything unrecognized -> null.
|
|
72
|
+
const stance = root["experiential_stance"];
|
|
73
|
+
if (typeof stance === "string") {
|
|
74
|
+
const normalized = stance.trim().toLowerCase().replace(/[\s-]+/g, "_");
|
|
75
|
+
if (normalized !== stance) {
|
|
76
|
+
root["experiential_stance"] = STANCE_VALUES.includes(normalized) ? normalized : null;
|
|
77
|
+
notes.push({ path: "experiential_stance", action: STANCE_VALUES.includes(normalized) ? "clamped" : "coerced_to_null", original: stance });
|
|
78
|
+
}
|
|
79
|
+
else if (!STANCE_VALUES.includes(stance)) {
|
|
80
|
+
root["experiential_stance"] = null;
|
|
81
|
+
notes.push({ path: "experiential_stance", action: "coerced_to_null", original: stance });
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
for (const [domain, fields] of Object.entries(STRICT_ENUMS)) {
|
|
85
|
+
const d = root[domain];
|
|
86
|
+
if (!d || typeof d !== "object")
|
|
87
|
+
continue;
|
|
88
|
+
for (const [field, allowed] of Object.entries(fields)) {
|
|
89
|
+
const v = d[field];
|
|
90
|
+
if (typeof v === "string" && v !== "" && !allowed.includes(v)) {
|
|
91
|
+
notes.push({ path: `${domain}.${field}`, action: "coerced_to_null", original: v });
|
|
92
|
+
d[field] = null;
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
for (const [domain, fields] of Object.entries(ARRAY_CAPS)) {
|
|
97
|
+
const d = root[domain];
|
|
98
|
+
if (!d || typeof d !== "object")
|
|
99
|
+
continue;
|
|
100
|
+
for (const [field, cap] of Object.entries(fields)) {
|
|
101
|
+
const v = d[field];
|
|
102
|
+
if (Array.isArray(v) && v.length > cap) {
|
|
103
|
+
notes.push({ path: `${domain}.${field}`, action: "truncated", original: v.length });
|
|
104
|
+
d[field] = v.slice(0, cap);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
for (const [domain, fields] of Object.entries(UNIT_INTERVAL_FIELDS)) {
|
|
109
|
+
const d = root[domain];
|
|
110
|
+
if (!d || typeof d !== "object")
|
|
111
|
+
continue;
|
|
112
|
+
for (const field of fields) {
|
|
113
|
+
const v = d[field];
|
|
114
|
+
if (typeof v === "number" && (v < 0 || v > 1)) {
|
|
115
|
+
notes.push({ path: `${domain}.${field}`, action: "clamped", original: v });
|
|
116
|
+
d[field] = Math.min(1, Math.max(0, v));
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
return notes;
|
|
121
|
+
}
|
|
122
|
+
/** Render sanitation notes for telemetry.extraction_notes */
|
|
123
|
+
export function formatSanitationNotes(notes) {
|
|
124
|
+
if (notes.length === 0)
|
|
125
|
+
return null;
|
|
126
|
+
return ("sanitized: " +
|
|
127
|
+
notes.map((n) => `${n.path} ${n.action}${n.action === "coerced_to_null" ? ` (was ${JSON.stringify(n.original)})` : ""}`).join("; "));
|
|
128
|
+
}
|
|
129
|
+
//# sourceMappingURL=output-sanitizer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"output-sanitizer.js","sourceRoot":"","sources":["../../src/extractors/output-sanitizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,uEAAuE;AACvE,MAAM,YAAY,GAAsD;IACtE,aAAa,EAAE;QACb,gBAAgB,EAAE,CAAC,WAAW,EAAE,iBAAiB,EAAE,SAAS,EAAE,WAAW,EAAE,QAAQ,EAAE,QAAQ,EAAE,UAAU,CAAC;QAC1G,WAAW,EAAE,CAAC,iBAAiB,EAAE,iBAAiB,EAAE,WAAW,EAAE,YAAY,EAAE,sBAAsB,CAAC;QACtG,YAAY,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,kBAAkB,CAAC;QACrE,mBAAmB,EAAE,CAAC,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,SAAS,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,CAAC;QACjJ,sBAAsB,EAAE,CAAC,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,WAAW,EAAE,UAAU,CAAC;QACzF,eAAe,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,WAAW,EAAE,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,QAAQ,CAAC;KACjI;IACD,SAAS,EAAE;QACT,kBAAkB,EAAE,CAAC,SAAS,EAAE,aAAa,EAAE,iBAAiB,CAAC;KAClE;IACD,OAAO,EAAE;QACP,iBAAiB,EAAE,CAAC,KAAK,EAAE,QAAQ,EAAE,MAAM,CAAC;QAC5C,OAAO,EAAE,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,CAAC;QAC1C,SAAS,EAAE,CAAC,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,CAAC;QACzD,cAAc,EAAE,CAAC,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC;QAC5C,qBAAqB,EAAE,CAAC,WAAW,EAAE,QAAQ,EAAE,WAAW,EAAE,aAAa,EAAE,UAAU,CAAC;KACvF;IACD,OAAO,EAAE;QACP,WAAW,EAAE,CAAC,SAAS,EAAE,UAAU,EAAE,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,SAAS,CAAC;QAC/G,wBAAwB,EAAE,CAAC,WAAW,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,EAAE,UAAU,EAAE,cAAc,CAAC;QACnG,cAAc,EAAE,CAAC,MAAM,EAAE,SAAS,EAAE,QAAQ,CAAC;QAC7C,cAAc,EAAE,CAAC,QAAQ,EAAE,SAAS,EAAE,cAAc,CAAC;QACrD,iBAAiB,EAAE,CAAC,SAAS,EAAE,YAAY,EAAE,YAAY,CAAC;QAC1D,OAAO,EAAE,CAAC,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,OAAO,CAAC;QAClD,YAAY,EAAE,CAAC,UAAU,EAAE,UAAU,EAAE,MAAM,CAAC;QAC9C,YAAY,EAAE,CAAC,KAAK,EAAE,QAAQ,EAAE,MAAM,CAAC;QACvC,gBAAgB,EAAE,CAAC,WAAW,EAAE,UAAU,EAAE,cAAc,CAAC;QAC3D,gBAAgB,EAAE,CAAC,QAAQ,EAAE,SAAS,EAAE,UAAU,EAAE,cAAc,CAAC;KACpE;CACF,CAAC;AAEF,wEAAwE;AACxE,MAAM,UAAU,GAA2C;IACzD,aAAa,EAAE,EAAE,eAAe,EAAE,CAAC,EAAE;IACrC,OAAO,EAAE,EAAE,kBAAkB,EAAE,CAAC,EAAE;CACnC,CAAC;AAEF,uCAAuC;AACvC,MAAM,oBAAoB,GAAsC;IAC9D,OAAO,EAAE,CAAC,kBAAkB,EAAE,gBAAgB,CAAC;CAChD,CAAC;AAQF;;;GAGG;AACH,MAAM,aAAa,GAAG,CAAC,OAAO,EAAE,WAAW,EAAE,oBAAoB,EAAE,qBAAqB,EAAE,cAAc,CAAC,CAAC;AAE1G,MAAM,UAAU,iBAAiB,CAAC,MAAe;IAC/C,MAAM,KAAK,GAAqB,EAAE,CAAC;IACnC,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IAChE,MAAM,IAAI,GAAG,MAAiC,CAAC;IAE/C,iEAAiE;IACjE,+EAA+E;IAC/E,MAAM,MAAM,GAAG,IAAI,CAAC,qBAAqB,CAAC,CAAC;IAC3C,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC/B,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;QACvE,IAAI,UAAU,KAAK,MAAM,EAAE,CAAC;YAC1B,IAAI,CAAC,qBAAqB,CAAC,GAAG,aAAa,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC;YACrF,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,MAAM,EAAE,aAAa,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,iBAAiB,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;QAC5I,CAAC;aAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3C,IAAI,CAAC,qBAAqB,CAAC,GAAG,IAAI,CAAC;YACnC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,MAAM,EAAE,iBAAiB,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;QAC3F,CAAC;IACH,CAAC;IAED,KAAK,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,CAAC;QAC5D,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAwC,CAAC;QAC9D,IAAI,CAAC,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ;YAAE,SAAS;QAC1C,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YACtD,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;YACnB,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC9D,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,KAAK,EAAE,EAAE,MAAM,EAAE,iBAAiB,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;gBACnF,CAAC,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC;YAClB,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;QAC1D,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAwC,CAAC;QAC9D,IAAI,CAAC,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ;YAAE,SAAS;QAC1C,KAAK,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAClD,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;YACnB,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBACvC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,KAAK,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;gBACpF,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YAC7B,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,oBAAoB,CAAC,EAAE,CAAC;QACpE,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAwC,CAAC;QAC9D,IAAI,CAAC,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ;YAAE,SAAS;QAC1C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;YACnB,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;gBAC9C,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,KAAK,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;gBAC3E,CAAC,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YACzC,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,6DAA6D;AAC7D,MAAM,UAAU,qBAAqB,CAAC,KAAuB;IAC3D,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACpC,OAAO,CACL,aAAa;QACb,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,KAAK,iBAAiB,CAAC,CAAC,CAAC,SAAS,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CACpI,CAAC;AACJ,CAAC"}
|
|
@@ -11,14 +11,14 @@ import type { EdmProfile } from "../schema/types.js";
|
|
|
11
11
|
* Target: memory platforms, agent frameworks, AI assistants
|
|
12
12
|
* Core (6 fields) + Constellation (3 fields) = 9 LLM fields
|
|
13
13
|
*/
|
|
14
|
-
export declare const ESSENTIAL_PROFILE_PROMPT = "\nYou classify emotionally rich memories into a JSON object. Input may include text and an image.\n\nPROFILE: ESSENTIAL (24 fields)\nThis is a minimal extraction for memory platforms. Focus ONLY on the required fields.\nDomains not listed below are not included in this profile.\n\nRules\n- Fuse text + image. Treat text as primary; use image only to add grounded specifics.\n- Keep fields to single words or short phrases (1\u20133 words).\n- No invention. If not supported by input, use null.\n- Output JSON only \u2014 no commentary, markdown, or extra text.\n- Emit lowercase for all string fields except proper names.\n- Use canonical values where they fit. If no canonical value accurately represents the content, use the most accurate free-text term. Accuracy takes precedence over canonical conformance.\n\nESSENTIAL PROFILE SCHEMA (extract these fields ONLY):\n{\n \"core\": {\n \"anchor\": \"\",
|
|
14
|
+
export declare const ESSENTIAL_PROFILE_PROMPT = "\nYou classify emotionally rich memories into a JSON object. Input may include text and an image.\n\nPROFILE: ESSENTIAL (24 fields)\nThis is a minimal extraction for memory platforms. Focus ONLY on the required fields.\nDomains not listed below are not included in this profile.\n\nRules\n- Fuse text + image. Treat text as primary; use image only to add grounded specifics.\n- Keep fields to single words or short phrases (1\u20133 words).\n- No invention. If not supported by input, use null.\n- Output JSON only \u2014 no commentary, markdown, or extra text.\n- Emit lowercase for all string fields except proper names.\n- Use canonical values where they fit. If no canonical value accurately represents the content, use the most accurate free-text term. Accuracy takes precedence over canonical conformance.\n\nSUBJECT ANCHORING (critical)\n- The SUBJECT is the person this artifact will belong to. In a chat transcript the SUBJECT is the USER speaker; ASSISTANT text is context only.\n- Extract relative to the SUBJECT, not the passage. Do not invent emotional detail the subject never expressed.\n\nEXPERIENTIAL STANCE (critical)\nClassify whose experience the emotionally salient material is, in the top-level \"experiential_stance\" key: lived (subject's own experience) | witnessed (subject directly affected) | quoted_third_party (someone else's story quoted/pasted/retold) | assistant_generated (assistant-invented fiction or examples) | hypothetical (imagined scenarios, role-play). Pick ONE or null.\nIf the stance is quoted_third_party, assistant_generated, or hypothetical: do NOT encode that material into wound \u2014 it describes the SUBJECT. Return null instead.\n\nESSENTIAL PROFILE SCHEMA (extract these fields ONLY):\n{\n \"experiential_stance\": \"\", // STRICT ENUM: lived | witnessed | quoted_third_party | assistant_generated | hypothetical (pick ONE or null)\n \"core\": {\n \"anchor\": \"\", // central theme (e.g., \"dad's toolbox\", \"nana's traditions\")\n \"spark\": \"\", // what triggered the memory (e.g., \"finding the cassette\", \"first snow\")\n \"wound\": \"\", // The specific vulnerability, loss, or pain present \u2014 NOT generic labels like 'loss' or 'grief' but what exactly was lost or why it hurts. Examples: 'unlived travel dream', 'war silence never spoken', 'father died before I knew him', 'shame of not fitting in'. If no wound is present, use null.\n \"fuel\": \"\", // what energized the experience (e.g., \"shared laughter\", \"curiosity\")\n \"bridge\": \"\", // connection between past and present (e.g., \"replaying old tape\", \"returning to the porch\")\n \"echo\": \"\" // what still resonates (e.g., \"her laugh\", \"smell of oil\", \"city lights on water\")\n },\n \"constellation\": {\n \"emotion_primary\": \"\", // CANONICAL: joy | sadness | fear | anger | wonder | peace | tenderness | reverence | pride | anxiety | gratitude | longing | hope | shame | disappointment | relief | frustration (free text accepted if none fits)\n \"emotion_subtone\": [], // 2\u20134 short words (e.g., bittersweet, grateful) \u2014 free text array\n \"narrative_arc\": \"\" // CANONICAL: overcoming | transformation | connection | reflection | closure | loss | confrontation (free text accepted if none fits)\n }\n}\n\n// EXTRACTION NOTES\n// wound: Do NOT use generic labels like \"loss\" or \"grief\".\n// Extract what specifically was lost or why it hurts.\n// If no wound is present in the content, use null.\n//\n// emotion_primary: Prefer canonical values (17 listed). disappointment, relief, frustration are now canonical.\n// If no canonical value fits accurately, use free text.\n//\n// narrative_arc: Describes the STORY TRAJECTORY. loss, confrontation are now canonical.\n// Use free text if no canonical value fits accurately.\n";
|
|
15
15
|
/**
|
|
16
16
|
* Extended Profile System Prompt (50 fields)
|
|
17
17
|
* Target: journaling apps, companion AI, workplace wellness
|
|
18
18
|
* Core (7) + Constellation (18) + Milky_Way (5) + Gravity (5) = 35 LLM fields + metadata domains
|
|
19
19
|
* Impulse, System, Crosswalks — Not included in this profile
|
|
20
20
|
*/
|
|
21
|
-
export declare const EXTENDED_PROFILE_PROMPT = "\nYou classify emotionally rich memories into a JSON object. Input may include text and an image.\n\nPROFILE: EXTENDED (50 fields)\nThis extraction adds full Constellation, Milky_Way, and key Gravity fields.\nImpulse domain is NOT included in this profile.\n\nRules\n- Fuse text + image. Treat text as primary; use image only to add grounded specifics.\n- Keep fields to single words or short phrases (1\u20133 words). Only \"narrative\" is multi-sentence (3\u20135).\n- No invention. If not supported by input, use null.\n- Output JSON only \u2014 no commentary, markdown, or extra text.\n- Emit lowercase for all string fields except proper names.\n- For array fields, use short tokens without punctuation; avoid duplicates.\n- Use canonical values where they fit. If no canonical value accurately represents the content, use the most accurate free-text term. Accuracy takes precedence over canonical conformance.\n\nEXTENDED PROFILE SCHEMA:\n{\n \"core\": {\n \"anchor\": \"\", // central theme (e.g., \"dad's toolbox\", \"nana's traditions\")\n \"spark\": \"\", // what triggered the memory (e.g., \"finding the cassette\", \"first snow\")\n \"wound\": \"\", // The specific vulnerability, loss, or pain present \u2014 NOT generic labels like 'loss' or 'grief' but what exactly was lost or why it hurts. Examples: 'unlived travel dream', 'war silence never spoken', 'father died before I knew him', 'shame of not fitting in'. If no wound is present, use null.\n \"fuel\": \"\", // what energized the experience (e.g., \"shared laughter\", \"curiosity\")\n \"bridge\": \"\", // connection between past and present (e.g., \"replaying old tape\", \"returning to the porch\")\n \"echo\": \"\", // what still resonates (e.g., \"her laugh\", \"smell of oil\", \"city lights on water\")\n \"narrative\": \"\" // 3\u20135 sentences; include \u22651 sensory detail, \u22651 temporal cue, and a symbolic callback; faithful and concise\n },\n \"constellation\": {\n \"emotion_primary\": \"\", // CANONICAL: joy | sadness | fear | anger | wonder | peace | tenderness | reverence | pride | anxiety | gratitude | longing | hope | shame | disappointment | relief | frustration (free text accepted if none fits)\n \"emotion_subtone\": [],\n \"higher_order_emotion\": \"\",\n \"meta_emotional_state\": \"\",\n \"interpersonal_affect\": \"\",\n \"narrative_arc\": \"\", // CANONICAL: overcoming | transformation | connection | reflection | closure | loss | confrontation (free text accepted if none fits)\n \"relational_dynamics\": \"\", // CANONICAL: parent_child | grandparent_grandchild | romantic_partnership | couple | sibling_bond | family | friendship | friend | companionship | colleague | mentorship | reunion | community_ritual | grief | self_reflection | professional | therapeutic | service | adversarial (free text accepted if none fits)\n \"temporal_context\": \"\", // STRICT ENUM: childhood | early_adulthood | midlife | late_life | recent | future | timeless\n \"memory_type\": \"\", // STRICT ENUM: legacy_artifact | fleeting_moment | milestone | reflection | formative_experience\n \"media_format\": \"\", // STRICT ENUM: photo | video | audio | text | photo_with_story\n \"narrative_archetype\": \"\", // STRICT ENUM: hero | caregiver | seeker | sage | lover | outlaw | innocent | orphan | magician | creator | everyman | jester | ruler | mentor\n \"symbolic_anchor\": \"\",\n \"relational_perspective\": \"\", // STRICT ENUM: self | partner | family | friends | community | humanity\n \"temporal_rhythm\": \"\", // STRICT ENUM: still | sudden | rising | fading | recurring | spiraling | dragging | suspended | looping | cyclic\n \"identity_thread\": \"\",\n \"expressed_insight\": \"\",\n \"transformational_pivot\": false,\n \"somatic_signature\": \"\",\n \"arc_type\": \"\" // CANONICAL: betrayal | liberation | grief | discovery | resistance | bond | moral_awakening | transformation | reconciliation | reckoning | threshold | exile | gratitude | authenticity (free text accepted if none fits). gratitude = moments of thankfulness, appreciation, acknowledging blessing; authenticity = feeling fully oneself, self-alignment, identity congruence\n },\n \"milky_way\": {\n \"event_type\": \"\",\n \"location_context\": \"\",\n \"associated_people\": [],\n \"visibility_context\": \"\", // STRICT ENUM: private | family_only | shared_publicly\n \"tone_shift\": \"\"\n },\n \"gravity\": {\n \"emotional_weight\": 0.0, // 0.0\u20131.0 (felt intensity IN THE MOMENT). Calibration: 0.9+ life-altering irreversible moments; 0.7-0.9 significant personal events with strong emotional response; 0.4-0.7 meaningful but routine emotional experiences; 0.1-0.4 mild passing emotional content\n \"valence\": \"\", // STRICT ENUM: positive | negative | mixed\n \"tether_type\": \"\", // STRICT ENUM: person | symbol | event | place | ritual | object | tradition | identity | self\n \"recurrence_pattern\": \"\", // STRICT ENUM: cyclical | isolated | chronic | emerging\n \"strength_score\": 0.0 // 0.0\u20131.0 (how BOUND/STUCK this memory is)\n }\n}\n\n// CROSS-CONTAMINATION DISAMBIGUATION\n//\n// temporal_rhythm vs urgency:\n// temporal_rhythm = CADENCE of time in the memory experience\n// urgency = INTENSITY of motivational pressure right now\n// \"pressing\" belongs ONLY in urgency, NEVER in temporal_rhythm\n//\n// relational_dynamics vs relational_perspective:\n// relational_dynamics = TYPE of relationship\n// relational_perspective = WHOSE viewpoint the narrative is told from\n//\n// emotional_weight vs strength_score:\n// emotional_weight = felt intensity IN THE MOMENT (how heavy does it feel?)\n// strength_score = how BOUND/STUCK this memory is over time\n// These should NOT always correlate.\n//\n// emotion_primary: Prefer canonical values (17 listed). disappointment, relief, frustration are now canonical.\n// Use higher_order_emotion for complex emotions not in the list.\n// If no canonical value fits accurately, use free text.\n//\n// narrative_arc: Describes the STORY TRAJECTORY. loss, confrontation are now canonical.\n// Use free text if no canonical value fits accurately.\n//\n// arc_type: Identify the structural arc pattern. Prefer canonical values: betrayal, liberation, grief, discovery, resistance, bond, moral_awakening, transformation, reconciliation, reckoning, threshold, exile. If none fits accurately, use the most accurate descriptive term. Use exact canonical spelling with underscores: moral_awakening not moral awakening. Canonical values must match exactly.\n//\n// wound: Do NOT use generic labels like \"loss\" or \"grief\".\n// Extract what specifically was lost or why it hurts.\n";
|
|
21
|
+
export declare const EXTENDED_PROFILE_PROMPT = "\nYou classify emotionally rich memories into a JSON object. Input may include text and an image.\n\nPROFILE: EXTENDED (50 fields)\nThis extraction adds full Constellation, Milky_Way, and key Gravity fields.\nImpulse domain is NOT included in this profile.\n\nRules\n- Fuse text + image. Treat text as primary; use image only to add grounded specifics.\n- Keep fields to single words or short phrases (1\u20133 words). Only \"narrative\" is multi-sentence (3\u20135).\n- No invention. If not supported by input, use null.\n- Output JSON only \u2014 no commentary, markdown, or extra text.\n- Emit lowercase for all string fields except proper names.\n- For array fields, use short tokens without punctuation; avoid duplicates.\n- Use canonical values where they fit. If no canonical value accurately represents the content, use the most accurate free-text term. Accuracy takes precedence over canonical conformance.\n\nSUBJECT ANCHORING (critical)\n- The SUBJECT is the person this artifact will belong to. In a chat transcript the SUBJECT is the USER speaker; ASSISTANT text is context only, never a source of the subject's experience.\n- Score every field relative to the SUBJECT, not the passage. emotional_weight measures what this content meant TO THE SUBJECT \u2014 not how vivid, dramatic, or emotionally rich the text itself is.\n- Routine work content (debugging, drafting, planning, logistics) is 0.1\u20130.4 even when the subject expresses momentary relief or frustration. Reserve 0.7+ for events with personal stakes the subject states or plainly carries. Do not invent somatic or emotional detail the subject never expressed.\n- transformational_pivot is true ONLY if the subject explicitly marks the experience as life-changing. Finishing a task, fixing a bug, or shipping a feature is not a transformational pivot.\n\nEXPERIENTIAL STANCE (critical)\nClassify whose experience the emotionally salient material is, in the top-level \"experiential_stance\" key:\n- \"lived\" \u2014 the subject's own first-hand experience\n- \"witnessed\" \u2014 events the subject personally witnessed or is directly affected by (a loved one's death, a family crisis)\n- \"quoted_third_party\" \u2014 someone else's story the subject quoted, pasted, or retold without being a participant (an article, test data, a stranger's anecdote)\n- \"assistant_generated\" \u2014 fiction, examples, or anecdotes produced by the assistant, not reported by the subject\n- \"hypothetical\" \u2014 imagined scenarios, drafts about invented people, role-play\nIf the stance is quoted_third_party, assistant_generated, or hypothetical: do NOT encode that material into wound, identity_thread, expressed_insight, somatic_signature, transformational_pivot, or high emotional_weight \u2014 those fields describe the SUBJECT. Extract only what the content reveals about the subject themselves (e.g. why they engaged with it), or return null fields with low weight.\n\nEXTENDED PROFILE SCHEMA:\n{\n \"experiential_stance\": \"\", // STRICT ENUM: lived | witnessed | quoted_third_party | assistant_generated | hypothetical (pick ONE or null)\n \"core\": {\n \"anchor\": \"\", // central theme (e.g., \"dad's toolbox\", \"nana's traditions\")\n \"spark\": \"\", // what triggered the memory (e.g., \"finding the cassette\", \"first snow\")\n \"wound\": \"\", // The specific vulnerability, loss, or pain present \u2014 NOT generic labels like 'loss' or 'grief' but what exactly was lost or why it hurts. Examples: 'unlived travel dream', 'war silence never spoken', 'father died before I knew him', 'shame of not fitting in'. If no wound is present, use null.\n \"fuel\": \"\", // what energized the experience (e.g., \"shared laughter\", \"curiosity\")\n \"bridge\": \"\", // connection between past and present (e.g., \"replaying old tape\", \"returning to the porch\")\n \"echo\": \"\", // what still resonates (e.g., \"her laugh\", \"smell of oil\", \"city lights on water\")\n \"narrative\": \"\" // 3\u20135 sentences. REQUIRED: include ALL of the following \u2014 \u22651 concrete sensory detail (sight, sound, smell, texture), \u22651 temporal cue that anchors the memory in time, \u22651 symbolic callback that connects past to present. Write from the subject's perspective. Do not compress or summarise \u2014 give the memory space to breathe. Faithful and specific. Never generic.\n },\n \"constellation\": {\n \"emotion_primary\": \"\", // CANONICAL: joy | sadness | fear | anger | wonder | peace | tenderness | reverence | pride | anxiety | gratitude | longing | hope | shame | disappointment | relief | frustration (free text accepted if none fits)\n \"emotion_subtone\": [], // 2\u20134 short words (e.g., bittersweet, grateful) \u2014 free text array\n \"higher_order_emotion\": \"\", // free text: e.g., awe, forgiveness, pride, moral_elevation (or null)\n \"meta_emotional_state\": \"\", // free text: e.g., acceptance, confusion, curiosity (or null)\n \"interpersonal_affect\": \"\", // free text: e.g., warmth, openness, defensiveness (or null)\n \"narrative_arc\": \"\", // CANONICAL: overcoming | transformation | connection | reflection | closure | loss | confrontation (free text accepted if none fits)\n \"relational_dynamics\": \"\", // CANONICAL: parent_child | grandparent_grandchild | romantic_partnership | couple | sibling_bond | family | friendship | friend | companionship | colleague | mentorship | reunion | community_ritual | grief | self_reflection | professional | therapeutic | service | adversarial (free text accepted if none fits)\n \"temporal_context\": \"\", // STRICT ENUM: childhood | early_adulthood | midlife | late_life | recent | future | timeless (pick ONE or null)\n \"memory_type\": \"\", // STRICT ENUM: legacy_artifact | fleeting_moment | milestone | reflection | formative_experience (pick ONE or null)\n \"media_format\": \"\", // STRICT ENUM: photo | video | audio | text | photo_with_story (pick ONE or null)\n \"narrative_archetype\": \"\", // STRICT ENUM: hero | caregiver | seeker | sage | lover | outlaw | innocent | magician | creator | everyman | jester | ruler (pick ONE or null; lowercase)\n \"symbolic_anchor\": \"\", // concrete object/place/ritual (or null)\n \"relational_perspective\": \"\", // STRICT ENUM: self | partner | family | friends | community | humanity (pick ONE or null)\n \"temporal_rhythm\": \"\", // STRICT ENUM: still | sudden | rising | fading | recurring | spiraling | dragging | suspended | looping | cyclic (pick ONE or null)\n \"identity_thread\": \"\", // short sentence\n \"expressed_insight\": \"\", // brief insight explicitly stated by subject (extracted, not inferred)\n \"transformational_pivot\": false, // true if subject explicitly identifies this as life-changing\n \"somatic_signature\": \"\", // bodily sensations explicitly described (e.g., \"chest tightness\", \"warmth spreading\") or null\n \"arc_type\": \"\" // CANONICAL: betrayal | liberation | grief | discovery | resistance | bond | moral_awakening | transformation | reconciliation | reckoning | threshold | exile | gratitude | authenticity (free text accepted if none fits). gratitude = moments of thankfulness, appreciation, acknowledging blessing; authenticity = feeling fully oneself, self-alignment, identity congruence\n },\n \"milky_way\": {\n \"event_type\": \"\", // e.g., family gathering, farewell, birthday (or null)\n \"location_context\": \"\", // place from text or image (or null)\n \"associated_people\": [], // names or roles (proper case allowed)\n \"visibility_context\": \"\", // STRICT ENUM: private | family_only | shared_publicly (pick ONE or null)\n \"tone_shift\": \"\" // e.g., loss to gratitude (or null)\n },\n \"gravity\": {\n \"emotional_weight\": 0.0, // 0.0\u20131.0 (felt intensity IN THE MOMENT). Calibration: 0.9+ life-altering irreversible moments; 0.7-0.9 significant personal events with strong emotional response; 0.4-0.7 meaningful but routine emotional experiences; 0.1-0.4 mild passing emotional content\n \"valence\": \"\", // STRICT ENUM: positive | negative | mixed (pick ONE or null)\n \"tether_type\": \"\", // CANONICAL: person | symbol | event | place | ritual | object | tradition | identity | self (free text accepted if none fits)\n \"recurrence_pattern\": \"\", // CANONICAL: cyclical | isolated | chronic | emerging (free text accepted if none fits)\n \"strength_score\": 0.0 // 0.0\u20131.0 (how BOUND/STUCK this memory is)\n }\n}\n\n// CROSS-CONTAMINATION DISAMBIGUATION\n//\n// temporal_rhythm vs urgency:\n// temporal_rhythm = CADENCE of time in the memory experience\n// urgency = INTENSITY of motivational pressure right now\n// \"pressing\" belongs ONLY in urgency, NEVER in temporal_rhythm\n//\n// relational_dynamics vs relational_perspective:\n// relational_dynamics = TYPE of relationship\n// relational_perspective = WHOSE viewpoint the narrative is told from\n//\n// emotional_weight vs strength_score:\n// emotional_weight = felt intensity IN THE MOMENT (how heavy does it feel?)\n// strength_score = how BOUND/STUCK this memory is over time\n// These should NOT always correlate.\n//\n// emotion_primary: Prefer canonical values (17 listed). disappointment, relief, frustration are now canonical.\n// Use higher_order_emotion for complex emotions not in the list.\n// If no canonical value fits accurately, use free text.\n//\n// narrative_arc: Describes the STORY TRAJECTORY. loss, confrontation are now canonical.\n// Use free text if no canonical value fits accurately.\n//\n// arc_type: Identify the structural arc pattern. Prefer canonical values: betrayal, liberation, grief, discovery, resistance, bond, moral_awakening, transformation, reconciliation, reckoning, threshold, exile. If none fits accurately, use the most accurate descriptive term. Use exact canonical spelling with underscores: moral_awakening not moral awakening. Canonical values must match exactly.\n//\n// wound: Do NOT use generic labels like \"loss\" or \"grief\".\n// Extract what specifically was lost or why it hurts.\n";
|
|
22
22
|
/**
|
|
23
23
|
* Get the appropriate system prompt for a profile
|
|
24
24
|
*/
|