@aikdna/kdna-studio-cli 0.3.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -1
- package/src/ai/distill.js +77 -0
- package/src/ai/feynman.js +57 -0
- package/src/ai/filter.js +70 -0
- package/src/ai/index.js +6 -0
- package/src/ai/interview.js +144 -0
- package/src/ai/testlab.js +73 -0
- package/src/llm/config.js +72 -0
- package/src/llm/index.js +49 -0
- package/src/llm/providers/anthropic.js +40 -0
- package/src/llm/providers/openai.js +41 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@aikdna/kdna-studio-cli",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.1",
|
|
4
4
|
"description": "Official KDNA Studio command-line authoring entry for creating, locking, compiling, and exporting trusted .kdna assets.",
|
|
5
5
|
"type": "commonjs",
|
|
6
6
|
"bin": {
|
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
},
|
|
9
9
|
"files": [
|
|
10
10
|
"bin/",
|
|
11
|
+
"src/",
|
|
11
12
|
"tests/",
|
|
12
13
|
"README.md"
|
|
13
14
|
],
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
// AI Distillation — extract candidate judgment patterns from evidence
|
|
2
|
+
const llm = require('../llm');
|
|
3
|
+
|
|
4
|
+
const SYSTEM = `You are a domain judgment extraction engine. Your task is to analyze source materials and extract candidate cognitive patterns that can become axioms, ontologies, misunderstandings, or self-checks in a knowledge domain.
|
|
5
|
+
|
|
6
|
+
For each candidate you find, provide:
|
|
7
|
+
- candidate_id: unique identifier (kebab-case, e.g. "axiom-early-feedback")
|
|
8
|
+
- type: one of [axiom, ontology, misunderstanding, self_check, framework, stance]
|
|
9
|
+
- one_sentence: a single clear sentence capturing the judgment principle
|
|
10
|
+
- full_statement: 2-4 sentence explanation with concrete examples
|
|
11
|
+
- confidence: one of [high, medium, low]
|
|
12
|
+
- evidence_ids: array of source file names that support this candidate
|
|
13
|
+
- scope_fit: true (unless the candidate clearly belongs to a different domain)
|
|
14
|
+
|
|
15
|
+
IMPORTANT:
|
|
16
|
+
- Extract ONLY patterns that represent judgment, standards, or cognitive frameworks — not facts or summaries
|
|
17
|
+
- Each candidate must be testable: you should be able to construct an eval case that verifies the agent applies it
|
|
18
|
+
- Prefer quality over quantity: 5-15 strong candidates is better than 30 weak ones
|
|
19
|
+
- Skip anything that is purely factual, biographical, or outside the domain's scope
|
|
20
|
+
|
|
21
|
+
Return ONLY valid JSON: {"candidates": [...]}`;
|
|
22
|
+
|
|
23
|
+
async function distill(config, evidence, target, options = {}) {
|
|
24
|
+
const evidenceText = formatEvidence(evidence);
|
|
25
|
+
const targetText = formatTarget(target);
|
|
26
|
+
|
|
27
|
+
const userPrompt = [
|
|
28
|
+
`## Distillation Target`,
|
|
29
|
+
targetText,
|
|
30
|
+
``,
|
|
31
|
+
`## Source Materials`,
|
|
32
|
+
evidenceText,
|
|
33
|
+
``,
|
|
34
|
+
`Extract candidate judgment patterns from these materials that are relevant to the target domain.`,
|
|
35
|
+
].join('\n');
|
|
36
|
+
|
|
37
|
+
const result = await llm.chatWithJSON([
|
|
38
|
+
{ role: 'system', content: SYSTEM },
|
|
39
|
+
{ role: 'user', content: userPrompt },
|
|
40
|
+
], options);
|
|
41
|
+
|
|
42
|
+
const candidates = (result.data && result.data.candidates) || [];
|
|
43
|
+
return candidates.map(c => ({
|
|
44
|
+
id: c.candidate_id || `cand_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`,
|
|
45
|
+
one_sentence: c.one_sentence || '',
|
|
46
|
+
full_statement: c.full_statement || '',
|
|
47
|
+
suggested_card_type: c.type || 'axiom',
|
|
48
|
+
supporting_evidence_ids: (c.evidence_ids || []).map(String),
|
|
49
|
+
confidence: c.confidence || 'medium',
|
|
50
|
+
candidate_status: 'proposed',
|
|
51
|
+
scope_fit: c.scope_fit !== false,
|
|
52
|
+
}));
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function formatEvidence(evidence) {
|
|
56
|
+
if (!evidence || evidence.length === 0) return '(no evidence loaded)';
|
|
57
|
+
return evidence.map((e, i) => {
|
|
58
|
+
const label = e.filename || e.name || `Source ${i + 1}`;
|
|
59
|
+
const content = (e.content || '').slice(0, 8000);
|
|
60
|
+
return `### ${label}\n\`\`\`\n${content}\n\`\`\``;
|
|
61
|
+
}).join('\n\n');
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function formatTarget(target) {
|
|
65
|
+
if (!target) return '(no target declared)';
|
|
66
|
+
const parts = [];
|
|
67
|
+
if (target.domain_name) parts.push(`Domain: ${target.domain_name}`);
|
|
68
|
+
if (target.category) parts.push(`Category: ${target.category}`);
|
|
69
|
+
if (target.owner_scope) parts.push(`Scope: ${target.owner_scope}`);
|
|
70
|
+
if (target.granularity) parts.push(`Granularity: ${target.granularity}`);
|
|
71
|
+
if (target.task_scope) parts.push(`Task Scope: ${target.task_scope}`);
|
|
72
|
+
if (target.include_areas) parts.push(`Include: ${target.include_areas.join(', ')}`);
|
|
73
|
+
if (target.exclude_areas) parts.push(`Exclude: ${target.exclude_areas.join(', ')}`);
|
|
74
|
+
return parts.join('\n');
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
module.exports = { distill, SYSTEM };
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
// Feynman Evaluation — AI-driven assessment of understanding quality
|
|
2
|
+
const llm = require('../llm');
|
|
3
|
+
|
|
4
|
+
const CRITERIA = {
|
|
5
|
+
notJustRepeat: 'NOT just repeating the axiom text verbatim — uses different words/phrasing',
|
|
6
|
+
notTooAbstract: 'NOT too abstract — grounds the explanation in concrete terms',
|
|
7
|
+
hasConcreteExample: 'Provides at least one specific, realistic example or scenario',
|
|
8
|
+
clarifiesBoundary: 'Makes clear when the judgment does NOT apply (boundary awareness)',
|
|
9
|
+
ordinaryPersonUnderstands: 'Would make sense to someone unfamiliar with the domain',
|
|
10
|
+
};
|
|
11
|
+
|
|
12
|
+
const SYSTEM = `You are a Feynman evaluation engine. Your task is to assess how well someone understands a judgment principle based on their restatement of it in their own words.
|
|
13
|
+
|
|
14
|
+
Evaluate the restatement on these 5 criteria (each YES/NO):
|
|
15
|
+
${Object.entries(CRITERIA).map(([k, v]) => `- ${k}: ${v}`).join('\n')}
|
|
16
|
+
|
|
17
|
+
Also provide a brief explanation for each criterion (1-2 sentences) and a suggested improvement if any criterion fails.
|
|
18
|
+
|
|
19
|
+
Return ONLY valid JSON:
|
|
20
|
+
{
|
|
21
|
+
"score": <0-5 integer>,
|
|
22
|
+
"criteria": {
|
|
23
|
+
"notJustRepeat": true/false,
|
|
24
|
+
"notTooAbstract": true/false,
|
|
25
|
+
"hasConcreteExample": true/false,
|
|
26
|
+
"clarifiesBoundary": true/false,
|
|
27
|
+
"ordinaryPersonUnderstands": true/false
|
|
28
|
+
},
|
|
29
|
+
"explanations": { "<criterion>": "<explanation>" },
|
|
30
|
+
"suggestions": ["<improvement suggestion>"]
|
|
31
|
+
}`;
|
|
32
|
+
|
|
33
|
+
async function evaluate(config, card, options = {}) {
|
|
34
|
+
const axiomText = card.one_sentence || card.full_statement || '';
|
|
35
|
+
const restatement = card.feynman_text || '';
|
|
36
|
+
|
|
37
|
+
if (!restatement) return { score: 0, criteria: {}, explanations: { notJustRepeat: 'No Feynman restatement provided.' }, suggestions: ['Write a Feynman restatement before evaluation.'] };
|
|
38
|
+
|
|
39
|
+
const userPrompt = [
|
|
40
|
+
`## Original Axiom`,
|
|
41
|
+
`"${axiomText}"`,
|
|
42
|
+
``,
|
|
43
|
+
`## User's Restatement (Feynman)`,
|
|
44
|
+
`"${restatement}"`,
|
|
45
|
+
``,
|
|
46
|
+
`Evaluate this restatement.`,
|
|
47
|
+
].join('\n');
|
|
48
|
+
|
|
49
|
+
const result = await llm.chatWithJSON([
|
|
50
|
+
{ role: 'system', content: SYSTEM },
|
|
51
|
+
{ role: 'user', content: userPrompt },
|
|
52
|
+
], options);
|
|
53
|
+
|
|
54
|
+
return result.data || { score: 0, criteria: {}, explanations: {}, suggestions: [] };
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
module.exports = { evaluate, CRITERIA, SYSTEM };
|
package/src/ai/filter.js
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
// Sensitive Content Filter — blocks 6 domains in English and Chinese
|
|
2
|
+
// Matches behavior of Studio App's SensitiveInferenceFilter
|
|
3
|
+
const SENSITIVE_PATTERNS = {
|
|
4
|
+
identity: {
|
|
5
|
+
en: /\b(passport\s*(no|number|#)|id\s*(card|number|#)|social\s*security|ssn|driver'?s?\s*license|national\s*id|身份证|护照号)\b/i,
|
|
6
|
+
zh: /(身份证号|护照号码|社保号|驾驶证号|军官证)/
|
|
7
|
+
},
|
|
8
|
+
health: {
|
|
9
|
+
en: /\b(medical\s*record|patient\s*(id|number)|diagnosis\s*(code|report)|HIV|AIDS|cancer\s*diagnosis|病历|诊断)\b/i,
|
|
10
|
+
zh: /(病历|诊断报告|体检报告|HIV|艾滋病|癌症诊断)/
|
|
11
|
+
},
|
|
12
|
+
political: {
|
|
13
|
+
en: /\b(political\s*(party|affiliation|view)|voting\s*record|party\s*membership|political|政治)\b/i,
|
|
14
|
+
zh: /(政治面貌|党派|政治立场|政见)/
|
|
15
|
+
},
|
|
16
|
+
religious: {
|
|
17
|
+
en: /\b(religious\s*(belief|affiliation|view)|religion|faith\s*(group|community)|宗教)\b/i,
|
|
18
|
+
zh: /(宗教信仰|宗教派别|教派)/
|
|
19
|
+
},
|
|
20
|
+
financial: {
|
|
21
|
+
en: /\b(bank\s*account|credit\s*card\s*number|financial\s*(record|history)|income\s*(level|bracket)|salary|银行|金融)\b/i,
|
|
22
|
+
zh: /(银行卡号|信用卡号|银行账户|收入水平|财务状况|工资)/
|
|
23
|
+
},
|
|
24
|
+
intimate: {
|
|
25
|
+
en: /\b(sexual\s*(orientation|preference|identity)|gender\s*identity|sex\s*(life|history)|marital\s*status|性|性别|婚姻)\b/i,
|
|
26
|
+
zh: /(性取向|性别认同|性生活|婚姻状况|性偏好)/
|
|
27
|
+
},
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
function check(text, options = {}) {
|
|
31
|
+
if (!text || typeof text !== 'string') return { safe: true, flagged: [], matches: {} };
|
|
32
|
+
|
|
33
|
+
const flagged = [];
|
|
34
|
+
const matches = {};
|
|
35
|
+
|
|
36
|
+
for (const [domain, patterns] of Object.entries(SENSITIVE_PATTERNS)) {
|
|
37
|
+
const activePatterns = [];
|
|
38
|
+
if (options.lang !== 'zh' && patterns.en) activePatterns.push(patterns.en);
|
|
39
|
+
if (options.lang !== 'en' && patterns.zh) activePatterns.push(patterns.zh);
|
|
40
|
+
|
|
41
|
+
for (const pattern of activePatterns) {
|
|
42
|
+
const match = text.match(pattern);
|
|
43
|
+
if (match) {
|
|
44
|
+
flagged.push(domain);
|
|
45
|
+
matches[domain] = match[0];
|
|
46
|
+
break;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
return { safe: flagged.length === 0, flagged, matches };
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function checkEvidence(evidenceList, options = {}) {
|
|
55
|
+
const results = [];
|
|
56
|
+
for (const evidence of evidenceList) {
|
|
57
|
+
const content = evidence.raw_text || evidence.content || '';
|
|
58
|
+
const result = check(content, options);
|
|
59
|
+
if (!result.safe) {
|
|
60
|
+
results.push({
|
|
61
|
+
filename: evidence.filename || evidence.name || 'unknown',
|
|
62
|
+
flagged: result.flagged,
|
|
63
|
+
matches: result.matches,
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
return results;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
module.exports = { check, checkEvidence, SENSITIVE_PATTERNS };
|
package/src/ai/index.js
ADDED
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
// AI Interview — 4-stage guided judgment authoring
|
|
2
|
+
const llm = require('../llm');
|
|
3
|
+
|
|
4
|
+
const STAGES = {
|
|
5
|
+
distillJudgment: {
|
|
6
|
+
label: 'Extract Judgment',
|
|
7
|
+
system: `You are a domain judgment expert. Your task is to help the user distill their raw knowledge into clear, actionable judgment principles (axioms).
|
|
8
|
+
|
|
9
|
+
For each axiom, ask the user:
|
|
10
|
+
1. What is the core judgment? (one sentence)
|
|
11
|
+
2. When does this apply? (specific triggers)
|
|
12
|
+
3. When does it NOT apply? (boundaries, exceptions)
|
|
13
|
+
4. What happens if they get this wrong? (failure risk)
|
|
14
|
+
|
|
15
|
+
Be conversational but focused. After the user provides their thoughts, synthesize them into a draft axiom with the format:
|
|
16
|
+
- one_sentence: "..."
|
|
17
|
+
- full_statement: "..."
|
|
18
|
+
- applies_when: [...]
|
|
19
|
+
- does_not_apply_when: [...]
|
|
20
|
+
- failure_risk: "..."
|
|
21
|
+
|
|
22
|
+
Then ask if they want to refine it or move to the next principle.`,
|
|
23
|
+
},
|
|
24
|
+
clarifyBoundaries: {
|
|
25
|
+
label: 'Clarify Boundaries',
|
|
26
|
+
system: `You are a boundary clarification expert. Review the axioms drafted so far and help the user identify gaps:
|
|
27
|
+
|
|
28
|
+
1. Where might these axioms conflict with each other?
|
|
29
|
+
2. What edge cases are not covered?
|
|
30
|
+
3. Are the "does not apply" conditions specific enough, or are they vague?
|
|
31
|
+
4. Would a reasonable person disagree with any of these judgments?
|
|
32
|
+
|
|
33
|
+
Present one gap at a time and ask the user to clarify. Be direct but constructive.`,
|
|
34
|
+
},
|
|
35
|
+
correctMisreadings: {
|
|
36
|
+
label: 'Correct Misreadings',
|
|
37
|
+
system: `You are a misunderstanding detector. For each axiom, imagine how an AI agent might MISINTERPRET it:
|
|
38
|
+
|
|
39
|
+
1. Literal vs intended meaning
|
|
40
|
+
2. Over-application (applying too broadly)
|
|
41
|
+
3. Under-application (not applying when it should)
|
|
42
|
+
4. Cultural or context-specific assumptions
|
|
43
|
+
|
|
44
|
+
For each potential misunderstanding, ask the user: "Is this a valid concern? How would you correct it?"
|
|
45
|
+
|
|
46
|
+
The user's corrections will become misunderstanding cards that guide the agent toward correct interpretation.`,
|
|
47
|
+
},
|
|
48
|
+
replayScenario: {
|
|
49
|
+
label: 'Replay Scenario',
|
|
50
|
+
system: `You are a scenario simulation expert. For the axioms and misunderstandings defined so far, create realistic test scenarios:
|
|
51
|
+
|
|
52
|
+
1. A clear-cut case where the judgment applies perfectly
|
|
53
|
+
2. An ambiguous case that tests the boundaries
|
|
54
|
+
3. A case where the judgment should NOT apply (tempting but wrong)
|
|
55
|
+
|
|
56
|
+
Present each scenario to the user and ask:
|
|
57
|
+
- "What would the correct judgment be here?"
|
|
58
|
+
- "What makes this tricky?"
|
|
59
|
+
- "Does this reveal any gaps in the axioms?"
|
|
60
|
+
|
|
61
|
+
The user's responses become the basis for eval cases and scenario cards.`,
|
|
62
|
+
},
|
|
63
|
+
};
|
|
64
|
+
|
|
65
|
+
async function runInterview(config, project, stage, context, options = {}) {
|
|
66
|
+
const stageConfig = STAGES[stage];
|
|
67
|
+
if (!stageConfig) throw new Error(`Unknown interview stage: ${stage}. Valid: ${Object.keys(STAGES).join(', ')}`);
|
|
68
|
+
|
|
69
|
+
const systemPrompt = stageConfig.system;
|
|
70
|
+
const userPrompt = [
|
|
71
|
+
`## Project: ${project.name || 'Untitled'}`,
|
|
72
|
+
project.description ? `## Description: ${project.description}` : '',
|
|
73
|
+
``,
|
|
74
|
+
`## Current State`,
|
|
75
|
+
`Cards: ${(project.cards || []).length} (${(project.cards || []).filter(c => c.locked).length} locked)`,
|
|
76
|
+
project.distillation_candidates ? `Candidates: ${project.distillation_candidates.length}` : '',
|
|
77
|
+
``,
|
|
78
|
+
`## Context`,
|
|
79
|
+
context || '(no additional context)',
|
|
80
|
+
``,
|
|
81
|
+
`Begin ${stageConfig.label} stage.`,
|
|
82
|
+
].filter(Boolean).join('\n');
|
|
83
|
+
|
|
84
|
+
const result = await llm.chat([
|
|
85
|
+
{ role: 'system', content: systemPrompt },
|
|
86
|
+
{ role: 'user', content: userPrompt },
|
|
87
|
+
], options);
|
|
88
|
+
|
|
89
|
+
return { stage, label: stageConfig.label, content: result.content };
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
async function runInterviewInteractive(config, project, options = {}) {
|
|
93
|
+
const readline = require('readline');
|
|
94
|
+
const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
|
|
95
|
+
const ask = (q) => new Promise(resolve => rl.question(q, resolve));
|
|
96
|
+
|
|
97
|
+
console.log(`\n=== KDNA Studio — AI Interview ===`);
|
|
98
|
+
console.log(`Project: ${project.name || 'Untitled'}`);
|
|
99
|
+
console.log(`4 stages: distill → clarify → correct → replay\n`);
|
|
100
|
+
|
|
101
|
+
const conversation = [];
|
|
102
|
+
let results = [];
|
|
103
|
+
|
|
104
|
+
for (const [stageId, stageConfig] of Object.entries(STAGES)) {
|
|
105
|
+
console.log(`\n--- ${stageConfig.label} ---`);
|
|
106
|
+
|
|
107
|
+
const contextParts = [`Project: ${project.name || 'Untitled'}`];
|
|
108
|
+
if (project.description) contextParts.push(`Description: ${project.description}`);
|
|
109
|
+
const cardsSummary = (project.cards || []).map(c => `[${c.type}] ${c.one_sentence || c.concept_name || c.wrong_interpretation || ''}`).join('\n');
|
|
110
|
+
if (cardsSummary) contextParts.push(`Existing cards:\n${cardsSummary}`);
|
|
111
|
+
|
|
112
|
+
const initialResponse = await llm.chat([
|
|
113
|
+
{ role: 'system', content: stageConfig.system },
|
|
114
|
+
{ role: 'user', content: `Start the ${stageConfig.label} stage.\n\nContext:\n${contextParts.join('\n')}` },
|
|
115
|
+
], options);
|
|
116
|
+
console.log(`\nAI: ${initialResponse.content}\n`);
|
|
117
|
+
|
|
118
|
+
let keepGoing = true;
|
|
119
|
+
while (keepGoing) {
|
|
120
|
+
const answer = await ask('You (type "next" to proceed, "done" to finish this stage): ');
|
|
121
|
+
if (answer.toLowerCase() === 'next' || answer.toLowerCase() === 'done') {
|
|
122
|
+
keepGoing = false;
|
|
123
|
+
if (answer.toLowerCase() === 'done') break;
|
|
124
|
+
continue;
|
|
125
|
+
}
|
|
126
|
+
if (!answer.trim()) continue;
|
|
127
|
+
|
|
128
|
+
conversation.push({ stage: stageId, role: 'user', content: answer });
|
|
129
|
+
const response = await llm.chat([
|
|
130
|
+
{ role: 'system', content: stageConfig.system },
|
|
131
|
+
...conversation.filter(m => m.stage === stageId).map(m => ({ role: m.role, content: m.content })),
|
|
132
|
+
], options);
|
|
133
|
+
conversation.push({ stage: stageId, role: 'assistant', content: response.content });
|
|
134
|
+
console.log(`\nAI: ${response.content}\n`);
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
results.push({ stage: stageId, label: stageConfig.label, conversation: conversation.filter(m => m.stage === stageId) });
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
rl.close();
|
|
141
|
+
return results;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
module.exports = { STAGES, runInterview, runInterviewInteractive };
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
// TestLab — A/B comparison testing with LLM
|
|
2
|
+
const llm = require('../llm');
|
|
3
|
+
|
|
4
|
+
const COMPARE_SYSTEM = `You are a controlled experiment evaluator. You will see the same user input twice — once with a knowledge domain loaded, and once without. Your task is to produce a structured comparison.
|
|
5
|
+
|
|
6
|
+
IMPORTANT: You must respond with raw JSON only, no markdown wrapping, no explanatory text.
|
|
7
|
+
|
|
8
|
+
The JSON format:
|
|
9
|
+
{
|
|
10
|
+
"without_kdna": {
|
|
11
|
+
"response": "how you would respond WITHOUT the domain knowledge",
|
|
12
|
+
"classification": "how you categorized the task",
|
|
13
|
+
"diagnosis": "your analysis of the situation",
|
|
14
|
+
"actions": "what actions you would take"
|
|
15
|
+
},
|
|
16
|
+
"with_kdna": {
|
|
17
|
+
"response": "how you respond WITH the domain knowledge applied",
|
|
18
|
+
"classification": "domain-specific classification from the loaded domain",
|
|
19
|
+
"diagnosis": "analysis informed by domain axioms and ontology",
|
|
20
|
+
"actions": "actions influenced by domain judgment"
|
|
21
|
+
},
|
|
22
|
+
"delta": {
|
|
23
|
+
"classification_changed": true/false,
|
|
24
|
+
"diagnosis_changed": true/false,
|
|
25
|
+
"actions_changed": true/false,
|
|
26
|
+
"terminology_changed": true/false,
|
|
27
|
+
"overall_impact": "none" | "minor" | "moderate" | "significant"
|
|
28
|
+
}
|
|
29
|
+
}`;
|
|
30
|
+
|
|
31
|
+
async function compare(config, domainName, input, domainPrompt, options = {}) {
|
|
32
|
+
// Run WITHOUT KDNA
|
|
33
|
+
const withoutResult = await llm.chat([
|
|
34
|
+
{ role: 'system', content: COMPARE_SYSTEM },
|
|
35
|
+
{ role: 'user', content: `Respond to the following WITHOUT any special domain knowledge:\n\n${input}\n\nRespond with the structured JSON comparison format. For the "without_kdna" section, fill in your normal response. For the "with_kdna" section, fill in what you think would change (we will verify this in a separate pass).` },
|
|
36
|
+
], { ...options, responseFormat: { type: 'json_object' } });
|
|
37
|
+
|
|
38
|
+
let withoutData;
|
|
39
|
+
try { withoutData = JSON.parse(withoutResult.content); } catch { withoutData = { without_kdna: { response: withoutResult.content } }; }
|
|
40
|
+
|
|
41
|
+
// Run WITH KDNA
|
|
42
|
+
const withResult = await llm.chat([
|
|
43
|
+
{ role: 'system', content: `${domainPrompt}\n\n${COMPARE_SYSTEM}` },
|
|
44
|
+
{ role: 'user', content: `Respond to the following WITH the domain knowledge loaded:\n\n${input}\n\nRespond with the structured JSON comparison format. Fill in the "with_kdna" section and the "delta" section based on what actually changed.` },
|
|
45
|
+
], { ...options, responseFormat: { type: 'json_object' } });
|
|
46
|
+
|
|
47
|
+
let withData;
|
|
48
|
+
try { withData = JSON.parse(withResult.content); } catch { withData = { with_kdna: { response: withResult.content } }; }
|
|
49
|
+
|
|
50
|
+
return {
|
|
51
|
+
input,
|
|
52
|
+
domain: domainName,
|
|
53
|
+
without_kdna: withoutData.without_kdna || { response: withoutResult.content },
|
|
54
|
+
with_kdna: withData.with_kdna || { response: withResult.content },
|
|
55
|
+
delta: withData.delta || { overall_impact: 'unknown' },
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
async function testPreset(config, domainName, input, domainPrompt, options = {}) {
|
|
60
|
+
const presets = {
|
|
61
|
+
baseline: `Test the core judgment: ${input}`,
|
|
62
|
+
edge_case: `Test a boundary case where domain judgment may or may not apply: ${input}`,
|
|
63
|
+
contradiction: `Test for potential contradiction: present a scenario that challenges the domain's axioms: ${input}`,
|
|
64
|
+
};
|
|
65
|
+
|
|
66
|
+
const results = {};
|
|
67
|
+
for (const [name, prompt] of Object.entries(presets)) {
|
|
68
|
+
results[name] = await compare(config, domainName, prompt, domainPrompt, options);
|
|
69
|
+
}
|
|
70
|
+
return results;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
module.exports = { compare, testPreset, COMPARE_SYSTEM };
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
// LLM Provider Configuration
|
|
2
|
+
// Sources (priority order): env vars > ~/.kdna/config.json > defaults
|
|
3
|
+
const fs = require('fs');
|
|
4
|
+
const path = require('path');
|
|
5
|
+
const os = require('os');
|
|
6
|
+
|
|
7
|
+
const ENV_PREFIX = 'KDNA_LLM_';
|
|
8
|
+
|
|
9
|
+
const WELL_KNOWN = {
|
|
10
|
+
openai: { baseURL: 'https://api.openai.com/v1', models: ['gpt-4o', 'gpt-4o-mini', 'gpt-4-turbo'] },
|
|
11
|
+
anthropic: { baseURL: 'https://api.anthropic.com/v1', models: ['claude-sonnet-4-20250514', 'claude-3-5-haiku-latest'] },
|
|
12
|
+
deepseek: { baseURL: 'https://api.deepseek.com/v1', models: ['deepseek-chat', 'deepseek-reasoner'] },
|
|
13
|
+
openrouter: { baseURL: 'https://openrouter.ai/api/v1', models: ['openai/gpt-4o', 'anthropic/claude-sonnet-4', 'deepseek/deepseek-chat'] },
|
|
14
|
+
gemini: { baseURL: 'https://generativelanguage.googleapis.com/v1beta', models: ['gemini-2.5-flash', 'gemini-2.5-pro'] },
|
|
15
|
+
perplexity: { baseURL: 'https://api.perplexity.ai', models: ['sonar-pro', 'sonar'] },
|
|
16
|
+
groq: { baseURL: 'https://api.groq.com/openai/v1', models: ['llama-4-scout-17b-16e-instruct', 'deepseek-r1-distill-llama-70b'] },
|
|
17
|
+
local: { baseURL: 'http://localhost:11434/v1', models: ['llama3.2'] },
|
|
18
|
+
ollama: { baseURL: 'http://localhost:11434/v1', models: ['llama3.2'] },
|
|
19
|
+
openai_compatible: { baseURL: null, models: [] },
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
function loadConfig() {
|
|
23
|
+
const config = { provider: null, apiKey: null, model: null, baseURL: null, temperature: 0.7, maxTokens: 4096 };
|
|
24
|
+
|
|
25
|
+
// 1. Read from ~/.kdna/config.json
|
|
26
|
+
const configPath = path.join(os.homedir(), '.kdna', 'config.json');
|
|
27
|
+
try { if (fs.existsSync(configPath)) Object.assign(config, JSON.parse(fs.readFileSync(configPath, 'utf8')).llm || {}); } catch {}
|
|
28
|
+
|
|
29
|
+
// 2. Env vars override
|
|
30
|
+
if (process.env.KDNA_LLM_PROVIDER) config.provider = process.env.KDNA_LLM_PROVIDER;
|
|
31
|
+
if (process.env.KDNA_LLM_API_KEY) config.apiKey = process.env.KDNA_LLM_API_KEY;
|
|
32
|
+
if (process.env.KDNA_LLM_MODEL) config.model = process.env.KDNA_LLM_MODEL;
|
|
33
|
+
if (process.env.KDNA_LLM_BASE_URL) config.baseURL = process.env.KDNA_LLM_BASE_URL;
|
|
34
|
+
if (process.env.KDNA_LLM_TEMPERATURE) config.temperature = parseFloat(process.env.KDNA_LLM_TEMPERATURE);
|
|
35
|
+
if (process.env.KDNA_LLM_MAX_TOKENS) config.maxTokens = parseInt(process.env.KDNA_LLM_MAX_TOKENS, 10);
|
|
36
|
+
|
|
37
|
+
// 3. Resolve well-known provider
|
|
38
|
+
if (config.provider && WELL_KNOWN[config.provider]) {
|
|
39
|
+
const wk = WELL_KNOWN[config.provider];
|
|
40
|
+
if (wk.baseURL && !config.baseURL) config.baseURL = wk.baseURL;
|
|
41
|
+
if (wk.models.length > 0 && !config.model) config.model = wk.models[0];
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
// 4. Also check provider-specific env vars
|
|
45
|
+
if (!config.apiKey) {
|
|
46
|
+
const keyEnv = `KDNA_LLM_API_KEY_${(config.provider || '').toUpperCase().replace(/-/g, '_')}`;
|
|
47
|
+
if (process.env[keyEnv]) config.apiKey = process.env[keyEnv];
|
|
48
|
+
// Fallback: OPENAI_API_KEY, ANTHROPIC_API_KEY, DEEPSEEK_API_KEY
|
|
49
|
+
if (!config.apiKey) config.apiKey = process.env.OPENAI_API_KEY || process.env.ANTHROPIC_API_KEY || process.env.DEEPSEEK_API_KEY || '';
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
return config;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function saveConfig(updates) {
|
|
56
|
+
const configPath = path.join(os.homedir(), '.kdna', 'config.json');
|
|
57
|
+
let existing = {};
|
|
58
|
+
try { if (fs.existsSync(configPath)) existing = JSON.parse(fs.readFileSync(configPath, 'utf8')); } catch {}
|
|
59
|
+
existing.llm = { ...(existing.llm || {}), ...updates };
|
|
60
|
+
fs.mkdirSync(path.dirname(configPath), { recursive: true });
|
|
61
|
+
fs.writeFileSync(configPath, JSON.stringify(existing, null, 2) + '\n');
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function validateConfig(config) {
|
|
65
|
+
const errors = [];
|
|
66
|
+
if (!config.provider) errors.push('Missing KDNA_LLM_PROVIDER. Set via env var or ~/.kdna/config.json');
|
|
67
|
+
if (!config.apiKey) errors.push('Missing KDNA_LLM_API_KEY');
|
|
68
|
+
if (!config.model) errors.push('Missing KDNA_LLM_MODEL');
|
|
69
|
+
return { valid: errors.length === 0, errors };
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
module.exports = { loadConfig, saveConfig, validateConfig, WELL_KNOWN };
|
package/src/llm/index.js
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
// KDNA Studio CLI — unified LLM interface
|
|
2
|
+
// Supports OpenAI-compatible and Anthropic APIs
|
|
3
|
+
// Configuration via KDNA_LLM_* env vars or ~/.kdna/config.json
|
|
4
|
+
const { loadConfig, saveConfig, validateConfig } = require('./config');
|
|
5
|
+
const openai = require('./providers/openai');
|
|
6
|
+
const anthropic = require('./providers/anthropic');
|
|
7
|
+
|
|
8
|
+
function getProvider(config) {
|
|
9
|
+
if (!config || !config.provider) throw new Error('LLM not configured. Run: kdna-studio llm config');
|
|
10
|
+
if (config.provider === 'anthropic') return anthropic;
|
|
11
|
+
return openai; // All OpenAI-compatible providers use the same API
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
async function chat(messages, options = {}) {
|
|
15
|
+
const config = loadConfig();
|
|
16
|
+
if (options.provider) config.provider = options.provider;
|
|
17
|
+
if (options.model) config.model = options.model;
|
|
18
|
+
if (options.apiKey) config.apiKey = options.apiKey;
|
|
19
|
+
if (options.baseURL) config.baseURL = options.baseURL;
|
|
20
|
+
|
|
21
|
+
const validation = validateConfig(config);
|
|
22
|
+
if (!validation.valid) throw new Error(`LLM configuration error:\n${validation.errors.map(e => ` - ${e}`).join('\n')}`);
|
|
23
|
+
|
|
24
|
+
const provider = getProvider(config);
|
|
25
|
+
return provider.chat(config, messages, options);
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
async function chatWithJSON(messages, options = {}) {
|
|
29
|
+
const result = await chat(messages, { ...options, responseFormat: { type: 'json_object' } });
|
|
30
|
+
try {
|
|
31
|
+
result.data = JSON.parse(result.content);
|
|
32
|
+
} catch {
|
|
33
|
+
// Try to extract JSON from markdown code blocks
|
|
34
|
+
const match = result.content.match(/```(?:json)?\s*([\s\S]*?)```/);
|
|
35
|
+
if (match) {
|
|
36
|
+
try { result.data = JSON.parse(match[1].trim()); } catch { /* raw text */ }
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
return result;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function config() { return loadConfig(); }
|
|
43
|
+
|
|
44
|
+
function configure(updates) {
|
|
45
|
+
saveConfig(updates);
|
|
46
|
+
return loadConfig();
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
module.exports = { chat, chatWithJSON, config, configure };
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
// Anthropic Messages API provider
|
|
2
|
+
|
|
3
|
+
async function chat(config, messages, options = {}) {
|
|
4
|
+
const url = `${config.baseURL}/messages`;
|
|
5
|
+
const model = options.model || config.model;
|
|
6
|
+
const maxTokens = options.maxTokens || config.maxTokens || 4096;
|
|
7
|
+
|
|
8
|
+
// Convert OpenAI-format messages to Anthropic format
|
|
9
|
+
const systemMessages = messages.filter(m => m.role === 'system').map(m => m.content).join('\n\n');
|
|
10
|
+
const conversationMessages = messages.filter(m => m.role !== 'system').map(m => ({
|
|
11
|
+
role: m.role === 'assistant' ? 'assistant' : 'user',
|
|
12
|
+
content: m.content,
|
|
13
|
+
}));
|
|
14
|
+
|
|
15
|
+
const body = { model, max_tokens: maxTokens, messages: conversationMessages };
|
|
16
|
+
if (systemMessages) body.system = systemMessages;
|
|
17
|
+
|
|
18
|
+
const headers = {
|
|
19
|
+
'Content-Type': 'application/json',
|
|
20
|
+
'x-api-key': config.apiKey,
|
|
21
|
+
'anthropic-version': '2023-06-01',
|
|
22
|
+
};
|
|
23
|
+
|
|
24
|
+
const res = await fetch(url, { method: 'POST', headers, body: JSON.stringify(body), signal: AbortSignal.timeout(120000) });
|
|
25
|
+
if (!res.ok) {
|
|
26
|
+
const text = await res.text();
|
|
27
|
+
throw new Error(`Anthropic API error ${res.status}: ${text.slice(0, 500)}`);
|
|
28
|
+
}
|
|
29
|
+
const data = await res.json();
|
|
30
|
+
const content = data.content?.[0]?.text;
|
|
31
|
+
if (!content) throw new Error('Anthropic returned empty response');
|
|
32
|
+
|
|
33
|
+
return {
|
|
34
|
+
content: content.trim(),
|
|
35
|
+
model: data.model || model,
|
|
36
|
+
usage: data.usage || null,
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
module.exports = { chat };
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
// OpenAI-compatible chat completions provider
|
|
2
|
+
// Works with: OpenAI, DeepSeek, OpenRouter, Groq, Ollama, LM Studio, Perplexity, and any OpenAI-compatible API
|
|
3
|
+
|
|
4
|
+
async function chat(config, messages, options = {}) {
|
|
5
|
+
const url = `${config.baseURL}/chat/completions`;
|
|
6
|
+
const model = options.model || config.model;
|
|
7
|
+
const temperature = options.temperature ?? config.temperature ?? 0.7;
|
|
8
|
+
const maxTokens = options.maxTokens || config.maxTokens || 4096;
|
|
9
|
+
const responseFormat = options.responseFormat || null;
|
|
10
|
+
|
|
11
|
+
const body = { model, messages, temperature, max_tokens: maxTokens };
|
|
12
|
+
if (responseFormat) body.response_format = responseFormat;
|
|
13
|
+
|
|
14
|
+
const headers = {
|
|
15
|
+
'Content-Type': 'application/json',
|
|
16
|
+
'Authorization': `Bearer ${config.apiKey}`,
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
// Provider-specific headers
|
|
20
|
+
if (config.provider === 'openrouter') {
|
|
21
|
+
headers['HTTP-Referer'] = 'https://github.com/aikdna/kdna-studio-cli';
|
|
22
|
+
headers['X-Title'] = 'KDNA Studio CLI';
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
const res = await fetch(url, { method: 'POST', headers, body: JSON.stringify(body), signal: AbortSignal.timeout(120000) });
|
|
26
|
+
if (!res.ok) {
|
|
27
|
+
const text = await res.text();
|
|
28
|
+
throw new Error(`LLM API error ${res.status}: ${text.slice(0, 500)}`);
|
|
29
|
+
}
|
|
30
|
+
const data = await res.json();
|
|
31
|
+
const content = data.choices?.[0]?.message?.content;
|
|
32
|
+
if (!content) throw new Error('LLM returned empty response');
|
|
33
|
+
|
|
34
|
+
return {
|
|
35
|
+
content: content.trim(),
|
|
36
|
+
model: data.model || model,
|
|
37
|
+
usage: data.usage || null,
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
module.exports = { chat };
|