@aikdna/kdna-studio-cli 0.10.2 → 0.13.0-rc.components.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -161
- package/bin/kdna-studio.js +6 -2951
- package/docs/CREATION_COMMAND_CONTRACT.md +30 -0
- package/docs/TERMINAL_AGENT_CREATION.md +36 -0
- package/package.json +35 -24
- package/src/component-operations.js +13 -0
- package/src/public-bindings.json +8788 -0
- package/src/terminal-workspace.js +507 -0
- package/src/ai/distill.js +0 -77
- package/src/ai/feynman.js +0 -120
- package/src/ai/filter.js +0 -70
- package/src/ai/index.js +0 -6
- package/src/ai/interview.js +0 -144
- package/src/ai/testlab.js +0 -73
- package/src/llm/config.js +0 -85
- package/src/llm/index.js +0 -50
- package/src/llm/providers/anthropic.js +0 -40
- package/src/llm/providers/openai.js +0 -41
package/src/ai/feynman.js
DELETED
|
@@ -1,120 +0,0 @@
|
|
|
1
|
-
// Feynman Evaluation — AI-driven assessment of understanding quality
|
|
2
|
-
const llm = require('../llm');
|
|
3
|
-
|
|
4
|
-
const CRITERIA = {
|
|
5
|
-
notJustRepeat: 'NOT just repeating the axiom text verbatim — uses different words/phrasing',
|
|
6
|
-
notTooAbstract: 'NOT too abstract — grounds the explanation in concrete terms',
|
|
7
|
-
hasConcreteExample: 'Provides at least one specific, realistic example or scenario',
|
|
8
|
-
clarifiesBoundary: 'Makes clear when the judgment does NOT apply (boundary awareness)',
|
|
9
|
-
ordinaryPersonUnderstands: 'Would make sense to someone unfamiliar with the domain',
|
|
10
|
-
};
|
|
11
|
-
|
|
12
|
-
const SYSTEM = `You are a Feynman evaluation engine. Your task is to assess how well someone understands a judgment principle based on their restatement of it in their own words.
|
|
13
|
-
|
|
14
|
-
Evaluate the restatement on these 5 criteria (each YES/NO):
|
|
15
|
-
${Object.entries(CRITERIA).map(([k, v]) => `- ${k}: ${v}`).join('\n')}
|
|
16
|
-
|
|
17
|
-
Also provide a brief explanation for each criterion (1-2 sentences) and a suggested improvement if any criterion fails.
|
|
18
|
-
|
|
19
|
-
Return ONLY valid JSON:
|
|
20
|
-
{
|
|
21
|
-
"score": <0-5 integer>,
|
|
22
|
-
"criteria": {
|
|
23
|
-
"notJustRepeat": true/false,
|
|
24
|
-
"notTooAbstract": true/false,
|
|
25
|
-
"hasConcreteExample": true/false,
|
|
26
|
-
"clarifiesBoundary": true/false,
|
|
27
|
-
"ordinaryPersonUnderstands": true/false
|
|
28
|
-
},
|
|
29
|
-
"explanations": { "<criterion>": "<explanation>" },
|
|
30
|
-
"suggestions": ["<improvement suggestion>"]
|
|
31
|
-
}`;
|
|
32
|
-
|
|
33
|
-
async function evaluate(config, card, options = {}) {
|
|
34
|
-
// Bug: prior version read `card.feynman_text` (a field nothing in the
|
|
35
|
-
// codebase ever wrote) and flattened `card.one_sentence` /
|
|
36
|
-
// `card.full_statement` — the schema actually nests those under
|
|
37
|
-
// `card.fields`. The AI evaluator therefore always received an empty
|
|
38
|
-
// restatement and a blank axiom, no matter what was in the project.
|
|
39
|
-
// The fix reads the canonical schema field names and the nested
|
|
40
|
-
// `fields` object, with a fallback for callers that pass the flattened
|
|
41
|
-
// shape directly.
|
|
42
|
-
const fields = (card && card.fields) || card || {};
|
|
43
|
-
const axiomText = fields.one_sentence || fields.full_statement || fields.question || '';
|
|
44
|
-
const restatement = card.feynman_restatement
|
|
45
|
-
|| fields.feynman_restatement
|
|
46
|
-
// NOTE: fields.feynman_text was previously in this fallback chain
|
|
47
|
-
// but is intentionally excluded — no code path in @aikdna writes
|
|
48
|
-
// feynman_text. The canonical field is feynman_restatement.
|
|
49
|
-
|| '';
|
|
50
|
-
|
|
51
|
-
if (!restatement) {
|
|
52
|
-
// Bug (#62): prior version returned score 0 with the explanation
|
|
53
|
-
// "No Feynman restatement provided", but nothing in the CLI ever
|
|
54
|
-
// *writes* a feynman_restatement on a card. Every `kdna-studio
|
|
55
|
-
// feynman` call hit this path, every call returned 0, and the
|
|
56
|
-
// fynchmann command was effectively dead.
|
|
57
|
-
//
|
|
58
|
-
// The fix synthesises a first-cut restatement from the card's own
|
|
59
|
-
// text (so the LLM has *something* to evaluate), and returns a
|
|
60
|
-
// structured note in `suggestions` so the caller knows the score
|
|
61
|
-
// is on a machine-generated restatement, not their own. The
|
|
62
|
-
// caller can then `card update --field feynman_restatement='...'`
|
|
63
|
-
// to lock in a human-written version.
|
|
64
|
-
const synthesised = synthFeynmanRestatement(fields);
|
|
65
|
-
return {
|
|
66
|
-
score: 0,
|
|
67
|
-
criteria: {},
|
|
68
|
-
explanations: {
|
|
69
|
-
notJustRepeat: 'No human-written Feynman restatement — evaluated a synthesised one.',
|
|
70
|
-
notTooAbstract: 'Review the synthesised restatement before trusting the score.',
|
|
71
|
-
},
|
|
72
|
-
suggestions: [
|
|
73
|
-
'No feynman_restatement found on this card. The score below is on an auto-generated restatement.',
|
|
74
|
-
`Set a human-written restatement: kdna-studio card update <project> ${card.id || '<card-id>'} --field feynman_restatement='{"text":"<your plain-language restatement>"}'`,
|
|
75
|
-
`Synthesised restatement: "${synthesised}"`,
|
|
76
|
-
],
|
|
77
|
-
synthesised_restatement: synthesised,
|
|
78
|
-
};
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
const userPrompt = [
|
|
82
|
-
`## Original Axiom`,
|
|
83
|
-
`"${axiomText}"`,
|
|
84
|
-
``,
|
|
85
|
-
`## User's Restatement (Feynman)`,
|
|
86
|
-
`"${restatement}"`,
|
|
87
|
-
``,
|
|
88
|
-
`Evaluate this restatement.`,
|
|
89
|
-
].join('\n');
|
|
90
|
-
|
|
91
|
-
const result = await llm.chatWithJSON([
|
|
92
|
-
{ role: 'system', content: SYSTEM },
|
|
93
|
-
{ role: 'user', content: userPrompt },
|
|
94
|
-
], options);
|
|
95
|
-
|
|
96
|
-
return result.data || { score: 0, criteria: {}, explanations: {}, suggestions: [] };
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
// Synthesise a first-cut restatement from the card's own fields. The
|
|
100
|
-
// output is intentionally plain — Feynman is "explain it like I'm
|
|
101
|
-
// five", so we strip axiom-speak and surface only the human-readable
|
|
102
|
-
// core.
|
|
103
|
-
function synthFeynmanRestatement(fields) {
|
|
104
|
-
const text = fields.one_sentence
|
|
105
|
-
|| fields.full_statement
|
|
106
|
-
|| fields.question
|
|
107
|
-
|| fields.essence
|
|
108
|
-
|| fields.scope
|
|
109
|
-
|| '(no text to restate)';
|
|
110
|
-
// Strip the leading "always/never/must" markers that signal axiom
|
|
111
|
-
// language. The LLM evaluator will then score the result on its
|
|
112
|
-
// own merits, but the human reader can already see plain English.
|
|
113
|
-
return String(text)
|
|
114
|
-
.replace(/^(always|never|must|should|shall)\s+/i, '')
|
|
115
|
-
.replace(/\s+always\s+/gi, ' ')
|
|
116
|
-
.replace(/\s+never\s+/gi, ' ')
|
|
117
|
-
.trim();
|
|
118
|
-
}
|
|
119
|
-
|
|
120
|
-
module.exports = { evaluate, CRITERIA, SYSTEM, synthFeynmanRestatement };
|
package/src/ai/filter.js
DELETED
|
@@ -1,70 +0,0 @@
|
|
|
1
|
-
// Sensitive Content Filter — blocks 6 domains in English and Chinese
|
|
2
|
-
// Matches behavior of Studio App's SensitiveInferenceFilter
|
|
3
|
-
const SENSITIVE_PATTERNS = {
|
|
4
|
-
identity: {
|
|
5
|
-
en: /\b(passport\s*(no|number|#)|id\s*(card|number|#)|social\s*security|ssn|driver'?s?\s*license|national\s*id|身份证|护照号)\b/i,
|
|
6
|
-
zh: /(身份证号|护照号码|社保号|驾驶证号|军官证)/
|
|
7
|
-
},
|
|
8
|
-
health: {
|
|
9
|
-
en: /\b(medical\s*record|patient\s*(id|number)|diagnosis\s*(code|report)|HIV|AIDS|cancer\s*diagnosis|病历|诊断)\b/i,
|
|
10
|
-
zh: /(病历|诊断报告|体检报告|HIV|艾滋病|癌症诊断)/
|
|
11
|
-
},
|
|
12
|
-
political: {
|
|
13
|
-
en: /\b(political\s*(party|affiliation|view)|voting\s*record|party\s*membership|political|政治)\b/i,
|
|
14
|
-
zh: /(政治面貌|党派|政治立场|政见)/
|
|
15
|
-
},
|
|
16
|
-
religious: {
|
|
17
|
-
en: /\b(religious\s*(belief|affiliation|view)|religion|faith\s*(group|community)|宗教)\b/i,
|
|
18
|
-
zh: /(宗教信仰|宗教派别|教派)/
|
|
19
|
-
},
|
|
20
|
-
financial: {
|
|
21
|
-
en: /\b(bank\s*account|credit\s*card\s*number|financial\s*(record|history)|income\s*(level|bracket)|salary|银行|金融)\b/i,
|
|
22
|
-
zh: /(银行卡号|信用卡号|银行账户|收入水平|财务状况|工资)/
|
|
23
|
-
},
|
|
24
|
-
intimate: {
|
|
25
|
-
en: /\b(sexual\s*(orientation|preference|identity)|gender\s*identity|sex\s*(life|history)|marital\s*status|性|性别|婚姻)\b/i,
|
|
26
|
-
zh: /(性取向|性别认同|性生活|婚姻状况|性偏好)/
|
|
27
|
-
},
|
|
28
|
-
};
|
|
29
|
-
|
|
30
|
-
function check(text, options = {}) {
|
|
31
|
-
if (!text || typeof text !== 'string') return { safe: true, flagged: [], matches: {} };
|
|
32
|
-
|
|
33
|
-
const flagged = [];
|
|
34
|
-
const matches = {};
|
|
35
|
-
|
|
36
|
-
for (const [domain, patterns] of Object.entries(SENSITIVE_PATTERNS)) {
|
|
37
|
-
const activePatterns = [];
|
|
38
|
-
if (options.lang !== 'zh' && patterns.en) activePatterns.push(patterns.en);
|
|
39
|
-
if (options.lang !== 'en' && patterns.zh) activePatterns.push(patterns.zh);
|
|
40
|
-
|
|
41
|
-
for (const pattern of activePatterns) {
|
|
42
|
-
const match = text.match(pattern);
|
|
43
|
-
if (match) {
|
|
44
|
-
flagged.push(domain);
|
|
45
|
-
matches[domain] = match[0];
|
|
46
|
-
break;
|
|
47
|
-
}
|
|
48
|
-
}
|
|
49
|
-
}
|
|
50
|
-
|
|
51
|
-
return { safe: flagged.length === 0, flagged, matches };
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
function checkEvidence(evidenceList, options = {}) {
|
|
55
|
-
const results = [];
|
|
56
|
-
for (const evidence of evidenceList) {
|
|
57
|
-
const content = evidence.raw_text || evidence.content || '';
|
|
58
|
-
const result = check(content, options);
|
|
59
|
-
if (!result.safe) {
|
|
60
|
-
results.push({
|
|
61
|
-
filename: evidence.filename || evidence.name || 'unknown',
|
|
62
|
-
flagged: result.flagged,
|
|
63
|
-
matches: result.matches,
|
|
64
|
-
});
|
|
65
|
-
}
|
|
66
|
-
}
|
|
67
|
-
return results;
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
module.exports = { check, checkEvidence, SENSITIVE_PATTERNS };
|
package/src/ai/index.js
DELETED
package/src/ai/interview.js
DELETED
|
@@ -1,144 +0,0 @@
|
|
|
1
|
-
// AI Interview — 4-stage guided judgment authoring
|
|
2
|
-
const llm = require('../llm');
|
|
3
|
-
|
|
4
|
-
const STAGES = {
|
|
5
|
-
distillJudgment: {
|
|
6
|
-
label: 'Extract Judgment',
|
|
7
|
-
system: `You are a domain judgment expert. Your task is to help the user distill their raw knowledge into clear, actionable judgment principles (axioms).
|
|
8
|
-
|
|
9
|
-
For each axiom, ask the user:
|
|
10
|
-
1. What is the core judgment? (one sentence)
|
|
11
|
-
2. When does this apply? (specific triggers)
|
|
12
|
-
3. When does it NOT apply? (boundaries, exceptions)
|
|
13
|
-
4. What happens if they get this wrong? (failure risk)
|
|
14
|
-
|
|
15
|
-
Be conversational but focused. After the user provides their thoughts, synthesize them into a draft axiom with the format:
|
|
16
|
-
- one_sentence: "..."
|
|
17
|
-
- full_statement: "..."
|
|
18
|
-
- applies_when: [...]
|
|
19
|
-
- does_not_apply_when: [...]
|
|
20
|
-
- failure_risk: "..."
|
|
21
|
-
|
|
22
|
-
Then ask if they want to refine it or move to the next principle.`,
|
|
23
|
-
},
|
|
24
|
-
clarifyBoundaries: {
|
|
25
|
-
label: 'Clarify Boundaries',
|
|
26
|
-
system: `You are a boundary clarification expert. Review the axioms drafted so far and help the user identify gaps:
|
|
27
|
-
|
|
28
|
-
1. Where might these axioms conflict with each other?
|
|
29
|
-
2. What edge cases are not covered?
|
|
30
|
-
3. Are the "does not apply" conditions specific enough, or are they vague?
|
|
31
|
-
4. Would a reasonable person disagree with any of these judgments?
|
|
32
|
-
|
|
33
|
-
Present one gap at a time and ask the user to clarify. Be direct but constructive.`,
|
|
34
|
-
},
|
|
35
|
-
correctMisreadings: {
|
|
36
|
-
label: 'Correct Misreadings',
|
|
37
|
-
system: `You are a misunderstanding detector. For each axiom, imagine how an AI agent might MISINTERPRET it:
|
|
38
|
-
|
|
39
|
-
1. Literal vs intended meaning
|
|
40
|
-
2. Over-application (applying too broadly)
|
|
41
|
-
3. Under-application (not applying when it should)
|
|
42
|
-
4. Cultural or context-specific assumptions
|
|
43
|
-
|
|
44
|
-
For each potential misunderstanding, ask the user: "Is this a valid concern? How would you correct it?"
|
|
45
|
-
|
|
46
|
-
The user's corrections will become misunderstanding cards that guide the agent toward correct interpretation.`,
|
|
47
|
-
},
|
|
48
|
-
replayScenario: {
|
|
49
|
-
label: 'Replay Scenario',
|
|
50
|
-
system: `You are a scenario simulation expert. For the axioms and misunderstandings defined so far, create realistic test scenarios:
|
|
51
|
-
|
|
52
|
-
1. A clear-cut case where the judgment applies perfectly
|
|
53
|
-
2. An ambiguous case that tests the boundaries
|
|
54
|
-
3. A case where the judgment should NOT apply (tempting but wrong)
|
|
55
|
-
|
|
56
|
-
Present each scenario to the user and ask:
|
|
57
|
-
- "What would the correct judgment be here?"
|
|
58
|
-
- "What makes this tricky?"
|
|
59
|
-
- "Does this reveal any gaps in the axioms?"
|
|
60
|
-
|
|
61
|
-
The user's responses become the basis for eval cases and scenario cards.`,
|
|
62
|
-
},
|
|
63
|
-
};
|
|
64
|
-
|
|
65
|
-
async function runInterview(config, project, stage, context, options = {}) {
|
|
66
|
-
const stageConfig = STAGES[stage];
|
|
67
|
-
if (!stageConfig) throw new Error(`Unknown interview stage: ${stage}. Valid: ${Object.keys(STAGES).join(', ')}`);
|
|
68
|
-
|
|
69
|
-
const systemPrompt = stageConfig.system;
|
|
70
|
-
const userPrompt = [
|
|
71
|
-
`## Project: ${project.name || 'Untitled'}`,
|
|
72
|
-
project.description ? `## Description: ${project.description}` : '',
|
|
73
|
-
``,
|
|
74
|
-
`## Current State`,
|
|
75
|
-
`Cards: ${(project.cards || []).length} (${(project.cards || []).filter(c => c.locked).length} locked)`,
|
|
76
|
-
project.distillation_candidates ? `Candidates: ${project.distillation_candidates.length}` : '',
|
|
77
|
-
``,
|
|
78
|
-
`## Context`,
|
|
79
|
-
context || '(no additional context)',
|
|
80
|
-
``,
|
|
81
|
-
`Begin ${stageConfig.label} stage.`,
|
|
82
|
-
].filter(Boolean).join('\n');
|
|
83
|
-
|
|
84
|
-
const result = await llm.chat([
|
|
85
|
-
{ role: 'system', content: systemPrompt },
|
|
86
|
-
{ role: 'user', content: userPrompt },
|
|
87
|
-
], options);
|
|
88
|
-
|
|
89
|
-
return { stage, label: stageConfig.label, content: result.content };
|
|
90
|
-
}
|
|
91
|
-
|
|
92
|
-
async function runInterviewInteractive(config, project, options = {}) {
|
|
93
|
-
const readline = require('readline');
|
|
94
|
-
const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
|
|
95
|
-
const ask = (q) => new Promise(resolve => rl.question(q, resolve));
|
|
96
|
-
|
|
97
|
-
console.log(`\n=== KDNA Studio — AI Interview ===`);
|
|
98
|
-
console.log(`Project: ${project.name || 'Untitled'}`);
|
|
99
|
-
console.log(`4 stages: distill → clarify → correct → replay\n`);
|
|
100
|
-
|
|
101
|
-
const conversation = [];
|
|
102
|
-
let results = [];
|
|
103
|
-
|
|
104
|
-
for (const [stageId, stageConfig] of Object.entries(STAGES)) {
|
|
105
|
-
console.log(`\n--- ${stageConfig.label} ---`);
|
|
106
|
-
|
|
107
|
-
const contextParts = [`Project: ${project.name || 'Untitled'}`];
|
|
108
|
-
if (project.description) contextParts.push(`Description: ${project.description}`);
|
|
109
|
-
const cardsSummary = (project.cards || []).map(c => `[${c.type}] ${c.one_sentence || c.concept_name || c.wrong_interpretation || ''}`).join('\n');
|
|
110
|
-
if (cardsSummary) contextParts.push(`Existing cards:\n${cardsSummary}`);
|
|
111
|
-
|
|
112
|
-
const initialResponse = await llm.chat([
|
|
113
|
-
{ role: 'system', content: stageConfig.system },
|
|
114
|
-
{ role: 'user', content: `Start the ${stageConfig.label} stage.\n\nContext:\n${contextParts.join('\n')}` },
|
|
115
|
-
], options);
|
|
116
|
-
console.log(`\nAI: ${initialResponse.content}\n`);
|
|
117
|
-
|
|
118
|
-
let keepGoing = true;
|
|
119
|
-
while (keepGoing) {
|
|
120
|
-
const answer = await ask('You (type "next" to proceed, "done" to finish this stage): ');
|
|
121
|
-
if (answer.toLowerCase() === 'next' || answer.toLowerCase() === 'done') {
|
|
122
|
-
keepGoing = false;
|
|
123
|
-
if (answer.toLowerCase() === 'done') break;
|
|
124
|
-
continue;
|
|
125
|
-
}
|
|
126
|
-
if (!answer.trim()) continue;
|
|
127
|
-
|
|
128
|
-
conversation.push({ stage: stageId, role: 'user', content: answer });
|
|
129
|
-
const response = await llm.chat([
|
|
130
|
-
{ role: 'system', content: stageConfig.system },
|
|
131
|
-
...conversation.filter(m => m.stage === stageId).map(m => ({ role: m.role, content: m.content })),
|
|
132
|
-
], options);
|
|
133
|
-
conversation.push({ stage: stageId, role: 'assistant', content: response.content });
|
|
134
|
-
console.log(`\nAI: ${response.content}\n`);
|
|
135
|
-
}
|
|
136
|
-
|
|
137
|
-
results.push({ stage: stageId, label: stageConfig.label, conversation: conversation.filter(m => m.stage === stageId) });
|
|
138
|
-
}
|
|
139
|
-
|
|
140
|
-
rl.close();
|
|
141
|
-
return results;
|
|
142
|
-
}
|
|
143
|
-
|
|
144
|
-
module.exports = { STAGES, runInterview, runInterviewInteractive };
|
package/src/ai/testlab.js
DELETED
|
@@ -1,73 +0,0 @@
|
|
|
1
|
-
// TestLab — A/B comparison testing with LLM
|
|
2
|
-
const llm = require('../llm');
|
|
3
|
-
|
|
4
|
-
const COMPARE_SYSTEM = `You are a controlled experiment evaluator. You will see the same user input twice — once with a knowledge domain loaded, and once without. Your task is to produce a structured comparison.
|
|
5
|
-
|
|
6
|
-
IMPORTANT: You must respond with raw JSON only, no markdown wrapping, no explanatory text.
|
|
7
|
-
|
|
8
|
-
The JSON format:
|
|
9
|
-
{
|
|
10
|
-
"without_kdna": {
|
|
11
|
-
"response": "how you would respond WITHOUT the domain knowledge",
|
|
12
|
-
"classification": "how you categorized the task",
|
|
13
|
-
"diagnosis": "your analysis of the situation",
|
|
14
|
-
"actions": "what actions you would take"
|
|
15
|
-
},
|
|
16
|
-
"with_kdna": {
|
|
17
|
-
"response": "how you respond WITH the domain knowledge applied",
|
|
18
|
-
"classification": "domain-specific classification from the loaded domain",
|
|
19
|
-
"diagnosis": "analysis informed by domain axioms and ontology",
|
|
20
|
-
"actions": "actions influenced by domain judgment"
|
|
21
|
-
},
|
|
22
|
-
"delta": {
|
|
23
|
-
"classification_changed": true/false,
|
|
24
|
-
"diagnosis_changed": true/false,
|
|
25
|
-
"actions_changed": true/false,
|
|
26
|
-
"terminology_changed": true/false,
|
|
27
|
-
"overall_impact": "none" | "minor" | "moderate" | "significant"
|
|
28
|
-
}
|
|
29
|
-
}`;
|
|
30
|
-
|
|
31
|
-
async function compare(config, domainName, input, domainPrompt, options = {}) {
|
|
32
|
-
// Run WITHOUT KDNA
|
|
33
|
-
const withoutResult = await llm.chat([
|
|
34
|
-
{ role: 'system', content: COMPARE_SYSTEM },
|
|
35
|
-
{ role: 'user', content: `Respond to the following WITHOUT any special domain knowledge:\n\n${input}\n\nRespond with the structured JSON comparison format. For the "without_kdna" section, fill in your normal response. For the "with_kdna" section, fill in what you think would change (we will verify this in a separate pass).` },
|
|
36
|
-
], { ...options, responseFormat: { type: 'json_object' } });
|
|
37
|
-
|
|
38
|
-
let withoutData;
|
|
39
|
-
try { withoutData = JSON.parse(withoutResult.content); } catch { withoutData = { without_kdna: { response: withoutResult.content } }; }
|
|
40
|
-
|
|
41
|
-
// Run WITH KDNA
|
|
42
|
-
const withResult = await llm.chat([
|
|
43
|
-
{ role: 'system', content: `${domainPrompt}\n\n${COMPARE_SYSTEM}` },
|
|
44
|
-
{ role: 'user', content: `Respond to the following WITH the domain knowledge loaded:\n\n${input}\n\nRespond with the structured JSON comparison format. Fill in the "with_kdna" section and the "delta" section based on what actually changed.` },
|
|
45
|
-
], { ...options, responseFormat: { type: 'json_object' } });
|
|
46
|
-
|
|
47
|
-
let withData;
|
|
48
|
-
try { withData = JSON.parse(withResult.content); } catch { withData = { with_kdna: { response: withResult.content } }; }
|
|
49
|
-
|
|
50
|
-
return {
|
|
51
|
-
input,
|
|
52
|
-
domain: domainName,
|
|
53
|
-
without_kdna: withoutData.without_kdna || { response: withoutResult.content },
|
|
54
|
-
with_kdna: withData.with_kdna || { response: withResult.content },
|
|
55
|
-
delta: withData.delta || { overall_impact: 'unknown' },
|
|
56
|
-
};
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
async function testPreset(config, domainName, input, domainPrompt, options = {}) {
|
|
60
|
-
const presets = {
|
|
61
|
-
baseline: `Test the core judgment: ${input}`,
|
|
62
|
-
edge_case: `Test a boundary case where domain judgment may or may not apply: ${input}`,
|
|
63
|
-
contradiction: `Test for potential contradiction: present a scenario that challenges the domain's axioms: ${input}`,
|
|
64
|
-
};
|
|
65
|
-
|
|
66
|
-
const results = {};
|
|
67
|
-
for (const [name, prompt] of Object.entries(presets)) {
|
|
68
|
-
results[name] = await compare(config, domainName, prompt, domainPrompt, options);
|
|
69
|
-
}
|
|
70
|
-
return results;
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
module.exports = { compare, testPreset, COMPARE_SYSTEM };
|
package/src/llm/config.js
DELETED
|
@@ -1,85 +0,0 @@
|
|
|
1
|
-
// LLM Provider Configuration
|
|
2
|
-
// Sources (priority order): env vars > ~/.kdna/config.json > defaults
|
|
3
|
-
const fs = require('fs');
|
|
4
|
-
const path = require('path');
|
|
5
|
-
const os = require('os');
|
|
6
|
-
|
|
7
|
-
const ENV_PREFIX = 'KDNA_LLM_';
|
|
8
|
-
|
|
9
|
-
const WELL_KNOWN = {
|
|
10
|
-
openai: { baseURL: 'https://api.openai.com/v1', models: ['gpt-4o', 'gpt-4o-mini', 'gpt-4-turbo'] },
|
|
11
|
-
anthropic: { baseURL: 'https://api.anthropic.com/v1', models: ['claude-sonnet-4-20250514', 'claude-3-5-haiku-latest'] },
|
|
12
|
-
deepseek: { baseURL: 'https://api.deepseek.com/v1', models: ['deepseek-chat', 'deepseek-reasoner'] },
|
|
13
|
-
openrouter: { baseURL: 'https://openrouter.ai/api/v1', models: ['openai/gpt-4o', 'anthropic/claude-sonnet-4', 'deepseek/deepseek-chat'] },
|
|
14
|
-
gemini: { baseURL: 'https://generativelanguage.googleapis.com/v1beta', models: ['gemini-2.5-flash', 'gemini-2.5-pro'] },
|
|
15
|
-
perplexity: { baseURL: 'https://api.perplexity.ai', models: ['sonar-pro', 'sonar'] },
|
|
16
|
-
groq: { baseURL: 'https://api.groq.com/openai/v1', models: ['llama-4-scout-17b-16e-instruct', 'deepseek-r1-distill-llama-70b'] },
|
|
17
|
-
local: { baseURL: 'http://localhost:11434/v1', models: ['llama3.2'] },
|
|
18
|
-
ollama: { baseURL: 'http://localhost:11434/v1', models: ['llama3.2'] },
|
|
19
|
-
openai_compatible: { baseURL: null, models: [] },
|
|
20
|
-
};
|
|
21
|
-
|
|
22
|
-
function loadConfig() {
|
|
23
|
-
const config = { provider: null, apiKey: null, model: null, baseURL: null, temperature: 0.7, maxTokens: 4096 };
|
|
24
|
-
|
|
25
|
-
// 1. Read from ~/.kdna/config.json
|
|
26
|
-
const configPath = path.join(os.homedir(), '.kdna', 'config.json');
|
|
27
|
-
try { if (fs.existsSync(configPath)) Object.assign(config, JSON.parse(fs.readFileSync(configPath, 'utf8')).llm || {}); } catch (e) { console.error('Error reading LLM config file:', e.message); }
|
|
28
|
-
|
|
29
|
-
// 2. Env vars override
|
|
30
|
-
if (process.env.KDNA_LLM_PROVIDER) config.provider = process.env.KDNA_LLM_PROVIDER;
|
|
31
|
-
if (process.env.KDNA_LLM_API_KEY) config.apiKey = process.env.KDNA_LLM_API_KEY;
|
|
32
|
-
if (process.env.KDNA_LLM_MODEL) config.model = process.env.KDNA_LLM_MODEL;
|
|
33
|
-
if (process.env.KDNA_LLM_BASE_URL) config.baseURL = process.env.KDNA_LLM_BASE_URL;
|
|
34
|
-
if (process.env.KDNA_LLM_TEMPERATURE) config.temperature = parseFloat(process.env.KDNA_LLM_TEMPERATURE);
|
|
35
|
-
if (process.env.KDNA_LLM_MAX_TOKENS) config.maxTokens = parseInt(process.env.KDNA_LLM_MAX_TOKENS, 10);
|
|
36
|
-
|
|
37
|
-
// 3. Resolve well-known provider
|
|
38
|
-
if (config.provider && WELL_KNOWN[config.provider]) {
|
|
39
|
-
const wk = WELL_KNOWN[config.provider];
|
|
40
|
-
if (wk.baseURL && !config.baseURL) config.baseURL = wk.baseURL;
|
|
41
|
-
if (wk.models.length > 0 && !config.model) config.model = wk.models[0];
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
// 4. Provider-specific env var (KDNA_LLM_API_KEY_<PROVIDER>)
|
|
45
|
-
if (!config.apiKey && config.provider) {
|
|
46
|
-
const keyEnv = `KDNA_LLM_API_KEY_${config.provider.toUpperCase().replace(/-/g, '_')}`;
|
|
47
|
-
if (process.env[keyEnv]) config.apiKey = process.env[keyEnv];
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
// 5. Provider-native fallback — each provider only uses its own native env var
|
|
51
|
-
if (!config.apiKey && config.provider) {
|
|
52
|
-
const PROVIDER_NATIVE_KEYS = {
|
|
53
|
-
openai: 'OPENAI_API_KEY',
|
|
54
|
-
anthropic: 'ANTHROPIC_API_KEY',
|
|
55
|
-
deepseek: 'DEEPSEEK_API_KEY',
|
|
56
|
-
groq: 'GROQ_API_KEY',
|
|
57
|
-
gemini: 'GEMINI_API_KEY',
|
|
58
|
-
perplexity: 'PERPLEXITY_API_KEY',
|
|
59
|
-
openrouter: 'OPENROUTER_API_KEY',
|
|
60
|
-
};
|
|
61
|
-
const nativeEnv = PROVIDER_NATIVE_KEYS[config.provider];
|
|
62
|
-
if (nativeEnv && process.env[nativeEnv]) config.apiKey = process.env[nativeEnv];
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
return config;
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
function saveConfig(updates) {
|
|
69
|
-
const configPath = path.join(os.homedir(), '.kdna', 'config.json');
|
|
70
|
-
let existing = {};
|
|
71
|
-
try { if (fs.existsSync(configPath)) existing = JSON.parse(fs.readFileSync(configPath, 'utf8')); } catch (e) { console.error('Error reading config file for save:', e.message); }
|
|
72
|
-
existing.llm = { ...(existing.llm || {}), ...updates };
|
|
73
|
-
fs.mkdirSync(path.dirname(configPath), { recursive: true });
|
|
74
|
-
fs.writeFileSync(configPath, JSON.stringify(existing, null, 2) + '\n');
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
function validateConfig(config) {
|
|
78
|
-
const errors = [];
|
|
79
|
-
if (!config.provider) errors.push('Missing KDNA_LLM_PROVIDER. Set via env var or ~/.kdna/config.json');
|
|
80
|
-
if (!config.apiKey) errors.push('Missing KDNA_LLM_API_KEY');
|
|
81
|
-
if (!config.model) errors.push('Missing KDNA_LLM_MODEL');
|
|
82
|
-
return { valid: errors.length === 0, errors };
|
|
83
|
-
}
|
|
84
|
-
|
|
85
|
-
module.exports = { loadConfig, saveConfig, validateConfig, WELL_KNOWN };
|
package/src/llm/index.js
DELETED
|
@@ -1,50 +0,0 @@
|
|
|
1
|
-
// KDNA Studio CLI — unified LLM interface
|
|
2
|
-
// Supports OpenAI-compatible and Anthropic APIs
|
|
3
|
-
// Configuration via KDNA_LLM_* env vars or ~/.kdna/config.json
|
|
4
|
-
const { loadConfig, saveConfig, validateConfig } = require('./config');
|
|
5
|
-
const openai = require('./providers/openai');
|
|
6
|
-
const anthropic = require('./providers/anthropic');
|
|
7
|
-
|
|
8
|
-
function getProvider(config) {
|
|
9
|
-
if (!config || !config.provider) throw new Error('LLM not configured. Run: kdna-studio llm config');
|
|
10
|
-
if (config.provider === 'anthropic') return anthropic;
|
|
11
|
-
return openai; // All OpenAI-compatible providers use the same API
|
|
12
|
-
}
|
|
13
|
-
|
|
14
|
-
async function chat(messages, options = {}) {
|
|
15
|
-
const config = loadConfig();
|
|
16
|
-
if (options.provider) config.provider = options.provider;
|
|
17
|
-
if (options.model) config.model = options.model;
|
|
18
|
-
if (options.apiKey) config.apiKey = options.apiKey;
|
|
19
|
-
if (options.baseURL) config.baseURL = options.baseURL;
|
|
20
|
-
|
|
21
|
-
const validation = validateConfig(config);
|
|
22
|
-
if (!validation.valid) throw new Error(`LLM configuration error:\n${validation.errors.map(e => ` - ${e}`).join('\n')}`);
|
|
23
|
-
|
|
24
|
-
const provider = getProvider(config);
|
|
25
|
-
return provider.chat(config, messages, options);
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
async function chatWithJSON(messages, options = {}) {
|
|
29
|
-
const result = await chat(messages, { ...options, responseFormat: { type: 'json_object' } });
|
|
30
|
-
try {
|
|
31
|
-
result.data = JSON.parse(result.content);
|
|
32
|
-
} catch (e) {
|
|
33
|
-
console.error('Error parsing LLM JSON response:', e.message);
|
|
34
|
-
// Try to extract JSON from markdown code blocks
|
|
35
|
-
const match = result.content.match(/```(?:json)?\s*([\s\S]*?)```/);
|
|
36
|
-
if (match) {
|
|
37
|
-
try { result.data = JSON.parse(match[1].trim()); } catch (e2) { console.error('Error parsing JSON from markdown code block:', e2.message); }
|
|
38
|
-
}
|
|
39
|
-
}
|
|
40
|
-
return result;
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
function config() { return loadConfig(); }
|
|
44
|
-
|
|
45
|
-
function configure(updates) {
|
|
46
|
-
saveConfig(updates);
|
|
47
|
-
return loadConfig();
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
module.exports = { chat, chatWithJSON, config, configure };
|
|
@@ -1,40 +0,0 @@
|
|
|
1
|
-
// Anthropic Messages API provider
|
|
2
|
-
|
|
3
|
-
async function chat(config, messages, options = {}) {
|
|
4
|
-
const url = `${config.baseURL}/messages`;
|
|
5
|
-
const model = options.model || config.model;
|
|
6
|
-
const maxTokens = options.maxTokens || config.maxTokens || 4096;
|
|
7
|
-
|
|
8
|
-
// Convert OpenAI-format messages to Anthropic format
|
|
9
|
-
const systemMessages = messages.filter(m => m.role === 'system').map(m => m.content).join('\n\n');
|
|
10
|
-
const conversationMessages = messages.filter(m => m.role !== 'system').map(m => ({
|
|
11
|
-
role: m.role === 'assistant' ? 'assistant' : 'user',
|
|
12
|
-
content: m.content,
|
|
13
|
-
}));
|
|
14
|
-
|
|
15
|
-
const body = { model, max_tokens: maxTokens, messages: conversationMessages };
|
|
16
|
-
if (systemMessages) body.system = systemMessages;
|
|
17
|
-
|
|
18
|
-
const headers = {
|
|
19
|
-
'Content-Type': 'application/json',
|
|
20
|
-
'x-api-key': config.apiKey,
|
|
21
|
-
'anthropic-version': '2023-06-01',
|
|
22
|
-
};
|
|
23
|
-
|
|
24
|
-
const res = await fetch(url, { method: 'POST', headers, body: JSON.stringify(body), signal: AbortSignal.timeout(120000) });
|
|
25
|
-
if (!res.ok) {
|
|
26
|
-
const text = await res.text();
|
|
27
|
-
throw new Error(`Anthropic API error ${res.status}: ${text.slice(0, 500)}`);
|
|
28
|
-
}
|
|
29
|
-
const data = await res.json();
|
|
30
|
-
const content = data.content?.[0]?.text;
|
|
31
|
-
if (!content) throw new Error('Anthropic returned empty response');
|
|
32
|
-
|
|
33
|
-
return {
|
|
34
|
-
content: content.trim(),
|
|
35
|
-
model: data.model || model,
|
|
36
|
-
usage: data.usage || null,
|
|
37
|
-
};
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
module.exports = { chat };
|
|
@@ -1,41 +0,0 @@
|
|
|
1
|
-
// OpenAI-compatible chat completions provider
|
|
2
|
-
// Works with: OpenAI, DeepSeek, OpenRouter, Groq, Ollama, LM Studio, Perplexity, and any OpenAI-compatible API
|
|
3
|
-
|
|
4
|
-
async function chat(config, messages, options = {}) {
|
|
5
|
-
const url = `${config.baseURL}/chat/completions`;
|
|
6
|
-
const model = options.model || config.model;
|
|
7
|
-
const temperature = options.temperature ?? config.temperature ?? 0.7;
|
|
8
|
-
const maxTokens = options.maxTokens || config.maxTokens || 4096;
|
|
9
|
-
const responseFormat = options.responseFormat || null;
|
|
10
|
-
|
|
11
|
-
const body = { model, messages, temperature, max_tokens: maxTokens };
|
|
12
|
-
if (responseFormat) body.response_format = responseFormat;
|
|
13
|
-
|
|
14
|
-
const headers = {
|
|
15
|
-
'Content-Type': 'application/json',
|
|
16
|
-
'Authorization': `Bearer ${config.apiKey}`,
|
|
17
|
-
};
|
|
18
|
-
|
|
19
|
-
// Provider-specific headers
|
|
20
|
-
if (config.provider === 'openrouter') {
|
|
21
|
-
headers['HTTP-Referer'] = 'https://github.com/aikdna/kdna-studio-cli';
|
|
22
|
-
headers['X-Title'] = 'KDNA Studio CLI';
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
const res = await fetch(url, { method: 'POST', headers, body: JSON.stringify(body), signal: AbortSignal.timeout(120000) });
|
|
26
|
-
if (!res.ok) {
|
|
27
|
-
const text = await res.text();
|
|
28
|
-
throw new Error(`LLM API error ${res.status}: ${text.slice(0, 500)}`);
|
|
29
|
-
}
|
|
30
|
-
const data = await res.json();
|
|
31
|
-
const content = data.choices?.[0]?.message?.content;
|
|
32
|
-
if (!content) throw new Error('LLM returned empty response');
|
|
33
|
-
|
|
34
|
-
return {
|
|
35
|
-
content: content.trim(),
|
|
36
|
-
model: data.model || model,
|
|
37
|
-
usage: data.usage || null,
|
|
38
|
-
};
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
module.exports = { chat };
|