daftari 1.15.0 → 1.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -0
- package/README.md +8 -2
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +6 -0
- package/dist/cli.js.map +1 -1
- package/dist/eval/generate.d.ts +12 -0
- package/dist/eval/generate.d.ts.map +1 -0
- package/dist/eval/generate.js +221 -0
- package/dist/eval/generate.js.map +1 -0
- package/dist/eval/index.d.ts +2 -0
- package/dist/eval/index.d.ts.map +1 -0
- package/dist/eval/index.js +311 -0
- package/dist/eval/index.js.map +1 -0
- package/dist/eval/llm.d.ts +47 -0
- package/dist/eval/llm.d.ts.map +1 -0
- package/dist/eval/llm.js +165 -0
- package/dist/eval/llm.js.map +1 -0
- package/dist/eval/prompts.d.ts +5 -0
- package/dist/eval/prompts.d.ts.map +1 -0
- package/dist/eval/prompts.js +44 -0
- package/dist/eval/prompts.js.map +1 -0
- package/dist/eval/run.d.ts +13 -0
- package/dist/eval/run.d.ts.map +1 -0
- package/dist/eval/run.js +78 -0
- package/dist/eval/run.js.map +1 -0
- package/dist/eval/score.d.ts +12 -0
- package/dist/eval/score.d.ts.map +1 -0
- package/dist/eval/score.js +154 -0
- package/dist/eval/score.js.map +1 -0
- package/dist/eval/storage.d.ts +10 -0
- package/dist/eval/storage.d.ts.map +1 -0
- package/dist/eval/storage.js +69 -0
- package/dist/eval/storage.js.map +1 -0
- package/dist/eval/subgraph.d.ts +17 -0
- package/dist/eval/subgraph.d.ts.map +1 -0
- package/dist/eval/subgraph.js +214 -0
- package/dist/eval/subgraph.js.map +1 -0
- package/dist/eval/tool-surface.d.ts +7 -0
- package/dist/eval/tool-surface.d.ts.map +1 -0
- package/dist/eval/tool-surface.js +160 -0
- package/dist/eval/tool-surface.js.map +1 -0
- package/dist/eval/types.d.ts +173 -0
- package/dist/eval/types.d.ts.map +1 -0
- package/dist/eval/types.js +44 -0
- package/dist/eval/types.js.map +1 -0
- package/package.json +2 -1
package/dist/eval/run.js
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
// src/eval/run.ts
|
|
2
|
+
// The answerer loop. For each question, run the answerer LLM k independent
|
|
3
|
+
// times against the in-process tool surface, recording a full trace per run.
|
|
4
|
+
// Results are keyed by `"${question_index}:${k_index}"` so --resume can skip
|
|
5
|
+
// pairs already marked complete and re-run only the rest.
|
|
6
|
+
import { err, ok } from "../frontmatter/types.js";
|
|
7
|
+
import { ANSWERER_SYSTEM_PROMPT, PROMPT_VERSION } from "./prompts.js";
|
|
8
|
+
import { buildToolSurface } from "./tool-surface.js";
|
|
9
|
+
export async function runAnswerer(questions, vaultRoot, llm, opts) {
|
|
10
|
+
const ts = opts.timestamp ?? "2026-01-01T00:00:00Z";
|
|
11
|
+
const id = opts.resumeFrom?.id ?? opts.runId ?? `${questions.id}-${opts.model}-${ts}`;
|
|
12
|
+
const runs = { ...(opts.resumeFrom?.runs ?? {}) };
|
|
13
|
+
// Builds the current EvalRun from the live `runs` map. Snapshotting after
|
|
14
|
+
// each (q,k) status change lets the caller persist partial progress, so a
|
|
15
|
+
// failure/crash leaves a resumable file on disk (see --resume).
|
|
16
|
+
const snapshot = () => ({
|
|
17
|
+
id,
|
|
18
|
+
questions_id: questions.id,
|
|
19
|
+
answerer_model: opts.model,
|
|
20
|
+
prompt_version: PROMPT_VERSION,
|
|
21
|
+
timestamp: ts,
|
|
22
|
+
k: opts.k,
|
|
23
|
+
runs,
|
|
24
|
+
});
|
|
25
|
+
const tools = buildToolSurface(vaultRoot);
|
|
26
|
+
const toolDefs = tools.defs;
|
|
27
|
+
for (let qi = 0; qi < questions.questions.length; qi++) {
|
|
28
|
+
const q = questions.questions[qi];
|
|
29
|
+
for (let k = 0; k < opts.k; k++) {
|
|
30
|
+
const key = `${qi}:${k}`;
|
|
31
|
+
if (runs[key]?.status === "complete")
|
|
32
|
+
continue;
|
|
33
|
+
const t0 = Date.now();
|
|
34
|
+
const r = await llm.completeWithTools({
|
|
35
|
+
model: opts.model,
|
|
36
|
+
system: ANSWERER_SYSTEM_PROMPT,
|
|
37
|
+
user: q.question,
|
|
38
|
+
tools: toolDefs,
|
|
39
|
+
toolHandler: tools.handler,
|
|
40
|
+
});
|
|
41
|
+
const wall_ms = Date.now() - t0;
|
|
42
|
+
if (!r.ok) {
|
|
43
|
+
// Mark this pair incomplete, persist the partial run (completed pairs
|
|
44
|
+
// plus this incomplete one), then surface the error. Persisting here
|
|
45
|
+
// is what makes --resume work: progress made before the failure is
|
|
46
|
+
// saved, so a re-run skips the completed pairs and retries the rest.
|
|
47
|
+
runs[key] = {
|
|
48
|
+
question_id: q.id,
|
|
49
|
+
question_index: qi,
|
|
50
|
+
k_index: k,
|
|
51
|
+
status: "incomplete",
|
|
52
|
+
trace: null,
|
|
53
|
+
};
|
|
54
|
+
await opts.persist?.(snapshot());
|
|
55
|
+
return err(r.error);
|
|
56
|
+
}
|
|
57
|
+
const trace = {
|
|
58
|
+
tool_calls: r.value.tool_calls,
|
|
59
|
+
final_answer: r.value.text,
|
|
60
|
+
total_tool_calls: r.value.tool_calls.length,
|
|
61
|
+
input_tokens: r.value.input_tokens,
|
|
62
|
+
output_tokens: r.value.output_tokens,
|
|
63
|
+
wall_ms,
|
|
64
|
+
stop_reason: r.value.stop_reason,
|
|
65
|
+
};
|
|
66
|
+
runs[key] = {
|
|
67
|
+
question_id: q.id,
|
|
68
|
+
question_index: qi,
|
|
69
|
+
k_index: k,
|
|
70
|
+
status: "complete",
|
|
71
|
+
trace,
|
|
72
|
+
};
|
|
73
|
+
await opts.persist?.(snapshot());
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
return ok(snapshot());
|
|
77
|
+
}
|
|
78
|
+
//# sourceMappingURL=run.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"run.js","sourceRoot":"","sources":["../../src/eval/run.ts"],"names":[],"mappings":"AAAA,kBAAkB;AAClB,2EAA2E;AAC3E,6EAA6E;AAC7E,6EAA6E;AAC7E,0DAA0D;AAE1D,OAAO,EAAE,GAAG,EAAE,EAAE,EAAe,MAAM,yBAAyB,CAAC;AAE/D,OAAO,EAAE,sBAAsB,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AACtE,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAYrD,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,SAAsB,EACtB,SAAiB,EACjB,GAAc,EACd,IAAgB;IAEhB,MAAM,EAAE,GAAG,IAAI,CAAC,SAAS,IAAI,sBAAsB,CAAC;IACpD,MAAM,EAAE,GAAG,IAAI,CAAC,UAAU,EAAE,EAAE,IAAI,IAAI,CAAC,KAAK,IAAI,GAAG,SAAS,CAAC,EAAE,IAAI,IAAI,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;IACtF,MAAM,IAAI,GAAiC,EAAE,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,EAAE,CAAC;IAEhF,0EAA0E;IAC1E,0EAA0E;IAC1E,gEAAgE;IAChE,MAAM,QAAQ,GAAG,GAAY,EAAE,CAAC,CAAC;QAC/B,EAAE;QACF,YAAY,EAAE,SAAS,CAAC,EAAE;QAC1B,cAAc,EAAE,IAAI,CAAC,KAAK;QAC1B,cAAc,EAAE,cAAc;QAC9B,SAAS,EAAE,EAAE;QACb,CAAC,EAAE,IAAI,CAAC,CAAC;QACT,IAAI;KACL,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAC;IAC1C,MAAM,QAAQ,GAAc,KAAK,CAAC,IAAI,CAAC;IAEvC,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,SAAS,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC;QACvD,MAAM,CAAC,GAAG,SAAS,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAChC,MAAM,GAAG,GAAG,GAAG,EAAE,IAAI,CAAC,EAAE,CAAC;YACzB,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,MAAM,KAAK,UAAU;gBAAE,SAAS;YAE/C,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACtB,MAAM,CAAC,GAAG,MAAM,GAAG,CAAC,iBAAiB,CAAC;gBACpC,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,MAAM,EAAE,sBAAsB;gBAC9B,IAAI,EAAE,CAAC,CAAC,QAAQ;gBAChB,KAAK,EAAE,QAAQ;gBACf,WAAW,EAAE,KAAK,CAAC,OAAO;aAC3B,CAAC,CAAC;YACH,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;YAChC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;gBACV,sEAAsE;gBACtE,qEAAqE;gBACrE,mEAAmE;gBACnE,qEAAqE;gBACrE,IAAI,CAAC,GAAG,CAAC,GAAG;oBACV,WAAW,EAAE,CAAC,CAAC,EAAE;oBACjB,cAAc,EAAE,EAAE;oBAClB,OAAO,EAAE,CAAC;oBACV,MAAM,EAAE,YAAY;oBACpB,KAAK,EAAE,IAAI;iBACZ,CAAC;gBACF,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,CAAC;gBACjC,OAAO,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YACtB,CAAC;YAED,MAAM,KAAK,GAAU;gBACnB,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,UAAU;gBAC9B,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI;gBAC1B,gBAAgB,EAAE,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM;gBAC3C,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,YAAY;gBAClC,aAAa,EAAE,CAAC,CAAC,KAAK,CAAC,aAAa;gBACpC,OAAO;gBACP,WAAW,EAAE,CAAC,CAAC,KAAK,CAAC,WAAW;aACjC,CAAC;YACF,IAAI,CAAC,GAAG,CAAC,GAAG;gBACV,WAAW,EAAE,CAAC,CAAC,EAAE;gBACjB,cAAc,EAAE,EAAE;gBAClB,OAAO,EAAE,CAAC;gBACV,MAAM,EAAE,UAAU;gBAClB,KAAK;aACN,CAAC;YACF,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,CAAC;QACnC,CAAC;IACH,CAAC;IAED,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,CAAC;AACxB,CAAC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { type Result } from "../frontmatter/types.js";
|
|
2
|
+
import type { LlmClient } from "./llm.js";
|
|
3
|
+
import { type CortexEvalError, type Grade, type Question, type Score, type Trace } from "./types.js";
|
|
4
|
+
export interface AggregateOptions {
|
|
5
|
+
traces: Map<string, Trace>;
|
|
6
|
+
}
|
|
7
|
+
export declare function aggregateScore(grades: Grade[], questions: Question[], opts: AggregateOptions): Score;
|
|
8
|
+
export interface GradeOptions {
|
|
9
|
+
model: string;
|
|
10
|
+
}
|
|
11
|
+
export declare function gradeAnswer(question: Question, questionIndex: number, kIndex: number, trace: Trace, llm: LlmClient, opts: GradeOptions): Promise<Result<Grade, CortexEvalError>>;
|
|
12
|
+
//# sourceMappingURL=score.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"score.d.ts","sourceRoot":"","sources":["../../src/eval/score.ts"],"names":[],"mappings":"AAKA,OAAO,EAAM,KAAK,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAC1D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAE1C,OAAO,EACL,KAAK,eAAe,EACpB,KAAK,KAAK,EAEV,KAAK,QAAQ,EACb,KAAK,KAAK,EAKV,KAAK,KAAK,EACX,MAAM,YAAY,CAAC;AAEpB,MAAM,WAAW,gBAAgB;IAE/B,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;CAC5B;AAUD,wBAAgB,cAAc,CAC5B,MAAM,EAAE,KAAK,EAAE,EACf,SAAS,EAAE,QAAQ,EAAE,EACrB,IAAI,EAAE,gBAAgB,GACrB,KAAK,CAmFP;AAiCD,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAC;CACf;AAED,wBAAsB,WAAW,CAC/B,QAAQ,EAAE,QAAQ,EAClB,aAAa,EAAE,MAAM,EACrB,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,KAAK,EACZ,GAAG,EAAE,SAAS,EACd,IAAI,EAAE,YAAY,GACjB,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,eAAe,CAAC,CAAC,CAiCzC"}
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
// src/eval/score.ts
|
|
2
|
+
// Aggregation of per-(question, k) grades into the headline tier-weighted
|
|
3
|
+
// score. Pure math. The LLM grader is added in Task 10; this v1 only
|
|
4
|
+
// computes scores from already-graded inputs.
|
|
5
|
+
import { ok } from "../frontmatter/types.js";
|
|
6
|
+
import { GRADER_PROMPT } from "./prompts.js";
|
|
7
|
+
import { TIER_WEIGHT, TIERS, } from "./types.js";
|
|
8
|
+
// Maps verdict to numeric value; null means excluded from aggregate.
|
|
9
|
+
const VERDICT_VALUE = {
|
|
10
|
+
yes: 1.0,
|
|
11
|
+
partial: 0.5,
|
|
12
|
+
no: 0.0,
|
|
13
|
+
ungraded: null,
|
|
14
|
+
};
|
|
15
|
+
export function aggregateScore(grades, questions, opts) {
|
|
16
|
+
const byTier = blankByTier();
|
|
17
|
+
// Group grades by question_id without non-null assertions.
|
|
18
|
+
const byQuestion = new Map();
|
|
19
|
+
for (const grade of grades) {
|
|
20
|
+
pushTo(byQuestion, grade.question_id, grade);
|
|
21
|
+
}
|
|
22
|
+
for (const tier of TIERS) {
|
|
23
|
+
const tierQuestions = questions.filter((q) => q.tier === tier);
|
|
24
|
+
const perQuestionMeans = [];
|
|
25
|
+
const efficiencyHits = [];
|
|
26
|
+
for (const q of tierQuestions) {
|
|
27
|
+
// Only include grades with a numeric verdict value (excludes ungraded).
|
|
28
|
+
const qGrades = (byQuestion.get(q.id) ?? []).filter((grade) => {
|
|
29
|
+
const val = VERDICT_VALUE[grade.verdict];
|
|
30
|
+
return val !== null;
|
|
31
|
+
});
|
|
32
|
+
if (qGrades.length === 0)
|
|
33
|
+
continue;
|
|
34
|
+
const values = qGrades.map((grade) => {
|
|
35
|
+
// Safe: we already filtered nulls above.
|
|
36
|
+
const val = VERDICT_VALUE[grade.verdict];
|
|
37
|
+
return val !== null ? val : 0;
|
|
38
|
+
});
|
|
39
|
+
const mean = avg(values);
|
|
40
|
+
perQuestionMeans.push(mean);
|
|
41
|
+
for (const grade of qGrades) {
|
|
42
|
+
const val = VERDICT_VALUE[grade.verdict];
|
|
43
|
+
if (val !== null && val > 0) {
|
|
44
|
+
const t = opts.traces.get(`${grade.question_id}:${grade.k_index}`);
|
|
45
|
+
if (t)
|
|
46
|
+
efficiencyHits.push(t.total_tool_calls);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
byTier[tier] = {
|
|
51
|
+
mean: perQuestionMeans.length > 0 ? avg(perQuestionMeans) : 0,
|
|
52
|
+
std: perQuestionMeans.length > 0 ? stddev(perQuestionMeans) : 0,
|
|
53
|
+
n: perQuestionMeans.length,
|
|
54
|
+
trace_efficiency: efficiencyHits.length > 0 ? avg(efficiencyHits) : 0,
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
// Weighted aggregate: sum(weight * tier_mean * tier_n) / sum(weight * tier_n)
|
|
58
|
+
let num = 0;
|
|
59
|
+
let denom = 0;
|
|
60
|
+
for (const tier of TIERS) {
|
|
61
|
+
const w = TIER_WEIGHT[tier];
|
|
62
|
+
const ts = byTier[tier];
|
|
63
|
+
num += w * ts.mean * ts.n;
|
|
64
|
+
denom += w * ts.n;
|
|
65
|
+
}
|
|
66
|
+
const score = denom > 0 ? num / denom : 0;
|
|
67
|
+
const scoreStd = denom > 0
|
|
68
|
+
? Math.sqrt(TIERS.reduce((acc, t) => {
|
|
69
|
+
const w = TIER_WEIGHT[t];
|
|
70
|
+
const ts = byTier[t];
|
|
71
|
+
return acc + ((w * ts.n) / denom) * ts.std ** 2;
|
|
72
|
+
}, 0))
|
|
73
|
+
: 0;
|
|
74
|
+
return {
|
|
75
|
+
score,
|
|
76
|
+
score_std: scoreStd,
|
|
77
|
+
by_tier: byTier,
|
|
78
|
+
models: { generator: "", answerer: "", grader: "" },
|
|
79
|
+
prompt_version: 0,
|
|
80
|
+
spec_version: 0,
|
|
81
|
+
questions_id: "",
|
|
82
|
+
results_id: "",
|
|
83
|
+
vault_hash: "",
|
|
84
|
+
k: 0,
|
|
85
|
+
n: 0,
|
|
86
|
+
timestamp: "",
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
function blankByTier() {
|
|
90
|
+
return {
|
|
91
|
+
retrieval: { mean: 0, std: 0, n: 0, trace_efficiency: 0 },
|
|
92
|
+
cross_reference: { mean: 0, std: 0, n: 0, trace_efficiency: 0 },
|
|
93
|
+
contradiction: { mean: 0, std: 0, n: 0, trace_efficiency: 0 },
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
// Appends `value` to the array at `key`, creating the array on first use.
|
|
97
|
+
// Mirrors the pushTo pattern from subgraph.ts to avoid non-null assertions.
|
|
98
|
+
function pushTo(m, key, value) {
|
|
99
|
+
const arr = m.get(key);
|
|
100
|
+
if (arr) {
|
|
101
|
+
arr.push(value);
|
|
102
|
+
}
|
|
103
|
+
else {
|
|
104
|
+
m.set(key, [value]);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
function avg(xs) {
|
|
108
|
+
return xs.reduce((a, b) => a + b, 0) / xs.length;
|
|
109
|
+
}
|
|
110
|
+
function stddev(xs) {
|
|
111
|
+
if (xs.length < 2)
|
|
112
|
+
return 0;
|
|
113
|
+
const m = avg(xs);
|
|
114
|
+
return Math.sqrt(xs.reduce((acc, x) => acc + (x - m) ** 2, 0) / xs.length);
|
|
115
|
+
}
|
|
116
|
+
export async function gradeAnswer(question, questionIndex, kIndex, trace, llm, opts) {
|
|
117
|
+
const cited = extractCitations(trace.final_answer);
|
|
118
|
+
const user = GRADER_PROMPT.replace("{{QUESTION}}", question.question)
|
|
119
|
+
.replace("{{EXPECTED_ANSWER}}", question.expected_answer)
|
|
120
|
+
.replace("{{EXPECTED_SOURCES}}", question.expected_sources.join(", "))
|
|
121
|
+
.replace("{{CLAIMED_ANSWER}}", trace.final_answer)
|
|
122
|
+
.replace("{{CITED_SOURCES}}", cited.join(", "));
|
|
123
|
+
const schema = {
|
|
124
|
+
type: "object",
|
|
125
|
+
required: ["correct", "reasoning"],
|
|
126
|
+
properties: {
|
|
127
|
+
correct: { enum: ["yes", "partial", "no"] },
|
|
128
|
+
reasoning: { type: "string" },
|
|
129
|
+
},
|
|
130
|
+
};
|
|
131
|
+
const r = await llm.completeJson({ model: opts.model, system: "", user, schema });
|
|
132
|
+
if (!r.ok)
|
|
133
|
+
return r;
|
|
134
|
+
// biome-ignore lint/suspicious/noExplicitAny: parsed JSON
|
|
135
|
+
const parsed = r.value.parsed;
|
|
136
|
+
const verdict = parsed?.correct === "yes" || parsed?.correct === "partial" || parsed?.correct === "no"
|
|
137
|
+
? parsed.correct
|
|
138
|
+
: "ungraded";
|
|
139
|
+
return ok({
|
|
140
|
+
question_id: question.id,
|
|
141
|
+
question_index: questionIndex,
|
|
142
|
+
k_index: kIndex,
|
|
143
|
+
verdict,
|
|
144
|
+
reasoning: typeof parsed?.reasoning === "string" ? parsed.reasoning : "",
|
|
145
|
+
grader_model: opts.model,
|
|
146
|
+
});
|
|
147
|
+
}
|
|
148
|
+
function extractCitations(answer) {
|
|
149
|
+
const out = [];
|
|
150
|
+
for (const m of answer.matchAll(/\[([^\]]+\.md)\]/g))
|
|
151
|
+
out.push(m[1]);
|
|
152
|
+
return out;
|
|
153
|
+
}
|
|
154
|
+
//# sourceMappingURL=score.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"score.js","sourceRoot":"","sources":["../../src/eval/score.ts"],"names":[],"mappings":"AAAA,oBAAoB;AACpB,0EAA0E;AAC1E,qEAAqE;AACrE,8CAA8C;AAE9C,OAAO,EAAE,EAAE,EAAe,MAAM,yBAAyB,CAAC;AAE1D,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,OAAO,EAML,WAAW,EACX,KAAK,GAIN,MAAM,YAAY,CAAC;AAOpB,qEAAqE;AACrE,MAAM,aAAa,GAA4C;IAC7D,GAAG,EAAE,GAAG;IACR,OAAO,EAAE,GAAG;IACZ,EAAE,EAAE,GAAG;IACP,QAAQ,EAAE,IAAI;CACf,CAAC;AAEF,MAAM,UAAU,cAAc,CAC5B,MAAe,EACf,SAAqB,EACrB,IAAsB;IAEtB,MAAM,MAAM,GAA4B,WAAW,EAAE,CAAC;IAEtD,2DAA2D;IAC3D,MAAM,UAAU,GAAG,IAAI,GAAG,EAAmB,CAAC;IAC9C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,CAAC,UAAU,EAAE,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;IAC/C,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,aAAa,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;QAC/D,MAAM,gBAAgB,GAAa,EAAE,CAAC;QACtC,MAAM,cAAc,GAAa,EAAE,CAAC;QAEpC,KAAK,MAAM,CAAC,IAAI,aAAa,EAAE,CAAC;YAC9B,wEAAwE;YACxE,MAAM,OAAO,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE;gBAC5D,MAAM,GAAG,GAAG,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;gBACzC,OAAO,GAAG,KAAK,IAAI,CAAC;YACtB,CAAC,CAAC,CAAC;YACH,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAEnC,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;gBACnC,yCAAyC;gBACzC,MAAM,GAAG,GAAG,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;gBACzC,OAAO,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YAChC,CAAC,CAAC,CAAC;YACH,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC;YACzB,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAE5B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC5B,MAAM,GAAG,GAAG,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;gBACzC,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC;oBAC5B,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;oBACnE,IAAI,CAAC;wBAAE,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC;gBACjD,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,GAAG;YACb,IAAI,EAAE,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC;YAC7D,GAAG,EAAE,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/D,CAAC,EAAE,gBAAgB,CAAC,MAAM;YAC1B,gBAAgB,EAAE,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC;SACtE,CAAC;IACJ,CAAC;IAED,8EAA8E;IAC9E,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,CAAC,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QAC5B,MAAM,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QACxB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC;QAC1B,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;IACpB,CAAC;IACD,MAAM,KAAK,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAE1C,MAAM,QAAQ,GACZ,KAAK,GAAG,CAAC;QACP,CAAC,CAAC,IAAI,CAAC,IAAI,CACP,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE;YACtB,MAAM,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;YACzB,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YACrB,OAAO,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;QAClD,CAAC,EAAE,CAAC,CAAC,CACN;QACH,CAAC,CAAC,CAAC,CAAC;IAER,OAAO;QACL,KAAK;QACL,SAAS,EAAE,QAAQ;QACnB,OAAO,EAAE,MAAM;QACf,MAAM,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE;QACnD,cAAc,EAAE,CAAC;QACjB,YAAY,EAAE,CAAC;QACf,YAAY,EAAE,EAAE;QAChB,UAAU,EAAE,EAAE;QACd,UAAU,EAAE,EAAE;QACd,CAAC,EAAE,CAAC;QACJ,CAAC,EAAE,CAAC;QACJ,SAAS,EAAE,EAAE;KACd,CAAC;AACJ,CAAC;AAED,SAAS,WAAW;IAClB,OAAO;QACL,SAAS,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE;QACzD,eAAe,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE;QAC/D,aAAa,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE;KAC9D,CAAC;AACJ,CAAC;AAED,0EAA0E;AAC1E,4EAA4E;AAC5E,SAAS,MAAM,CAAI,CAAmB,EAAE,GAAW,EAAE,KAAQ;IAC3D,MAAM,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACvB,IAAI,GAAG,EAAE,CAAC;QACR,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClB,CAAC;SAAM,CAAC;QACN,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;IACtB,CAAC;AACH,CAAC;AAED,SAAS,GAAG,CAAC,EAAY;IACvB,OAAO,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC;AACnD,CAAC;AAED,SAAS,MAAM,CAAC,EAAY;IAC1B,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IAC5B,MAAM,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC;IAClB,OAAO,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,CAAC;AAC7E,CAAC;AAQD,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,QAAkB,EAClB,aAAqB,EACrB,MAAc,EACd,KAAY,EACZ,GAAc,EACd,IAAkB;IAElB,MAAM,KAAK,GAAG,gBAAgB,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,IAAI,GAAG,aAAa,CAAC,OAAO,CAAC,cAAc,EAAE,QAAQ,CAAC,QAAQ,CAAC;SAClE,OAAO,CAAC,qBAAqB,EAAE,QAAQ,CAAC,eAAe,CAAC;SACxD,OAAO,CAAC,sBAAsB,EAAE,QAAQ,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;SACrE,OAAO,CAAC,oBAAoB,EAAE,KAAK,CAAC,YAAY,CAAC;SACjD,OAAO,CAAC,mBAAmB,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAElD,MAAM,MAAM,GAAG;QACb,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,CAAC,SAAS,EAAE,WAAW,CAAC;QAClC,UAAU,EAAE;YACV,OAAO,EAAE,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC,EAAE;YAC3C,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;SAC9B;KACO,CAAC;IAEX,MAAM,CAAC,GAAG,MAAM,GAAG,CAAC,YAAY,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;IAClF,IAAI,CAAC,CAAC,CAAC,EAAE;QAAE,OAAO,CAAC,CAAC;IACpB,0DAA0D;IAC1D,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,MAAa,CAAC;IACrC,MAAM,OAAO,GACX,MAAM,EAAE,OAAO,KAAK,KAAK,IAAI,MAAM,EAAE,OAAO,KAAK,SAAS,IAAI,MAAM,EAAE,OAAO,KAAK,IAAI;QACpF,CAAC,CAAC,MAAM,CAAC,OAAO;QAChB,CAAC,CAAC,UAAU,CAAC;IACjB,OAAO,EAAE,CAAC;QACR,WAAW,EAAE,QAAQ,CAAC,EAAE;QACxB,cAAc,EAAE,aAAa;QAC7B,OAAO,EAAE,MAAM;QACf,OAAO;QACP,SAAS,EAAE,OAAO,MAAM,EAAE,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE;QACxE,YAAY,EAAE,IAAI,CAAC,KAAK;KACzB,CAAC,CAAC;AACL,CAAC;AAED,SAAS,gBAAgB,CAAC,MAAc;IACtC,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,mBAAmB,CAAC;QAAE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACrE,OAAO,GAAG,CAAC;AACb,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { type Result } from "../frontmatter/types.js";
|
|
2
|
+
import { type CortexEvalError, type EvalRun, type HistoryEntry, type HistoryFile, type QuestionSet, type Score } from "./types.js";
|
|
3
|
+
export declare function writeQuestionSet(vault: string, qs: QuestionSet): Promise<void>;
|
|
4
|
+
export declare function readQuestionSet(vault: string, id: string): Promise<Result<QuestionSet, CortexEvalError>>;
|
|
5
|
+
export declare function writeResults(vault: string, run: EvalRun): Promise<void>;
|
|
6
|
+
export declare function readResults(vault: string, id: string): Promise<Result<EvalRun, CortexEvalError>>;
|
|
7
|
+
export declare function writeScore(vault: string, score: Score): Promise<void>;
|
|
8
|
+
export declare function appendHistory(vault: string, entry: HistoryEntry): Promise<void>;
|
|
9
|
+
export declare function readHistory(vault: string): Promise<Result<HistoryFile, CortexEvalError>>;
|
|
10
|
+
//# sourceMappingURL=storage.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"storage.d.ts","sourceRoot":"","sources":["../../src/eval/storage.ts"],"names":[],"mappings":"AAOA,OAAO,EAAW,KAAK,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EACL,KAAK,eAAe,EACpB,KAAK,OAAO,EAEZ,KAAK,YAAY,EACjB,KAAK,WAAW,EAChB,KAAK,WAAW,EAChB,KAAK,KAAK,EACX,MAAM,YAAY,CAAC;AA0BpB,wBAAsB,gBAAgB,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAGpF;AAED,wBAAgB,eAAe,CAC7B,KAAK,EAAE,MAAM,EACb,EAAE,EAAE,MAAM,GACT,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,eAAe,CAAC,CAAC,CAE/C;AAED,wBAAsB,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAG7E;AAED,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC,CAEhG;AAED,wBAAsB,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAG3E;AAED,wBAAsB,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAOrF;AAED,wBAAsB,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,eAAe,CAAC,CAAC,CAY9F"}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
// src/eval/storage.ts
|
|
2
|
+
// JSON I/O under .daftari/eval/. No business logic — just paths, schemas,
|
|
3
|
+
// rotation. Read paths are read-only-compatible per spec §12 resolution 5.
|
|
4
|
+
import { existsSync } from "node:fs";
|
|
5
|
+
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
|
6
|
+
import { join } from "node:path";
|
|
7
|
+
import { err, ok } from "../frontmatter/types.js";
|
|
8
|
+
import { HISTORY_RETENTION, } from "./types.js";
|
|
9
|
+
const EVAL_DIR = (vault) => join(vault, ".daftari", "eval");
|
|
10
|
+
const QS_DIR = (vault) => join(EVAL_DIR(vault), "questions");
|
|
11
|
+
const RES_DIR = (vault) => join(EVAL_DIR(vault), "results");
|
|
12
|
+
const SCORE_DIR = (vault) => join(EVAL_DIR(vault), "scores");
|
|
13
|
+
const HIST_FILE = (vault) => join(EVAL_DIR(vault), "history.json");
|
|
14
|
+
async function ensureDir(p) {
|
|
15
|
+
await mkdir(p, { recursive: true });
|
|
16
|
+
}
|
|
17
|
+
function writeJson(path, value) {
|
|
18
|
+
return writeFile(path, `${JSON.stringify(value, null, 2)}\n`, "utf8");
|
|
19
|
+
}
|
|
20
|
+
async function readJson(path) {
|
|
21
|
+
try {
|
|
22
|
+
const raw = await readFile(path, "utf8");
|
|
23
|
+
return ok(JSON.parse(raw));
|
|
24
|
+
}
|
|
25
|
+
catch (e) {
|
|
26
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
27
|
+
return err({ kind: "runtime", message: `read ${path}: ${msg}` });
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
export async function writeQuestionSet(vault, qs) {
|
|
31
|
+
await ensureDir(QS_DIR(vault));
|
|
32
|
+
await writeJson(join(QS_DIR(vault), `${qs.id}.json`), qs);
|
|
33
|
+
}
|
|
34
|
+
export function readQuestionSet(vault, id) {
|
|
35
|
+
return readJson(join(QS_DIR(vault), `${id}.json`));
|
|
36
|
+
}
|
|
37
|
+
export async function writeResults(vault, run) {
|
|
38
|
+
await ensureDir(RES_DIR(vault));
|
|
39
|
+
await writeJson(join(RES_DIR(vault), `${run.id}.json`), run);
|
|
40
|
+
}
|
|
41
|
+
export function readResults(vault, id) {
|
|
42
|
+
return readJson(join(RES_DIR(vault), `${id}.json`));
|
|
43
|
+
}
|
|
44
|
+
export async function writeScore(vault, score) {
|
|
45
|
+
await ensureDir(SCORE_DIR(vault));
|
|
46
|
+
await writeJson(join(SCORE_DIR(vault), `${score.results_id}.json`), score);
|
|
47
|
+
}
|
|
48
|
+
export async function appendHistory(vault, entry) {
|
|
49
|
+
await ensureDir(EVAL_DIR(vault));
|
|
50
|
+
const current = await readHistory(vault);
|
|
51
|
+
const runs = current.ok ? [...current.value.runs, entry] : [entry];
|
|
52
|
+
const trimmed = runs.slice(-HISTORY_RETENTION);
|
|
53
|
+
const out = { version: 1, runs: trimmed };
|
|
54
|
+
await writeJson(HIST_FILE(vault), out);
|
|
55
|
+
}
|
|
56
|
+
export async function readHistory(vault) {
|
|
57
|
+
const path = HIST_FILE(vault);
|
|
58
|
+
if (!existsSync(path))
|
|
59
|
+
return ok({ version: 1, runs: [] });
|
|
60
|
+
const r = await readJson(path);
|
|
61
|
+
if (!r.ok)
|
|
62
|
+
return r;
|
|
63
|
+
if (typeof r.value.version === "number" && r.value.version > 1) {
|
|
64
|
+
process.stderr.write(`daftari eval: history.json version ${r.value.version} is newer than supported (1); leaving untouched\n`);
|
|
65
|
+
return ok({ version: 1, runs: [] });
|
|
66
|
+
}
|
|
67
|
+
return r;
|
|
68
|
+
}
|
|
69
|
+
//# sourceMappingURL=storage.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"storage.js","sourceRoot":"","sources":["../../src/eval/storage.ts"],"names":[],"mappings":"AAAA,sBAAsB;AACtB,0EAA0E;AAC1E,2EAA2E;AAE3E,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,GAAG,EAAE,EAAE,EAAe,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAGL,iBAAiB,GAKlB,MAAM,YAAY,CAAC;AAEpB,MAAM,QAAQ,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;AACpE,MAAM,MAAM,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,WAAW,CAAC,CAAC;AACrE,MAAM,OAAO,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,CAAC,CAAC;AACpE,MAAM,SAAS,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,QAAQ,CAAC,CAAC;AACrE,MAAM,SAAS,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,cAAc,CAAC,CAAC;AAE3E,KAAK,UAAU,SAAS,CAAC,CAAS;IAChC,MAAM,KAAK,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;AACtC,CAAC;AAED,SAAS,SAAS,CAAI,IAAY,EAAE,KAAQ;IAC1C,OAAO,SAAS,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;AACxE,CAAC;AAED,KAAK,UAAU,QAAQ,CAAI,IAAY;IACrC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACzC,OAAO,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAM,CAAC,CAAC;IAClC,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,GAAG,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACvD,OAAO,GAAG,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,IAAI,KAAK,GAAG,EAAE,EAAE,CAAC,CAAC;IACnE,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,KAAa,EAAE,EAAe;IACnE,MAAM,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IAC/B,MAAM,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;AAC5D,CAAC;AAED,MAAM,UAAU,eAAe,CAC7B,KAAa,EACb,EAAU;IAEV,OAAO,QAAQ,CAAc,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC;AAClE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,KAAa,EAAE,GAAY;IAC5D,MAAM,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;IAChC,MAAM,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,GAAG,GAAG,CAAC,EAAE,OAAO,CAAC,EAAE,GAAG,CAAC,CAAC;AAC/D,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,KAAa,EAAE,EAAU;IACnD,OAAO,QAAQ,CAAU,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC;AAC/D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,KAAa,EAAE,KAAY;IAC1D,MAAM,SAAS,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;IAClC,MAAM,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,GAAG,KAAK,CAAC,UAAU,OAAO,CAAC,EAAE,KAAK,CAAC,CAAC;AAC7E,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,KAAa,EAAE,KAAmB;IACpE,MAAM,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;IACjC,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,KAAK,CAAC,CAAC;IACzC,MAAM,IAAI,GAAmB,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IACnF,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,iBAAiB,CAAC,CAAC;IAC/C,MAAM,GAAG,GAAgB,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;IACvD,MAAM,SAAS,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC;AACzC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,KAAa;IAC7C,MAAM,IAAI,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;IAC9B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;IAC3D,MAAM,CAAC,GAAG,MAAM,QAAQ,CAAc,IAAI,CAAC,CAAC;IAC5C,IAAI,CAAC,CAAC,CAAC,EAAE;QAAE,OAAO,CAAC,CAAC;IACpB,IAAI,OAAO,CAAC,CAAC,KAAK,CAAC,OAAO,KAAK,QAAQ,IAAI,CAAC,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;QAC/D,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,sCAAsC,CAAC,CAAC,KAAK,CAAC,OAAO,mDAAmD,CACzG,CAAC;QACF,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;IACtC,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { type Result } from "../frontmatter/types.js";
|
|
2
|
+
import type { CortexEvalError, SubgraphEdge } from "./types.js";
|
|
3
|
+
export interface SubgraphOptions {
|
|
4
|
+
maxNodes?: number;
|
|
5
|
+
}
|
|
6
|
+
export interface SubgraphNode {
|
|
7
|
+
path: string;
|
|
8
|
+
body: string;
|
|
9
|
+
frontmatter: Record<string, unknown>;
|
|
10
|
+
}
|
|
11
|
+
export interface Subgraph {
|
|
12
|
+
seed_doc: string;
|
|
13
|
+
nodes: SubgraphNode[];
|
|
14
|
+
edges: SubgraphEdge[];
|
|
15
|
+
}
|
|
16
|
+
export declare function sampleSubgraph(vaultRoot: string, seed: string, opts?: SubgraphOptions): Promise<Result<Subgraph, CortexEvalError>>;
|
|
17
|
+
//# sourceMappingURL=subgraph.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"subgraph.d.ts","sourceRoot":"","sources":["../../src/eval/subgraph.ts"],"names":[],"mappings":"AAmBA,OAAO,EAAW,KAAK,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAE/D,OAAO,KAAK,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAEhE,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACtC;AAED,MAAM,WAAW,QAAQ;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,YAAY,EAAE,CAAC;IACtB,KAAK,EAAE,YAAY,EAAE,CAAC;CACvB;AAED,wBAAsB,cAAc,CAClC,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,MAAM,EACZ,IAAI,GAAE,eAAoB,GACzB,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC,CA6D5C"}
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
// Deterministic subgraph sampling for the cortex quality metric.
|
|
2
|
+
//
|
|
3
|
+
// Given a vault and a seed string, derive a small connected subgraph rooted at
|
|
4
|
+
// one "seed doc". The seed→doc mapping is stratified by collection so a vault
|
|
5
|
+
// with one dense collection cannot dominate every sample. From the seed we hop
|
|
6
|
+
// along four edge kinds — frontmatter `sources`, in-body markdown links, logged
|
|
7
|
+
// tensions, and `superseded_by` revision links — collecting neighbours until a
|
|
8
|
+
// node cap is reached. In Daftari's data model `sources:` holds external
|
|
9
|
+
// citation slugs (not in-vault paths), so the real in-vault doc→doc revision
|
|
10
|
+
// edge lives in `superseded_by:`, walked bidirectionally below.
|
|
11
|
+
//
|
|
12
|
+
// Pure given (vault state + seed): the same inputs always yield the same
|
|
13
|
+
// subgraph. All randomness is replaced by SHA-256 indexing over sorted inputs.
|
|
14
|
+
import { createHash } from "node:crypto";
|
|
15
|
+
import { readFile } from "node:fs/promises";
|
|
16
|
+
import { resolve } from "node:path";
|
|
17
|
+
import { listTensions } from "../curation/tension.js";
|
|
18
|
+
import { parseDocument } from "../frontmatter/parser.js";
|
|
19
|
+
import { err, ok } from "../frontmatter/types.js";
|
|
20
|
+
import { openIndexForActiveProvider } from "../tools/search.js";
|
|
21
|
+
export async function sampleSubgraph(vaultRoot, seed, opts = {}) {
|
|
22
|
+
const maxNodes = opts.maxNodes ?? 5;
|
|
23
|
+
const indexRes = openIndexForActiveProvider(vaultRoot);
|
|
24
|
+
if (!indexRes.ok) {
|
|
25
|
+
return err({ kind: "runtime", message: `vault index unavailable: ${indexRes.error.message}` });
|
|
26
|
+
}
|
|
27
|
+
const db = indexRes.value;
|
|
28
|
+
let docs;
|
|
29
|
+
try {
|
|
30
|
+
docs = db.prepare("SELECT path, superseded_by FROM documents").all();
|
|
31
|
+
}
|
|
32
|
+
finally {
|
|
33
|
+
db.close();
|
|
34
|
+
}
|
|
35
|
+
if (docs.length === 0) {
|
|
36
|
+
return err({ kind: "runtime", message: "vault has no indexed documents" });
|
|
37
|
+
}
|
|
38
|
+
const strata = stratifyByCollection(docs.map((d) => d.path));
|
|
39
|
+
const stratumNames = [...strata.keys()].sort();
|
|
40
|
+
const stratumIdx = hashToIndex(`${seed}:stratum`, stratumNames.length);
|
|
41
|
+
const stratumName = stratumNames[stratumIdx];
|
|
42
|
+
const stratumDocs = [...(strata.get(stratumName) ?? [])].sort();
|
|
43
|
+
const seedIdx = hashToIndex(`${seed}:doc`, stratumDocs.length);
|
|
44
|
+
const seedDoc = stratumDocs[seedIdx];
|
|
45
|
+
const visited = new Map();
|
|
46
|
+
const edges = [];
|
|
47
|
+
const tensionsByDoc = await loadTensionEdges(vaultRoot);
|
|
48
|
+
// Bidirectional supersede map, built from the already-materialized SQL rows
|
|
49
|
+
// (no extra file reads). Bidirectional so a seed landing on EITHER the old or
|
|
50
|
+
// the new doc reaches its counterpart.
|
|
51
|
+
const supersededByDoc = new Map();
|
|
52
|
+
for (const d of docs) {
|
|
53
|
+
if (typeof d.superseded_by === "string" && d.superseded_by.length > 0) {
|
|
54
|
+
pushTo(supersededByDoc, d.path, { other: d.superseded_by });
|
|
55
|
+
pushTo(supersededByDoc, d.superseded_by, { other: d.path });
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
await loadNode(vaultRoot, seedDoc, visited);
|
|
59
|
+
await walkHop(vaultRoot, seedDoc, visited, edges, tensionsByDoc, supersededByDoc);
|
|
60
|
+
const firstHopNeighbors = [...visited.keys()].filter((p) => p !== seedDoc);
|
|
61
|
+
for (const n of firstHopNeighbors) {
|
|
62
|
+
if (visited.size >= maxNodes)
|
|
63
|
+
break;
|
|
64
|
+
await walkHop(vaultRoot, n, visited, edges, tensionsByDoc, supersededByDoc);
|
|
65
|
+
}
|
|
66
|
+
const nodes = trimToCap(seedDoc, visited, edges, maxNodes);
|
|
67
|
+
const nodePaths = new Set(nodes.map((n) => n.path));
|
|
68
|
+
return ok({
|
|
69
|
+
seed_doc: seedDoc,
|
|
70
|
+
nodes,
|
|
71
|
+
edges: edges.filter((e) => keepEdge(e, nodePaths)),
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
// An edge is kept when its `from` survived the node cap, and:
|
|
75
|
+
// - for `link`/`tension`/`superseded` edges, the `to` is also a retained
|
|
76
|
+
// in-vault node — these reference other vault documents, so a dangling
|
|
77
|
+
// target means the neighbour was trimmed or never existed and the edge is
|
|
78
|
+
// meaningless.
|
|
79
|
+
// - for `sources` edges, the `to` is a provenance citation that is, by the
|
|
80
|
+
// vault's frontmatter convention, an external source slug rather than an
|
|
81
|
+
// in-vault `.md` path. Such an edge records real provenance off the seed
|
|
82
|
+
// doc and is retained even though the cited source is not itself a node.
|
|
83
|
+
function keepEdge(e, nodePaths) {
|
|
84
|
+
if (!nodePaths.has(e.from))
|
|
85
|
+
return false;
|
|
86
|
+
if (e.kind === "sources")
|
|
87
|
+
return true;
|
|
88
|
+
return nodePaths.has(e.to);
|
|
89
|
+
}
|
|
90
|
+
async function loadTensionEdges(vaultRoot) {
|
|
91
|
+
const tensionsByDoc = new Map();
|
|
92
|
+
const tensionsRes = await listTensions(vaultRoot);
|
|
93
|
+
if (!tensionsRes.ok)
|
|
94
|
+
return tensionsByDoc;
|
|
95
|
+
for (const t of tensionsRes.value) {
|
|
96
|
+
pushTo(tensionsByDoc, t.sourceA, { other: t.sourceB });
|
|
97
|
+
pushTo(tensionsByDoc, t.sourceB, { other: t.sourceA });
|
|
98
|
+
}
|
|
99
|
+
return tensionsByDoc;
|
|
100
|
+
}
|
|
101
|
+
function stratifyByCollection(paths) {
|
|
102
|
+
const m = new Map();
|
|
103
|
+
for (const p of paths) {
|
|
104
|
+
const collection = p.split("/")[0] || "_root";
|
|
105
|
+
pushTo(m, collection, p);
|
|
106
|
+
}
|
|
107
|
+
return m;
|
|
108
|
+
}
|
|
109
|
+
// Appends `value` to the array at `key`, creating the array on first use.
|
|
110
|
+
// Replaces the `map.get(key)!.push(...)` non-null-assertion idiom that biome's
|
|
111
|
+
// lint/style/noNonNullAssertion rule forbids.
|
|
112
|
+
function pushTo(m, key, value) {
|
|
113
|
+
const arr = m.get(key);
|
|
114
|
+
if (arr)
|
|
115
|
+
arr.push(value);
|
|
116
|
+
else
|
|
117
|
+
m.set(key, [value]);
|
|
118
|
+
}
|
|
119
|
+
function hashToIndex(input, mod) {
|
|
120
|
+
if (mod <= 0)
|
|
121
|
+
throw new Error("mod must be positive");
|
|
122
|
+
const h = createHash("sha256").update(input).digest();
|
|
123
|
+
const n = h.readUInt32BE(0);
|
|
124
|
+
return n % mod;
|
|
125
|
+
}
|
|
126
|
+
// Loads a document's frontmatter + body into `visited`. Uses the project's
|
|
127
|
+
// canonical parser; on malformed YAML it falls back to empty frontmatter so the
|
|
128
|
+
// walk continues. A missing file is silently skipped.
|
|
129
|
+
async function loadNode(vaultRoot, path, visited) {
|
|
130
|
+
if (visited.has(path))
|
|
131
|
+
return;
|
|
132
|
+
let raw;
|
|
133
|
+
try {
|
|
134
|
+
raw = await readFile(resolve(vaultRoot, path), "utf8");
|
|
135
|
+
}
|
|
136
|
+
catch {
|
|
137
|
+
return; // Missing doc — silently skip.
|
|
138
|
+
}
|
|
139
|
+
const parsed = parseDocument(raw);
|
|
140
|
+
if (parsed.ok) {
|
|
141
|
+
visited.set(path, { path, body: parsed.value.content, frontmatter: parsed.value.raw });
|
|
142
|
+
}
|
|
143
|
+
else {
|
|
144
|
+
visited.set(path, { path, body: raw, frontmatter: {} });
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
async function walkHop(vaultRoot, from, visited, edges, tensionsByDoc, supersededByDoc) {
|
|
148
|
+
const node = visited.get(from);
|
|
149
|
+
if (!node)
|
|
150
|
+
return;
|
|
151
|
+
const sources = Array.isArray(node.frontmatter.sources)
|
|
152
|
+
? node.frontmatter.sources
|
|
153
|
+
: [];
|
|
154
|
+
for (const s of sources) {
|
|
155
|
+
if (typeof s !== "string")
|
|
156
|
+
continue;
|
|
157
|
+
edges.push({ from, to: s, kind: "sources" });
|
|
158
|
+
await loadNode(vaultRoot, s, visited);
|
|
159
|
+
}
|
|
160
|
+
const links = extractInVaultLinks(node.body);
|
|
161
|
+
for (const l of links) {
|
|
162
|
+
edges.push({ from, to: l, kind: "link" });
|
|
163
|
+
await loadNode(vaultRoot, l, visited);
|
|
164
|
+
}
|
|
165
|
+
const tensions = tensionsByDoc.get(from) ?? [];
|
|
166
|
+
for (const t of tensions) {
|
|
167
|
+
edges.push({ from, to: t.other, kind: "tension" });
|
|
168
|
+
await loadNode(vaultRoot, t.other, visited);
|
|
169
|
+
}
|
|
170
|
+
const superseded = supersededByDoc.get(from) ?? [];
|
|
171
|
+
for (const s of superseded) {
|
|
172
|
+
edges.push({ from, to: s.other, kind: "superseded" });
|
|
173
|
+
await loadNode(vaultRoot, s.other, visited);
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
function trimToCap(seed, visited, edges, cap) {
|
|
177
|
+
if (visited.size <= cap)
|
|
178
|
+
return [...visited.values()];
|
|
179
|
+
const degree = new Map();
|
|
180
|
+
for (const e of edges) {
|
|
181
|
+
degree.set(e.from, (degree.get(e.from) ?? 0) + 1);
|
|
182
|
+
degree.set(e.to, (degree.get(e.to) ?? 0) + 1);
|
|
183
|
+
}
|
|
184
|
+
const ranked = [...visited.entries()]
|
|
185
|
+
.map(([path, node]) => ({ path, node, degree: degree.get(path) ?? 0 }))
|
|
186
|
+
.sort((a, b) => {
|
|
187
|
+
if (a.path === seed)
|
|
188
|
+
return -1;
|
|
189
|
+
if (b.path === seed)
|
|
190
|
+
return 1;
|
|
191
|
+
// Higher degree wins; path tiebreak keeps the cut deterministic
|
|
192
|
+
// independent of engine sort stability.
|
|
193
|
+
return b.degree - a.degree || a.path.localeCompare(b.path);
|
|
194
|
+
});
|
|
195
|
+
return ranked.slice(0, cap).map((r) => r.node);
|
|
196
|
+
}
|
|
197
|
+
// Deliberately matches only standard markdown links to in-vault `.md` files —
|
|
198
|
+
// `[text](path.md)` / `[text](path.md#anchor)`, skipping web/mailto and
|
|
199
|
+
// absolute-rooted hrefs. Wiki-style `[[links]]` are not a Daftari vault
|
|
200
|
+
// convention and are intentionally unsupported.
|
|
201
|
+
function extractInVaultLinks(body) {
|
|
202
|
+
const out = [];
|
|
203
|
+
const re = /\[[^\]]*\]\(([^)]+\.md)(?:#[^)]*)?\)/g;
|
|
204
|
+
for (const m of body.matchAll(re)) {
|
|
205
|
+
const href = m[1];
|
|
206
|
+
if (/^https?:|^mailto:/i.test(href))
|
|
207
|
+
continue;
|
|
208
|
+
if (href.startsWith("/"))
|
|
209
|
+
continue;
|
|
210
|
+
out.push(href);
|
|
211
|
+
}
|
|
212
|
+
return out;
|
|
213
|
+
}
|
|
214
|
+
//# sourceMappingURL=subgraph.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"subgraph.js","sourceRoot":"","sources":["../../src/eval/subgraph.ts"],"names":[],"mappings":"AAAA,iEAAiE;AACjE,EAAE;AACF,+EAA+E;AAC/E,8EAA8E;AAC9E,+EAA+E;AAC/E,gFAAgF;AAChF,+EAA+E;AAC/E,yEAAyE;AACzE,6EAA6E;AAC7E,gEAAgE;AAChE,EAAE;AACF,yEAAyE;AACzE,+EAA+E;AAE/E,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACzD,OAAO,EAAE,GAAG,EAAE,EAAE,EAAe,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,0BAA0B,EAAE,MAAM,oBAAoB,CAAC;AAmBhE,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,SAAiB,EACjB,IAAY,EACZ,OAAwB,EAAE;IAE1B,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;IACpC,MAAM,QAAQ,GAAG,0BAA0B,CAAC,SAAS,CAAC,CAAC;IACvD,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,OAAO,GAAG,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,4BAA4B,QAAQ,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IACjG,CAAC;IACD,MAAM,EAAE,GAAG,QAAQ,CAAC,KAAK,CAAC;IAE1B,IAAI,IAAsD,CAAC;IAC3D,IAAI,CAAC;QACH,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC,2CAA2C,CAAC,CAAC,GAAG,EAG/D,CAAC;IACN,CAAC;YAAS,CAAC;QACT,EAAE,CAAC,KAAK,EAAE,CAAC;IACb,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,OAAO,GAAG,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,gCAAgC,EAAE,CAAC,CAAC;IAC7E,CAAC;IAED,MAAM,MAAM,GAAG,oBAAoB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAC7D,MAAM,YAAY,GAAG,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/C,MAAM,UAAU,GAAG,WAAW,CAAC,GAAG,IAAI,UAAU,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC;IACvE,MAAM,WAAW,GAAG,YAAY,CAAC,UAAU,CAAC,CAAC;IAC7C,MAAM,WAAW,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAChE,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,IAAI,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IAC/D,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;IAErC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAwB,CAAC;IAChD,MAAM,KAAK,GAAmB,EAAE,CAAC;IAEjC,MAAM,aAAa,GAAG,MAAM,gBAAgB,CAAC,SAAS,CAAC,CAAC;IAExD,4EAA4E;IAC5E,8EAA8E;IAC9E,uCAAuC;IACvC,MAAM,eAAe,GAAG,IAAI,GAAG,EAAoC,CAAC;IACpE,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,IAAI,OAAO,CAAC,CAAC,aAAa,KAAK,QAAQ,IAAI,CAAC,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtE,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,aAAa,EAAE,CAAC,CAAC;YAC5D,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC,aAAa,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,CAAC,SAAS,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAC5C,MAAM,OAAO,CAAC,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,EAAE,eAAe,CAAC,CAAC;IAClF,MAAM,iBAAiB,GAAG,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,OAAO,CAAC,CAAC;IAC3E,KAAK,MAAM,CAAC,IAAI,iBAAiB,EAAE,CAAC;QAClC,IAAI,OAAO,CAAC,IAAI,IAAI,QAAQ;YAAE,MAAM;QACpC,MAAM,OAAO,CAAC,SAAS,EAAE,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,EAAE,eAAe,CAAC,CAAC;IAC9E,CAAC;IAED,MAAM,KAAK,GAAG,SAAS,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;IAC3D,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAEpD,OAAO,EAAE,CAAC;QACR,QAAQ,EAAE,OAAO;QACjB,KAAK;QACL,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;KACnD,CAAC,CAAC;AACL,CAAC;AAED,8DAA8D;AAC9D,0EAA0E;AAC1E,0EAA0E;AAC1E,6EAA6E;AAC7E,kBAAkB;AAClB,4EAA4E;AAC5E,4EAA4E;AAC5E,4EAA4E;AAC5E,4EAA4E;AAC5E,SAAS,QAAQ,CAAC,CAAe,EAAE,SAAsB;IACvD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IACzC,IAAI,CAAC,CAAC,IAAI,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IACtC,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;AAC7B,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,SAAiB;IAC/C,MAAM,aAAa,GAAG,IAAI,GAAG,EAAoC,CAAC;IAClE,MAAM,WAAW,GAAG,MAAM,YAAY,CAAC,SAAS,CAAC,CAAC;IAClD,IAAI,CAAC,WAAW,CAAC,EAAE;QAAE,OAAO,aAAa,CAAC;IAC1C,KAAK,MAAM,CAAC,IAAI,WAAW,CAAC,KAAK,EAAE,CAAC;QAClC,MAAM,CAAC,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QACvD,MAAM,CAAC,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;IACzD,CAAC;IACD,OAAO,aAAa,CAAC;AACvB,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAe;IAC3C,MAAM,CAAC,GAAG,IAAI,GAAG,EAAoB,CAAC;IACtC,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC;QAC9C,MAAM,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC;IAC3B,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAED,0EAA0E;AAC1E,+EAA+E;AAC/E,8CAA8C;AAC9C,SAAS,MAAM,CAAI,CAAmB,EAAE,GAAW,EAAE,KAAQ;IAC3D,MAAM,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACvB,IAAI,GAAG;QAAE,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;;QACpB,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;AAC3B,CAAC;AAED,SAAS,WAAW,CAAC,KAAa,EAAE,GAAW;IAC7C,IAAI,GAAG,IAAI,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;IACtD,MAAM,CAAC,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC;IACtD,MAAM,CAAC,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IAC5B,OAAO,CAAC,GAAG,GAAG,CAAC;AACjB,CAAC;AAED,2EAA2E;AAC3E,gFAAgF;AAChF,sDAAsD;AACtD,KAAK,UAAU,QAAQ,CACrB,SAAiB,EACjB,IAAY,EACZ,OAAkC;IAElC,IAAI,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC;QAAE,OAAO;IAC9B,IAAI,GAAW,CAAC;IAChB,IAAI,CAAC;QACH,GAAG,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC;IACzD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,+BAA+B;IACzC,CAAC;IACD,MAAM,MAAM,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;IAClC,IAAI,MAAM,CAAC,EAAE,EAAE,CAAC;QACd,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC;IACzF,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC,CAAC;IAC1D,CAAC;AACH,CAAC;AAED,KAAK,UAAU,OAAO,CACpB,SAAiB,EACjB,IAAY,EACZ,OAAkC,EAClC,KAAqB,EACrB,aAAoD,EACpD,eAAsD;IAEtD,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC/B,IAAI,CAAC,IAAI;QAAE,OAAO;IAElB,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC;QACrD,CAAC,CAAE,IAAI,CAAC,WAAW,CAAC,OAAqB;QACzC,CAAC,CAAC,EAAE,CAAC;IACP,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,IAAI,OAAO,CAAC,KAAK,QAAQ;YAAE,SAAS;QACpC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;QAC7C,MAAM,QAAQ,CAAC,SAAS,EAAE,CAAC,EAAE,OAAO,CAAC,CAAC;IACxC,CAAC;IAED,MAAM,KAAK,GAAG,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7C,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;QAC1C,MAAM,QAAQ,CAAC,SAAS,EAAE,CAAC,EAAE,OAAO,CAAC,CAAC;IACxC,CAAC;IAED,MAAM,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;IAC/C,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;QACnD,MAAM,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM,UAAU,GAAG,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;IACnD,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC,CAAC;QACtD,MAAM,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAC9C,CAAC;AACH,CAAC;AAED,SAAS,SAAS,CAChB,IAAY,EACZ,OAAkC,EAClC,KAAqB,EACrB,GAAW;IAEX,IAAI,OAAO,CAAC,IAAI,IAAI,GAAG;QAAE,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACtD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAClD,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAChD,CAAC;IACD,MAAM,MAAM,GAAG,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;SAClC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;SACtE,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACb,IAAI,CAAC,CAAC,IAAI,KAAK,IAAI;YAAE,OAAO,CAAC,CAAC,CAAC;QAC/B,IAAI,CAAC,CAAC,IAAI,KAAK,IAAI;YAAE,OAAO,CAAC,CAAC;QAC9B,gEAAgE;QAChE,wCAAwC;QACxC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC7D,CAAC,CAAC,CAAC;IACL,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;AACjD,CAAC;AAED,8EAA8E;AAC9E,wEAAwE;AACxE,wEAAwE;AACxE,gDAAgD;AAChD,SAAS,mBAAmB,CAAC,IAAY;IACvC,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,MAAM,EAAE,GAAG,uCAAuC,CAAC;IACnD,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC;QAClC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAClB,IAAI,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,SAAS;QAC9C,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,SAAS;QACnC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACjB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { ToolDef } from "./llm.js";
|
|
2
|
+
export interface ToolSurface {
|
|
3
|
+
defs: ToolDef[];
|
|
4
|
+
handler: (name: string, input: unknown) => Promise<unknown>;
|
|
5
|
+
}
|
|
6
|
+
export declare function buildToolSurface(vaultRoot: string): ToolSurface;
|
|
7
|
+
//# sourceMappingURL=tool-surface.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tool-surface.d.ts","sourceRoot":"","sources":["../../src/eval/tool-surface.ts"],"names":[],"mappings":"AAiBA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,UAAU,CAAC;AAExC,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,OAAO,EAAE,CAAC;IAChB,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;CAC7D;AAiID,wBAAgB,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,WAAW,CAyB/D"}
|