@multileaf/ai-workflow 0.2.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/dist/decision.d.ts +49 -0
- package/dist/decision.js +105 -0
- package/dist/semantic.d.ts +39 -0
- package/dist/semantic.js +160 -0
- package/dist/telemetry.js +2 -0
- package/dist/workflow.js +127 -1
- package/docs/backlog.md +2 -0
- package/docs/decision-model-evaluation.md +42 -0
- package/docs/semantic-mapping.md +39 -0
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,19 @@ All notable changes to this project are documented in this file. The format foll
|
|
|
4
4
|
|
|
5
5
|
## [Unreleased]
|
|
6
6
|
|
|
7
|
+
## [0.3.0] - 2026-10-08
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
|
|
11
|
+
- Optional semantic mapping in `aiw scan`: when `.aiw/semantic.yml` enables a local embedding model (for example Ollama `embeddinggemma-2`), the scan verifies model availability, embeds line-aligned file chunks, and writes a versioned index to `.aiw/semantic/index.json`.
|
|
12
|
+
- `aiw semantic query` and `aiw semantic status` commands for cosine-similarity retrieval over the semantic index, gated by the same configuration flag.
|
|
13
|
+
- Advisory decision triage via a local decision model (`jev/clef-flash` through Ollama `/v1/systemone`): flag-gated by `.aiw/decision.yml`, `aiw decision status` and `aiw decision triage --artifact=<path>` return advisory evidence-completeness and gate decisions with confidence, never replacing human approval.
|
|
14
|
+
- Documentation for semantic mapping and an evaluation of the `jev/clef` decision model for advisory flow decisions.
|
|
15
|
+
|
|
16
|
+
### Fixed
|
|
17
|
+
|
|
18
|
+
- Removed high-severity transitive vulnerabilities (`brace-expansion`, `source-map-js`) reported by `npm audit`.
|
|
19
|
+
|
|
7
20
|
## [0.2.1] - 2026-09-13
|
|
8
21
|
|
|
9
22
|
### Changed
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import type { FileSystem } from "./types.js";
|
|
2
|
+
export type DecisionConfig = {
|
|
3
|
+
enabled: boolean;
|
|
4
|
+
model: string;
|
|
5
|
+
baseUrl: string;
|
|
6
|
+
};
|
|
7
|
+
export type DecisionQuestion = {
|
|
8
|
+
type: "noul";
|
|
9
|
+
instructions: string;
|
|
10
|
+
criteria?: {
|
|
11
|
+
true: string;
|
|
12
|
+
false: string;
|
|
13
|
+
};
|
|
14
|
+
} | {
|
|
15
|
+
type: "choice";
|
|
16
|
+
instructions: string;
|
|
17
|
+
criteria: Record<string, string | null>;
|
|
18
|
+
} | {
|
|
19
|
+
type: "score";
|
|
20
|
+
instructions: string;
|
|
21
|
+
criteria: string[];
|
|
22
|
+
};
|
|
23
|
+
export type DecisionAnswer = {
|
|
24
|
+
type: "noul" | "choice" | "score";
|
|
25
|
+
noul?: number;
|
|
26
|
+
choice?: string;
|
|
27
|
+
score?: number;
|
|
28
|
+
probabilities?: Record<string, number>;
|
|
29
|
+
confidence?: number;
|
|
30
|
+
};
|
|
31
|
+
export type DecisionRequest = {
|
|
32
|
+
state: string;
|
|
33
|
+
questions: Record<string, DecisionQuestion>;
|
|
34
|
+
};
|
|
35
|
+
export type DecisionResponse = {
|
|
36
|
+
model: string;
|
|
37
|
+
answers: Record<string, DecisionAnswer>;
|
|
38
|
+
};
|
|
39
|
+
export type DecisionProvider = {
|
|
40
|
+
available(): Promise<boolean>;
|
|
41
|
+
decide(request: DecisionRequest): Promise<DecisionResponse>;
|
|
42
|
+
};
|
|
43
|
+
export declare const DECISION_CONFIG_PATH = "decision.yml";
|
|
44
|
+
export declare const DEFAULT_DECISION_CONFIG: DecisionConfig;
|
|
45
|
+
export declare function parseDecisionConfig(text: string): DecisionConfig;
|
|
46
|
+
export declare function loadDecisionConfig(fs: FileSystem, aiwPath: string): DecisionConfig | undefined;
|
|
47
|
+
export declare function parseDecisionResponse(text: string): DecisionResponse;
|
|
48
|
+
export declare function createOllamaDecisionProvider(config: DecisionConfig): DecisionProvider;
|
|
49
|
+
export declare function buildEvidenceTriageRequest(artifact: string): DecisionRequest;
|
package/dist/decision.js
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import { parseDocument } from "yaml";
|
|
2
|
+
export const DECISION_CONFIG_PATH = "decision.yml";
|
|
3
|
+
export const DEFAULT_DECISION_CONFIG = {
|
|
4
|
+
enabled: false,
|
|
5
|
+
model: "clef-flash:latest",
|
|
6
|
+
baseUrl: "http://localhost:11434",
|
|
7
|
+
};
|
|
8
|
+
export function parseDecisionConfig(text) {
|
|
9
|
+
const document = parseDocument(text);
|
|
10
|
+
const node = document.get("decision");
|
|
11
|
+
if (document.errors.length > 0 || !node || typeof node !== "object")
|
|
12
|
+
throw new Error("Decision configuration must be a YAML object with a `decision` mapping");
|
|
13
|
+
const raw = node.toJSON();
|
|
14
|
+
const enabled = raw.enabled;
|
|
15
|
+
const model = raw.model;
|
|
16
|
+
const baseUrl = raw.baseUrl;
|
|
17
|
+
if (typeof enabled !== "boolean")
|
|
18
|
+
throw new Error("decision.enabled must be a boolean");
|
|
19
|
+
if (!enabled)
|
|
20
|
+
return { ...DEFAULT_DECISION_CONFIG, enabled: false };
|
|
21
|
+
if (typeof model !== "string" || model.trim() === "")
|
|
22
|
+
throw new Error("decision.model must be a non-empty string when decision mapping is enabled");
|
|
23
|
+
if (typeof baseUrl !== "string" || !/^https?:\/\//i.test(baseUrl))
|
|
24
|
+
throw new Error("decision.baseUrl must be an http(s) URL when decision mapping is enabled");
|
|
25
|
+
return { enabled: true, model, baseUrl };
|
|
26
|
+
}
|
|
27
|
+
export function loadDecisionConfig(fs, aiwPath) {
|
|
28
|
+
const configPath = `${aiwPath}/${DECISION_CONFIG_PATH}`;
|
|
29
|
+
if (!fs.exists(configPath))
|
|
30
|
+
return undefined;
|
|
31
|
+
return parseDecisionConfig(fs.read(configPath));
|
|
32
|
+
}
|
|
33
|
+
function parseAnswer(value) {
|
|
34
|
+
if (!value || typeof value !== "object")
|
|
35
|
+
throw new Error("Invalid decision answer");
|
|
36
|
+
const raw = value;
|
|
37
|
+
if (raw.type !== "noul" && raw.type !== "choice" && raw.type !== "score")
|
|
38
|
+
throw new Error("Invalid decision answer type");
|
|
39
|
+
return { ...raw, type: raw.type };
|
|
40
|
+
}
|
|
41
|
+
export function parseDecisionResponse(text) {
|
|
42
|
+
const parsed = JSON.parse(text);
|
|
43
|
+
const answers = parsed.answers;
|
|
44
|
+
if (typeof parsed.model !== "string" || !answers || typeof answers !== "object")
|
|
45
|
+
throw new Error("Invalid decision response");
|
|
46
|
+
const result = {};
|
|
47
|
+
for (const [key, value] of Object.entries(answers))
|
|
48
|
+
result[key] = parseAnswer(value);
|
|
49
|
+
return { model: parsed.model, answers: result };
|
|
50
|
+
}
|
|
51
|
+
export function createOllamaDecisionProvider(config) {
|
|
52
|
+
return {
|
|
53
|
+
async available() {
|
|
54
|
+
try {
|
|
55
|
+
const response = await fetch(`${config.baseUrl}/api/tags`);
|
|
56
|
+
if (!response.ok)
|
|
57
|
+
return false;
|
|
58
|
+
const body = (await response.json());
|
|
59
|
+
const names = new Set((body.models ?? []).flatMap((entry) => typeof entry?.name === "string" && typeof entry?.model === "string"
|
|
60
|
+
? [entry.name, entry.model]
|
|
61
|
+
: []));
|
|
62
|
+
return names.has(config.model);
|
|
63
|
+
}
|
|
64
|
+
catch {
|
|
65
|
+
return false;
|
|
66
|
+
}
|
|
67
|
+
},
|
|
68
|
+
async decide(request) {
|
|
69
|
+
const questionCount = Object.keys(request.questions).length;
|
|
70
|
+
if (questionCount === 0 || questionCount > 64)
|
|
71
|
+
throw new Error("Decision requests must contain between 1 and 64 questions");
|
|
72
|
+
const response = await fetch(`${config.baseUrl}/v1/systemone`, {
|
|
73
|
+
method: "POST",
|
|
74
|
+
headers: { "content-type": "application/json" },
|
|
75
|
+
body: JSON.stringify({
|
|
76
|
+
model: config.model,
|
|
77
|
+
state: request.state,
|
|
78
|
+
questions: request.questions,
|
|
79
|
+
}),
|
|
80
|
+
});
|
|
81
|
+
if (!response.ok)
|
|
82
|
+
throw new Error(`Decision request failed for model ${config.model}: ${await response.text()}`);
|
|
83
|
+
return parseDecisionResponse(await response.text());
|
|
84
|
+
},
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
export function buildEvidenceTriageRequest(artifact) {
|
|
88
|
+
return {
|
|
89
|
+
state: artifact,
|
|
90
|
+
questions: {
|
|
91
|
+
evidence_complete: {
|
|
92
|
+
type: "noul",
|
|
93
|
+
instructions: "Does this artifact include concrete verification evidence for every stated requirement?",
|
|
94
|
+
},
|
|
95
|
+
gate: {
|
|
96
|
+
type: "choice",
|
|
97
|
+
instructions: "Should this artifact advance to human review?",
|
|
98
|
+
criteria: {
|
|
99
|
+
ready: "Every requirement has matching evidence and validation commands",
|
|
100
|
+
"needs-work": "Evidence is missing, incomplete, or does not cover all requirements",
|
|
101
|
+
},
|
|
102
|
+
},
|
|
103
|
+
},
|
|
104
|
+
};
|
|
105
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import type { FileSystem } from "./types.js";
|
|
2
|
+
export type SemanticConfig = {
|
|
3
|
+
enabled: boolean;
|
|
4
|
+
model: string;
|
|
5
|
+
baseUrl: string;
|
|
6
|
+
chunkChars: number;
|
|
7
|
+
topK: number;
|
|
8
|
+
};
|
|
9
|
+
export type SemanticIndexChunk = {
|
|
10
|
+
file: string;
|
|
11
|
+
text: string;
|
|
12
|
+
vector: number[];
|
|
13
|
+
};
|
|
14
|
+
export type SemanticIndex = {
|
|
15
|
+
schema: 1;
|
|
16
|
+
model: string;
|
|
17
|
+
chunks: SemanticIndexChunk[];
|
|
18
|
+
};
|
|
19
|
+
export type SemanticMatch = {
|
|
20
|
+
file: string;
|
|
21
|
+
score: number;
|
|
22
|
+
excerpt: string;
|
|
23
|
+
};
|
|
24
|
+
export type EmbeddingProvider = {
|
|
25
|
+
available(): Promise<boolean>;
|
|
26
|
+
embed(texts: string[]): Promise<number[][]>;
|
|
27
|
+
};
|
|
28
|
+
export declare const SEMANTIC_CONFIG_PATH = "semantic.yml";
|
|
29
|
+
export declare const SEMANTIC_INDEX_PATH = "semantic/index.json";
|
|
30
|
+
export declare const DEFAULT_SEMANTIC_CONFIG: SemanticConfig;
|
|
31
|
+
export declare function parseSemanticConfig(text: string): SemanticConfig;
|
|
32
|
+
export declare function loadSemanticConfig(fs: FileSystem, aiwPath: string): SemanticConfig | undefined;
|
|
33
|
+
export declare function createOllamaEmbeddingProvider(config: SemanticConfig): EmbeddingProvider;
|
|
34
|
+
export declare function chunkText(content: string, chunkChars: number): string[];
|
|
35
|
+
export declare function cosineSimilarity(a: number[], b: number[]): number;
|
|
36
|
+
export declare function buildSemanticIndex(files: string[], readContent: (file: string) => Promise<string | undefined>, provider: EmbeddingProvider, config: SemanticConfig): Promise<SemanticIndex>;
|
|
37
|
+
export declare function serializeSemanticIndex(index: SemanticIndex): string;
|
|
38
|
+
export declare function parseSemanticIndex(text: string): SemanticIndex;
|
|
39
|
+
export declare function querySemanticIndex(index: SemanticIndex, query: string, provider: EmbeddingProvider, config: SemanticConfig): Promise<SemanticMatch[]>;
|
package/dist/semantic.js
ADDED
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
import { parseDocument } from "yaml";
|
|
2
|
+
export const SEMANTIC_CONFIG_PATH = "semantic.yml";
|
|
3
|
+
export const SEMANTIC_INDEX_PATH = "semantic/index.json";
|
|
4
|
+
export const DEFAULT_SEMANTIC_CONFIG = {
|
|
5
|
+
enabled: false,
|
|
6
|
+
model: "embeddinggemma-2:latest",
|
|
7
|
+
baseUrl: "http://localhost:11434",
|
|
8
|
+
chunkChars: 1200,
|
|
9
|
+
topK: 8,
|
|
10
|
+
};
|
|
11
|
+
export function parseSemanticConfig(text) {
|
|
12
|
+
const document = parseDocument(text);
|
|
13
|
+
const node = document.get("semantic");
|
|
14
|
+
if (document.errors.length > 0 || !node || typeof node !== "object")
|
|
15
|
+
throw new Error("Semantic configuration must be a YAML object with a `semantic` mapping");
|
|
16
|
+
const raw = node.toJSON();
|
|
17
|
+
const enabled = raw.enabled;
|
|
18
|
+
const model = raw.model;
|
|
19
|
+
const baseUrl = raw.baseUrl;
|
|
20
|
+
const chunkChars = raw.chunkChars;
|
|
21
|
+
const topK = raw.topK;
|
|
22
|
+
if (typeof enabled !== "boolean")
|
|
23
|
+
throw new Error("semantic.enabled must be a boolean");
|
|
24
|
+
if (!enabled)
|
|
25
|
+
return { ...DEFAULT_SEMANTIC_CONFIG, enabled: false };
|
|
26
|
+
if (typeof model !== "string" || model.trim() === "")
|
|
27
|
+
throw new Error("semantic.model must be a non-empty string when semantic mapping is enabled");
|
|
28
|
+
if (typeof baseUrl !== "string" || !/^https?:\/\//i.test(baseUrl))
|
|
29
|
+
throw new Error("semantic.baseUrl must be an http(s) URL when semantic mapping is enabled");
|
|
30
|
+
const parsedChunk = chunkChars === undefined ? DEFAULT_SEMANTIC_CONFIG.chunkChars : chunkChars;
|
|
31
|
+
const parsedTopK = topK === undefined ? DEFAULT_SEMANTIC_CONFIG.topK : topK;
|
|
32
|
+
if (typeof parsedChunk !== "number" || !Number.isInteger(parsedChunk) || parsedChunk < 200)
|
|
33
|
+
throw new Error("semantic.chunkChars must be an integer of at least 200");
|
|
34
|
+
if (typeof parsedTopK !== "number" || !Number.isInteger(parsedTopK) || parsedTopK < 1)
|
|
35
|
+
throw new Error("semantic.topK must be a positive integer");
|
|
36
|
+
return { enabled: true, model, baseUrl, chunkChars: parsedChunk, topK: parsedTopK };
|
|
37
|
+
}
|
|
38
|
+
export function loadSemanticConfig(fs, aiwPath) {
|
|
39
|
+
const configPath = `${aiwPath}/${SEMANTIC_CONFIG_PATH}`;
|
|
40
|
+
if (!fs.exists(configPath))
|
|
41
|
+
return undefined;
|
|
42
|
+
return parseSemanticConfig(fs.read(configPath));
|
|
43
|
+
}
|
|
44
|
+
export function createOllamaEmbeddingProvider(config) {
|
|
45
|
+
const fetchImpl = (url, init) => fetch(url, init);
|
|
46
|
+
return {
|
|
47
|
+
async available() {
|
|
48
|
+
try {
|
|
49
|
+
const response = await fetchImpl(`${config.baseUrl}/api/tags`);
|
|
50
|
+
if (!response.ok)
|
|
51
|
+
return false;
|
|
52
|
+
const body = (await response.json());
|
|
53
|
+
const names = new Set((body.models ?? []).flatMap((entry) => typeof entry?.name === "string" && typeof entry?.model === "string"
|
|
54
|
+
? [entry.name, entry.model]
|
|
55
|
+
: []));
|
|
56
|
+
return names.has(config.model);
|
|
57
|
+
}
|
|
58
|
+
catch {
|
|
59
|
+
return false;
|
|
60
|
+
}
|
|
61
|
+
},
|
|
62
|
+
async embed(texts) {
|
|
63
|
+
const vectors = [];
|
|
64
|
+
for (const text of texts) {
|
|
65
|
+
const response = await fetchImpl(`${config.baseUrl}/api/embeddings`, {
|
|
66
|
+
method: "POST",
|
|
67
|
+
headers: { "content-type": "application/json" },
|
|
68
|
+
body: JSON.stringify({ model: config.model, prompt: text }),
|
|
69
|
+
});
|
|
70
|
+
if (!response.ok)
|
|
71
|
+
throw new Error(`Embedding request failed for model ${config.model}`);
|
|
72
|
+
const body = (await response.json());
|
|
73
|
+
if (!Array.isArray(body.embedding) || body.embedding.some((v) => typeof v !== "number"))
|
|
74
|
+
throw new Error(`Embedding model ${config.model} returned an invalid vector`);
|
|
75
|
+
vectors.push(body.embedding);
|
|
76
|
+
}
|
|
77
|
+
return vectors;
|
|
78
|
+
},
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
export function chunkText(content, chunkChars) {
|
|
82
|
+
const lines = content.split("\n");
|
|
83
|
+
const chunks = [];
|
|
84
|
+
let current = [];
|
|
85
|
+
let length = 0;
|
|
86
|
+
for (const line of lines) {
|
|
87
|
+
if (length + line.length + 1 > chunkChars && current.length > 0) {
|
|
88
|
+
chunks.push(current.join("\n"));
|
|
89
|
+
current = [];
|
|
90
|
+
length = 0;
|
|
91
|
+
}
|
|
92
|
+
current.push(line);
|
|
93
|
+
length += line.length + 1;
|
|
94
|
+
}
|
|
95
|
+
if (current.length > 0 && current.join("").trim().length > 0)
|
|
96
|
+
chunks.push(current.join("\n"));
|
|
97
|
+
return chunks;
|
|
98
|
+
}
|
|
99
|
+
export function cosineSimilarity(a, b) {
|
|
100
|
+
let dot = 0;
|
|
101
|
+
let normA = 0;
|
|
102
|
+
let normB = 0;
|
|
103
|
+
for (let index = 0; index < a.length && index < b.length; index += 1) {
|
|
104
|
+
dot += a[index] * b[index];
|
|
105
|
+
normA += a[index] * a[index];
|
|
106
|
+
normB += b[index] * b[index];
|
|
107
|
+
}
|
|
108
|
+
if (normA === 0 || normB === 0)
|
|
109
|
+
return 0;
|
|
110
|
+
return dot / (Math.sqrt(normA) * Math.sqrt(normB));
|
|
111
|
+
}
|
|
112
|
+
export async function buildSemanticIndex(files, readContent, provider, config) {
|
|
113
|
+
const entries = [];
|
|
114
|
+
for (const file of files) {
|
|
115
|
+
const content = await readContent(file);
|
|
116
|
+
if (content === undefined)
|
|
117
|
+
continue;
|
|
118
|
+
for (const chunk of chunkText(content, config.chunkChars))
|
|
119
|
+
entries.push({ file, text: chunk });
|
|
120
|
+
}
|
|
121
|
+
if (entries.length === 0)
|
|
122
|
+
return { schema: 1, model: config.model, chunks: [] };
|
|
123
|
+
const vectors = await provider.embed(entries.map((entry) => entry.text));
|
|
124
|
+
return {
|
|
125
|
+
schema: 1,
|
|
126
|
+
model: config.model,
|
|
127
|
+
chunks: entries.map((entry, index) => ({ ...entry, vector: vectors[index] })),
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
export function serializeSemanticIndex(index) {
|
|
131
|
+
return JSON.stringify(index);
|
|
132
|
+
}
|
|
133
|
+
export function parseSemanticIndex(text) {
|
|
134
|
+
const parsed = JSON.parse(text);
|
|
135
|
+
if (parsed.schema !== 1 || typeof parsed.model !== "string" || !Array.isArray(parsed.chunks))
|
|
136
|
+
throw new Error("Invalid semantic index");
|
|
137
|
+
for (const chunk of parsed.chunks) {
|
|
138
|
+
if (typeof chunk.file !== "string" ||
|
|
139
|
+
typeof chunk.text !== "string" ||
|
|
140
|
+
!Array.isArray(chunk.vector) ||
|
|
141
|
+
chunk.vector.some((value) => typeof value !== "number"))
|
|
142
|
+
throw new Error("Invalid semantic index chunk");
|
|
143
|
+
}
|
|
144
|
+
return parsed;
|
|
145
|
+
}
|
|
146
|
+
export async function querySemanticIndex(index, query, provider, config) {
|
|
147
|
+
if (index.model !== config.model)
|
|
148
|
+
throw new Error(`Semantic index was built with ${index.model} but the active semantic model is ${config.model}`);
|
|
149
|
+
if (index.chunks.length === 0)
|
|
150
|
+
return [];
|
|
151
|
+
const [vector] = await provider.embed([query]);
|
|
152
|
+
return index.chunks
|
|
153
|
+
.map((chunk) => ({
|
|
154
|
+
file: chunk.file,
|
|
155
|
+
score: cosineSimilarity(vector, chunk.vector),
|
|
156
|
+
excerpt: chunk.text.split("\n").slice(0, 3).join("\n").slice(0, 200),
|
|
157
|
+
}))
|
|
158
|
+
.sort((a, b) => b.score - a.score)
|
|
159
|
+
.slice(0, config.topK);
|
|
160
|
+
}
|
package/dist/telemetry.js
CHANGED
|
@@ -13,6 +13,7 @@ const TELEMETRY_COMMANDS = new Set([
|
|
|
13
13
|
"context",
|
|
14
14
|
"context-quality",
|
|
15
15
|
"context-summary",
|
|
16
|
+
"decision",
|
|
16
17
|
"doctor",
|
|
17
18
|
"gate",
|
|
18
19
|
"generate",
|
|
@@ -29,6 +30,7 @@ const TELEMETRY_COMMANDS = new Set([
|
|
|
29
30
|
"rollback",
|
|
30
31
|
"scan",
|
|
31
32
|
"self-validate",
|
|
33
|
+
"semantic",
|
|
32
34
|
"skills",
|
|
33
35
|
"spec",
|
|
34
36
|
"status",
|
package/dist/workflow.js
CHANGED
|
@@ -21,6 +21,8 @@ import { validatePackageContract } from "./package-contract.js";
|
|
|
21
21
|
import { parseLock, resolvePackage, serializeLock } from "./lockfile.js";
|
|
22
22
|
import { executeVercelSkills, installVercelSkill, nodeCommandExecutor } from "./vercel-skills.js";
|
|
23
23
|
import { scanProject } from "./scanner.js";
|
|
24
|
+
import { buildEvidenceTriageRequest, createOllamaDecisionProvider, loadDecisionConfig, DECISION_CONFIG_PATH, } from "./decision.js";
|
|
25
|
+
import { buildSemanticIndex, createOllamaEmbeddingProvider, loadSemanticConfig, parseSemanticIndex, querySemanticIndex, SEMANTIC_INDEX_PATH, serializeSemanticIndex, } from "./semantic.js";
|
|
24
26
|
import { assertKnownPermissions, assertPermissions, assertPackagePermissions, serializePermissionReview, } from "./package-audit.js";
|
|
25
27
|
import { planPackageUpdate } from "./package-updates.js";
|
|
26
28
|
import { searchRegistry, serializeRegistry } from "./registry.js";
|
|
@@ -426,8 +428,126 @@ export async function runCommand(args, root, fs, services = {}) {
|
|
|
426
428
|
.map((fact) => ` - key: ${fact.key}\n value: ${fact.value}\n state: ${fact.state}\n method: ${fact.method}\n confidence: ${fact.confidence}${fact.requiresConfirmation === undefined ? "" : `\n requires_confirmation: ${fact.requiresConfirmation}`}\n evidence: [${fact.evidence.map((item) => `"${item.source}"`).join(", ")}]`)
|
|
427
429
|
.join("\n");
|
|
428
430
|
fs.write(join(aiw, "profile.yml"), `schema: 1\nstatus: scanned\nfacts:\n${evidence}\nruntime:\n languages: [${profile.runtime.languages.join(", ")}]\nframeworks: [${profile.frameworks.join(", ")}]\npackage_manager: ${profile.packageManager}\nquality:\n linter: ${profile.quality.linter?.name ?? "unknown"}\n linter_command: ${profile.quality.linter?.command ?? "unknown"}\n formatter: ${profile.quality.formatter?.name ?? "unknown"}\n formatter_command: ${profile.quality.formatter?.command ?? "unknown"}\n typecheck: ${profile.quality.typecheck?.name ?? "unknown"}\n typecheck_command: ${profile.quality.typecheck?.command ?? "unknown"}\ntesting: ${profile.testing?.name ?? "unknown"}\ntesting_command: ${profile.testing?.command ?? "unknown"}\nci: [${profile.ci.join(", ")}]\nworkspaces: [${profile.workspaces.join(", ")}]\nproject_modules: ${JSON.stringify(profile.modules ?? [])}\npolicies:\n artifact_language: en\n`);
|
|
431
|
+
const semanticConfig = loadSemanticConfig(fs, aiw);
|
|
432
|
+
if (semanticConfig?.enabled) {
|
|
433
|
+
const provider = createOllamaEmbeddingProvider(semanticConfig);
|
|
434
|
+
if (!(await provider.available()))
|
|
435
|
+
return {
|
|
436
|
+
exitCode: 1,
|
|
437
|
+
error: `Semantic model ${semanticConfig.model} is not available at ${semanticConfig.baseUrl}.`,
|
|
438
|
+
};
|
|
439
|
+
const scanFiles = await (services.scanner?.scan(root) ?? scanProject(root));
|
|
440
|
+
const index = await buildSemanticIndex(scanFiles.files, async (file) => {
|
|
441
|
+
const absolute = join(root, file);
|
|
442
|
+
try {
|
|
443
|
+
return new TextDecoder("utf-8", { fatal: true }).decode(fs.readBytes?.(absolute) ?? Buffer.from(fs.read(absolute), "utf8"));
|
|
444
|
+
}
|
|
445
|
+
catch {
|
|
446
|
+
return undefined;
|
|
447
|
+
}
|
|
448
|
+
}, provider, semanticConfig);
|
|
449
|
+
fs.mkdir(join(aiw, "semantic"));
|
|
450
|
+
fs.write(join(aiw, SEMANTIC_INDEX_PATH), serializeSemanticIndex(index));
|
|
451
|
+
}
|
|
429
452
|
return { exitCode: 0, output: "Project profile generated." };
|
|
430
453
|
}
|
|
454
|
+
if (command === "decision") {
|
|
455
|
+
const usage = "Usage: aiw decision triage --artifact=<path> | aiw decision status";
|
|
456
|
+
const action = args[1];
|
|
457
|
+
const config = loadDecisionConfig(fs, aiw);
|
|
458
|
+
if (action === "status") {
|
|
459
|
+
if (!config)
|
|
460
|
+
return {
|
|
461
|
+
exitCode: 0,
|
|
462
|
+
output: `Decision model is not configured. Create .aiw/${DECISION_CONFIG_PATH} to enable it.`,
|
|
463
|
+
};
|
|
464
|
+
return {
|
|
465
|
+
exitCode: 0,
|
|
466
|
+
output: `Decision model ${config.enabled ? "enabled" : "disabled"}: ${config.model} at ${config.baseUrl}`,
|
|
467
|
+
};
|
|
468
|
+
}
|
|
469
|
+
if (action === "triage") {
|
|
470
|
+
const parsed = parseNamedOptions(args.slice(2), ["artifact"]);
|
|
471
|
+
if (!parsed?.artifact)
|
|
472
|
+
return { exitCode: 1, error: usage };
|
|
473
|
+
if (!config?.enabled)
|
|
474
|
+
return {
|
|
475
|
+
exitCode: 1,
|
|
476
|
+
error: "Decision model is disabled. Set decision.enabled=true in .aiw/decision.yml.",
|
|
477
|
+
};
|
|
478
|
+
const artifactPath = safeProjectFile(root, parsed.artifact, fs);
|
|
479
|
+
if (!fs.exists(artifactPath))
|
|
480
|
+
return { exitCode: 1, error: `Artifact not found: ${parsed.artifact}` };
|
|
481
|
+
const provider = createOllamaDecisionProvider(config);
|
|
482
|
+
if (!(await provider.available()))
|
|
483
|
+
return {
|
|
484
|
+
exitCode: 1,
|
|
485
|
+
error: `Decision model ${config.model} is not available at ${config.baseUrl}.`,
|
|
486
|
+
};
|
|
487
|
+
const decision = await provider.decide(buildEvidenceTriageRequest(fs.read(artifactPath)));
|
|
488
|
+
const gate = decision.answers.gate;
|
|
489
|
+
const evidence = decision.answers.evidence_complete;
|
|
490
|
+
return {
|
|
491
|
+
exitCode: 0,
|
|
492
|
+
output: [
|
|
493
|
+
"Advisory decision (never replaces human approval):",
|
|
494
|
+
` gate: ${gate?.choice ?? "unknown"} (confidence ${formatConfidence(gate?.confidence)})`,
|
|
495
|
+
` evidence_complete: ${formatNoul(evidence?.noul)}`,
|
|
496
|
+
].join("\n"),
|
|
497
|
+
};
|
|
498
|
+
}
|
|
499
|
+
return { exitCode: 1, error: usage };
|
|
500
|
+
}
|
|
501
|
+
if (command === "semantic") {
|
|
502
|
+
const usage = "Usage: aiw semantic query --text=<query> | aiw semantic status";
|
|
503
|
+
const action = args[1];
|
|
504
|
+
if (action === "status") {
|
|
505
|
+
const config = loadSemanticConfig(fs, aiw);
|
|
506
|
+
if (!config)
|
|
507
|
+
return {
|
|
508
|
+
exitCode: 0,
|
|
509
|
+
output: `Semantic mapping is not configured. Create .aiw/${"semantic.yml"} to enable it.`,
|
|
510
|
+
};
|
|
511
|
+
const indexPath = join(aiw, SEMANTIC_INDEX_PATH);
|
|
512
|
+
const built = fs.exists(indexPath)
|
|
513
|
+
? `indexed chunks: ${parseSemanticIndex(fs.read(indexPath)).chunks.length}`
|
|
514
|
+
: "index: not built (run aiw scan)";
|
|
515
|
+
return {
|
|
516
|
+
exitCode: 0,
|
|
517
|
+
output: `Semantic mapping ${config.enabled ? "enabled" : "disabled"}: model ${config.model} at ${config.baseUrl}\n${built}`,
|
|
518
|
+
};
|
|
519
|
+
}
|
|
520
|
+
if (action === "query") {
|
|
521
|
+
const parsed = parseNamedOptions(args.slice(2), ["text"]);
|
|
522
|
+
if (!parsed?.text)
|
|
523
|
+
return { exitCode: 1, error: usage };
|
|
524
|
+
const config = loadSemanticConfig(fs, aiw);
|
|
525
|
+
if (!config?.enabled)
|
|
526
|
+
return {
|
|
527
|
+
exitCode: 1,
|
|
528
|
+
error: "Semantic mapping is disabled. Set semantic.enabled=true in .aiw/semantic.yml.",
|
|
529
|
+
};
|
|
530
|
+
const indexPath = join(aiw, SEMANTIC_INDEX_PATH);
|
|
531
|
+
if (!fs.exists(indexPath))
|
|
532
|
+
return { exitCode: 1, error: "Semantic index is missing. Run `aiw scan` first." };
|
|
533
|
+
const provider = createOllamaEmbeddingProvider(config);
|
|
534
|
+
if (!(await provider.available()))
|
|
535
|
+
return {
|
|
536
|
+
exitCode: 1,
|
|
537
|
+
error: `Semantic model ${config.model} is not available at ${config.baseUrl}.`,
|
|
538
|
+
};
|
|
539
|
+
const matches = await querySemanticIndex(parseSemanticIndex(fs.read(indexPath)), parsed.text, provider, config);
|
|
540
|
+
return {
|
|
541
|
+
exitCode: 0,
|
|
542
|
+
output: matches.length === 0
|
|
543
|
+
? "No semantic matches."
|
|
544
|
+
: matches
|
|
545
|
+
.map((match) => `- ${match.file} (score ${match.score.toFixed(3)})\n ${match.excerpt}`)
|
|
546
|
+
.join("\n"),
|
|
547
|
+
};
|
|
548
|
+
}
|
|
549
|
+
return { exitCode: 1, error: usage };
|
|
550
|
+
}
|
|
431
551
|
if (command === "target") {
|
|
432
552
|
const target = args[1];
|
|
433
553
|
if (!target)
|
|
@@ -1586,7 +1706,7 @@ export async function runCommand(args, root, fs, services = {}) {
|
|
|
1586
1706
|
}
|
|
1587
1707
|
return {
|
|
1588
1708
|
exitCode: 0,
|
|
1589
|
-
output: "aiw install [--target target] | workflow <start|status|complete> [--id=id] | approve <stage> | reject <stage> --reason=text | skip <optional-stage> --reason=text | ui [--port=number] | orchestrate --plan=path [--execute] [--max-parallel=number] | telemetry <status|enable|disable> [privacy options] | plugin <create|validate> [options] | scan | self-validate --ticket=TYPE-000 | organization-policy --file=path [--replace] | policy-check --package=path [--package=path] | preset --file=path [--replace] | context | context-summary | context-quality | capabilities [--target=target] | token-usage [--stage=name --budget=number --used=number] | registry [--search=query] [--private=name] | registry configure --file=path [--replace] | skills <search|inspect|install|check|update> [options] | audit-package --package=path | verify-package --package=path --checksum=sha256 | brainstorm [--title=title] | spec [--title=title] | adr [--id=id --title=title] | plan [--title=title] | verify | trace | gate <stage> | recommend [--select=id,id] | status | doctor | repair | uninstall [--dry-run] | target <target> | rollback | confirm | resolve --package=path | update --package=path --target=target | validate [--package=path]",
|
|
1709
|
+
output: "aiw install [--target target] | workflow <start|status|complete> [--id=id] | approve <stage> | reject <stage> --reason=text | skip <optional-stage> --reason=text | ui [--port=number] | orchestrate --plan=path [--execute] [--max-parallel=number] | telemetry <status|enable|disable> [privacy options] | plugin <create|validate> [options] | scan | semantic <status|query> | decision <status|triage> | self-validate --ticket=TYPE-000 | organization-policy --file=path [--replace] | policy-check --package=path [--package=path] | preset --file=path [--replace] | context | context-summary | context-quality | capabilities [--target=target] | token-usage [--stage=name --budget=number --used=number] | registry [--search=query] [--private=name] | registry configure --file=path [--replace] | skills <search|inspect|install|check|update> [options] | audit-package --package=path | verify-package --package=path --checksum=sha256 | brainstorm [--title=title] | spec [--title=title] | adr [--id=id --title=title] | plan [--title=title] | verify | trace | gate <stage> | recommend [--select=id,id] | status | doctor | repair | uninstall [--dry-run] | target <target> | rollback | confirm | resolve --package=path | update --package=path --target=target | validate [--package=path]",
|
|
1590
1710
|
};
|
|
1591
1711
|
}
|
|
1592
1712
|
catch (error) {
|
|
@@ -1661,6 +1781,12 @@ function approvalArtifactPath(aiw, stage) {
|
|
|
1661
1781
|
};
|
|
1662
1782
|
return resolve(aiw, paths[stage]);
|
|
1663
1783
|
}
|
|
1784
|
+
function formatConfidence(confidence) {
|
|
1785
|
+
return confidence === undefined ? "unknown" : confidence.toFixed(3);
|
|
1786
|
+
}
|
|
1787
|
+
function formatNoul(noul) {
|
|
1788
|
+
return noul === undefined ? "unknown" : noul.toFixed(3);
|
|
1789
|
+
}
|
|
1664
1790
|
function parseNamedOptions(args, allowed) {
|
|
1665
1791
|
const entries = args.map((arg) => arg.match(/^--([a-z-]+)=(.+)$/)?.slice(1));
|
|
1666
1792
|
if (args.some((arg) => arg.indexOf("=") !== arg.lastIndexOf("=")) ||
|
package/docs/backlog.md
CHANGED
|
@@ -68,6 +68,8 @@ Priority levels: **P0** blocks the product or makes it unsafe; **P1** is require
|
|
|
68
68
|
|
|
69
69
|
## Context and cost — P2
|
|
70
70
|
|
|
71
|
+
- [x] **SEM-001** Add flag-gated semantic mapping to scan and retrieval. _Acceptance:_ `.aiw/semantic.yml` with `enabled: true` verifies model availability before indexing, builds a versioned chunk index during `aiw scan`, and `aiw semantic query` returns ranked matches; missing or disabled configuration keeps scans fully deterministic and offline; semantic results are retrieval aids, never confirmed facts.
|
|
72
|
+
- [x] **SEM-002** Integrate a decision model (jev/clef-flash via Ollama `/v1/systemone`) for advisory gate triage. _Acceptance:_ `.aiw/decision.yml` with `enabled: true` verifies model availability before use; `aiw decision triage` returns advisory `evidence_complete` and `gate` decisions with confidence; disabled or missing configuration makes no network request; advisory output never replaces human approval. See `docs/decision-model-evaluation.md`.
|
|
71
73
|
- [x] **CTX-001** Build layered project context store.
|
|
72
74
|
- [x] **CTX-002** Add task-scoped context retrieval and delta loading.
|
|
73
75
|
- [x] **CTX-003** Add summary cache and freshness invalidation.
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# Decision Model Evaluation: jev/clef (clef-flash)
|
|
2
|
+
|
|
3
|
+
Status: implemented 2026-10-08 as an advisory triage layer behind a flag (`.aiw/decision.yml`, commands `aiw decision status` / `aiw decision triage`). It uses the `/v1/systemone` endpoint.
|
|
4
|
+
|
|
5
|
+
## What the model offers
|
|
6
|
+
|
|
7
|
+
Local inspection (`ollama show clef-flash:latest`, Ollama 0.40.1, MLX runner):
|
|
8
|
+
|
|
9
|
+
| Property | Value |
|
|
10
|
+
| ------------------------- | -------------------------------------------- |
|
|
11
|
+
| Architecture | `clef` (purpose-built decision model by jev) |
|
|
12
|
+
| Parameters / quantization | 9.5B, mxfp8, ~12 GB |
|
|
13
|
+
| Context | 262,144 tokens |
|
|
14
|
+
| Capabilities | `decision`, `vision` |
|
|
15
|
+
| License | Apache 2.0 |
|
|
16
|
+
|
|
17
|
+
A dedicated decision model is architecturally attractive for AI Workflow: the SDD flow already separates automated quality gates from human approval, and an advisory triage layer between them could pre-screen evidence completeness, rank recommendations, and classify drift before a human reviews it.
|
|
18
|
+
|
|
19
|
+
## API
|
|
20
|
+
|
|
21
|
+
Clef Flash answers typed questions over a supplied state in a single non-autoregressive forward pass (Ollama `/v1/systemone`). Question types used by AI Workflow:
|
|
22
|
+
|
|
23
|
+
- `noul` — yes/no with the probability that the answer is true (`evidence_complete`).
|
|
24
|
+
- `choice` — ordered options with per-option probabilities (`gate`: `ready` / `needs-work`).
|
|
25
|
+
|
|
26
|
+
## Implemented integration
|
|
27
|
+
|
|
28
|
+
The earlier blocker (no exposed endpoint in Ollama 0.40.1) is resolved by the `/v1/systemone` endpoint documented at <https://ollama.com/library/clef-flash>. AI Workflow now ships the advisory layer:
|
|
29
|
+
|
|
30
|
+
- `src/decision.ts`: `DecisionConfig` (`.aiw/decision.yml`, flag-gated: `enabled` / `model` / `baseUrl`), a `DecisionProvider` with availability verification (`/api/tags`), typed question/response contracts, and `buildEvidenceTriageRequest` for pre-gate triage.
|
|
31
|
+
- `aiw decision status` and `aiw decision triage --artifact=<path>`: advisory `evidence_complete` (noul) and `gate` (`ready` / `needs-work`) decisions over any project artifact, with explicit confidence reporting.
|
|
32
|
+
|
|
33
|
+
Observed behavior on real plans: an evidence-free plan scored `gate: ready` with confidence 0.044 and `evidence_complete` 0.260, while a fully evidenced plan scored confidence 0.718 and `evidence_complete` 0.924 — low confidence signals an inconclusive decision rather than approval.
|
|
34
|
+
|
|
35
|
+
## Future integration points (advisory)
|
|
36
|
+
|
|
37
|
+
- **Recommendation ranking**: order capability recommendations by decision-model relevance to the confirmed profile.
|
|
38
|
+
- **Semantic re-ranking**: combine embedding recall (`aiw semantic query`) with clef re-ranking for sharper context selection during interpretation.
|
|
39
|
+
|
|
40
|
+
## Hard boundary (unchanged)
|
|
41
|
+
|
|
42
|
+
Decision-model output is `inferred` evidence with confidence, never `confirmed`. It can move an artifact to _pending review_, never to _approved_, and cannot replace the fingerprinted human approval ledger. This matches the existing approval contract and keeps the audit trail human-owned.
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# Semantic Mapping
|
|
2
|
+
|
|
3
|
+
AI Workflow can build a semantic index of the scanned project with a local embedding model such as `embeddinggemma-2` served by Ollama. Semantic mapping is opt-in and flag-gated: it activates only when a valid configuration exists, and every scan and flow query re-reads the flag.
|
|
4
|
+
|
|
5
|
+
## Configuration
|
|
6
|
+
|
|
7
|
+
Create `.aiw/semantic.yml`:
|
|
8
|
+
|
|
9
|
+
```yaml
|
|
10
|
+
semantic:
|
|
11
|
+
enabled: true
|
|
12
|
+
model: embeddinggemma-2:latest
|
|
13
|
+
baseUrl: http://localhost:11434
|
|
14
|
+
chunkChars: 1200 # optional, minimum 200
|
|
15
|
+
topK: 8 # optional, results per query
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
- `enabled: false` (or a missing file) disables semantic mapping entirely; scans stay deterministic and make no network request.
|
|
19
|
+
- When enabled, `aiw scan` verifies that `model` is present at `baseUrl` (`GET /api/tags`) before indexing. A missing model fails the scan with an actionable error instead of producing a partial index.
|
|
20
|
+
|
|
21
|
+
## Indexing during scan
|
|
22
|
+
|
|
23
|
+
When semantic mapping is enabled, `aiw scan` additionally:
|
|
24
|
+
|
|
25
|
+
1. Reads the scanned files with strict UTF-8 decoding (unreadable files are skipped).
|
|
26
|
+
2. Splits each file into line-aligned chunks of at most `chunkChars` characters.
|
|
27
|
+
3. Embeds each chunk with `POST /api/embeddings`.
|
|
28
|
+
4. Writes the versioned index to `.aiw/semantic/index.json` (`schema: 1`).
|
|
29
|
+
|
|
30
|
+
The index stores the model name; queries against an index built with a different model fail explicitly.
|
|
31
|
+
|
|
32
|
+
## Queries during the flow
|
|
33
|
+
|
|
34
|
+
`aiw semantic query --text="<natural language>"` ranks indexed chunks by cosine similarity and returns the top `topK` matches with file, score, and excerpt. `aiw semantic status` reports whether mapping is configured and whether the index is built.
|
|
35
|
+
|
|
36
|
+
## Boundaries
|
|
37
|
+
|
|
38
|
+
- Semantic results are retrieval aids, not evidence. They never create confirmed project facts and never bypass human approval gates.
|
|
39
|
+
- The index is derived state; delete and rebuild it with `aiw scan` after structural changes.
|