@team-harness/memory-algorithms 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +27 -0
- package/README.md +203 -0
- package/dist/contracts.d.ts +178 -0
- package/dist/contracts.js +1 -0
- package/dist/index.d.ts +14 -0
- package/dist/index.js +6 -0
- package/dist/runtime/documents.d.ts +15 -0
- package/dist/runtime/documents.js +209 -0
- package/dist/runtime/l1.d.ts +10 -0
- package/dist/runtime/l1.js +172 -0
- package/dist/runtime/run.d.ts +44 -0
- package/dist/runtime/run.js +191 -0
- package/dist/runtime/skill-workspace.d.ts +70 -0
- package/dist/runtime/skill-workspace.js +156 -0
- package/dist/runtime/skills.d.ts +9 -0
- package/dist/runtime/skills.js +48 -0
- package/dist/runtime/telemetry.d.ts +10 -0
- package/dist/runtime/telemetry.js +5 -0
- package/dist/runtime/tools.d.ts +15 -0
- package/dist/runtime/tools.js +5 -0
- package/dist/upstream/config.d.ts +1 -0
- package/dist/upstream/config.js +1 -0
- package/dist/upstream/core/conversation/l0-recorder.d.ts +6 -0
- package/dist/upstream/core/conversation/l0-recorder.js +1 -0
- package/dist/upstream/core/memory-prompt/composer.d.ts +6 -0
- package/dist/upstream/core/memory-prompt/composer.js +33 -0
- package/dist/upstream/core/memory-prompt/types.d.ts +103 -0
- package/dist/upstream/core/memory-prompt/types.js +21 -0
- package/dist/upstream/core/prompts/l1-dedup.d.ts +33 -0
- package/dist/upstream/core/prompts/l1-dedup.js +202 -0
- package/dist/upstream/core/prompts/l1-extraction.d.ts +24 -0
- package/dist/upstream/core/prompts/l1-extraction.js +400 -0
- package/dist/upstream/core/prompts/persona-generation.d.ts +29 -0
- package/dist/upstream/core/prompts/persona-generation.js +284 -0
- package/dist/upstream/core/prompts/scene-extraction.d.ts +40 -0
- package/dist/upstream/core/prompts/scene-extraction.js +534 -0
- package/dist/upstream/core/record/l1-dedup.d.ts +10 -0
- package/dist/upstream/core/record/l1-dedup.js +108 -0
- package/dist/upstream/core/record/l1-extractor.d.ts +33 -0
- package/dist/upstream/core/record/l1-extractor.js +128 -0
- package/dist/upstream/core/record/l1-writer.d.ts +95 -0
- package/dist/upstream/core/record/l1-writer.js +1 -0
- package/dist/upstream/core/scene/filename-normalizer.d.ts +6 -0
- package/dist/upstream/core/scene/filename-normalizer.js +30 -0
- package/dist/upstream/core/scene/scene-format.d.ts +26 -0
- package/dist/upstream/core/scene/scene-format.js +53 -0
- package/dist/upstream/core/scene/scene-index.d.ts +7 -0
- package/dist/upstream/core/scene/scene-index.js +1 -0
- package/dist/upstream/core/scene/scene-navigation.d.ts +66 -0
- package/dist/upstream/core/scene/scene-navigation.js +107 -0
- package/dist/upstream/core/skill/conversation-add/message-compressor.d.ts +47 -0
- package/dist/upstream/core/skill/conversation-add/message-compressor.js +58 -0
- package/dist/upstream/core/skill/conversation-add/oversize-strategy.d.ts +41 -0
- package/dist/upstream/core/skill/conversation-add/oversize-strategy.js +100 -0
- package/dist/upstream/core/skill/prompts/skill-review-prompt.d.ts +39 -0
- package/dist/upstream/core/skill/prompts/skill-review-prompt.js +197 -0
- package/dist/upstream/core/skill/skill-extractor.d.ts +146 -0
- package/dist/upstream/core/skill/skill-extractor.js +432 -0
- package/dist/upstream/core/skill/skill-format.d.ts +46 -0
- package/dist/upstream/core/skill/skill-format.js +191 -0
- package/dist/upstream/core/skill/skill-tools.d.ts +75 -0
- package/dist/upstream/core/skill/skill-tools.js +193 -0
- package/dist/upstream/core/skill/types.d.ts +324 -0
- package/dist/upstream/core/skill/types.js +7 -0
- package/dist/upstream/utils/sanitize.d.ts +96 -0
- package/dist/upstream/utils/sanitize.js +359 -0
- package/package.json +28 -0
- package/upstream/baseline.json +426 -0
- package/upstream/changes.md +81 -0
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { Atom, Continuation, Dependencies, ExtractionResult, Message, RunInput } from "../contracts.js";
|
|
2
|
+
export interface L1Input extends RunInput {
|
|
3
|
+
messages: Message[];
|
|
4
|
+
continuation?: Continuation;
|
|
5
|
+
maxNewMessages?: number;
|
|
6
|
+
maxBackgroundMessages?: number;
|
|
7
|
+
conflictRecallTopK?: number;
|
|
8
|
+
}
|
|
9
|
+
export declare function validateMessages(messages: Message[]): void;
|
|
10
|
+
export declare function extractL1(deps: Dependencies, input: L1Input): Promise<ExtractionResult<Atom>>;
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
import { formatExtractionPrompt, getExtractMemoriesSystemPrompt } from "../upstream/core/prompts/l1-extraction.js";
|
|
2
|
+
import { formatBatchConflictPrompt, getConflictDetectionSystemPrompt } from "../upstream/core/prompts/l1-dedup.js";
|
|
3
|
+
import { parseExtractionResult, normalizeType } from "../upstream/core/record/l1-extractor.js";
|
|
4
|
+
import { parseBatchResult } from "../upstream/core/record/l1-dedup.js";
|
|
5
|
+
import { composeMemorySystemPrompt } from "../upstream/core/memory-prompt/composer.js";
|
|
6
|
+
import { shouldExtractL1 } from "../upstream/utils/sanitize.js";
|
|
7
|
+
import { ALGORITHM_VERSION, Run, coverage, hash, uniqueRefs } from "./run.js";
|
|
8
|
+
// Port of l1-candidate-recall.ts: native hybrid first, otherwise parallel legs + RRF(k=60).
|
|
9
|
+
async function recall(run, query, topK) {
|
|
10
|
+
const search = run.deps.memories;
|
|
11
|
+
if (!search)
|
|
12
|
+
return [];
|
|
13
|
+
if (++run.recallQueries > run.limits.maxRecallQueries)
|
|
14
|
+
throw new Error("Candidate recall budget exhausted");
|
|
15
|
+
const leg = async (fn) => {
|
|
16
|
+
try {
|
|
17
|
+
return await run.wait(fn());
|
|
18
|
+
}
|
|
19
|
+
catch {
|
|
20
|
+
run.check();
|
|
21
|
+
run.diagnostics.push("Candidate recall leg failed; dedup may be incomplete");
|
|
22
|
+
return [];
|
|
23
|
+
}
|
|
24
|
+
};
|
|
25
|
+
let lists;
|
|
26
|
+
if (search.hybrid)
|
|
27
|
+
lists = [await leg(() => search.hybrid(query, run.input.scopeKey, topK, run.signal))];
|
|
28
|
+
else {
|
|
29
|
+
lists = await Promise.all([
|
|
30
|
+
search.fts ? leg(() => search.fts(query, run.input.scopeKey, topK, run.signal)) : Promise.resolve([]),
|
|
31
|
+
search.embed && search.vector ? leg(async () => search.vector(await run.wait(search.embed(query, run.signal)), run.input.scopeKey, topK, run.signal)) : Promise.resolve([]),
|
|
32
|
+
]);
|
|
33
|
+
}
|
|
34
|
+
const ranked = new Map();
|
|
35
|
+
for (const list of lists) {
|
|
36
|
+
const seen = new Set();
|
|
37
|
+
for (const [rank, value] of list.entries()) {
|
|
38
|
+
const atom = run.read(value);
|
|
39
|
+
if (seen.has(atom.id))
|
|
40
|
+
continue;
|
|
41
|
+
seen.add(atom.id);
|
|
42
|
+
const old = ranked.get(atom.id);
|
|
43
|
+
ranked.set(atom.id, { atom, score: (old?.score ?? 0) + 1 / (60 + rank + 1) });
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
return [...ranked.values()].sort((a, b) => b.score - a.score).slice(0, topK).map(r => r.atom);
|
|
47
|
+
}
|
|
48
|
+
export function validateMessages(messages) {
|
|
49
|
+
const ids = new Set();
|
|
50
|
+
for (const m of messages) {
|
|
51
|
+
if (!m.id || ids.has(m.id) || !Number.isFinite(m.timestamp) || !Array.isArray(m.evidence) || !m.evidence.length)
|
|
52
|
+
throw new Error("Invalid or duplicate source message");
|
|
53
|
+
if (!["user", "assistant", "tool_call", "tool_result"].includes(m.role) || typeof m.content !== "string")
|
|
54
|
+
throw new Error("Invalid message content");
|
|
55
|
+
for (const ref of m.evidence)
|
|
56
|
+
if (!ref.id || ref.version === undefined || ref.version === null)
|
|
57
|
+
throw new Error("Missing evidence version");
|
|
58
|
+
ids.add(m.id);
|
|
59
|
+
}
|
|
60
|
+
uniqueRefs(messages.flatMap(m => m.evidence));
|
|
61
|
+
}
|
|
62
|
+
export async function extractL1(deps, input) {
|
|
63
|
+
input = { ...input, messages: structuredClone(input.messages), strategy: structuredClone(input.strategy) };
|
|
64
|
+
const inputHash = hash({ scopeKey: input.scopeKey, mode: input.mode ?? "code", messages: input.messages, strategy: input.strategy,
|
|
65
|
+
maxNewMessages: input.maxNewMessages ?? 10, maxBackgroundMessages: input.maxBackgroundMessages ?? 5, conflictRecallTopK: input.conflictRecallTopK ?? 5 });
|
|
66
|
+
const run = new Run(deps, input, "l1", inputHash);
|
|
67
|
+
const cov = coverage(input.messages.map(m => m.id));
|
|
68
|
+
try {
|
|
69
|
+
run.check();
|
|
70
|
+
validateMessages(input.messages);
|
|
71
|
+
const start = input.continuation?.nextIndex ?? 0;
|
|
72
|
+
if (input.continuation && (input.continuation.algorithmVersion !== ALGORITHM_VERSION || input.continuation.inputHash !== inputHash))
|
|
73
|
+
throw new Error("Stale continuation");
|
|
74
|
+
const count = input.maxNewMessages ?? 10, bgCount = input.maxBackgroundMessages ?? 5, topK = input.conflictRecallTopK ?? 5;
|
|
75
|
+
if (![start, count, bgCount, topK].every(Number.isSafeInteger) || start < 0 || start > input.messages.length || count < 1 || bgCount < 0 || topK < 1)
|
|
76
|
+
throw new Error("Invalid window options");
|
|
77
|
+
const end = Math.min(start + count, input.messages.length);
|
|
78
|
+
const window = input.messages.slice(start, end);
|
|
79
|
+
const fresh = window.filter(m => shouldExtractL1(m.content));
|
|
80
|
+
const background = bgCount === 0 ? [] : input.messages.slice(0, start).filter(m => shouldExtractL1(m.content)).slice(-bgCount);
|
|
81
|
+
cov.processed = window.map(m => m.id);
|
|
82
|
+
cov.filtered = window.filter(m => !fresh.includes(m)).map(m => m.id);
|
|
83
|
+
cov.background = background.map(m => m.id);
|
|
84
|
+
cov.truncated = [...window, ...background].filter(m => m.completeness === "truncated").map(m => m.id);
|
|
85
|
+
cov.remaining = input.messages.slice(end).map(m => m.id);
|
|
86
|
+
const extracted = [];
|
|
87
|
+
let sceneNames = [];
|
|
88
|
+
if (fresh.length) {
|
|
89
|
+
const system = composeMemorySystemPrompt(getExtractMemoriesSystemPrompt(input.mode ?? "code"), input.strategy);
|
|
90
|
+
const raw = await run.model(system, formatExtractionPrompt({ newMessages: fresh, backgroundMessages: background, previousSceneName: input.continuation?.previousSceneName }));
|
|
91
|
+
const parsed = parseExtractionResult(raw);
|
|
92
|
+
if (parsed.emptyReason && parsed.emptyReason !== "empty_scenes")
|
|
93
|
+
throw new Error(`L1 parse failed: ${parsed.emptyReason}`);
|
|
94
|
+
sceneNames = parsed.scenes.map(s => s.scene_name);
|
|
95
|
+
const allowed = new Set(fresh.map(m => m.id));
|
|
96
|
+
for (const scene of parsed.scenes) {
|
|
97
|
+
if (scene.message_ids.some(id => !allowed.has(id)))
|
|
98
|
+
throw new Error("Scene cites evidence outside new messages");
|
|
99
|
+
for (const memory of scene.memories) {
|
|
100
|
+
const type = normalizeType(memory.type);
|
|
101
|
+
if (!type)
|
|
102
|
+
throw new Error("Invalid memory type");
|
|
103
|
+
if (!memory.source_message_ids.length || memory.source_message_ids.some(id => !allowed.has(id)))
|
|
104
|
+
throw new Error("Memory cites evidence outside new messages");
|
|
105
|
+
if (!Number.isFinite(memory.priority) || memory.priority < -1 || memory.priority > 100)
|
|
106
|
+
throw new Error("Invalid priority");
|
|
107
|
+
extracted.push({ ...memory, metadata: memory.metadata, type, scene_name: scene.scene_name, record_id: run.id() });
|
|
108
|
+
if (extracted.length > run.limits.maxExtractedMemories)
|
|
109
|
+
throw new Error("Extracted memory budget exhausted");
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
if (new Set(extracted.map(m => m.record_id)).size !== extracted.length)
|
|
114
|
+
throw new Error("Duplicate generated ID");
|
|
115
|
+
const matches = [];
|
|
116
|
+
for (const memory of extracted)
|
|
117
|
+
matches.push({ newMemory: memory, candidates: await recall(run, memory.content, topK) });
|
|
118
|
+
let decisions = extracted.map(m => ({ record_id: m.record_id, action: "store", target_ids: [] }));
|
|
119
|
+
if (matches.some(m => m.candidates.length)) {
|
|
120
|
+
const raw = await run.model(getConflictDetectionSystemPrompt(input.mode ?? "code"), formatBatchConflictPrompt(matches));
|
|
121
|
+
decisions = parseBatchResult(raw, extracted, { warn: () => run.diagnostics.push("Dedup parser degraded; conflict status unverified"), debug: () => run.diagnostics.push("Missing dedup decision; stored without conflict verification") });
|
|
122
|
+
}
|
|
123
|
+
const candidates = new Map(matches.flatMap(m => m.candidates).map(m => [m.id, m]));
|
|
124
|
+
if (extracted.some(m => candidates.has(m.record_id)))
|
|
125
|
+
throw new Error("Generated ID collides with existing memory");
|
|
126
|
+
const usedTargets = new Set(), usedDecisions = new Set();
|
|
127
|
+
const changes = [];
|
|
128
|
+
for (const d of decisions) {
|
|
129
|
+
const memory = extracted.find(m => m.record_id === d.record_id);
|
|
130
|
+
if (!memory || usedDecisions.has(d.record_id))
|
|
131
|
+
throw new Error("Unknown or duplicate dedup decision");
|
|
132
|
+
usedDecisions.add(d.record_id);
|
|
133
|
+
const targetIds = [...new Set(d.target_ids)];
|
|
134
|
+
for (const id of targetIds)
|
|
135
|
+
if (!candidates.has(id) || usedTargets.has(id))
|
|
136
|
+
throw new Error("Invalid or overlapping dedup targets");
|
|
137
|
+
const old = targetIds.map(id => candidates.get(id));
|
|
138
|
+
if ((d.action === "merge" || d.action === "update") && (!old.length || !d.merged_content?.trim()))
|
|
139
|
+
throw new Error("Incomplete dedup update");
|
|
140
|
+
if (d.action === "store" && old.length)
|
|
141
|
+
throw new Error("Store cannot replace existing memory");
|
|
142
|
+
if (d.action !== "skip")
|
|
143
|
+
targetIds.forEach(id => usedTargets.add(id));
|
|
144
|
+
const before = old.map(m => ({ id: m.id, version: m.version }));
|
|
145
|
+
if (d.action === "skip") {
|
|
146
|
+
changes.push({ action: "skip", before });
|
|
147
|
+
continue;
|
|
148
|
+
}
|
|
149
|
+
const isMerge = d.action === "merge" || d.action === "update";
|
|
150
|
+
if (d.merged_priority !== undefined && (!Number.isFinite(d.merged_priority) || d.merged_priority < -1 || d.merged_priority > 100))
|
|
151
|
+
throw new Error("Invalid merged priority");
|
|
152
|
+
const evidence = uniqueRefs([...fresh.filter(m => memory.source_message_ids.includes(m.id)).flatMap(m => m.evidence.map(ref => m.completeness === "truncated" ? { ...ref, completeness: "truncated" } : ref)), ...old.flatMap(m => m.evidence)]);
|
|
153
|
+
changes.push({ action: d.action === "store" ? "create" : d.action, before, after: {
|
|
154
|
+
id: memory.record_id, content: isMerge ? d.merged_content : memory.content,
|
|
155
|
+
type: d.merged_type ?? memory.type, priority: d.merged_priority ?? memory.priority,
|
|
156
|
+
scene_name: memory.scene_name, source_message_ids: [...new Set([...memory.source_message_ids, ...old.flatMap(m => m.source_message_ids)])],
|
|
157
|
+
metadata: memory.metadata, timestamps: [...new Set([run.time, ...old.flatMap(m => m.timestamps), ...(d.merged_timestamps ?? [])])].sort(),
|
|
158
|
+
createdAt: run.time, updatedAt: run.time, version: Math.max(0, ...old.map(m => m.version)) + 1,
|
|
159
|
+
sessionKey: input.scopeKey, sessionId: input.runId, scopeKey: input.scopeKey, evidence,
|
|
160
|
+
} });
|
|
161
|
+
}
|
|
162
|
+
run.check();
|
|
163
|
+
const result = run.result(changes, cov, end < input.messages.length ? "partial" : "completed");
|
|
164
|
+
result.sceneNames = sceneNames;
|
|
165
|
+
if (end < input.messages.length)
|
|
166
|
+
result.continuation = { algorithmVersion: ALGORITHM_VERSION, inputHash, nextIndex: end, previousSceneName: sceneNames.at(-1) ?? input.continuation?.previousSceneName };
|
|
167
|
+
return result;
|
|
168
|
+
}
|
|
169
|
+
catch (error) {
|
|
170
|
+
return run.failure(error, cov);
|
|
171
|
+
}
|
|
172
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import type { Change, Coverage, Dependencies, ExtractionResult, Limits, RunInput, Stage, VersionRef } from "../contracts.js";
|
|
2
|
+
import type { LocalTool } from "./tools.js";
|
|
3
|
+
export declare const ALGORITHM_VERSION = "0.1.0";
|
|
4
|
+
export declare const UPSTREAM_COMMIT = "906b5823b5106eed8f842b62f16d23228838149a";
|
|
5
|
+
export declare function hash(value: unknown): string;
|
|
6
|
+
export declare function uniqueRefs<T extends {
|
|
7
|
+
id: string;
|
|
8
|
+
version: string | number;
|
|
9
|
+
hash?: string;
|
|
10
|
+
completeness?: "full" | "truncated";
|
|
11
|
+
}>(refs: T[]): T[];
|
|
12
|
+
export declare class Run {
|
|
13
|
+
readonly deps: Dependencies;
|
|
14
|
+
readonly input: RunInput;
|
|
15
|
+
readonly stage: Stage;
|
|
16
|
+
readonly inputHash: string;
|
|
17
|
+
readonly limits: Limits;
|
|
18
|
+
readonly signal: AbortSignal;
|
|
19
|
+
readonly time: string;
|
|
20
|
+
readonly diagnostics: string[];
|
|
21
|
+
readonly readSet: Map<string, VersionRef>;
|
|
22
|
+
private readonly readHashes;
|
|
23
|
+
readonly promptHashes: string[];
|
|
24
|
+
calls: number;
|
|
25
|
+
toolCalls: number;
|
|
26
|
+
recallQueries: number;
|
|
27
|
+
private integrityError?;
|
|
28
|
+
private usageKnown;
|
|
29
|
+
private usage;
|
|
30
|
+
constructor(deps: Dependencies, input: RunInput, stage: Stage, inputHash: string);
|
|
31
|
+
id(): string;
|
|
32
|
+
check(): void;
|
|
33
|
+
invalidate(message: string): never;
|
|
34
|
+
wait<T>(promise: Promise<T>): Promise<T>;
|
|
35
|
+
read<T extends {
|
|
36
|
+
id: string;
|
|
37
|
+
version: number;
|
|
38
|
+
scopeKey: string;
|
|
39
|
+
}>(value: T): T;
|
|
40
|
+
model(systemPrompt: string, prompt: string, tools?: Record<string, LocalTool<any>>, maxIterations?: number, maxOutputTokens?: number): Promise<string>;
|
|
41
|
+
result<T>(changes: Change<T>[], coverage: Coverage, status?: ExtractionResult<T>["status"]): ExtractionResult<T>;
|
|
42
|
+
failure<T>(error: unknown, coverage: Coverage): ExtractionResult<T>;
|
|
43
|
+
}
|
|
44
|
+
export declare function coverage(ids?: string[]): Coverage;
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
+
import { Ajv } from "ajv";
|
|
3
|
+
export const ALGORITHM_VERSION = "0.1.0";
|
|
4
|
+
export const UPSTREAM_COMMIT = "906b5823b5106eed8f842b62f16d23228838149a";
|
|
5
|
+
export function hash(value) {
|
|
6
|
+
return createHash("sha256").update(JSON.stringify(value, (_key, item) => {
|
|
7
|
+
if (item && typeof item === "object" && !Array.isArray(item))
|
|
8
|
+
return Object.fromEntries(Object.keys(item).sort().map(key => [key, item[key]]));
|
|
9
|
+
return item;
|
|
10
|
+
})).digest("hex");
|
|
11
|
+
}
|
|
12
|
+
export function uniqueRefs(refs) {
|
|
13
|
+
const result = new Map();
|
|
14
|
+
for (const ref of refs) {
|
|
15
|
+
const key = JSON.stringify([ref.id, ref.version]), old = result.get(key);
|
|
16
|
+
if (old?.hash && ref.hash && old.hash !== ref.hash)
|
|
17
|
+
throw new Error("Conflicting evidence hashes for the same version");
|
|
18
|
+
const merged = { ...old, ...ref };
|
|
19
|
+
if (old?.hash)
|
|
20
|
+
merged.hash = old.hash;
|
|
21
|
+
if (old?.completeness === "truncated")
|
|
22
|
+
merged.completeness = "truncated";
|
|
23
|
+
result.set(key, merged);
|
|
24
|
+
}
|
|
25
|
+
return [...result.values()];
|
|
26
|
+
}
|
|
27
|
+
const defaults = {
|
|
28
|
+
maxCalls: 20, maxIterations: 16, maxInputChars: 120_000,
|
|
29
|
+
maxOutputTokens: 8192, timeoutMs: 180_000, maxToolCalls: 64, maxWorkspaceChars: 1_000_000,
|
|
30
|
+
maxRecallQueries: 100, maxExtractedMemories: 100,
|
|
31
|
+
};
|
|
32
|
+
export class Run {
|
|
33
|
+
deps;
|
|
34
|
+
input;
|
|
35
|
+
stage;
|
|
36
|
+
inputHash;
|
|
37
|
+
limits;
|
|
38
|
+
signal;
|
|
39
|
+
time;
|
|
40
|
+
diagnostics = [];
|
|
41
|
+
readSet = new Map();
|
|
42
|
+
readHashes = new Map();
|
|
43
|
+
promptHashes = [];
|
|
44
|
+
calls = 0;
|
|
45
|
+
toolCalls = 0;
|
|
46
|
+
recallQueries = 0;
|
|
47
|
+
integrityError;
|
|
48
|
+
usageKnown = true;
|
|
49
|
+
usage = { inputTokens: 0, outputTokens: 0 };
|
|
50
|
+
constructor(deps, input, stage, inputHash) {
|
|
51
|
+
this.deps = deps;
|
|
52
|
+
this.input = input;
|
|
53
|
+
this.stage = stage;
|
|
54
|
+
this.inputHash = inputHash;
|
|
55
|
+
if (!input.runId || !input.scopeKey)
|
|
56
|
+
throw new Error("runId and scopeKey are required");
|
|
57
|
+
if (input.strategy && input.strategy.layer !== stage)
|
|
58
|
+
throw new Error("Strategy layer mismatch");
|
|
59
|
+
this.limits = { ...defaults, ...Object.fromEntries(Object.entries(input.limits ?? {}).filter(([, value]) => value !== undefined)) };
|
|
60
|
+
for (const [key, value] of Object.entries(this.limits)) {
|
|
61
|
+
if (!Number.isSafeInteger(value) || value <= 0)
|
|
62
|
+
throw new Error(`Invalid limit: ${key}`);
|
|
63
|
+
}
|
|
64
|
+
const timeout = AbortSignal.timeout(this.limits.timeoutMs);
|
|
65
|
+
this.signal = input.signal ? AbortSignal.any([input.signal, timeout]) : timeout;
|
|
66
|
+
this.time = (deps.now?.() ?? new Date()).toISOString();
|
|
67
|
+
}
|
|
68
|
+
id() { return this.deps.createId?.() ?? randomUUID(); }
|
|
69
|
+
check() { this.signal.throwIfAborted(); if (this.integrityError)
|
|
70
|
+
throw this.integrityError; }
|
|
71
|
+
invalidate(message) { this.integrityError = new Error(message); throw this.integrityError; }
|
|
72
|
+
async wait(promise) {
|
|
73
|
+
this.check();
|
|
74
|
+
return new Promise((resolve, reject) => {
|
|
75
|
+
const abort = () => reject(this.signal.reason);
|
|
76
|
+
this.signal.addEventListener("abort", abort, { once: true });
|
|
77
|
+
promise.then(value => {
|
|
78
|
+
this.signal.removeEventListener("abort", abort);
|
|
79
|
+
if (this.signal.aborted)
|
|
80
|
+
reject(this.signal.reason);
|
|
81
|
+
else
|
|
82
|
+
resolve(value);
|
|
83
|
+
}, error => { this.signal.removeEventListener("abort", abort); reject(error); });
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
read(value) {
|
|
87
|
+
if (!value.id || value.scopeKey !== this.input.scopeKey || !Number.isSafeInteger(value.version) || value.version < 1) {
|
|
88
|
+
this.invalidate("Read outside requested scope or invalid entity version");
|
|
89
|
+
}
|
|
90
|
+
const prior = this.readSet.get(value.id);
|
|
91
|
+
if (prior && prior.version !== value.version)
|
|
92
|
+
this.invalidate("Snapshot changed during analysis");
|
|
93
|
+
const digest = hash(value);
|
|
94
|
+
if (this.readHashes.has(value.id) && this.readHashes.get(value.id) !== digest)
|
|
95
|
+
this.invalidate("Entity content changed without a version change");
|
|
96
|
+
this.readSet.set(value.id, { id: value.id, version: value.version });
|
|
97
|
+
this.readHashes.set(value.id, digest);
|
|
98
|
+
return structuredClone(value);
|
|
99
|
+
}
|
|
100
|
+
async model(systemPrompt, prompt, tools = {}, maxIterations = this.limits.maxIterations, maxOutputTokens = this.limits.maxOutputTokens) {
|
|
101
|
+
const ajv = new Ajv({ strict: false, allErrors: true });
|
|
102
|
+
const validators = new Map(Object.entries(tools).map(([name, t]) => [name, ajv.compile(t.inputSchema)]));
|
|
103
|
+
const messages = [{ role: "user", content: prompt }];
|
|
104
|
+
const textParts = [];
|
|
105
|
+
this.promptHashes.push(hash({ systemPrompt, prompt, tools: Object.entries(tools).map(([name, t]) => ({ name, inputSchema: t.inputSchema })) }));
|
|
106
|
+
for (let step = 0; step < Math.min(maxIterations, this.limits.maxIterations); step++) {
|
|
107
|
+
this.check();
|
|
108
|
+
if (this.calls >= this.limits.maxCalls)
|
|
109
|
+
throw new Error("Model call budget exhausted");
|
|
110
|
+
const definitions = Object.entries(tools).map(([name, t]) => ({ name, description: t.description, inputSchema: t.inputSchema }));
|
|
111
|
+
if (systemPrompt.length + JSON.stringify(messages).length + JSON.stringify(definitions).length > this.limits.maxInputChars)
|
|
112
|
+
throw new Error("Model input budget exhausted");
|
|
113
|
+
this.calls++;
|
|
114
|
+
const response = await this.wait(this.deps.model.complete({ stage: this.stage, systemPrompt,
|
|
115
|
+
messages: structuredClone(messages), tools: definitions,
|
|
116
|
+
maxOutputTokens: Math.min(maxOutputTokens, this.limits.maxOutputTokens), signal: this.signal }));
|
|
117
|
+
textParts.push(response.text);
|
|
118
|
+
if (response.usage) {
|
|
119
|
+
for (const n of Object.values(response.usage))
|
|
120
|
+
if (!Number.isFinite(n) || n < 0)
|
|
121
|
+
throw new Error("Invalid model usage");
|
|
122
|
+
this.usage.inputTokens += response.usage.inputTokens;
|
|
123
|
+
this.usage.outputTokens += response.usage.outputTokens;
|
|
124
|
+
}
|
|
125
|
+
else
|
|
126
|
+
this.usageKnown = false;
|
|
127
|
+
if (response.finishReason === "length" || response.finishReason === "error")
|
|
128
|
+
throw new Error(`Model stopped: ${response.finishReason}`);
|
|
129
|
+
if (JSON.stringify(response).length > this.limits.maxWorkspaceChars)
|
|
130
|
+
throw new Error("Model response too large");
|
|
131
|
+
const calls = response.toolCalls ?? [];
|
|
132
|
+
if (!calls.length) {
|
|
133
|
+
if (response.finishReason !== "stop")
|
|
134
|
+
throw new Error("Missing tool calls");
|
|
135
|
+
return textParts.join("\n");
|
|
136
|
+
}
|
|
137
|
+
if (response.finishReason !== "tool_calls")
|
|
138
|
+
throw new Error("Unexpected tool calls");
|
|
139
|
+
if (new Set(calls.map(c => c.id)).size !== calls.length || calls.some(c => !c.id))
|
|
140
|
+
throw new Error("Invalid tool call IDs");
|
|
141
|
+
messages.push({ role: "assistant", content: response.text, toolCalls: calls });
|
|
142
|
+
for (const call of calls) {
|
|
143
|
+
this.check();
|
|
144
|
+
if (++this.toolCalls > this.limits.maxToolCalls)
|
|
145
|
+
throw new Error("Tool call budget exhausted");
|
|
146
|
+
let result;
|
|
147
|
+
const validate = validators.get(call.name);
|
|
148
|
+
if (!validate || !validate(call.arguments))
|
|
149
|
+
result = JSON.stringify({ error: "INVALID_TOOL_ARGUMENTS" });
|
|
150
|
+
else {
|
|
151
|
+
try {
|
|
152
|
+
result = await this.wait(tools[call.name].execute(call.arguments));
|
|
153
|
+
}
|
|
154
|
+
catch (error) {
|
|
155
|
+
this.check();
|
|
156
|
+
result = JSON.stringify({ error: error instanceof Error ? error.message : "Tool failed" });
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
try {
|
|
160
|
+
const outcome = JSON.parse(result);
|
|
161
|
+
if (outcome && typeof outcome === "object" && outcome.error)
|
|
162
|
+
this.diagnostics.push(`Tool ${call.name} rejected an operation`);
|
|
163
|
+
}
|
|
164
|
+
catch { /* Document reads need not contain JSON. */ }
|
|
165
|
+
const next = { role: "tool", toolCallId: call.id, content: result };
|
|
166
|
+
if (systemPrompt.length + JSON.stringify([...messages, next]).length + JSON.stringify(definitions).length > this.limits.maxInputChars) {
|
|
167
|
+
result = JSON.stringify({ error: "TOOL_RESULT_TOO_LARGE", message: "Result exceeds remaining context budget. Use a smaller read or finish the current task." });
|
|
168
|
+
this.diagnostics.push(`Tool ${call.name} result exceeded context budget`);
|
|
169
|
+
}
|
|
170
|
+
messages.push({ role: "tool", toolCallId: call.id, content: result });
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
throw new Error("Tool iteration budget exhausted");
|
|
174
|
+
}
|
|
175
|
+
result(changes, coverage, status = "completed") {
|
|
176
|
+
return { status, changes, readSet: [...this.readSet.values()], coverage, diagnostics: [...this.diagnostics], provenance: {
|
|
177
|
+
algorithmVersion: ALGORITHM_VERSION, upstreamCommit: UPSTREAM_COMMIT, profile: "bounded-analysis-v1",
|
|
178
|
+
evidenceGranularity: this.stage === "l1" ? "record" : "run",
|
|
179
|
+
runId: this.input.runId, scopeKey: this.input.scopeKey, stage: this.stage, model: this.deps.model.id,
|
|
180
|
+
inputHash: this.inputHash, promptHashes: this.promptHashes, calls: this.calls,
|
|
181
|
+
usage: this.usageKnown ? { ...this.usage } : null,
|
|
182
|
+
} };
|
|
183
|
+
}
|
|
184
|
+
failure(error, coverage) {
|
|
185
|
+
this.diagnostics.push(error instanceof Error ? error.message : "Analysis failed");
|
|
186
|
+
return this.result([], { ...coverage, processed: [], remaining: [...new Set([...coverage.processed, ...coverage.remaining])] }, this.input.signal?.aborted ? "cancelled" : "failed");
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
export function coverage(ids = []) {
|
|
190
|
+
return { processed: [], background: [], filtered: [], truncated: [], remaining: ids };
|
|
191
|
+
}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import type { Change, Skill, SkillResource } from "../contracts.js";
|
|
2
|
+
import { Run } from "./run.js";
|
|
3
|
+
export declare class SkillCoreError extends Error {
|
|
4
|
+
readonly code: string;
|
|
5
|
+
constructor(code: string, message: string);
|
|
6
|
+
}
|
|
7
|
+
type Identity = {
|
|
8
|
+
user_id: string;
|
|
9
|
+
team_id: string;
|
|
10
|
+
agent_id: string;
|
|
11
|
+
task_id?: string;
|
|
12
|
+
};
|
|
13
|
+
type SkillView = Skill & {
|
|
14
|
+
skill_id: string;
|
|
15
|
+
manifest: SkillResource[];
|
|
16
|
+
};
|
|
17
|
+
export declare function safePath(path: string): string;
|
|
18
|
+
export declare class SkillCore {
|
|
19
|
+
private run;
|
|
20
|
+
private originals;
|
|
21
|
+
private current;
|
|
22
|
+
private history;
|
|
23
|
+
constructor(run: Run);
|
|
24
|
+
private load;
|
|
25
|
+
private save;
|
|
26
|
+
list(p: Identity & {
|
|
27
|
+
pagination?: {
|
|
28
|
+
limit?: number;
|
|
29
|
+
offset?: number;
|
|
30
|
+
};
|
|
31
|
+
}): Promise<{
|
|
32
|
+
items: SkillView[];
|
|
33
|
+
total: number;
|
|
34
|
+
}>;
|
|
35
|
+
search(p: Identity & {
|
|
36
|
+
query: string;
|
|
37
|
+
top_k?: number;
|
|
38
|
+
}): Promise<{
|
|
39
|
+
skill: SkillView;
|
|
40
|
+
score: number;
|
|
41
|
+
}[]>;
|
|
42
|
+
get(p: Identity & {
|
|
43
|
+
skill_id: string;
|
|
44
|
+
version?: number;
|
|
45
|
+
}): Promise<SkillView>;
|
|
46
|
+
create(p: Identity & {
|
|
47
|
+
name: string;
|
|
48
|
+
content: string;
|
|
49
|
+
}): Promise<SkillView>;
|
|
50
|
+
private writable;
|
|
51
|
+
update(p: Identity & {
|
|
52
|
+
skill_id: string;
|
|
53
|
+
expected_version: number;
|
|
54
|
+
content: string;
|
|
55
|
+
}): Promise<SkillView>;
|
|
56
|
+
patch(p: Identity & {
|
|
57
|
+
skill_id: string;
|
|
58
|
+
expected_version: number;
|
|
59
|
+
old_string: string;
|
|
60
|
+
new_string: string;
|
|
61
|
+
replace_all?: boolean;
|
|
62
|
+
}): Promise<SkillView>;
|
|
63
|
+
writeFiles(p: Identity & {
|
|
64
|
+
skill_id: string;
|
|
65
|
+
expected_version: number;
|
|
66
|
+
files: SkillResource[];
|
|
67
|
+
}): Promise<SkillView>;
|
|
68
|
+
changes(): Change<Skill>[];
|
|
69
|
+
}
|
|
70
|
+
export {};
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
import { parseSkillFile, validateSkillFile } from "../upstream/core/skill/skill-format.js";
|
|
2
|
+
export class SkillCoreError extends Error {
|
|
3
|
+
code;
|
|
4
|
+
constructor(code, message) {
|
|
5
|
+
super(message);
|
|
6
|
+
this.code = code;
|
|
7
|
+
}
|
|
8
|
+
}
|
|
9
|
+
export function safePath(path) {
|
|
10
|
+
if (!path || path.startsWith("/") || path.includes("\\") || path.includes("\0") || path.includes(":"))
|
|
11
|
+
throw new Error("Invalid path");
|
|
12
|
+
if (path.split("/").some(part => !part || part === "." || part === ".."))
|
|
13
|
+
throw new Error("Invalid path");
|
|
14
|
+
return path;
|
|
15
|
+
}
|
|
16
|
+
function view(skill) { return { ...structuredClone(skill), skill_id: skill.id, manifest: structuredClone(skill.resources) }; }
|
|
17
|
+
function boundedLimit(value) {
|
|
18
|
+
if (!Number.isSafeInteger(value) || value < 1)
|
|
19
|
+
throw new Error("Invalid Skill query limit");
|
|
20
|
+
return Math.min(100, value);
|
|
21
|
+
}
|
|
22
|
+
function validated(content, name) {
|
|
23
|
+
const file = parseSkillFile(content);
|
|
24
|
+
validateSkillFile(file);
|
|
25
|
+
if (name && file.frontmatter.name !== name)
|
|
26
|
+
throw new Error("Skill name mismatch");
|
|
27
|
+
return file.frontmatter;
|
|
28
|
+
}
|
|
29
|
+
export class SkillCore {
|
|
30
|
+
run;
|
|
31
|
+
originals = new Map();
|
|
32
|
+
current = new Map();
|
|
33
|
+
history = new Map();
|
|
34
|
+
constructor(run) {
|
|
35
|
+
this.run = run;
|
|
36
|
+
}
|
|
37
|
+
load(skill) {
|
|
38
|
+
const checked = this.run.read(skill);
|
|
39
|
+
if (!this.originals.has(checked.id)) {
|
|
40
|
+
this.originals.set(checked.id, checked);
|
|
41
|
+
this.current.set(checked.id, structuredClone(checked));
|
|
42
|
+
}
|
|
43
|
+
return this.current.get(checked.id);
|
|
44
|
+
}
|
|
45
|
+
save(skill) {
|
|
46
|
+
this.run.check();
|
|
47
|
+
// Strip compatibility view fields before returning a public Skill entity.
|
|
48
|
+
skill = { id: skill.id, version: skill.version, scopeKey: skill.scopeKey,
|
|
49
|
+
filename: skill.filename, content: skill.content, evidence: skill.evidence,
|
|
50
|
+
name: skill.name, description: skill.description, resources: skill.resources, writable: skill.writable };
|
|
51
|
+
const others = [...this.current.values()].filter(s => s.id !== skill.id);
|
|
52
|
+
if (others.some(s => s.name === skill.name))
|
|
53
|
+
throw new Error("Skill name already exists");
|
|
54
|
+
if (JSON.stringify([...others, skill]).length > this.run.limits.maxWorkspaceChars)
|
|
55
|
+
throw new Error("Skill workspace budget exhausted");
|
|
56
|
+
this.current.set(skill.id, structuredClone(skill));
|
|
57
|
+
this.history.set(`${skill.id}:${skill.version}`, structuredClone(skill));
|
|
58
|
+
return view(skill);
|
|
59
|
+
}
|
|
60
|
+
async list(p) {
|
|
61
|
+
const limit = boundedLimit(p.pagination?.limit ?? 50);
|
|
62
|
+
const reader = this.run.deps.skills;
|
|
63
|
+
const page = reader ? await this.run.wait(reader.list(this.run.input.scopeKey, limit, this.run.signal)) : { items: [], total: 0 };
|
|
64
|
+
page.items.forEach(s => this.load(s));
|
|
65
|
+
const staged = [...this.current].filter(([id, skill]) => this.originals.get(id)?.version !== skill.version);
|
|
66
|
+
const items = [...new Map([...staged, ...page.items.map(s => [s.id, this.current.get(s.id)]), ...this.current]).values()];
|
|
67
|
+
const newCount = [...this.current.keys()].filter(id => !this.originals.has(id)).length;
|
|
68
|
+
return { items: items.slice(0, limit).map(view), total: Math.max(page.total + newCount, items.length) };
|
|
69
|
+
}
|
|
70
|
+
async search(p) {
|
|
71
|
+
const limit = boundedLimit(p.top_k ?? 10);
|
|
72
|
+
const hits = this.run.deps.skills ? await this.run.wait(this.run.deps.skills.search(this.run.input.scopeKey, p.query, limit, this.run.signal)) : [];
|
|
73
|
+
const results = new Map(hits.map(hit => [hit.skill.id, { skill: view(this.load(hit.skill)), score: hit.score }]));
|
|
74
|
+
const terms = p.query.toLocaleLowerCase().trim().split(/\s+/).filter(Boolean);
|
|
75
|
+
for (const skill of this.current.values()) {
|
|
76
|
+
const original = this.originals.get(skill.id);
|
|
77
|
+
if (original && original.version === skill.version)
|
|
78
|
+
continue;
|
|
79
|
+
const content = `${skill.name} ${skill.description} ${skill.content}`.toLocaleLowerCase();
|
|
80
|
+
if (terms.length && terms.every(term => content.includes(term)))
|
|
81
|
+
results.set(skill.id, { skill: view(skill), score: 1 });
|
|
82
|
+
else
|
|
83
|
+
results.delete(skill.id);
|
|
84
|
+
}
|
|
85
|
+
return [...results.values()].sort((a, b) => b.score - a.score).slice(0, limit);
|
|
86
|
+
}
|
|
87
|
+
async get(p) {
|
|
88
|
+
let skill = this.current.get(p.skill_id);
|
|
89
|
+
if (!skill) {
|
|
90
|
+
if (!this.run.deps.skills)
|
|
91
|
+
throw new Error("Skill not found");
|
|
92
|
+
const loaded = await this.run.wait(this.run.deps.skills.get(this.run.input.scopeKey, p.skill_id, this.run.signal));
|
|
93
|
+
if (loaded.id !== p.skill_id)
|
|
94
|
+
this.run.invalidate("Skill reader returned a different ID");
|
|
95
|
+
skill = this.load(loaded);
|
|
96
|
+
}
|
|
97
|
+
if (p.version !== undefined && p.version !== skill.version) {
|
|
98
|
+
const historical = this.history.get(`${p.skill_id}:${p.version}`) ?? this.originals.get(p.skill_id);
|
|
99
|
+
if (!historical || historical.version !== p.version)
|
|
100
|
+
throw new Error("Historical version unavailable");
|
|
101
|
+
return view(historical);
|
|
102
|
+
}
|
|
103
|
+
return view(skill);
|
|
104
|
+
}
|
|
105
|
+
async create(p) {
|
|
106
|
+
const fm = validated(p.content, p.name);
|
|
107
|
+
const id = this.run.id();
|
|
108
|
+
if (this.current.has(id))
|
|
109
|
+
throw new Error("Duplicate generated ID");
|
|
110
|
+
return this.save({ id, version: 1, scopeKey: this.run.input.scopeKey, filename: "SKILL.md",
|
|
111
|
+
name: fm.name, description: fm.description, content: p.content, resources: [], evidence: [], writable: true });
|
|
112
|
+
}
|
|
113
|
+
async writable(p) {
|
|
114
|
+
const skill = await this.get(p);
|
|
115
|
+
if (!skill.writable)
|
|
116
|
+
throw new SkillCoreError("FORBIDDEN", "Skill is read-only");
|
|
117
|
+
if (skill.version !== p.expected_version)
|
|
118
|
+
throw new SkillCoreError("VERSION_CONFLICT", "Skill version changed");
|
|
119
|
+
return skill;
|
|
120
|
+
}
|
|
121
|
+
async update(p) {
|
|
122
|
+
const skill = await this.writable(p);
|
|
123
|
+
const fm = validated(p.content, skill.name);
|
|
124
|
+
return this.save({ ...skill, version: skill.version + 1, content: p.content, description: fm.description });
|
|
125
|
+
}
|
|
126
|
+
async patch(p) {
|
|
127
|
+
const skill = await this.writable(p);
|
|
128
|
+
if (!p.old_string)
|
|
129
|
+
throw new Error("Empty patch target");
|
|
130
|
+
const count = skill.content.split(p.old_string).length - 1;
|
|
131
|
+
if (count === 0 || (count > 1 && !p.replace_all))
|
|
132
|
+
throw new Error("Patch target missing or ambiguous");
|
|
133
|
+
const content = p.replace_all ? skill.content.split(p.old_string).join(p.new_string) : skill.content.replace(p.old_string, () => p.new_string);
|
|
134
|
+
return this.update({ ...p, content });
|
|
135
|
+
}
|
|
136
|
+
async writeFiles(p) {
|
|
137
|
+
const skill = await this.writable(p);
|
|
138
|
+
const files = new Map(skill.resources.map(f => [f.path, f]));
|
|
139
|
+
for (const f of p.files) {
|
|
140
|
+
safePath(f.path);
|
|
141
|
+
if (f.encoding === "base64" && Buffer.from(f.content, "base64").toString("base64") !== f.content)
|
|
142
|
+
throw new Error("Invalid base64");
|
|
143
|
+
files.set(f.path, structuredClone(f));
|
|
144
|
+
}
|
|
145
|
+
return this.save({ ...skill, version: skill.version + 1, resources: [...files.values()] });
|
|
146
|
+
}
|
|
147
|
+
changes() {
|
|
148
|
+
return [...this.current.values()].flatMap(skill => {
|
|
149
|
+
const old = this.originals.get(skill.id);
|
|
150
|
+
if (old && JSON.stringify(old) === JSON.stringify(skill))
|
|
151
|
+
return [];
|
|
152
|
+
return [{ action: old ? "update" : "create",
|
|
153
|
+
before: old ? [{ id: old.id, version: old.version }] : [], after: structuredClone(skill) }];
|
|
154
|
+
});
|
|
155
|
+
}
|
|
156
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { Dependencies, ExtractionResult, Message, RunInput, Skill } from "../contracts.js";
|
|
2
|
+
export interface SkillInput extends Omit<RunInput, "strategy" | "mode"> {
|
|
3
|
+
messages: Message[];
|
|
4
|
+
prefixSkillsLimit?: number;
|
|
5
|
+
headChars?: number;
|
|
6
|
+
tailChars?: number;
|
|
7
|
+
reason?: string;
|
|
8
|
+
}
|
|
9
|
+
export declare function extractSkills(deps: Dependencies, input: SkillInput): Promise<ExtractionResult<Skill>>;
|