@knolo/core 3.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent.d.ts +53 -0
- package/dist/agent.js +175 -0
- package/dist/builder.d.ts +20 -0
- package/dist/builder.js +196 -0
- package/dist/index.d.ts +21 -0
- package/dist/index.js +13 -0
- package/dist/indexer.d.ts +23 -0
- package/dist/indexer.js +71 -0
- package/dist/pack.d.ts +35 -0
- package/dist/pack.js +175 -0
- package/dist/patch.d.ts +22 -0
- package/dist/patch.js +35 -0
- package/dist/quality/diversify.d.ts +13 -0
- package/dist/quality/diversify.js +41 -0
- package/dist/quality/proximity.d.ts +2 -0
- package/dist/quality/proximity.js +31 -0
- package/dist/quality/signature.d.ts +3 -0
- package/dist/quality/signature.js +24 -0
- package/dist/quality/similarity.d.ts +3 -0
- package/dist/quality/similarity.js +27 -0
- package/dist/query.d.ts +41 -0
- package/dist/query.js +463 -0
- package/dist/rank.d.ts +21 -0
- package/dist/rank.js +31 -0
- package/dist/router.d.ts +28 -0
- package/dist/router.js +74 -0
- package/dist/routing_profile.d.ts +19 -0
- package/dist/routing_profile.js +102 -0
- package/dist/semantic.d.ts +7 -0
- package/dist/semantic.js +98 -0
- package/dist/tokenize.d.ts +24 -0
- package/dist/tokenize.js +53 -0
- package/dist/tool_gate.d.ts +3 -0
- package/dist/tool_gate.js +8 -0
- package/dist/tool_parse.d.ts +2 -0
- package/dist/tool_parse.js +102 -0
- package/dist/tools.d.ts +27 -0
- package/dist/tools.js +34 -0
- package/dist/trace.d.ts +45 -0
- package/dist/trace.js +12 -0
- package/dist/utils/utf8.d.ts +8 -0
- package/dist/utils/utf8.js +72 -0
- package/package.json +39 -0
package/dist/agent.d.ts
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import type { Pack } from './pack.js';
|
|
2
|
+
import type { QueryOptions } from './query.js';
|
|
3
|
+
export type AgentPromptTemplate = string[] | {
|
|
4
|
+
format: 'markdown';
|
|
5
|
+
template: string;
|
|
6
|
+
};
|
|
7
|
+
export type AgentToolPolicy = {
|
|
8
|
+
mode: 'allow' | 'deny';
|
|
9
|
+
tools: string[];
|
|
10
|
+
};
|
|
11
|
+
export type AgentRetrievalDefaults = {
|
|
12
|
+
namespace: string[];
|
|
13
|
+
topK?: number;
|
|
14
|
+
queryExpansion?: QueryOptions['queryExpansion'];
|
|
15
|
+
semantic?: Omit<NonNullable<QueryOptions['semantic']>, 'queryEmbedding' | 'enabled' | 'force'> & {
|
|
16
|
+
enabled?: boolean;
|
|
17
|
+
};
|
|
18
|
+
minScore?: number;
|
|
19
|
+
requirePhrases?: string[];
|
|
20
|
+
source?: string[];
|
|
21
|
+
};
|
|
22
|
+
export type AgentDefinitionV1 = {
|
|
23
|
+
id: string;
|
|
24
|
+
version: 1;
|
|
25
|
+
name?: string;
|
|
26
|
+
description?: string;
|
|
27
|
+
systemPrompt: AgentPromptTemplate;
|
|
28
|
+
retrievalDefaults: AgentRetrievalDefaults;
|
|
29
|
+
toolPolicy?: AgentToolPolicy;
|
|
30
|
+
metadata?: Record<string, string | number | boolean | null>;
|
|
31
|
+
};
|
|
32
|
+
export type AgentRegistry = {
|
|
33
|
+
version: 1;
|
|
34
|
+
agents: AgentDefinitionV1[];
|
|
35
|
+
};
|
|
36
|
+
export type ResolveAgentInput = {
|
|
37
|
+
agentId: string;
|
|
38
|
+
query?: QueryOptions;
|
|
39
|
+
patch?: Record<string, string | number | boolean>;
|
|
40
|
+
};
|
|
41
|
+
export type ResolvedAgent = {
|
|
42
|
+
agent: AgentDefinitionV1;
|
|
43
|
+
systemPrompt: string;
|
|
44
|
+
retrievalOptions: QueryOptions;
|
|
45
|
+
};
|
|
46
|
+
export declare function validateAgentRegistry(reg: AgentRegistry): void;
|
|
47
|
+
export declare function validateAgentDefinition(agent: AgentDefinitionV1): void;
|
|
48
|
+
export declare function listAgents(pack: Pack): string[];
|
|
49
|
+
export declare function getAgent(pack: Pack, agentId: string): AgentDefinitionV1 | undefined;
|
|
50
|
+
export declare function buildSystemPrompt(agent: AgentDefinitionV1, patch?: Record<string, string | number | boolean>): string;
|
|
51
|
+
export declare function resolveAgent(pack: Pack, input: ResolveAgentInput): ResolvedAgent;
|
|
52
|
+
export declare function isToolAllowed(agent: AgentDefinitionV1, toolId: string): boolean;
|
|
53
|
+
export declare function assertToolAllowed(agent: AgentDefinitionV1, toolId: string): void;
|
package/dist/agent.js
ADDED
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
import { validateQueryOptions } from './query.js';
|
|
2
|
+
export function validateAgentRegistry(reg) {
|
|
3
|
+
if (!reg || typeof reg !== 'object') {
|
|
4
|
+
throw new Error('agent registry must be an object.');
|
|
5
|
+
}
|
|
6
|
+
if (reg.version !== 1) {
|
|
7
|
+
throw new Error('agent registry version must be 1.');
|
|
8
|
+
}
|
|
9
|
+
if (!Array.isArray(reg.agents)) {
|
|
10
|
+
throw new Error('agent registry agents must be an array.');
|
|
11
|
+
}
|
|
12
|
+
const seen = new Set();
|
|
13
|
+
for (const agent of reg.agents) {
|
|
14
|
+
validateAgentDefinition(agent);
|
|
15
|
+
if (seen.has(agent.id)) {
|
|
16
|
+
throw new Error(`agent id must be unique: ${agent.id}`);
|
|
17
|
+
}
|
|
18
|
+
seen.add(agent.id);
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
export function validateAgentDefinition(agent) {
|
|
22
|
+
if (!agent || typeof agent !== 'object') {
|
|
23
|
+
throw new Error('agent definition must be an object.');
|
|
24
|
+
}
|
|
25
|
+
if (typeof agent.id !== 'string' || !agent.id.trim()) {
|
|
26
|
+
throw new Error('agent id must be a non-empty string.');
|
|
27
|
+
}
|
|
28
|
+
if (!/^[a-z0-9]+(?:[._-][a-z0-9]+)*$/.test(agent.id)) {
|
|
29
|
+
throw new Error(`agent id must be slug-like: ${agent.id}`);
|
|
30
|
+
}
|
|
31
|
+
if (agent.version !== 1) {
|
|
32
|
+
throw new Error(`agent ${agent.id} version must be 1.`);
|
|
33
|
+
}
|
|
34
|
+
validateSystemPrompt(agent);
|
|
35
|
+
const defaults = agent.retrievalDefaults;
|
|
36
|
+
if (!defaults || typeof defaults !== 'object') {
|
|
37
|
+
throw new Error(`agent ${agent.id} retrievalDefaults must be an object.`);
|
|
38
|
+
}
|
|
39
|
+
if (!Array.isArray(defaults.namespace) ||
|
|
40
|
+
defaults.namespace.length === 0 ||
|
|
41
|
+
defaults.namespace.some((ns) => typeof ns !== 'string' || !ns.trim())) {
|
|
42
|
+
throw new Error(`agent ${agent.id} retrievalDefaults.namespace must be a non-empty string array.`);
|
|
43
|
+
}
|
|
44
|
+
if (defaults.topK !== undefined &&
|
|
45
|
+
(!Number.isInteger(defaults.topK) || defaults.topK < 1)) {
|
|
46
|
+
throw new Error(`agent ${agent.id} retrievalDefaults.topK must be a positive integer.`);
|
|
47
|
+
}
|
|
48
|
+
if (agent.toolPolicy) {
|
|
49
|
+
const { mode, tools } = agent.toolPolicy;
|
|
50
|
+
if (mode !== 'allow' && mode !== 'deny') {
|
|
51
|
+
throw new Error(`agent ${agent.id} toolPolicy.mode must be "allow" or "deny".`);
|
|
52
|
+
}
|
|
53
|
+
if (!Array.isArray(tools) ||
|
|
54
|
+
tools.some((tool) => typeof tool !== 'string' || !tool.trim())) {
|
|
55
|
+
throw new Error(`agent ${agent.id} toolPolicy.tools must be a string array.`);
|
|
56
|
+
}
|
|
57
|
+
if (new Set(tools).size !== tools.length) {
|
|
58
|
+
throw new Error(`agent ${agent.id} toolPolicy.tools must contain unique values.`);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
const syntheticOpts = {
|
|
62
|
+
namespace: defaults.namespace,
|
|
63
|
+
topK: defaults.topK,
|
|
64
|
+
queryExpansion: defaults.queryExpansion,
|
|
65
|
+
semantic: defaults.semantic,
|
|
66
|
+
minScore: defaults.minScore,
|
|
67
|
+
requirePhrases: defaults.requirePhrases,
|
|
68
|
+
source: defaults.source,
|
|
69
|
+
};
|
|
70
|
+
validateQueryOptions(syntheticOpts);
|
|
71
|
+
}
|
|
72
|
+
export function listAgents(pack) {
|
|
73
|
+
const reg = pack.meta.agents;
|
|
74
|
+
if (!reg?.agents?.length)
|
|
75
|
+
return [];
|
|
76
|
+
return reg.agents.map((agent) => agent.id);
|
|
77
|
+
}
|
|
78
|
+
export function getAgent(pack, agentId) {
|
|
79
|
+
return pack.meta.agents?.agents.find((agent) => agent.id === agentId);
|
|
80
|
+
}
|
|
81
|
+
export function buildSystemPrompt(agent, patch = {}) {
|
|
82
|
+
const template = agent.systemPrompt;
|
|
83
|
+
if (Array.isArray(template)) {
|
|
84
|
+
return template.join('\n');
|
|
85
|
+
}
|
|
86
|
+
const source = template.template;
|
|
87
|
+
const placeholders = Array.from(source.matchAll(/\{\{\s*([A-Za-z0-9_.-]+)\s*\}\}/g)).map((m) => m[1]);
|
|
88
|
+
for (const key of placeholders) {
|
|
89
|
+
if (!(key in patch)) {
|
|
90
|
+
throw new Error(`agent ${agent.id} system prompt missing patch value for placeholder: ${key}`);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
return source.replace(/\{\{\s*([A-Za-z0-9_.-]+)\s*\}\}/g, (_match, key) => String(patch[key]));
|
|
94
|
+
}
|
|
95
|
+
export function resolveAgent(pack, input) {
|
|
96
|
+
const agent = getAgent(pack, input.agentId);
|
|
97
|
+
if (!agent) {
|
|
98
|
+
throw new Error(`agent not found: ${input.agentId}`);
|
|
99
|
+
}
|
|
100
|
+
const defaults = {
|
|
101
|
+
namespace: agent.retrievalDefaults.namespace,
|
|
102
|
+
topK: agent.retrievalDefaults.topK,
|
|
103
|
+
queryExpansion: agent.retrievalDefaults.queryExpansion,
|
|
104
|
+
semantic: agent.retrievalDefaults.semantic,
|
|
105
|
+
minScore: agent.retrievalDefaults.minScore,
|
|
106
|
+
requirePhrases: agent.retrievalDefaults.requirePhrases,
|
|
107
|
+
source: agent.retrievalDefaults.source,
|
|
108
|
+
};
|
|
109
|
+
const caller = input.query ?? {};
|
|
110
|
+
const retrievalOptions = {
|
|
111
|
+
...defaults,
|
|
112
|
+
...caller,
|
|
113
|
+
namespace: defaults.namespace,
|
|
114
|
+
queryExpansion: {
|
|
115
|
+
...(defaults.queryExpansion ?? {}),
|
|
116
|
+
...(caller.queryExpansion ?? {}),
|
|
117
|
+
},
|
|
118
|
+
semantic: {
|
|
119
|
+
...(defaults.semantic ?? {}),
|
|
120
|
+
...(caller.semantic ?? {}),
|
|
121
|
+
blend: {
|
|
122
|
+
...(defaults.semantic?.blend ?? {}),
|
|
123
|
+
...(caller.semantic?.blend ?? {}),
|
|
124
|
+
},
|
|
125
|
+
},
|
|
126
|
+
};
|
|
127
|
+
if (!defaults.queryExpansion && !caller.queryExpansion)
|
|
128
|
+
delete retrievalOptions.queryExpansion;
|
|
129
|
+
if (!defaults.semantic && !caller.semantic)
|
|
130
|
+
delete retrievalOptions.semantic;
|
|
131
|
+
if (retrievalOptions.semantic &&
|
|
132
|
+
!defaults.semantic?.blend &&
|
|
133
|
+
!caller.semantic?.blend) {
|
|
134
|
+
delete retrievalOptions.semantic.blend;
|
|
135
|
+
}
|
|
136
|
+
validateQueryOptions(retrievalOptions);
|
|
137
|
+
return {
|
|
138
|
+
agent,
|
|
139
|
+
systemPrompt: buildSystemPrompt(agent, input.patch),
|
|
140
|
+
retrievalOptions,
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
export function isToolAllowed(agent, toolId) {
|
|
144
|
+
const policy = agent.toolPolicy;
|
|
145
|
+
if (!policy)
|
|
146
|
+
return true;
|
|
147
|
+
const hasTool = policy.tools.includes(toolId);
|
|
148
|
+
if (policy.mode === 'allow') {
|
|
149
|
+
return hasTool;
|
|
150
|
+
}
|
|
151
|
+
return !hasTool;
|
|
152
|
+
}
|
|
153
|
+
export function assertToolAllowed(agent, toolId) {
|
|
154
|
+
if (!isToolAllowed(agent, toolId)) {
|
|
155
|
+
throw new Error(`agent ${agent.id} does not allow tool: ${toolId}`);
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
function validateSystemPrompt(agent) {
|
|
159
|
+
const prompt = agent.systemPrompt;
|
|
160
|
+
if (Array.isArray(prompt)) {
|
|
161
|
+
if (!prompt.length || prompt.some((line) => typeof line !== 'string')) {
|
|
162
|
+
throw new Error(`agent ${agent.id} systemPrompt must be a non-empty string array.`);
|
|
163
|
+
}
|
|
164
|
+
if (!prompt.join('').trim()) {
|
|
165
|
+
throw new Error(`agent ${agent.id} systemPrompt must not be empty.`);
|
|
166
|
+
}
|
|
167
|
+
return;
|
|
168
|
+
}
|
|
169
|
+
if (!prompt ||
|
|
170
|
+
prompt.format !== 'markdown' ||
|
|
171
|
+
typeof prompt.template !== 'string' ||
|
|
172
|
+
!prompt.template.trim()) {
|
|
173
|
+
throw new Error(`agent ${agent.id} systemPrompt markdown template must be a non-empty string.`);
|
|
174
|
+
}
|
|
175
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { AgentDefinitionV1, AgentRegistry } from './agent.js';
|
|
2
|
+
export type BuildInputDoc = {
|
|
3
|
+
id?: string;
|
|
4
|
+
heading?: string;
|
|
5
|
+
namespace?: string;
|
|
6
|
+
text: string;
|
|
7
|
+
};
|
|
8
|
+
export type BuildPackOptions = {
|
|
9
|
+
agents?: AgentRegistry | AgentDefinitionV1[];
|
|
10
|
+
semantic?: {
|
|
11
|
+
enabled: boolean;
|
|
12
|
+
modelId: string;
|
|
13
|
+
embeddings: Float32Array[];
|
|
14
|
+
quantization?: {
|
|
15
|
+
type: 'int8_l2norm';
|
|
16
|
+
perVectorScale?: true;
|
|
17
|
+
};
|
|
18
|
+
};
|
|
19
|
+
};
|
|
20
|
+
export declare function buildPack(docs: BuildInputDoc[], opts?: BuildPackOptions): Promise<Uint8Array>;
|
package/dist/builder.js
ADDED
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* builder.ts
|
|
3
|
+
*
|
|
4
|
+
* Build `.knolo` packs from input docs. Persists headings/docIds/token lengths
|
|
5
|
+
* and stores avgBlockLen in meta for stable query-time normalization.
|
|
6
|
+
*/
|
|
7
|
+
import { buildIndex } from './indexer.js';
|
|
8
|
+
import { tokenize } from './tokenize.js';
|
|
9
|
+
import { getTextEncoder } from './utils/utf8.js';
|
|
10
|
+
import { encodeScaleF16, quantizeEmbeddingInt8L2Norm } from './semantic.js';
|
|
11
|
+
import { validateAgentRegistry } from './agent.js';
|
|
12
|
+
export async function buildPack(docs, opts = {}) {
|
|
13
|
+
const normalizedDocs = validateDocs(docs);
|
|
14
|
+
// Prepare blocks (strip MD) and carry heading/docId for optional boosts.
|
|
15
|
+
const blocks = normalizedDocs.map((d, i) => ({
|
|
16
|
+
id: i,
|
|
17
|
+
text: stripMd(d.text),
|
|
18
|
+
heading: d.heading,
|
|
19
|
+
}));
|
|
20
|
+
// Build index
|
|
21
|
+
const { lexicon, postings } = buildIndex(blocks);
|
|
22
|
+
const blockTokenLens = blocks.map((b) => tokenize(b.text).length);
|
|
23
|
+
const totalTokens = blockTokenLens.reduce((sum, len) => sum + len, 0);
|
|
24
|
+
const avgBlockLen = blocks.length ? totalTokens / blocks.length : 1;
|
|
25
|
+
const agents = normalizeAgents(opts.agents);
|
|
26
|
+
const meta = {
|
|
27
|
+
version: 3,
|
|
28
|
+
stats: {
|
|
29
|
+
docs: normalizedDocs.length,
|
|
30
|
+
blocks: blocks.length,
|
|
31
|
+
terms: lexicon.length,
|
|
32
|
+
avgBlockLen,
|
|
33
|
+
},
|
|
34
|
+
...(agents ? { agents } : {}),
|
|
35
|
+
};
|
|
36
|
+
// Persist blocks as objects to optionally carry heading/docId/token length.
|
|
37
|
+
const blocksPayload = blocks.map((b, i) => ({
|
|
38
|
+
text: b.text,
|
|
39
|
+
heading: b.heading ?? null,
|
|
40
|
+
docId: normalizedDocs[i]?.id ?? null,
|
|
41
|
+
namespace: normalizedDocs[i]?.namespace ?? null,
|
|
42
|
+
len: blockTokenLens[i] ?? 0,
|
|
43
|
+
}));
|
|
44
|
+
// Encode sections
|
|
45
|
+
const enc = getTextEncoder();
|
|
46
|
+
const metaBytes = enc.encode(JSON.stringify(meta));
|
|
47
|
+
const lexBytes = enc.encode(JSON.stringify(lexicon));
|
|
48
|
+
const blocksBytes = enc.encode(JSON.stringify(blocksPayload));
|
|
49
|
+
const semanticEnabled = Boolean(opts.semantic?.enabled);
|
|
50
|
+
const semanticSection = semanticEnabled && opts.semantic
|
|
51
|
+
? buildSemanticSection(blocks.length, opts.semantic)
|
|
52
|
+
: undefined;
|
|
53
|
+
const semBytes = semanticSection
|
|
54
|
+
? enc.encode(JSON.stringify(semanticSection.semJson))
|
|
55
|
+
: undefined;
|
|
56
|
+
const semBlob = semanticSection?.semBlob;
|
|
57
|
+
const totalLength = 4 +
|
|
58
|
+
metaBytes.length +
|
|
59
|
+
4 +
|
|
60
|
+
lexBytes.length +
|
|
61
|
+
4 +
|
|
62
|
+
postings.length * 4 +
|
|
63
|
+
4 +
|
|
64
|
+
blocksBytes.length +
|
|
65
|
+
(semanticEnabled && semBytes && semBlob
|
|
66
|
+
? 4 + semBytes.length + 4 + semBlob.length
|
|
67
|
+
: 0);
|
|
68
|
+
const out = new Uint8Array(totalLength);
|
|
69
|
+
const dv = new DataView(out.buffer);
|
|
70
|
+
let offset = 0;
|
|
71
|
+
// meta
|
|
72
|
+
dv.setUint32(offset, metaBytes.length, true);
|
|
73
|
+
offset += 4;
|
|
74
|
+
out.set(metaBytes, offset);
|
|
75
|
+
offset += metaBytes.length;
|
|
76
|
+
// lexicon
|
|
77
|
+
dv.setUint32(offset, lexBytes.length, true);
|
|
78
|
+
offset += 4;
|
|
79
|
+
out.set(lexBytes, offset);
|
|
80
|
+
offset += lexBytes.length;
|
|
81
|
+
// postings (alignment-safe via DataView)
|
|
82
|
+
dv.setUint32(offset, postings.length, true);
|
|
83
|
+
offset += 4;
|
|
84
|
+
for (let i = 0; i < postings.length; i++) {
|
|
85
|
+
dv.setUint32(offset, postings[i], true);
|
|
86
|
+
offset += 4;
|
|
87
|
+
}
|
|
88
|
+
// blocks
|
|
89
|
+
dv.setUint32(offset, blocksBytes.length, true);
|
|
90
|
+
offset += 4;
|
|
91
|
+
out.set(blocksBytes, offset);
|
|
92
|
+
offset += blocksBytes.length;
|
|
93
|
+
if (semanticEnabled && semBytes && semBlob) {
|
|
94
|
+
dv.setUint32(offset, semBytes.length, true);
|
|
95
|
+
offset += 4;
|
|
96
|
+
out.set(semBytes, offset);
|
|
97
|
+
offset += semBytes.length;
|
|
98
|
+
dv.setUint32(offset, semBlob.length, true);
|
|
99
|
+
offset += 4;
|
|
100
|
+
out.set(semBlob, offset);
|
|
101
|
+
}
|
|
102
|
+
return out;
|
|
103
|
+
}
|
|
104
|
+
function normalizeAgents(input) {
|
|
105
|
+
if (!input)
|
|
106
|
+
return undefined;
|
|
107
|
+
const registry = Array.isArray(input)
|
|
108
|
+
? { version: 1, agents: input }
|
|
109
|
+
: input;
|
|
110
|
+
validateAgentRegistry(registry);
|
|
111
|
+
return registry;
|
|
112
|
+
}
|
|
113
|
+
function buildSemanticSection(blockCount, semantic) {
|
|
114
|
+
const { embeddings } = semantic;
|
|
115
|
+
if (!Array.isArray(embeddings) || embeddings.length !== blockCount) {
|
|
116
|
+
throw new Error(`semantic.embeddings must be provided with one embedding per block (expected ${blockCount}).`);
|
|
117
|
+
}
|
|
118
|
+
const quantizationType = semantic.quantization?.type ?? 'int8_l2norm';
|
|
119
|
+
if (quantizationType !== 'int8_l2norm') {
|
|
120
|
+
throw new Error(`Unsupported semantic quantization type: ${quantizationType}`);
|
|
121
|
+
}
|
|
122
|
+
const dims = embeddings[0]?.length ?? 0;
|
|
123
|
+
if (!dims)
|
|
124
|
+
throw new Error('semantic.embeddings must contain vectors with non-zero dimensions.');
|
|
125
|
+
const vecs = new Int8Array(embeddings.length * dims);
|
|
126
|
+
const scales = new Uint16Array(embeddings.length);
|
|
127
|
+
for (let i = 0; i < embeddings.length; i++) {
|
|
128
|
+
const embedding = embeddings[i];
|
|
129
|
+
if (!(embedding instanceof Float32Array)) {
|
|
130
|
+
throw new Error(`semantic.embeddings[${i}] must be a Float32Array.`);
|
|
131
|
+
}
|
|
132
|
+
if (embedding.length !== dims) {
|
|
133
|
+
throw new Error(`semantic.embeddings[${i}] dims mismatch: expected ${dims}, got ${embedding.length}.`);
|
|
134
|
+
}
|
|
135
|
+
const { q, scale } = quantizeEmbeddingInt8L2Norm(embedding);
|
|
136
|
+
vecs.set(q, i * dims);
|
|
137
|
+
scales[i] = encodeScaleF16(scale);
|
|
138
|
+
}
|
|
139
|
+
const vecByteOffset = 0;
|
|
140
|
+
const vecByteLength = vecs.byteLength;
|
|
141
|
+
const scalesByteOffset = vecByteLength;
|
|
142
|
+
const scalesByteLength = scales.byteLength;
|
|
143
|
+
const semBlob = new Uint8Array(vecByteLength + scalesByteLength);
|
|
144
|
+
semBlob.set(new Uint8Array(vecs.buffer, vecs.byteOffset, vecByteLength), vecByteOffset);
|
|
145
|
+
semBlob.set(new Uint8Array(scales.buffer, scales.byteOffset, scalesByteLength), scalesByteOffset);
|
|
146
|
+
const semJson = {
|
|
147
|
+
version: 1,
|
|
148
|
+
modelId: semantic.modelId,
|
|
149
|
+
dims,
|
|
150
|
+
encoding: 'int8_l2norm',
|
|
151
|
+
perVectorScale: true,
|
|
152
|
+
blocks: {
|
|
153
|
+
vectors: { byteOffset: vecByteOffset, length: vecs.length },
|
|
154
|
+
scales: {
|
|
155
|
+
byteOffset: scalesByteOffset,
|
|
156
|
+
length: scales.length,
|
|
157
|
+
encoding: 'float16',
|
|
158
|
+
},
|
|
159
|
+
},
|
|
160
|
+
};
|
|
161
|
+
return { semJson, semBlob };
|
|
162
|
+
}
|
|
163
|
+
function validateDocs(docs) {
|
|
164
|
+
if (!Array.isArray(docs)) {
|
|
165
|
+
throw new Error('buildPack expects an array of docs: [{ text, id?, heading?, namespace? }, ...]');
|
|
166
|
+
}
|
|
167
|
+
return docs.map((doc, i) => {
|
|
168
|
+
if (!doc || typeof doc !== 'object') {
|
|
169
|
+
throw new Error(`Invalid doc at index ${i}: expected an object with a string "text" field.`);
|
|
170
|
+
}
|
|
171
|
+
if (typeof doc.text !== 'string' || !doc.text.trim()) {
|
|
172
|
+
throw new Error(`Invalid doc at index ${i}: "text" must be a non-empty string.`);
|
|
173
|
+
}
|
|
174
|
+
if (doc.id !== undefined && typeof doc.id !== 'string') {
|
|
175
|
+
throw new Error(`Invalid doc at index ${i}: "id" must be a string when provided.`);
|
|
176
|
+
}
|
|
177
|
+
if (doc.heading !== undefined && typeof doc.heading !== 'string') {
|
|
178
|
+
throw new Error(`Invalid doc at index ${i}: "heading" must be a string when provided.`);
|
|
179
|
+
}
|
|
180
|
+
if (doc.namespace !== undefined && typeof doc.namespace !== 'string') {
|
|
181
|
+
throw new Error(`Invalid doc at index ${i}: "namespace" must be a string when provided.`);
|
|
182
|
+
}
|
|
183
|
+
return doc;
|
|
184
|
+
});
|
|
185
|
+
}
|
|
186
|
+
/** Strip Markdown syntax with lightweight regexes (no deps). */
|
|
187
|
+
function stripMd(md) {
|
|
188
|
+
let text = md.replace(/```[\s\S]*?```/g, ' ');
|
|
189
|
+
text = text.replace(/`[^`]*`/g, ' ');
|
|
190
|
+
text = text.replace(/[\*_~]+/g, ' ');
|
|
191
|
+
text = text.replace(/^#+\s*/gm, '');
|
|
192
|
+
text = text.replace(/\[([^\]]+)\]\([^)]+\)/g, '$1');
|
|
193
|
+
text = text.replace(/[\[\]()]/g, ' ');
|
|
194
|
+
text = text.replace(/\s+/g, ' ').trim();
|
|
195
|
+
return text;
|
|
196
|
+
}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
export { mountPack, hasSemantic } from './pack.js';
|
|
2
|
+
export { query, lexConfidence, validateQueryOptions, validateSemanticQueryOptions, } from './query.js';
|
|
3
|
+
export { makeContextPatch } from './patch.js';
|
|
4
|
+
export { buildPack } from './builder.js';
|
|
5
|
+
export { quantizeEmbeddingInt8L2Norm, encodeScaleF16, decodeScaleF16, } from './semantic.js';
|
|
6
|
+
export { listAgents, getAgent, resolveAgent, buildSystemPrompt, isToolAllowed, assertToolAllowed, validateAgentRegistry, validateAgentDefinition, } from './agent.js';
|
|
7
|
+
export type { MountOptions, PackMeta, Pack } from './pack.js';
|
|
8
|
+
export type { QueryOptions, Hit } from './query.js';
|
|
9
|
+
export type { ContextPatch } from './patch.js';
|
|
10
|
+
export type { BuildInputDoc, BuildPackOptions } from './builder.js';
|
|
11
|
+
export type { AgentPromptTemplate, AgentToolPolicy, AgentRetrievalDefaults, AgentDefinitionV1, AgentRegistry, ResolveAgentInput, ResolvedAgent, } from './agent.js';
|
|
12
|
+
export { parseToolCallV1FromText } from './tool_parse.js';
|
|
13
|
+
export { nowIso, createTrace } from './trace.js';
|
|
14
|
+
export { assertToolCallAllowed } from './tool_gate.js';
|
|
15
|
+
export { getAgentRoutingProfileV1, getPackRoutingProfilesV1, } from './routing_profile.js';
|
|
16
|
+
export { isRouteDecisionV1, validateRouteDecisionV1, selectAgentIdFromRouteDecisionV1, } from './router.js';
|
|
17
|
+
export { isToolCallV1, isToolResultV1 } from './tools.js';
|
|
18
|
+
export type { ToolId, ToolCallV1, ToolResultErrorV1, ToolResultV1, ToolSpecV1, } from './tools.js';
|
|
19
|
+
export type { TraceEventV1 } from './trace.js';
|
|
20
|
+
export type { AgentRoutingProfileV1 } from './routing_profile.js';
|
|
21
|
+
export type { RouteCandidateV1, RouteDecisionV1 } from './router.js';
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
// src/index.ts
|
|
2
|
+
export { mountPack, hasSemantic } from './pack.js';
|
|
3
|
+
export { query, lexConfidence, validateQueryOptions, validateSemanticQueryOptions, } from './query.js';
|
|
4
|
+
export { makeContextPatch } from './patch.js';
|
|
5
|
+
export { buildPack } from './builder.js';
|
|
6
|
+
export { quantizeEmbeddingInt8L2Norm, encodeScaleF16, decodeScaleF16, } from './semantic.js';
|
|
7
|
+
export { listAgents, getAgent, resolveAgent, buildSystemPrompt, isToolAllowed, assertToolAllowed, validateAgentRegistry, validateAgentDefinition, } from './agent.js';
|
|
8
|
+
export { parseToolCallV1FromText } from './tool_parse.js';
|
|
9
|
+
export { nowIso, createTrace } from './trace.js';
|
|
10
|
+
export { assertToolCallAllowed } from './tool_gate.js';
|
|
11
|
+
export { getAgentRoutingProfileV1, getPackRoutingProfilesV1, } from './routing_profile.js';
|
|
12
|
+
export { isRouteDecisionV1, validateRouteDecisionV1, selectAgentIdFromRouteDecisionV1, } from './router.js';
|
|
13
|
+
export { isToolCallV1, isToolResultV1 } from './tools.js';
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
export type Block = {
|
|
2
|
+
id: number;
|
|
3
|
+
text: string;
|
|
4
|
+
heading?: string;
|
|
5
|
+
};
|
|
6
|
+
export type IndexBuildResult = {
|
|
7
|
+
lexicon: Array<[string, number]>;
|
|
8
|
+
postings: Uint32Array;
|
|
9
|
+
};
|
|
10
|
+
/**
|
|
11
|
+
* Build an inverted index from an array of blocks. The postings list format
|
|
12
|
+
* encodes, for each term, a header containing the termId, followed by
|
|
13
|
+
* sequences of blockId and positions for that term, with zeros as delimiters.
|
|
14
|
+
* The structure looks like:
|
|
15
|
+
*
|
|
16
|
+
* [termId, blockId+1, pos, pos, 0, blockId+1, pos, 0, 0, termId, ...]
|
|
17
|
+
*
|
|
18
|
+
* Block IDs are stored as bid+1 so that 0 can remain a sentinel delimiter.
|
|
19
|
+
* Each block section ends with a 0, and each term section ends with a 0. The
|
|
20
|
+
* entire array can be streamed sequentially without needing to know the sizes
|
|
21
|
+
* of individual lists ahead of time.
|
|
22
|
+
*/
|
|
23
|
+
export declare function buildIndex(blocks: Block[]): IndexBuildResult;
|
package/dist/indexer.js
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* indexer.ts
|
|
3
|
+
*
|
|
4
|
+
* Implements a basic inverted index builder. Given an array of blocks, it
|
|
5
|
+
* produces a lexicon mapping each unique term to a term identifier and a
|
|
6
|
+
* flattened postings array. This representation is intentionally naïve to
|
|
7
|
+
* prioritise clarity and portability over maximum compression. The pack
|
|
8
|
+
* builder can later swap this implementation for a more compact format.
|
|
9
|
+
*/
|
|
10
|
+
import { tokenize } from "./tokenize.js";
|
|
11
|
+
/**
|
|
12
|
+
* Build an inverted index from an array of blocks. The postings list format
|
|
13
|
+
* encodes, for each term, a header containing the termId, followed by
|
|
14
|
+
* sequences of blockId and positions for that term, with zeros as delimiters.
|
|
15
|
+
* The structure looks like:
|
|
16
|
+
*
|
|
17
|
+
* [termId, blockId+1, pos, pos, 0, blockId+1, pos, 0, 0, termId, ...]
|
|
18
|
+
*
|
|
19
|
+
* Block IDs are stored as bid+1 so that 0 can remain a sentinel delimiter.
|
|
20
|
+
* Each block section ends with a 0, and each term section ends with a 0. The
|
|
21
|
+
* entire array can be streamed sequentially without needing to know the sizes
|
|
22
|
+
* of individual lists ahead of time.
|
|
23
|
+
*/
|
|
24
|
+
export function buildIndex(blocks) {
|
|
25
|
+
// Map term to termId and interim map of termId -> blockId -> positions
|
|
26
|
+
const term2id = new Map();
|
|
27
|
+
const termBlockPositions = new Map();
|
|
28
|
+
const getTermId = (t) => {
|
|
29
|
+
let id = term2id.get(t);
|
|
30
|
+
if (id === undefined) {
|
|
31
|
+
id = term2id.size + 1; // term IDs start at 1
|
|
32
|
+
term2id.set(t, id);
|
|
33
|
+
}
|
|
34
|
+
return id;
|
|
35
|
+
};
|
|
36
|
+
// Build a local term frequency map per block, then populate the global map
|
|
37
|
+
for (const block of blocks) {
|
|
38
|
+
const toks = tokenize(block.text);
|
|
39
|
+
const perTermPositions = new Map();
|
|
40
|
+
for (const tk of toks) {
|
|
41
|
+
const id = getTermId(tk.term);
|
|
42
|
+
let positions = perTermPositions.get(id);
|
|
43
|
+
if (!positions) {
|
|
44
|
+
positions = [];
|
|
45
|
+
perTermPositions.set(id, positions);
|
|
46
|
+
}
|
|
47
|
+
positions.push(tk.pos);
|
|
48
|
+
}
|
|
49
|
+
// Merge into global structure
|
|
50
|
+
for (const [tid, positions] of perTermPositions) {
|
|
51
|
+
let blockMap = termBlockPositions.get(tid);
|
|
52
|
+
if (!blockMap) {
|
|
53
|
+
blockMap = new Map();
|
|
54
|
+
termBlockPositions.set(tid, blockMap);
|
|
55
|
+
}
|
|
56
|
+
blockMap.set(block.id, positions);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
// Flatten postings into a single Uint32Array
|
|
60
|
+
const postings = [];
|
|
61
|
+
for (const [tid, blockMap] of termBlockPositions) {
|
|
62
|
+
postings.push(tid);
|
|
63
|
+
for (const [bid, positions] of blockMap) {
|
|
64
|
+
postings.push(bid + 1, ...positions, 0);
|
|
65
|
+
}
|
|
66
|
+
postings.push(0); // end of term
|
|
67
|
+
}
|
|
68
|
+
// Convert lexicon to array for serialization
|
|
69
|
+
const lexicon = Array.from(term2id.entries());
|
|
70
|
+
return { lexicon, postings: new Uint32Array(postings) };
|
|
71
|
+
}
|
package/dist/pack.d.ts
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import type { AgentRegistry } from './agent.js';
|
|
2
|
+
export type MountOptions = {
|
|
3
|
+
src: string | ArrayBufferLike | Uint8Array;
|
|
4
|
+
};
|
|
5
|
+
export type PackMeta = {
|
|
6
|
+
version: number;
|
|
7
|
+
stats: {
|
|
8
|
+
docs: number;
|
|
9
|
+
blocks: number;
|
|
10
|
+
terms: number;
|
|
11
|
+
avgBlockLen?: number;
|
|
12
|
+
};
|
|
13
|
+
agents?: AgentRegistry;
|
|
14
|
+
};
|
|
15
|
+
export type Pack = {
|
|
16
|
+
meta: PackMeta;
|
|
17
|
+
lexicon: Map<string, number>;
|
|
18
|
+
postings: Uint32Array;
|
|
19
|
+
blocks: string[];
|
|
20
|
+
headings?: (string | null)[];
|
|
21
|
+
docIds?: (string | null)[];
|
|
22
|
+
namespaces?: (string | null)[];
|
|
23
|
+
blockTokenLens?: number[];
|
|
24
|
+
semantic?: {
|
|
25
|
+
version: 1;
|
|
26
|
+
modelId: string;
|
|
27
|
+
dims: number;
|
|
28
|
+
encoding: 'int8_l2norm';
|
|
29
|
+
perVectorScale: boolean;
|
|
30
|
+
vecs: Int8Array;
|
|
31
|
+
scales?: Uint16Array;
|
|
32
|
+
};
|
|
33
|
+
};
|
|
34
|
+
export declare function hasSemantic(pack: Pack): boolean;
|
|
35
|
+
export declare function mountPack(opts: MountOptions): Promise<Pack>;
|