knolo-core 0.2.3 → 3.1.0

package/bin/knolo.mjs

@@ -1,69 +1,387 @@
 #!/usr/bin/env node
 // Robust CLI that works with ESM or CJS builds and odd resolution cases.
 
-import { readFileSync, writeFileSync } from "node:fs";
-import path from "node:path";
-import { fileURLToPath, pathToFileURL } from "node:url";
-import { createRequire } from "node:module";
+import { readFileSync, writeFileSync, readdirSync, statSync } from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+import { createRequire } from 'node:module';
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 const require = createRequire(import.meta.url);
 
+function parseScalar(value) {
+  if (value === 'true') return true;
+  if (value === 'false') return false;
+  if (value === 'null') return null;
+  if (/^-?\d+(?:\.\d+)?$/.test(value)) return Number(value);
+  const quoted = value.match(/^("|')(.*)\1$/);
+  if (quoted) return quoted[2];
+  return value;
+}
+
+function parseSimpleYaml(content) {
+  const lines = content
+    .split(/\n/)
+    .map((line) => line.replace(/\r$/, ''))
+    .map((line) => line.replace(/\t/g, '  ').replace(/\s+#.*$/, ''));
+
+  const root = {};
+  const stack = [{ indent: -1, value: root }];
+
+  const nextMeaningfulLine = (from) => {
+    for (let i = from + 1; i < lines.length; i++) {
+      if (lines[i].trim()) return lines[i].trim();
+    }
+    return '';
+  };
+
+  for (let i = 0; i < lines.length; i++) {
+    const raw = lines[i];
+    if (!raw.trim()) continue;
+
+    const indent = raw.match(/^\s*/)[0].length;
+    const line = raw.trim();
+
+    while (stack.length > 1 && indent <= stack[stack.length - 1].indent) {
+      stack.pop();
+    }
+
+    const parent = stack[stack.length - 1].value;
+
+    if (line.startsWith('- ')) {
+      if (!Array.isArray(parent)) {
+        throw new Error('YAML array item found under non-array parent.');
+      }
+
+      const body = line.slice(2).trim();
+      const pair = body.match(/^([^:]+):(.*)$/);
+      if (!pair) {
+        parent.push(parseScalar(body));
+        continue;
+      }
+
+      const obj = {};
+      const key = pair[1].trim();
+      const rhs = pair[2].trim();
+      if (rhs) {
+        obj[key] = parseScalar(rhs);
+      } else {
+        const next = nextMeaningfulLine(i);
+        obj[key] = next.startsWith('- ') ? [] : {};
+        stack.push({ indent, value: obj[key] });
+      }
+      parent.push(obj);
+      continue;
+    }
+
+    const pair = line.match(/^([^:]+):(.*)$/);
+    if (!pair) {
+      throw new Error(`Unsupported YAML line: ${line}`);
+    }
+
+    const key = pair[1].trim();
+    const rhs = pair[2].trim();
+    if (Array.isArray(parent)) {
+      const obj = {};
+      parent.push(obj);
+      if (rhs) {
+        obj[key] = parseScalar(rhs);
+      } else {
+        const next = nextMeaningfulLine(i);
+        obj[key] = next.startsWith('- ') ? [] : {};
+        stack.push({ indent, value: obj[key] });
+      }
+      continue;
+    }
+
+    if (rhs) {
+      parent[key] = parseScalar(rhs);
+    } else {
+      const next = nextMeaningfulLine(i);
+      parent[key] = next.startsWith('- ') ? [] : {};
+      stack.push({ indent, value: parent[key] });
+    }
+  }
+
+  return root;
+}
+
 async function tryImport(filePath) {
-  // 1) ESM via file URL
   try {
     const url = pathToFileURL(filePath).href;
     return await import(url);
   } catch (_) {}
-  // 2) CJS via require
   try {
     return require(filePath);
   } catch (_) {}
   return null;
 }
 
-function getBuildPack(mod) {
-  if (!mod) return undefined;
-  // Named export (ESM)
-  if (typeof mod.buildPack === "function") return mod.buildPack;
-  // CJS default export object: { buildPack } or function
-  if (mod.default) {
-    if (typeof mod.default === "function") return mod.default;
-    if (typeof mod.default.buildPack === "function") return mod.default.buildPack;
-  }
-  // Some CJS setups export { buildPack } directly
-  if (typeof mod === "function") return mod;
-  if (typeof mod.buildPack === "function") return mod.buildPack;
-  return undefined;
+function pickBuildExports(mod) {
+  if (!mod) return null;
+  const root = mod.default && typeof mod.default === 'object' ? mod.default : mod;
+  const buildPack =
+    typeof mod.buildPack === 'function'
+      ? mod.buildPack
+      : typeof root.buildPack === 'function'
+        ? root.buildPack
+        : typeof root === 'function'
+          ? root
+          : undefined;
+  const validateAgentDefinition =
+    typeof mod.validateAgentDefinition === 'function'
+      ? mod.validateAgentDefinition
+      : root.validateAgentDefinition;
+  const validateAgentRegistry =
+    typeof mod.validateAgentRegistry === 'function'
+      ? mod.validateAgentRegistry
+      : root.validateAgentRegistry;
+
+  if (!buildPack) return null;
+  return { buildPack, validateAgentDefinition, validateAgentRegistry };
 }
 
-async function loadBuildPack() {
+async function loadBuildExports() {
   const candidates = [
-    path.resolve(__dirname, "../dist/index.js"),
-    path.resolve(__dirname, "../dist/builder.js"),
-    // Also try .cjs just in case someone built CJS
-    path.resolve(__dirname, "../dist/index.cjs"),
-    path.resolve(__dirname, "../dist/builder.cjs"),
+    path.resolve(__dirname, '../dist/index.js'),
+    path.resolve(__dirname, '../dist/builder.js'),
+    path.resolve(__dirname, '../dist/index.cjs'),
+    path.resolve(__dirname, '../dist/builder.cjs'),
   ];
   for (const p of candidates) {
     const mod = await tryImport(p);
-    const buildPack = getBuildPack(mod);
-    if (buildPack) return buildPack;
+    const exports = pickBuildExports(mod);
+    if (exports) return exports;
   }
-  throw new Error("Could not locate a buildPack function in dist/");
+  throw new Error('Could not locate a buildPack function in dist/');
 }
 
-const buildPack = await loadBuildPack();
+function validateCliDocs(raw) {
+  if (!Array.isArray(raw)) {
+    throw new Error(
+      'Input JSON must be an array of docs: [{ "text": "...", "id"?: "...", "heading"?: "..." }]'
+    );
+  }
+  for (let i = 0; i < raw.length; i++) {
+    const doc = raw[i];
+    if (!doc || typeof doc !== 'object') {
+      throw new Error(`Invalid doc at index ${i}: expected an object.`);
+    }
+    if (typeof doc.text !== 'string' || !doc.text.trim()) {
+      throw new Error(`Invalid doc at index ${i}: "text" must be a non-empty string.`);
+    }
+  }
+  return raw;
+}
+
+function parseArgs(argv) {
+  const positional = [];
+  const flags = { embeddingsPath: undefined, modelId: undefined, agentsDir: undefined };
+
+  for (let i = 0; i < argv.length; i++) {
+    const arg = argv[i];
+    if (!arg.startsWith('--')) {
+      positional.push(arg);
+      continue;
+    }
+    if (arg === '--embeddings') {
+      flags.embeddingsPath = argv[++i];
+      continue;
+    }
+    if (arg === '--model-id') {
+      flags.modelId = argv[++i];
+      continue;
+    }
+    if (arg === '--agents') {
+      flags.agentsDir = argv[++i];
+      continue;
+    }
+    if (arg === '--help' || arg === '-h') {
+      flags.help = true;
+      continue;
+    }
+    throw new Error(`Unknown flag: ${arg}`);
+  }
+  return { positional, flags };
+}
 
-const inFile = process.argv[2];
-const outFile = process.argv[3] || "knowledge.knolo";
+function parseAgentFileContent(content, filePath) {
+  const ext = path.extname(filePath).toLowerCase();
+  if (ext === '.json') return JSON.parse(content);
+  if (ext === '.yaml' || ext === '.yml') return parseSimpleYaml(content);
+  throw new Error(`Unsupported agent file extension: ${filePath}`);
+}
+
+function normalizeAgentFromFile(parsed, filePath) {
+  if (!parsed || typeof parsed !== 'object') {
+    throw new Error(`Invalid agent definition in ${filePath}: expected object.`);
+  }
+  if ('agent' in parsed && parsed.agent && typeof parsed.agent === 'object') {
+    return parsed.agent;
+  }
+  return parsed;
+}
+
+function loadAgentsFromDir(agentsDir, validators = {}) {
+  const { validateAgentDefinition, validateAgentRegistry } = validators;
+  const dirPath = path.resolve(agentsDir);
+  let entries;
+  try {
+    entries = readdirSync(dirPath, { withFileTypes: true });
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    throw new Error(`Unable to read agents directory ${agentsDir}: ${message}`);
+  }
+
+  const files = entries
+    .filter((entry) => entry.isFile())
+    .map((entry) => entry.name)
+    .filter((name) => ['.json', '.yaml', '.yml'].includes(path.extname(name).toLowerCase()))
+    .sort((a, b) => a.localeCompare(b));
+
+  const loaded = [];
+  for (const file of files) {
+    const fullPath = path.join(dirPath, file);
+    try {
+      const content = readFileSync(fullPath, 'utf8');
+      const parsed = parseAgentFileContent(content, fullPath);
+      const agent = normalizeAgentFromFile(parsed, fullPath);
+      if (typeof validateAgentDefinition === 'function') {
+        validateAgentDefinition(agent);
+      }
+      loaded.push({ file, agent });
+    } catch (err) {
+      const message = err instanceof Error ? err.message : String(err);
+      throw new Error(`Failed to load agent file ${fullPath}: ${message}`);
+    }
+  }
+
+  const duplicateById = new Map();
+  for (const item of loaded) {
+    const key = String(item.agent?.id ?? '');
+    if (!duplicateById.has(key)) duplicateById.set(key, []);
+    duplicateById.get(key).push(item.file);
+  }
+  for (const [id, fileNames] of duplicateById.entries()) {
+    if (fileNames.length > 1) {
+      throw new Error(
+        `Duplicate agent id "${id}" found in files: ${fileNames.sort((a, b) => a.localeCompare(b)).join(', ')}`
+      );
+    }
+  }
+
+  const agents = loaded
+    .map((item) => item.agent)
+    .sort((a, b) => String(a.id).localeCompare(String(b.id)));
+  const registry = { version: 1, agents };
+
+  if (typeof validateAgentRegistry === 'function') {
+    validateAgentRegistry(registry);
+  }
+
+  return registry;
+}
+
+function loadEmbeddingsFromJson(filePath, expectedCount) {
+  const parsed = JSON.parse(readFileSync(filePath, 'utf8'));
+  const vectors = Array.isArray(parsed?.embeddings) ? parsed.embeddings : parsed;
+  if (!Array.isArray(vectors)) {
+    throw new Error('Embeddings JSON must be either an array of vectors or { "embeddings": [...] }.');
+  }
+  if (vectors.length !== expectedCount) {
+    throw new Error(`Embeddings length mismatch: expected ${expectedCount}, got ${vectors.length}.`);
+  }
+
+  const first = vectors[0];
+  if (!Array.isArray(first) || first.length === 0) {
+    throw new Error('Embeddings must contain non-empty numeric vectors.');
+  }
+  const dims = first.length;
+
+  return vectors.map((entry, i) => {
+    if (!Array.isArray(entry)) {
+      throw new Error(`Embeddings[${i}] must be an array of numbers.`);
+    }
+    if (entry.length !== dims) {
+      throw new Error(`Embeddings[${i}] has dims ${entry.length}, expected ${dims}.`);
+    }
+    const vec = new Float32Array(dims);
+    for (let d = 0; d < dims; d++) {
+      const value = entry[d];
+      if (!Number.isFinite(value)) {
+        throw new Error(`Embeddings[${i}][${d}] must be a finite number.`);
+      }
+      vec[d] = value;
+    }
+    return vec;
+  });
+}
+
+function printUsage() {
+  console.log(
+    'Usage: knolo <input.json> [output.knolo] [--agents ./agents] [--embeddings embeddings.json --model-id model-name]'
+  );
+}
+
+const { buildPack, validateAgentDefinition, validateAgentRegistry } = await loadBuildExports();
+const { positional, flags } = parseArgs(process.argv.slice(2));
+
+if (flags.help) {
+  printUsage();
+  process.exit(0);
+}
+
+const inFile = positional[0];
+const outFile = positional[1] || 'knowledge.knolo';
 
 if (!inFile) {
-  console.log("Usage: knolo <input.json> [output.knolo]");
+  printUsage();
   process.exit(1);
 }
 
-const docs = JSON.parse(readFileSync(inFile, "utf8"));
-const bytes = await buildPack(docs);
-writeFileSync(outFile, Buffer.from(bytes));
-console.log(`wrote ${outFile}`);
+try {
+  const rawText = readFileSync(inFile, 'utf8');
+  const parsed = JSON.parse(rawText);
+  const docs = validateCliDocs(parsed);
+
+  const options = {};
+
+  if (flags.embeddingsPath || flags.modelId) {
+    if (!flags.embeddingsPath || !flags.modelId) {
+      throw new Error('Both --embeddings and --model-id are required when enabling semantic build output.');
+    }
+    const embeddings = loadEmbeddingsFromJson(flags.embeddingsPath, docs.length);
+    options.semantic = {
+      enabled: true,
+      modelId: flags.modelId,
+      embeddings,
+      quantization: { type: 'int8_l2norm', perVectorScale: true },
+    };
+  }
+
+  if (flags.agentsDir) {
+    let dirStats;
+    try {
+      dirStats = statSync(flags.agentsDir);
+    } catch (err) {
+      const message = err instanceof Error ? err.message : String(err);
+      throw new Error(`Unable to access --agents path ${flags.agentsDir}: ${message}`);
+    }
+    if (!dirStats.isDirectory()) {
+      throw new Error(`--agents path must be a directory: ${flags.agentsDir}`);
+    }
+
+    options.agents = loadAgentsFromDir(flags.agentsDir, {
+      validateAgentDefinition,
+      validateAgentRegistry,
+    });
+  }
+
+  const bytes = await buildPack(docs, options);
+  writeFileSync(outFile, Buffer.from(bytes));
+  console.log(`wrote ${outFile}`);
+} catch (err) {
+  const message = err instanceof Error ? err.message : String(err);
+  console.error(`knolo: ${message}`);
+  process.exit(1);
+}

package/dist/agent.d.ts

@@ -0,0 +1,53 @@
+import type { Pack } from './pack.js';
+import type { QueryOptions } from './query.js';
+export type AgentPromptTemplate = string[] | {
+    format: 'markdown';
+    template: string;
+};
+export type AgentToolPolicy = {
+    mode: 'allow' | 'deny';
+    tools: string[];
+};
+export type AgentRetrievalDefaults = {
+    namespace: string[];
+    topK?: number;
+    queryExpansion?: QueryOptions['queryExpansion'];
+    semantic?: Omit<NonNullable<QueryOptions['semantic']>, 'queryEmbedding' | 'enabled' | 'force'> & {
+        enabled?: boolean;
+    };
+    minScore?: number;
+    requirePhrases?: string[];
+    source?: string[];
+};
+export type AgentDefinitionV1 = {
+    id: string;
+    version: 1;
+    name?: string;
+    description?: string;
+    systemPrompt: AgentPromptTemplate;
+    retrievalDefaults: AgentRetrievalDefaults;
+    toolPolicy?: AgentToolPolicy;
+    metadata?: Record<string, string | number | boolean | null>;
+};
+export type AgentRegistry = {
+    version: 1;
+    agents: AgentDefinitionV1[];
+};
+export type ResolveAgentInput = {
+    agentId: string;
+    query?: QueryOptions;
+    patch?: Record<string, string | number | boolean>;
+};
+export type ResolvedAgent = {
+    agent: AgentDefinitionV1;
+    systemPrompt: string;
+    retrievalOptions: QueryOptions;
+};
+export declare function validateAgentRegistry(reg: AgentRegistry): void;
+export declare function validateAgentDefinition(agent: AgentDefinitionV1): void;
+export declare function listAgents(pack: Pack): string[];
+export declare function getAgent(pack: Pack, agentId: string): AgentDefinitionV1 | undefined;
+export declare function buildSystemPrompt(agent: AgentDefinitionV1, patch?: Record<string, string | number | boolean>): string;
+export declare function resolveAgent(pack: Pack, input: ResolveAgentInput): ResolvedAgent;
+export declare function isToolAllowed(agent: AgentDefinitionV1, toolId: string): boolean;
+export declare function assertToolAllowed(agent: AgentDefinitionV1, toolId: string): void;

package/dist/agent.js

@@ -0,0 +1,175 @@
+import { validateQueryOptions } from './query.js';
+export function validateAgentRegistry(reg) {
+    if (!reg || typeof reg !== 'object') {
+        throw new Error('agent registry must be an object.');
+    }
+    if (reg.version !== 1) {
+        throw new Error('agent registry version must be 1.');
+    }
+    if (!Array.isArray(reg.agents)) {
+        throw new Error('agent registry agents must be an array.');
+    }
+    const seen = new Set();
+    for (const agent of reg.agents) {
+        validateAgentDefinition(agent);
+        if (seen.has(agent.id)) {
+            throw new Error(`agent id must be unique: ${agent.id}`);
+        }
+        seen.add(agent.id);
+    }
+}
+export function validateAgentDefinition(agent) {
+    if (!agent || typeof agent !== 'object') {
+        throw new Error('agent definition must be an object.');
+    }
+    if (typeof agent.id !== 'string' || !agent.id.trim()) {
+        throw new Error('agent id must be a non-empty string.');
+    }
+    if (!/^[a-z0-9]+(?:[._-][a-z0-9]+)*$/.test(agent.id)) {
+        throw new Error(`agent id must be slug-like: ${agent.id}`);
+    }
+    if (agent.version !== 1) {
+        throw new Error(`agent ${agent.id} version must be 1.`);
+    }
+    validateSystemPrompt(agent);
+    const defaults = agent.retrievalDefaults;
+    if (!defaults || typeof defaults !== 'object') {
+        throw new Error(`agent ${agent.id} retrievalDefaults must be an object.`);
+    }
+    if (!Array.isArray(defaults.namespace) ||
+        defaults.namespace.length === 0 ||
+        defaults.namespace.some((ns) => typeof ns !== 'string' || !ns.trim())) {
+        throw new Error(`agent ${agent.id} retrievalDefaults.namespace must be a non-empty string array.`);
+    }
+    if (defaults.topK !== undefined &&
+        (!Number.isInteger(defaults.topK) || defaults.topK < 1)) {
+        throw new Error(`agent ${agent.id} retrievalDefaults.topK must be a positive integer.`);
+    }
+    if (agent.toolPolicy) {
+        const { mode, tools } = agent.toolPolicy;
+        if (mode !== 'allow' && mode !== 'deny') {
+            throw new Error(`agent ${agent.id} toolPolicy.mode must be "allow" or "deny".`);
+        }
+        if (!Array.isArray(tools) ||
+            tools.some((tool) => typeof tool !== 'string' || !tool.trim())) {
+            throw new Error(`agent ${agent.id} toolPolicy.tools must be a string array.`);
+        }
+        if (new Set(tools).size !== tools.length) {
+            throw new Error(`agent ${agent.id} toolPolicy.tools must contain unique values.`);
+        }
+    }
+    const syntheticOpts = {
+        namespace: defaults.namespace,
+        topK: defaults.topK,
+        queryExpansion: defaults.queryExpansion,
+        semantic: defaults.semantic,
+        minScore: defaults.minScore,
+        requirePhrases: defaults.requirePhrases,
+        source: defaults.source,
+    };
+    validateQueryOptions(syntheticOpts);
+}
+export function listAgents(pack) {
+    const reg = pack.meta.agents;
+    if (!reg?.agents?.length)
+        return [];
+    return reg.agents.map((agent) => agent.id);
+}
+export function getAgent(pack, agentId) {
+    return pack.meta.agents?.agents.find((agent) => agent.id === agentId);
+}
+export function buildSystemPrompt(agent, patch = {}) {
+    const template = agent.systemPrompt;
+    if (Array.isArray(template)) {
+        return template.join('\n');
+    }
+    const source = template.template;
+    const placeholders = Array.from(source.matchAll(/\{\{\s*([A-Za-z0-9_.-]+)\s*\}\}/g)).map((m) => m[1]);
+    for (const key of placeholders) {
+        if (!(key in patch)) {
+            throw new Error(`agent ${agent.id} system prompt missing patch value for placeholder: ${key}`);
+        }
+    }
+    return source.replace(/\{\{\s*([A-Za-z0-9_.-]+)\s*\}\}/g, (_match, key) => String(patch[key]));
+}
+export function resolveAgent(pack, input) {
+    const agent = getAgent(pack, input.agentId);
+    if (!agent) {
+        throw new Error(`agent not found: ${input.agentId}`);
+    }
+    const defaults = {
+        namespace: agent.retrievalDefaults.namespace,
+        topK: agent.retrievalDefaults.topK,
+        queryExpansion: agent.retrievalDefaults.queryExpansion,
+        semantic: agent.retrievalDefaults.semantic,
+        minScore: agent.retrievalDefaults.minScore,
+        requirePhrases: agent.retrievalDefaults.requirePhrases,
+        source: agent.retrievalDefaults.source,
+    };
+    const caller = input.query ?? {};
+    const retrievalOptions = {
+        ...defaults,
+        ...caller,
+        namespace: defaults.namespace,
+        queryExpansion: {
+            ...(defaults.queryExpansion ?? {}),
+            ...(caller.queryExpansion ?? {}),
+        },
+        semantic: {
+            ...(defaults.semantic ?? {}),
+            ...(caller.semantic ?? {}),
+            blend: {
+                ...(defaults.semantic?.blend ?? {}),
+                ...(caller.semantic?.blend ?? {}),
+            },
+        },
+    };
+    if (!defaults.queryExpansion && !caller.queryExpansion)
+        delete retrievalOptions.queryExpansion;
+    if (!defaults.semantic && !caller.semantic)
+        delete retrievalOptions.semantic;
+    if (retrievalOptions.semantic &&
+        !defaults.semantic?.blend &&
+        !caller.semantic?.blend) {
+        delete retrievalOptions.semantic.blend;
+    }
+    validateQueryOptions(retrievalOptions);
+    return {
+        agent,
+        systemPrompt: buildSystemPrompt(agent, input.patch),
+        retrievalOptions,
+    };
+}
+export function isToolAllowed(agent, toolId) {
+    const policy = agent.toolPolicy;
+    if (!policy)
+        return true;
+    const hasTool = policy.tools.includes(toolId);
+    if (policy.mode === 'allow') {
+        return hasTool;
+    }
+    return !hasTool;
+}
+export function assertToolAllowed(agent, toolId) {
+    if (!isToolAllowed(agent, toolId)) {
+        throw new Error(`agent ${agent.id} does not allow tool: ${toolId}`);
+    }
+}
+function validateSystemPrompt(agent) {
+    const prompt = agent.systemPrompt;
+    if (Array.isArray(prompt)) {
+        if (!prompt.length || prompt.some((line) => typeof line !== 'string')) {
+            throw new Error(`agent ${agent.id} systemPrompt must be a non-empty string array.`);
+        }
+        if (!prompt.join('').trim()) {
+            throw new Error(`agent ${agent.id} systemPrompt must not be empty.`);
+        }
+        return;
+    }
+    if (!prompt ||
+        prompt.format !== 'markdown' ||
+        typeof prompt.template !== 'string' ||
+        !prompt.template.trim()) {
+        throw new Error(`agent ${agent.id} systemPrompt markdown template must be a non-empty string.`);
+    }
+}

package/dist/builder.d.ts

@@ -1,6 +1,20 @@
+import type { AgentDefinitionV1, AgentRegistry } from './agent.js';
 export type BuildInputDoc = {
     id?: string;
     heading?: string;
+    namespace?: string;
     text: string;
 };
-export declare function buildPack(docs: BuildInputDoc[]): Promise<Uint8Array>;
+export type BuildPackOptions = {
+    agents?: AgentRegistry | AgentDefinitionV1[];
+    semantic?: {
+        enabled: boolean;
+        modelId: string;
+        embeddings: Float32Array[];
+        quantization?: {
+            type: 'int8_l2norm';
+            perVectorScale?: true;
+        };
+    };
+};
+export declare function buildPack(docs: BuildInputDoc[], opts?: BuildPackOptions): Promise<Uint8Array>;