langflower 0.0.10 → 0.1.1

package/vendor/server/skeleton/nodes/hello-embed/lib/search.ts

@@ -0,0 +1,254 @@
+import fs from 'node:fs';
+import { DatabaseSync } from 'node:sqlite';
+import type { TextEmbedder } from './text-embedder.ts';
+import { l2Normalize, loadVecIndex, topCosine } from './vectors.ts';
+
+export type SearchHit = {
+	readonly path: string;
+	readonly heading: string;
+	readonly score: number;
+	readonly text: string;
+};
+
+export type RunSearchOptions = {
+	readonly sqlitePath: string;
+	readonly query: string;
+	readonly topK: number;
+	readonly embedder: TextEmbedder;
+};
+
+type ChunkRow = {
+	readonly id: string;
+	readonly path: string;
+	readonly heading: string;
+	readonly text: string;
+};
+
+type RankedId = {
+	readonly id: string;
+};
+
+const RRF_K = 60;
+
+const FTS_TOKEN = /[\p{L}\p{N}_]+/gu;
+
+const asChunkRow = (row: unknown): ChunkRow => {
+	if (
+		row === null ||
+		typeof row !== 'object' ||
+		!('id' in row) ||
+		!('path' in row) ||
+		!('heading' in row) ||
+		!('text' in row)
+	) {
+		throw new Error('Unexpected chunk row');
+	}
+	const id = (row as { id: unknown }).id;
+	const path = (row as { path: unknown }).path;
+	const heading = (row as { heading: unknown }).heading;
+	const text = (row as { text: unknown }).text;
+	if (
+		typeof id !== 'string' ||
+		typeof path !== 'string' ||
+		typeof heading !== 'string' ||
+		typeof text !== 'string'
+	) {
+		throw new Error('chunk row has invalid types');
+	}
+	return { id, path, heading, text };
+};
+
+type VecRow = {
+	readonly id: string;
+	readonly dim: number;
+	readonly vector: Uint8Array;
+};
+
+const asVecRow = (row: unknown): VecRow => {
+	if (
+		row === null ||
+		typeof row !== 'object' ||
+		!('id' in row) ||
+		!('dim' in row) ||
+		!('vector' in row)
+	) {
+		throw new Error('Unexpected chunk_vec row');
+	}
+	const id = (row as { id: unknown }).id;
+	const dimRaw = (row as { dim: unknown }).dim;
+	const vector = (row as { vector: unknown }).vector;
+	const dim = typeof dimRaw === 'bigint' ? Number(dimRaw) : dimRaw;
+	if (
+		typeof id !== 'string' ||
+		typeof dim !== 'number' ||
+		!(vector instanceof Uint8Array)
+	) {
+		throw new Error('chunk_vec row has invalid types');
+	}
+	return { id, dim, vector };
+};
+
+const retrieveCandidateLimit = (topK: number): number => Math.max(topK * 4, 32);
+
+/** Quoted FTS5 tokens so chat text cannot inject MATCH operators. */
+export const ftsMatchQuery = (raw: string): string | undefined => {
+	const tokens = raw.match(FTS_TOKEN);
+	if (tokens === null || tokens.length === 0) {
+		return undefined;
+	}
+	return tokens.map((token) => `"${token.replaceAll('"', '')}"`).join(' ');
+};
+
+export const reciprocalRankFusion = (
+	lists: readonly (readonly RankedId[])[],
+	k: number,
+): readonly { readonly id: string; readonly score: number }[] => {
+	const scores = new Map<string, number>();
+	for (const list of lists) {
+		list.forEach((item, index) => {
+			const rank = index + 1;
+			scores.set(item.id, (scores.get(item.id) ?? 0) + 1 / (k + rank));
+		});
+	}
+	return [...scores.entries()]
+		.map(([id, score]) => ({ id, score }))
+		.sort((left, right) => right.score - left.score);
+};
+
+export const formatHitsText = (
+	query: string,
+	hits: readonly SearchHit[],
+): string => {
+	if (hits.length === 0) {
+		return `Question:\n${query}\n\nNo hits.`;
+	}
+	const blocks = hits.map((hit, index) => {
+		const heading = hit.heading.length > 0 ? hit.heading : '(intro)';
+		const score = hit.score.toFixed(2);
+		return `${String(index + 1)}. ${hit.path} — ${heading} (${score})\n${hit.text}`;
+	});
+	return `Question:\n${query}\n\nContext:\n${blocks.join('\n\n')}`;
+};
+
+const hasFtsTable = (db: DatabaseSync): boolean => {
+	const row = db
+		.prepare(
+			`SELECT 1 AS ok FROM sqlite_master WHERE type = 'table' AND name = 'chunk_fts'`,
+		)
+		.get() as { ok?: unknown } | undefined;
+	return row !== undefined;
+};
+
+const ftsRankedIds = (
+	db: DatabaseSync,
+	query: string,
+	limit: number,
+): readonly RankedId[] => {
+	if (!hasFtsTable(db)) {
+		return [];
+	}
+	const match = ftsMatchQuery(query);
+	if (match === undefined) {
+		return [];
+	}
+	try {
+		const rows = db
+			.prepare('SELECT id FROM chunk_fts WHERE chunk_fts MATCH ? LIMIT ?')
+			.all(match, limit);
+		return rows.flatMap((row) => {
+			if (
+				row === null ||
+				typeof row !== 'object' ||
+				!('id' in row) ||
+				typeof (row as { id: unknown }).id !== 'string'
+			) {
+				return [];
+			}
+			return [{ id: (row as { id: string }).id }];
+		});
+	} catch {
+		return [];
+	}
+};
+
+export const runSearch = async (
+	options: RunSearchOptions,
+): Promise<{
+	readonly hits: readonly SearchHit[];
+	readonly text: string;
+}> => {
+	const query = options.query.trim();
+	if (!fs.existsSync(options.sqlitePath)) {
+		throw new Error(
+			`SQLite DB not found: ${options.sqlitePath}. Run kb-ingest first.`,
+		);
+	}
+	if (query.length === 0) {
+		return { hits: [], text: formatHitsText(query, []) };
+	}
+
+	const db = new DatabaseSync(options.sqlitePath);
+	try {
+		const candidateLimit = retrieveCandidateLimit(options.topK);
+		const vecRows = db
+			.prepare('SELECT id, dim, vector FROM chunk_vec')
+			.all()
+			.map((row) => asVecRow(row));
+		const mtimeMs = fs.statSync(options.sqlitePath).mtimeMs;
+		const index = loadVecIndex(
+			options.sqlitePath,
+			vecRows.map((row) => ({
+				id: row.id,
+				dim: row.dim,
+				vector: row.vector,
+			})),
+			mtimeMs,
+		);
+		let cosineList: readonly RankedId[] = [];
+		if (index.dim > 0) {
+			const [raw] = await options.embedder.embedTexts([query]);
+			if (raw === undefined) {
+				throw new Error('Embedding returned no vector');
+			}
+			if (raw.length !== index.dim) {
+				throw new Error(
+					`Embedding dim ${String(raw.length)} does not match stored dim ${String(index.dim)}. Re-run ingest after switching models.`,
+				);
+			}
+			const queryVec = l2Normalize([...raw]);
+			cosineList = topCosine(index, queryVec, candidateLimit).map(
+				(item) => ({ id: item.id }),
+			);
+		}
+		const ftsList = ftsRankedIds(db, query, candidateLimit);
+		const fused = reciprocalRankFusion(
+			[cosineList, ftsList].filter((list) => list.length > 0),
+			RRF_K,
+		).slice(0, options.topK);
+		const byId = new Map(
+			db
+				.prepare('SELECT id, path, heading, text FROM chunk')
+				.all()
+				.map((row) => {
+					const parsed = asChunkRow(row);
+					return [parsed.id, parsed] as const;
+				}),
+		);
+		const hits: SearchHit[] = [];
+		for (const item of fused) {
+			const chunk = byId.get(item.id);
+			if (chunk === undefined) {
+				continue;
+			}
+			hits.push({
+				path: chunk.path,
+				heading: chunk.heading,
+				score: item.score,
+				text: chunk.text,
+			});
+		}
+		return { hits, text: formatHitsText(query, hits) };
+	} finally {
+		db.close();
+	}
+};

package/vendor/server/skeleton/nodes/hello-embed/lib/text-embedder.ts

@@ -0,0 +1,14 @@
+import type { EmbedHandle, EmbedTextRole } from '@langflower/node-sdk';
+
+export type TextEmbedder = {
+	readonly expectedDim?: number;
+	embedTexts: (texts: readonly string[]) => Promise<readonly Float32Array[]>;
+};
+
+export const textEmbedderFromHandle = (
+	handle: EmbedHandle,
+	role: EmbedTextRole,
+): TextEmbedder => ({
+	expectedDim: handle.dim,
+	embedTexts: (texts) => handle.embedTexts(texts, { role }),
+});

package/vendor/server/skeleton/nodes/hello-embed/lib/vectors.ts

@@ -0,0 +1,126 @@
+export const l2Normalize = (values: readonly number[]): Float32Array => {
+	const out = new Float32Array(values.length);
+	let sumSq = 0;
+	for (let i = 0; i < values.length; i += 1) {
+		const n = values[i] ?? 0;
+		out[i] = n;
+		sumSq += n * n;
+	}
+	const norm = Math.sqrt(sumSq);
+	if (norm === 0) {
+		return out;
+	}
+	const scale = 1 / norm;
+	for (let i = 0; i < out.length; i += 1) {
+		out[i] = (out[i] ?? 0) * scale;
+	}
+	return out;
+};
+
+export const float32ToBlob = (values: Float32Array): Uint8Array => {
+	const bytes = new Uint8Array(values.byteLength);
+	bytes.set(
+		new Uint8Array(values.buffer, values.byteOffset, values.byteLength),
+	);
+	return bytes;
+};
+
+export const blobToFloat32 = (blob: Uint8Array): Float32Array => {
+	const copy = new Uint8Array(blob.byteLength);
+	copy.set(blob);
+	return new Float32Array(copy.buffer);
+};
+
+export type VecIndex = {
+	readonly sqlitePath: string;
+	readonly mtimeMs: number;
+	readonly dim: number;
+	readonly ids: readonly string[];
+	readonly matrix: Float32Array;
+};
+
+const vecCache = new Map<string, VecIndex>();
+
+export const loadVecIndex = (
+	sqlitePath: string,
+	rows: readonly {
+		readonly id: string;
+		readonly dim: number;
+		readonly vector: Uint8Array;
+	}[],
+	mtimeMs: number,
+): VecIndex => {
+	const cached = vecCache.get(sqlitePath);
+	if (cached !== undefined && cached.mtimeMs === mtimeMs) {
+		return cached;
+	}
+	if (rows.length === 0) {
+		const empty: VecIndex = {
+			sqlitePath,
+			mtimeMs,
+			dim: 0,
+			ids: [],
+			matrix: new Float32Array(0),
+		};
+		vecCache.set(sqlitePath, empty);
+		return empty;
+	}
+	const dim = rows[0]?.dim ?? 0;
+	const ids: string[] = [];
+	const matrix = new Float32Array(rows.length * dim);
+	rows.forEach((row, index) => {
+		if (row.dim !== dim) {
+			throw new Error(
+				`chunk_vec dim mismatch: ${String(row.dim)} vs ${String(dim)} at ${row.id}`,
+			);
+		}
+		ids.push(row.id);
+		const floats = blobToFloat32(row.vector);
+		if (floats.length !== dim) {
+			throw new Error(
+				`chunk_vec blob length ${String(floats.length)} != dim ${String(dim)} at ${row.id}`,
+			);
+		}
+		matrix.set(floats, index * dim);
+	});
+	const index: VecIndex = {
+		sqlitePath,
+		mtimeMs,
+		dim,
+		ids,
+		matrix,
+	};
+	vecCache.set(sqlitePath, index);
+	return index;
+};
+
+export const topCosine = (
+	index: VecIndex,
+	query: Float32Array,
+	limit: number,
+): readonly { readonly id: string; readonly score: number }[] => {
+	if (index.dim === 0 || index.ids.length === 0) {
+		return [];
+	}
+	if (query.length !== index.dim) {
+		return [];
+	}
+	const scored: { id: string; score: number }[] = [];
+	for (let row = 0; row < index.ids.length; row += 1) {
+		const offset = row * index.dim;
+		let sum = 0;
+		for (let i = 0; i < index.dim; i += 1) {
+			sum += (query[i] ?? 0) * (index.matrix[offset + i] ?? 0);
+		}
+		const id = index.ids[row];
+		if (id !== undefined) {
+			scored.push({ id, score: sum });
+		}
+	}
+	scored.sort((a, b) => b.score - a.score);
+	return scored.slice(0, Math.max(0, limit));
+};
+
+export const clearVecCache = (): void => {
+	vecCache.clear();
+};

package/vendor/server/skeleton/nodes/hello-embed/lib/walk-markdown.test.ts

@@ -0,0 +1,62 @@
+import fs from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+import { walkMarkdown } from './walk-markdown.ts';
+
+describe('walkMarkdown', () => {
+	let root: string;
+
+	beforeEach(async () => {
+		root = await fs.mkdtemp(path.join(os.tmpdir(), 'hello-embed-walk-'));
+	});
+
+	afterEach(async () => {
+		await fs.rm(root, { recursive: true, force: true });
+	});
+
+	it('walks recursive md and skips node_modules, .git, and .langflower/.cache', async () => {
+		await fs.mkdir(path.join(root, 'docs'), { recursive: true });
+		await fs.mkdir(path.join(root, 'node_modules', 'pkg'), {
+			recursive: true,
+		});
+		await fs.mkdir(path.join(root, '.git'), { recursive: true });
+		await fs.mkdir(
+			path.join(root, '.langflower', '.cache', 'hello-embed'),
+			{ recursive: true },
+		);
+		await fs.mkdir(path.join(root, '.langflower', 'skills'), {
+			recursive: true,
+		});
+		await fs.writeFile(path.join(root, 'README.md'), '# Root\n', 'utf8');
+		await fs.writeFile(
+			path.join(root, 'docs', 'guide.md'),
+			'# Guide\n',
+			'utf8',
+		);
+		await fs.writeFile(
+			path.join(root, 'node_modules', 'pkg', 'secret.md'),
+			'nope',
+			'utf8',
+		);
+		await fs.writeFile(path.join(root, '.git', 'HEAD.md'), 'nope', 'utf8');
+		await fs.writeFile(
+			path.join(root, '.langflower', '.cache', 'hello-embed', 'x.md'),
+			'nope',
+			'utf8',
+		);
+		await fs.writeFile(
+			path.join(root, '.langflower', 'skills', 'SKILL.md'),
+			'# Skill\n',
+			'utf8',
+		);
+
+		const files = await walkMarkdown(root);
+		const rels = files.map((file) => file.relPath).sort();
+		expect(rels).toEqual([
+			'.langflower/skills/SKILL.md',
+			'README.md',
+			'docs/guide.md',
+		]);
+	});
+});

package/vendor/server/skeleton/nodes/hello-embed/lib/walk-markdown.ts

@@ -0,0 +1,67 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import {
+	asPosixRelative,
+	isCacheRelative,
+	MAX_FILE_BYTES,
+	SKIP_DIR_NAMES,
+} from './paths.ts';
+
+export type MarkdownFile = {
+	readonly absPath: string;
+	readonly relPath: string;
+	readonly text: string;
+	readonly skipped?: string;
+};
+
+const isMarkdownName = (name: string): boolean =>
+	name.toLowerCase().endsWith('.md');
+
+export const walkMarkdown = async (
+	rootDir: string,
+): Promise<readonly MarkdownFile[]> => {
+	const files: MarkdownFile[] = [];
+
+	const visit = async (absDir: string, relPosix: string): Promise<void> => {
+		const entries = await fs.readdir(absDir, { withFileTypes: true });
+		for (const entry of entries) {
+			const childRel =
+				relPosix.length === 0
+					? entry.name
+					: `${relPosix}/${entry.name}`;
+			const childAbs = path.join(absDir, entry.name);
+			if (entry.isDirectory()) {
+				if (SKIP_DIR_NAMES.has(entry.name)) {
+					continue;
+				}
+				if (isCacheRelative(asPosixRelative(childRel))) {
+					continue;
+				}
+				await visit(childAbs, asPosixRelative(childRel));
+				continue;
+			}
+			if (!entry.isFile() || !isMarkdownName(entry.name)) {
+				continue;
+			}
+			if (isCacheRelative(asPosixRelative(childRel))) {
+				continue;
+			}
+			const stat = await fs.stat(childAbs);
+			const relPath = asPosixRelative(childRel);
+			if (stat.size > MAX_FILE_BYTES) {
+				files.push({
+					absPath: childAbs,
+					relPath,
+					text: '',
+					skipped: `skipped ${relPath} (${String(stat.size)} bytes > ${String(MAX_FILE_BYTES)})`,
+				});
+				continue;
+			}
+			const text = await fs.readFile(childAbs, 'utf8');
+			files.push({ absPath: childAbs, relPath, text });
+		}
+	};
+
+	await visit(rootDir, '');
+	return files;
+};

package/vendor/server/skeleton/nodes/hello-embed/package.json

@@ -0,0 +1,11 @@
+{
+	"name": "hello-embed",
+	"version": "0.0.0",
+	"private": true,
+	"type": "module",
+	"peerDependencies": {
+		"@langflower/node-sdk": "0.1.0",
+		"@rx-evo/stateful-observable": "^0.6.0",
+		"rxjs": "^7.8.1"
+	}
+}

package/vendor/server/skeleton/nodes/hello-embed/search-handle.ts

@@ -0,0 +1,119 @@
+import {
+	defineReactiveNode,
+	EMBED_HANDLE_WIRE_TYPE,
+	isEmbedHandle,
+	TOOL_HANDLE_WIRE_TYPE,
+	type ToolHandle,
+} from '@langflower/node-sdk';
+import {
+	clampTopK,
+	DEFAULT_SEARCH_TOP_K,
+	DEFAULT_SQLITE_PATH,
+	resolveSqlitePath,
+} from './lib/paths.ts';
+import { runSearch } from './lib/search.ts';
+import { textEmbedderFromHandle } from './lib/text-embedder.ts';
+
+const asString = (value: unknown, fallback: string): string => {
+	if (typeof value === 'string') {
+		return value;
+	}
+	return fallback;
+};
+
+const buildSearchTool = (
+	embedInput: unknown,
+	sqlitePath: string,
+): readonly ToolHandle[] => {
+	if (!isEmbedHandle(embedInput)) {
+		throw new Error(
+			'hello-embed-search-handle requires a wired embed input from common-embed-provider.',
+		);
+	}
+	const embedder = textEmbedderFromHandle(embedInput, 'query');
+	return [
+		{
+			toolId: 'project_search',
+			name: 'project_search',
+			description:
+				'Hybrid search (vector + keyword) over the project markdown index from hello-embed-ingest. Returns Question + Context with full chunk bodies (path, heading, RRF score). Empty only if the index has no chunks or the query is empty.',
+			inputSchema: {
+				type: 'object',
+				properties: {
+					query: {
+						type: 'string',
+						description: 'Search phrase.',
+					},
+					topK: {
+						type: 'integer',
+						description: 'Max hits (default 8).',
+					},
+				},
+				required: ['query'],
+				additionalProperties: false,
+			},
+			invoke: async (args) => {
+				const query = typeof args.query === 'string' ? args.query : '';
+				const result = await runSearch({
+					sqlitePath,
+					query,
+					topK: clampTopK(args.topK, DEFAULT_SEARCH_TOP_K),
+					embedder,
+				});
+				return result.text;
+			},
+		},
+	];
+};
+
+/**
+ * LLM inventory: project_search over the hello-embed sqlite index.
+ */
+export default defineReactiveNode({
+	type: 'hello-embed-search-handle',
+	displayName: 'Hello Embed Search Handle',
+	category: 'Hello Embed',
+	description:
+		'Emits project_search (hybrid retrieve, full chunks) for an agent tools port. Wire embed from common-embed-provider. Uses the same sqlitePath default as ingest/search.',
+	uiSchema: [
+		{
+			field: 'sqlitePath',
+			type: 'string',
+			label: 'SQLite path',
+			default: DEFAULT_SQLITE_PATH,
+		},
+	] as const,
+	bind(ctx, { makeInput, configureOutput, combineInputs }) {
+		const embed = makeInput<unknown>('embed', {
+			name: 'embed',
+			wireType: EMBED_HANDLE_WIRE_TYPE,
+			required: true,
+			description: 'Wire from common-embed-provider (fan-out OK).',
+		});
+
+		const tools$ = combineInputs([embed, ctx], ([embedInput, ec]) => {
+			const projectDir = String(ec.projectDir ?? '');
+			if (projectDir.length === 0) {
+				throw new Error(
+					'hello-embed-search-handle requires ctx.projectDir.',
+				);
+			}
+			return buildSearchTool(
+				embedInput,
+				resolveSqlitePath(
+					projectDir,
+					asString(ec.params.sqlitePath, DEFAULT_SQLITE_PATH),
+				),
+			);
+		});
+
+		return {
+			inputs: [embed],
+			outputs: [
+				configureOutput('tools', tools$, {
+					wireType: TOOL_HANDLE_WIRE_TYPE,
+				}),
+			],
+		};
+	},
+});